From dfd2b1bd9002c3cbe38e2cac8014d8b2723c4137 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 5 Feb 2021 23:26:46 +0100 Subject: [PATCH 1/5] lib/sources: Internal representation for cleanSourceWith --- lib/sources.nix | 68 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/lib/sources.nix b/lib/sources.nix index 1a821f55056..cad3c5b1a14 100644 --- a/lib/sources.nix +++ b/lib/sources.nix @@ -17,8 +17,6 @@ let pathExists readFile ; -in -rec { # Returns the type of a path: regular (for file), symlink, or directory pathType = p: getAttr (baseNameOf p) (readDir (dirOf p)); @@ -84,16 +82,11 @@ rec { # cleanSourceWith = { filter ? _path: _type: true, src, name ? null }: let - isFiltered = src ? _isLibCleanSourceWith; - origSrc = if isFiltered then src.origSrc else src; - filter' = if isFiltered then name: type: filter name type && src.filter name type else filter; - name' = if name != null then name else if isFiltered then src.name else "source"; - in { - inherit origSrc; - filter = filter'; - outPath = builtins.path { filter = filter'; path = origSrc; name = name'; }; - _isLibCleanSourceWith = true; - name = name'; + orig = toSourceAttributes src; + in fromSourceAttributes { + inherit (orig) origSrc; + filter = path: type: filter path type && orig.filter path type; + name = if name != null then name else orig.name; }; # Filter sources by a list of regular expressions. @@ -177,4 +170,55 @@ rec { pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir); canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src)); + + # -------------------------------------------------------------------------- # + # Internal functions + # + + # toSourceAttributes : sourceLike -> SourceAttrs + # + # Convert any source-like object into a simple, singular representation. + # We don't expose this representation in order to avoid having a fifth path- + # like class of objects in the wild. + # (Existing ones being: paths, strings, sources and x//{outPath}) + # So instead of exposing internals, we build a library of combinator functions. + toSourceAttributes = src: + let + isFiltered = src ? _isLibCleanSourceWith; + in + { + # The original path + origSrc = if isFiltered then src.origSrc else src; + filter = if isFiltered then src.filter else _: _: true; + name = if isFiltered then src.name else "source"; + }; + + # fromSourceAttributes : SourceAttrs -> Source + # + # Inverse of toSourceAttributes for Source objects. + fromSourceAttributes = { origSrc, filter, name }: + { + _isLibCleanSourceWith = true; + inherit origSrc filter name; + outPath = builtins.path { inherit filter name; path = origSrc; }; + }; + +in { + inherit + pathType + pathIsDirectory + pathIsRegularFile + + pathIsGitRepo + commitIdFromGitRepo + + cleanSource + cleanSourceWith + cleanSourceFilter + pathHasContext + canCleanSource + + sourceByRegex + sourceFilesBySuffices + ; } From d14be76615299c55e8f239cd1ec95c8107f41b33 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 6 Feb 2021 00:15:33 +0100 Subject: [PATCH 2/5] lib/tests/sources.sh: init --- lib/sources.nix | 1 + lib/tests/release.nix | 4 +++ lib/tests/sources.sh | 59 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100755 lib/tests/sources.sh diff --git a/lib/sources.nix b/lib/sources.nix index cad3c5b1a14..c7a0c000de6 100644 --- a/lib/sources.nix +++ b/lib/sources.nix @@ -1,6 +1,7 @@ # Functions for copying sources to the Nix store. { lib }: +# Tested in lib/tests/sources.sh let inherit (builtins) hasContext diff --git a/lib/tests/release.nix b/lib/tests/release.nix index 800d8a65c14..c3b05251f70 100644 --- a/lib/tests/release.nix +++ b/lib/tests/release.nix @@ -26,7 +26,11 @@ pkgs.runCommandNoCC "nixpkgs-lib-tests" { nix-store --init cp -r ${../.} lib + echo "Running lib/tests/modules.sh" bash lib/tests/modules.sh + echo "Running lib/tests/sources.sh" + TEST_LIB=$PWD/lib bash lib/tests/sources.sh + touch $out '' diff --git a/lib/tests/sources.sh b/lib/tests/sources.sh new file mode 100755 index 00000000000..71fee719cb2 --- /dev/null +++ b/lib/tests/sources.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Use +# || die +die() { + echo >&2 "test case failed: " "$@" + exit 1 +} + +if test -n "${TEST_LIB:-}"; then + export NIX_PATH=nixpkgs="$(dirname "$TEST_LIB")" +else + export NIX_PATH=nixpkgs="$(cd $(dirname ${BASH_SOURCE[0]})/../..; pwd)" +fi + +work="$(mktemp -d)" +clean_up() { + rm -rf "$work" +} +trap clean_up EXIT +cd $work + +touch {README.md,module.o,foo.bar} + +# nix-instantiate doesn't write out the source, only computing the hash, so +# this uses the experimental nix command instead. + +dir="$(nix eval --raw '(with import ; "${ + cleanSource ./. +}")')" +(cd $dir; find) | sort -f | diff -U10 - <(cat <&2 tests ok From 970273023a51020611dc915198f965d5dd34574a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 28 Mar 2021 21:49:20 +0200 Subject: [PATCH 3/5] lib.sources.sourceFilesBySuffices: Improve doc --- lib/sources.nix | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/sources.nix b/lib/sources.nix index c7a0c000de6..407f9d21b8b 100644 --- a/lib/sources.nix +++ b/lib/sources.nix @@ -104,14 +104,26 @@ let inherit src; }; - # Get all files ending with the specified suffices from the given - # directory or its descendants. E.g. `sourceFilesBySuffices ./dir - # [".xml" ".c"]'. - sourceFilesBySuffices = path: exts: + /* + Get all files ending with the specified suffices from the given + source directory or its descendants, omitting files that do not match + any suffix. The result of the example below will include files like + `./dir/module.c` and `./dir/subdir/doc.xml` if present. + + Type: sourceLike -> [String] -> Source + + Example: + sourceFilesBySuffices ./. [ ".xml" ".c" ] + */ + sourceFilesBySuffices = + # Path or source containing the files to be returned + src: + # A list of file suffix strings + exts: let filter = name: type: let base = baseNameOf (toString name); in type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts; - in cleanSourceWith { inherit filter; src = path; }; + in cleanSourceWith { inherit filter src; }; pathIsGitRepo = path: (tryEval (commitIdFromGitRepo path)).success; From 4a025692d1daf90b3a9cefe2d996e163e1ad05d3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 6 Mar 2021 12:26:54 +0100 Subject: [PATCH 4/5] lib.sources: Generate docs --- doc/doc-support/lib-function-docs.nix | 1 + doc/functions/library.xml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/doc/doc-support/lib-function-docs.nix b/doc/doc-support/lib-function-docs.nix index 5199b949e7b..f6d613cac0b 100644 --- a/doc/doc-support/lib-function-docs.nix +++ b/doc/doc-support/lib-function-docs.nix @@ -22,5 +22,6 @@ with pkgs; stdenv.mkDerivation { docgen lists 'List manipulation functions' docgen debug 'Debugging functions' docgen options 'NixOS / nixpkgs option handling' + docgen sources 'Source filtering functions' ''; } diff --git a/doc/functions/library.xml b/doc/functions/library.xml index 6ffb944b5a6..21bcf5b88c9 100644 --- a/doc/functions/library.xml +++ b/doc/functions/library.xml @@ -25,4 +25,6 @@ + + From 4cf56e56405a226d8b21e8643e32a10b60930ae9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 21 Feb 2021 12:36:42 +0100 Subject: [PATCH 5/5] lib.sources.trace: init --- lib/sources.nix | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/lib/sources.nix b/lib/sources.nix index 407f9d21b8b..407829b547b 100644 --- a/lib/sources.nix +++ b/lib/sources.nix @@ -12,6 +12,7 @@ let tryEval ; inherit (lib) + boolToString filter getAttr isString @@ -90,6 +91,29 @@ let name = if name != null then name else orig.name; }; + /* + Add logging to a source, for troubleshooting the filtering behavior. + Type: + sources.trace :: sourceLike -> Source + */ + trace = + # Source to debug. The returned source will behave like this source, but also log its filter invocations. + src: + let + attrs = toSourceAttributes src; + in + fromSourceAttributes ( + attrs // { + filter = path: type: + let + r = attrs.filter path type; + in + builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r; + } + ) // { + satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant; + }; + # Filter sources by a list of regular expressions. # # E.g. `src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"]` @@ -233,5 +257,7 @@ in { sourceByRegex sourceFilesBySuffices + + trace ; }