diff --git a/doc/doc-support/lib-function-docs.nix b/doc/doc-support/lib-function-docs.nix
index 5199b949e7b..f6d613cac0b 100644
--- a/doc/doc-support/lib-function-docs.nix
+++ b/doc/doc-support/lib-function-docs.nix
@@ -22,5 +22,6 @@ with pkgs; stdenv.mkDerivation {
docgen lists 'List manipulation functions'
docgen debug 'Debugging functions'
docgen options 'NixOS / nixpkgs option handling'
+ docgen sources 'Source filtering functions'
'';
}
diff --git a/doc/functions/library.xml b/doc/functions/library.xml
index 6ffb944b5a6..21bcf5b88c9 100644
--- a/doc/functions/library.xml
+++ b/doc/functions/library.xml
@@ -25,4 +25,6 @@
+
+
diff --git a/lib/sources.nix b/lib/sources.nix
index 1a821f55056..407829b547b 100644
--- a/lib/sources.nix
+++ b/lib/sources.nix
@@ -1,6 +1,7 @@
# Functions for copying sources to the Nix store.
{ lib }:
+# Tested in lib/tests/sources.sh
let
inherit (builtins)
hasContext
@@ -11,14 +12,13 @@ let
tryEval
;
inherit (lib)
+ boolToString
filter
getAttr
isString
pathExists
readFile
;
-in
-rec {
# Returns the type of a path: regular (for file), symlink, or directory
pathType = p: getAttr (baseNameOf p) (readDir (dirOf p));
@@ -84,18 +84,36 @@ rec {
#
cleanSourceWith = { filter ? _path: _type: true, src, name ? null }:
let
- isFiltered = src ? _isLibCleanSourceWith;
- origSrc = if isFiltered then src.origSrc else src;
- filter' = if isFiltered then name: type: filter name type && src.filter name type else filter;
- name' = if name != null then name else if isFiltered then src.name else "source";
- in {
- inherit origSrc;
- filter = filter';
- outPath = builtins.path { filter = filter'; path = origSrc; name = name'; };
- _isLibCleanSourceWith = true;
- name = name';
+ orig = toSourceAttributes src;
+ in fromSourceAttributes {
+ inherit (orig) origSrc;
+ filter = path: type: filter path type && orig.filter path type;
+ name = if name != null then name else orig.name;
};
+ /*
+ Add logging to a source, for troubleshooting the filtering behavior.
+ Type:
+ sources.trace :: sourceLike -> Source
+ */
+ trace =
+ # Source to debug. The returned source will behave like this source, but also log its filter invocations.
+ src:
+ let
+ attrs = toSourceAttributes src;
+ in
+ fromSourceAttributes (
+ attrs // {
+ filter = path: type:
+ let
+ r = attrs.filter path type;
+ in
+ builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r;
+ }
+ ) // {
+ satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant;
+ };
+
# Filter sources by a list of regular expressions.
#
# E.g. `src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"]`
@@ -110,14 +128,26 @@ rec {
inherit src;
};
- # Get all files ending with the specified suffices from the given
- # directory or its descendants. E.g. `sourceFilesBySuffices ./dir
- # [".xml" ".c"]'.
- sourceFilesBySuffices = path: exts:
+ /*
+ Get all files ending with the specified suffices from the given
+ source directory or its descendants, omitting files that do not match
+ any suffix. The result of the example below will include files like
+ `./dir/module.c` and `./dir/subdir/doc.xml` if present.
+
+ Type: sourceLike -> [String] -> Source
+
+ Example:
+ sourceFilesBySuffices ./. [ ".xml" ".c" ]
+ */
+ sourceFilesBySuffices =
+ # Path or source containing the files to be returned
+ src:
+ # A list of file suffix strings
+ exts:
let filter = name: type:
let base = baseNameOf (toString name);
in type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts;
- in cleanSourceWith { inherit filter; src = path; };
+ in cleanSourceWith { inherit filter src; };
pathIsGitRepo = path: (tryEval (commitIdFromGitRepo path)).success;
@@ -177,4 +207,57 @@ rec {
pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir);
canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src));
+
+ # -------------------------------------------------------------------------- #
+ # Internal functions
+ #
+
+ # toSourceAttributes : sourceLike -> SourceAttrs
+ #
+ # Convert any source-like object into a simple, singular representation.
+ # We don't expose this representation in order to avoid having a fifth path-
+ # like class of objects in the wild.
+ # (Existing ones being: paths, strings, sources and x//{outPath})
+ # So instead of exposing internals, we build a library of combinator functions.
+ toSourceAttributes = src:
+ let
+ isFiltered = src ? _isLibCleanSourceWith;
+ in
+ {
+ # The original path
+ origSrc = if isFiltered then src.origSrc else src;
+ filter = if isFiltered then src.filter else _: _: true;
+ name = if isFiltered then src.name else "source";
+ };
+
+ # fromSourceAttributes : SourceAttrs -> Source
+ #
+ # Inverse of toSourceAttributes for Source objects.
+ fromSourceAttributes = { origSrc, filter, name }:
+ {
+ _isLibCleanSourceWith = true;
+ inherit origSrc filter name;
+ outPath = builtins.path { inherit filter name; path = origSrc; };
+ };
+
+in {
+ inherit
+ pathType
+ pathIsDirectory
+ pathIsRegularFile
+
+ pathIsGitRepo
+ commitIdFromGitRepo
+
+ cleanSource
+ cleanSourceWith
+ cleanSourceFilter
+ pathHasContext
+ canCleanSource
+
+ sourceByRegex
+ sourceFilesBySuffices
+
+ trace
+ ;
}
diff --git a/lib/tests/release.nix b/lib/tests/release.nix
index 800d8a65c14..c3b05251f70 100644
--- a/lib/tests/release.nix
+++ b/lib/tests/release.nix
@@ -26,7 +26,11 @@ pkgs.runCommandNoCC "nixpkgs-lib-tests" {
nix-store --init
cp -r ${../.} lib
+ echo "Running lib/tests/modules.sh"
bash lib/tests/modules.sh
+ echo "Running lib/tests/sources.sh"
+ TEST_LIB=$PWD/lib bash lib/tests/sources.sh
+
touch $out
''
diff --git a/lib/tests/sources.sh b/lib/tests/sources.sh
new file mode 100755
index 00000000000..71fee719cb2
--- /dev/null
+++ b/lib/tests/sources.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Use
+# || die
+die() {
+ echo >&2 "test case failed: " "$@"
+ exit 1
+}
+
+if test -n "${TEST_LIB:-}"; then
+ export NIX_PATH=nixpkgs="$(dirname "$TEST_LIB")"
+else
+ export NIX_PATH=nixpkgs="$(cd $(dirname ${BASH_SOURCE[0]})/../..; pwd)"
+fi
+
+work="$(mktemp -d)"
+clean_up() {
+ rm -rf "$work"
+}
+trap clean_up EXIT
+cd $work
+
+touch {README.md,module.o,foo.bar}
+
+# nix-instantiate doesn't write out the source, only computing the hash, so
+# this uses the experimental nix command instead.
+
+dir="$(nix eval --raw '(with import ; "${
+ cleanSource ./.
+}")')"
+(cd $dir; find) | sort -f | diff -U10 - <(cat <; "${
+ cleanSourceWith { src = '"$work"'; filter = path: type: ! hasSuffix ".bar" path; }
+}")')"
+(cd $dir; find) | sort -f | diff -U10 - <(cat <; "${
+ cleanSourceWith { src = cleanSource '"$work"'; filter = path: type: ! hasSuffix ".bar" path; }
+}")')"
+(cd $dir; find) | sort -f | diff -U10 - <(cat <&2 tests ok