Merge pull request #124875 from hercules-ci/lib-sources

lib.sources: docs, tests, refactoring
This commit is contained in:
Robert Hensing 2021-06-28 14:02:37 +02:00 committed by GitHub
commit 4c4c00e9f1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 166 additions and 17 deletions

View file

@ -22,5 +22,6 @@ with pkgs; stdenv.mkDerivation {
docgen lists 'List manipulation functions' docgen lists 'List manipulation functions'
docgen debug 'Debugging functions' docgen debug 'Debugging functions'
docgen options 'NixOS / nixpkgs option handling' docgen options 'NixOS / nixpkgs option handling'
docgen sources 'Source filtering functions'
''; '';
} }

View file

@ -25,4 +25,6 @@
<xi:include href="./library/generated/debug.xml" /> <xi:include href="./library/generated/debug.xml" />
<xi:include href="./library/generated/options.xml" /> <xi:include href="./library/generated/options.xml" />
<xi:include href="./library/generated/sources.xml" />
</section> </section>

View file

@ -1,6 +1,7 @@
# Functions for copying sources to the Nix store. # Functions for copying sources to the Nix store.
{ lib }: { lib }:
# Tested in lib/tests/sources.sh
let let
inherit (builtins) inherit (builtins)
hasContext hasContext
@ -11,14 +12,13 @@ let
tryEval tryEval
; ;
inherit (lib) inherit (lib)
boolToString
filter filter
getAttr getAttr
isString isString
pathExists pathExists
readFile readFile
; ;
in
rec {
# Returns the type of a path: regular (for file), symlink, or directory # Returns the type of a path: regular (for file), symlink, or directory
pathType = p: getAttr (baseNameOf p) (readDir (dirOf p)); pathType = p: getAttr (baseNameOf p) (readDir (dirOf p));
@ -84,18 +84,36 @@ rec {
# #
cleanSourceWith = { filter ? _path: _type: true, src, name ? null }: cleanSourceWith = { filter ? _path: _type: true, src, name ? null }:
let let
isFiltered = src ? _isLibCleanSourceWith; orig = toSourceAttributes src;
origSrc = if isFiltered then src.origSrc else src; in fromSourceAttributes {
filter' = if isFiltered then name: type: filter name type && src.filter name type else filter; inherit (orig) origSrc;
name' = if name != null then name else if isFiltered then src.name else "source"; filter = path: type: filter path type && orig.filter path type;
in { name = if name != null then name else orig.name;
inherit origSrc;
filter = filter';
outPath = builtins.path { filter = filter'; path = origSrc; name = name'; };
_isLibCleanSourceWith = true;
name = name';
}; };
/*
Add logging to a source, for troubleshooting the filtering behavior.
Type:
sources.trace :: sourceLike -> Source
*/
trace =
# Source to debug. The returned source will behave like this source, but also log its filter invocations.
src:
let
attrs = toSourceAttributes src;
in
fromSourceAttributes (
attrs // {
filter = path: type:
let
r = attrs.filter path type;
in
builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r;
}
) // {
satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant;
};
# Filter sources by a list of regular expressions. # Filter sources by a list of regular expressions.
# #
# E.g. `src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"]` # E.g. `src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"]`
@ -110,14 +128,26 @@ rec {
inherit src; inherit src;
}; };
# Get all files ending with the specified suffices from the given /*
# directory or its descendants. E.g. `sourceFilesBySuffices ./dir Get all files ending with the specified suffices from the given
# [".xml" ".c"]'. source directory or its descendants, omitting files that do not match
sourceFilesBySuffices = path: exts: any suffix. The result of the example below will include files like
`./dir/module.c` and `./dir/subdir/doc.xml` if present.
Type: sourceLike -> [String] -> Source
Example:
sourceFilesBySuffices ./. [ ".xml" ".c" ]
*/
sourceFilesBySuffices =
# Path or source containing the files to be returned
src:
# A list of file suffix strings
exts:
let filter = name: type: let filter = name: type:
let base = baseNameOf (toString name); let base = baseNameOf (toString name);
in type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts; in type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts;
in cleanSourceWith { inherit filter; src = path; }; in cleanSourceWith { inherit filter src; };
pathIsGitRepo = path: (tryEval (commitIdFromGitRepo path)).success; pathIsGitRepo = path: (tryEval (commitIdFromGitRepo path)).success;
@ -177,4 +207,57 @@ rec {
pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir); pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir);
canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src)); canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src));
# -------------------------------------------------------------------------- #
# Internal functions
#
# toSourceAttributes : sourceLike -> SourceAttrs
#
# Convert any source-like object into a simple, singular representation.
# We don't expose this representation in order to avoid having a fifth path-
# like class of objects in the wild.
# (Existing ones being: paths, strings, sources and x//{outPath})
# So instead of exposing internals, we build a library of combinator functions.
toSourceAttributes = src:
let
isFiltered = src ? _isLibCleanSourceWith;
in
{
# The original path
origSrc = if isFiltered then src.origSrc else src;
filter = if isFiltered then src.filter else _: _: true;
name = if isFiltered then src.name else "source";
};
# fromSourceAttributes : SourceAttrs -> Source
#
# Inverse of toSourceAttributes for Source objects.
fromSourceAttributes = { origSrc, filter, name }:
{
_isLibCleanSourceWith = true;
inherit origSrc filter name;
outPath = builtins.path { inherit filter name; path = origSrc; };
};
in {
inherit
pathType
pathIsDirectory
pathIsRegularFile
pathIsGitRepo
commitIdFromGitRepo
cleanSource
cleanSourceWith
cleanSourceFilter
pathHasContext
canCleanSource
sourceByRegex
sourceFilesBySuffices
trace
;
} }

View file

@ -26,7 +26,11 @@ pkgs.runCommandNoCC "nixpkgs-lib-tests" {
nix-store --init nix-store --init
cp -r ${../.} lib cp -r ${../.} lib
echo "Running lib/tests/modules.sh"
bash lib/tests/modules.sh bash lib/tests/modules.sh
echo "Running lib/tests/sources.sh"
TEST_LIB=$PWD/lib bash lib/tests/sources.sh
touch $out touch $out
'' ''

59
lib/tests/sources.sh Executable file
View file

@ -0,0 +1,59 @@
#!/usr/bin/env bash
set -euo pipefail
# Use
# || die
die() {
echo >&2 "test case failed: " "$@"
exit 1
}
if test -n "${TEST_LIB:-}"; then
export NIX_PATH=nixpkgs="$(dirname "$TEST_LIB")"
else
export NIX_PATH=nixpkgs="$(cd $(dirname ${BASH_SOURCE[0]})/../..; pwd)"
fi
work="$(mktemp -d)"
clean_up() {
rm -rf "$work"
}
trap clean_up EXIT
cd $work
touch {README.md,module.o,foo.bar}
# nix-instantiate doesn't write out the source, only computing the hash, so
# this uses the experimental nix command instead.
dir="$(nix eval --raw '(with import <nixpkgs/lib>; "${
cleanSource ./.
}")')"
(cd $dir; find) | sort -f | diff -U10 - <(cat <<EOF
.
./foo.bar
./README.md
EOF
) || die "cleanSource 1"
dir="$(nix eval --raw '(with import <nixpkgs/lib>; "${
cleanSourceWith { src = '"$work"'; filter = path: type: ! hasSuffix ".bar" path; }
}")')"
(cd $dir; find) | sort -f | diff -U10 - <(cat <<EOF
.
./module.o
./README.md
EOF
) || die "cleanSourceWith 1"
dir="$(nix eval --raw '(with import <nixpkgs/lib>; "${
cleanSourceWith { src = cleanSource '"$work"'; filter = path: type: ! hasSuffix ".bar" path; }
}")')"
(cd $dir; find) | sort -f | diff -U10 - <(cat <<EOF
.
./README.md
EOF
) || die "cleanSourceWith + cleanSource"
echo >&2 tests ok