Merge pull request #125973 from blaggacao/nixos-test-ref/02-normalise-test-driver-nix-expressions

nixos test ref/02 normalise test driver nix expressions
This commit is contained in:
Jacek Galowicz 2021-06-16 18:59:23 +02:00 committed by GitHub
commit a6d8488c58
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 135 additions and 122 deletions

View file

@ -973,7 +973,7 @@ def subtest(name: str) -> Iterator[None]:
if __name__ == "__main__":
arg_parser = argparse.ArgumentParser()
arg_parser = argparse.ArgumentParser(prog="nixos-test-driver")
arg_parser.add_argument(
"-K",
"--keep-vm-state",

View file

@ -16,13 +16,19 @@ rec {
inherit pkgs;
mkTestDriver =
# Reifies and correctly wraps the python test driver for
# the respective qemu version and with or without ocr support
pythonTestDriver = {
qemu_pkg ? pkgs.qemu_test
, enableOCR ? false
}:
let
testDriverScript = ./test-driver/test-driver.py;
in
qemu_pkg: stdenv.mkDerivation {
name = "nixos-test-driver";
testDriverScript = ./test-driver/test-driver.py;
ocrProg = tesseract4.override { enableLanguages = [ "eng" ]; };
imagemagick_tiff = imagemagick_light.override { inherit libtiff; };
in stdenv.mkDerivation {
inherit name;
nativeBuildInputs = [ makeWrapper ];
buildInputs = [ (python3.withPackages (p: [ p.ptpython p.colorama ])) ];
@ -35,7 +41,7 @@ rec {
buildPhase = ''
python <<EOF
from pydoc import importfile
with open('driver-exports', 'w') as fp:
with open('driver-symbols', 'w') as fp:
fp.write(','.join(dir(importfile('${testDriverScript}'))))
EOF
'';
@ -57,20 +63,17 @@ rec {
# TODO: copy user script part into this file (append)
wrapProgram $out/bin/nixos-test-driver \
--argv0 ${name} \
--prefix PATH : "${lib.makeBinPath [ qemu_pkg vde2 netpbm coreutils ]}" \
${lib.optionalString enableOCR
"--prefix PATH : '${ocrProg}/bin:${imagemagick_tiff}/bin'"} \
install -m 0644 -vD driver-exports $out/nix-support/driver-exports
install -m 0644 -vD driver-symbols $out/nix-support/driver-symbols
'';
};
# Run an automated test suite in the given virtual network.
runTests = {
# the script that runs the network
driver,
# a source position in the format of builtins.unsafeGetAttrPos
# for meta.position
pos,
}:
runTests = { driver, pos }:
stdenv.mkDerivation {
name = "vm-test-run-${driver.testName}";
@ -87,10 +90,99 @@ rec {
inherit driver;
};
inherit pos;
inherit pos; # for better debugging
};
# Generate convenience wrappers for running the test driver
# has vlans, vms and test script defaulted through env variables
# also instantiates test script with nodes, if it's a function (contract)
setupDriverForTest = {
testScript
, testName
, nodes
, qemu_pkg ? pkgs.qemu_test
, enableOCR ? false
, skipLint ? false
, passthru ? {}
}:
let
# FIXME: get this pkg from the module system
testDriver = pythonTestDriver { inherit qemu_pkg enableOCR;};
testDriverName =
let
# A standard store path to the vm monitor is built like this:
# /tmp/nix-build-vm-test-run-$name.drv-0/vm-state-machine/monitor
# The max filename length of a unix domain socket is 108 bytes.
# This means $name can at most be 50 bytes long.
maxTestNameLen = 50;
testNameLen = builtins.stringLength testName;
in with builtins;
if testNameLen > maxTestNameLen then
abort
("The name of the test '${testName}' must not be longer than ${toString maxTestNameLen} " +
"it's currently ${toString testNameLen} characters long.")
else
"nixos-test-driver-${testName}";
vlans = map (m: m.config.virtualisation.vlans) (lib.attrValues nodes);
vms = map (m: m.config.system.build.vm) (lib.attrValues nodes);
nodeHostNames = map (c: c.config.system.name) (lib.attrValues nodes);
invalidNodeNames = lib.filter
(node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null)
(builtins.attrNames nodes);
testScript' =
# Call the test script with the computed nodes.
if lib.isFunction testScript
then testScript { inherit nodes; }
else testScript;
in
if lib.length invalidNodeNames > 0 then
throw ''
Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})!
All machines are referenced as python variables in the testing framework which will break the
script when special characters are used.
Please stick to alphanumeric chars and underscores as separation.
''
else lib.warnIf skipLint "Linting is disabled" (runCommand testDriverName
{
inherit testName;
nativeBuildInputs = [ makeWrapper ];
testScript = testScript';
preferLocalBuild = true;
passthru = passthru // {
inherit nodes;
};
}
''
mkdir -p $out/bin
echo -n "$testScript" > $out/test-script
${lib.optionalString (!skipLint) ''
PYFLAKES_BUILTINS="$(
echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)},
< ${lib.escapeShellArg "${testDriver}/nix-support/driver-symbols"}
)" ${python3Packages.pyflakes}/bin/pyflakes $out/test-script
''}
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/
vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
wrapProgram $out/bin/nixos-test-driver \
--add-flags "''${vms[*]}" \
--run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\"" \
--set VLANS '${toString vlans}'
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms
wrapProgram $out/bin/nixos-run-vms \
--add-flags "''${vms[*]}" \
--set tests 'start_all(); join_all();' \
--set VLANS '${toString vlans}'
'');
# Make a full-blown test
makeTest =
{ testScript
, enableOCR ? false
@ -106,128 +198,47 @@ rec {
, ...
} @ t:
let
# A standard store path to the vm monitor is built like this:
# /tmp/nix-build-vm-test-run-$name.drv-0/vm-state-machine/monitor
# The max filename length of a unix domain socket is 108 bytes.
# This means $name can at most be 50 bytes long.
maxTestNameLen = 50;
testNameLen = builtins.stringLength name;
ocrProg = tesseract4.override { enableLanguages = [ "eng" ]; };
imagemagick_tiff = imagemagick_light.override { inherit libtiff; };
# Generate convenience wrappers for running the test driver
# interactively with the specified network, and for starting the
# VMs from the command line.
mkDriver = qemu_pkg:
nodes = qemu_pkg:
let
build-vms = import ./build-vms.nix {
inherit system pkgs minimal specialArgs;
extraConfigurations = extraConfigurations ++ (pkgs.lib.optional (qemu_pkg != null)
extraConfigurations = extraConfigurations ++ [(
{
virtualisation.qemu.package = qemu_pkg;
}
) ++ [(
{
# Ensure we do not use aliases. Ideally this is only set
# when the test framework is used by Nixpkgs NixOS tests.
nixpkgs.config.allowAliases = false;
}
)];
};
# FIXME: get this pkg from the module system
testDriver = mkTestDriver (if qemu_pkg == null then pkgs.qemu_test else qemu_pkg);
nodes = build-vms.buildVirtualNetwork (
t.nodes or (if t ? machine then { machine = t.machine; } else { })
);
vlans = map (m: m.config.virtualisation.vlans) (lib.attrValues nodes);
vms = map (m: m.config.system.build.vm) (lib.attrValues nodes);
testScript' =
# Call the test script with the computed nodes.
if lib.isFunction testScript
then testScript { inherit nodes; }
else testScript;
testDriverName = with builtins;
if testNameLen > maxTestNameLen then
abort
("The name of the test '${name}' must not be longer than ${toString maxTestNameLen} " +
"it's currently ${toString testNameLen} characters long.")
else
"nixos-test-driver-${name}";
in
lib.warnIf skipLint "Linting is disabled" (runCommand testDriverName
{
nativeBuildInputs = [ makeWrapper ];
testScript = testScript';
preferLocalBuild = true;
testName = name;
passthru = passthru // {
inherit nodes;
};
}
''
mkdir -p $out/bin
build-vms.buildVirtualNetwork (
t.nodes or (if t ? machine then { machine = t.machine; } else { })
);
echo -n "$testScript" > $out/test-script
${lib.optionalString (!skipLint) ''
PYFLAKES_BUILTINS="$(
echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)},
< ${lib.escapeShellArg "${testDriver}/nix-support/driver-exports"}
)" ${python3Packages.pyflakes}/bin/pyflakes $out/test-script
''}
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/
vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
wrapProgram $out/bin/nixos-test-driver \
--add-flags "''${vms[*]}" \
${lib.optionalString enableOCR
"--prefix PATH : '${ocrProg}/bin:${imagemagick_tiff}/bin'"} \
--run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\"" \
--set VLANS '${toString vlans}'
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms
wrapProgram $out/bin/nixos-run-vms \
--add-flags "''${vms[*]}" \
${lib.optionalString enableOCR "--prefix PATH : '${ocrProg}/bin'"} \
--set tests 'start_all(); join_all();' \
--set VLANS '${toString vlans}'
''); # "
passMeta = drv: drv // lib.optionalAttrs (t ? meta) {
meta = (drv.meta or { }) // t.meta;
driver = setupDriverForTest {
inherit testScript enableOCR skipLint;
testName = name;
qemu_pkg = pkgs.qemu_test;
nodes = nodes pkgs.qemu_test;
};
driverInteractive = setupDriverForTest {
inherit testScript enableOCR skipLint;
testName = name;
qemu_pkg = pkgs.qemu;
nodes = nodes pkgs.qemu;
};
driver = mkDriver null;
driverInteractive = mkDriver pkgs.qemu;
test = passMeta (runTests { inherit driver pos; });
nodeNames = builtins.attrNames driver.nodes;
invalidNodeNames = lib.filter
(node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null)
nodeNames;
nodeHostNames = map (c: c.config.system.name) (lib.attrValues driver.nodes);
test =
let
passMeta = drv: drv // lib.optionalAttrs (t ? meta) {
meta = (drv.meta or { }) // t.meta;
};
in passMeta (runTests { inherit driver pos; });
in
if lib.length invalidNodeNames > 0 then
throw ''
Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})!
All machines are referenced as python variables in the testing framework which will break the
script when special characters are used.
Please stick to alphanumeric chars and underscores as separation.
''
else
test // {
inherit test driver driverInteractive;
inherit (driver) nodes;
inherit test driver driverInteractive nodes;
};
runInMachine =
@ -235,7 +246,7 @@ rec {
, machine
, preBuild ? ""
, postBuild ? ""
, qemu ? pkgs.qemu_test
, qemu_pkg ? pkgs.qemu_test
, ... # ???
}:
let
@ -272,6 +283,8 @@ rec {
client.succeed("sync") # flush all data before pulling the plug
'';
testDriver = pythonTestDriver { inherit qemu_pkg; };
vmRunCommand = writeText "vm-run" ''
xchg=vm-state-client/xchg
${coreutils}/bin/mkdir $out
@ -290,7 +303,7 @@ rec {
unset xchg
export tests='${testScript}'
${mkTestDriver qemu}/bin/nixos-test-driver --keep-vm-state ${vm.config.system.build.vm}/bin/run-*-vm
${testDriver}/bin/nixos-test-driver --keep-vm-state ${vm.config.system.build.vm}/bin/run-*-vm
''; # */
in