From 25865688a729d15dbb2dc21ebd9fbf74e2cffc4b Mon Sep 17 00:00:00 2001 From: Parnell Springmeyer Date: Fri, 1 Dec 2017 21:00:52 -0600 Subject: [PATCH 1/3] docker: init fetchdocker nix code for docker2nix This change adds granular, non-docker daemon docker image fetchers and a docker image layer compositor to be used in conjunction with the `docker2nix` utility provided by the `haskellPackages.hocker` package. This change includes a hackage package version bump and updated sha256 for recent fixes released to `hocker` resulting from formulating this patch. --- .../build-support/fetchdocker/credentials.nix | 38 ++++++++ pkgs/build-support/fetchdocker/default.nix | 61 ++++++++++++ .../fetchdocker/fetchDockerConfig.nix | 13 +++ .../fetchdocker/fetchDockerLayer.nix | 13 +++ .../fetchdocker/fetchdocker-builder.sh | 28 ++++++ .../fetchdocker/generic-fetcher.nix | 97 +++++++++++++++++++ .../haskell-modules/hackage-packages.nix | 4 +- pkgs/top-level/all-packages.nix | 6 ++ 8 files changed, 258 insertions(+), 2 deletions(-) create mode 100644 pkgs/build-support/fetchdocker/credentials.nix create mode 100644 pkgs/build-support/fetchdocker/default.nix create mode 100644 pkgs/build-support/fetchdocker/fetchDockerConfig.nix create mode 100644 pkgs/build-support/fetchdocker/fetchDockerLayer.nix create mode 100644 pkgs/build-support/fetchdocker/fetchdocker-builder.sh create mode 100644 pkgs/build-support/fetchdocker/generic-fetcher.nix diff --git a/pkgs/build-support/fetchdocker/credentials.nix b/pkgs/build-support/fetchdocker/credentials.nix new file mode 100644 index 00000000000..c01288dbf53 --- /dev/null +++ b/pkgs/build-support/fetchdocker/credentials.nix @@ -0,0 +1,38 @@ +# We provide three paths to get the credentials into the builder's +# environment: +# +# 1. Via impureEnvVars. This method is difficult for multi-user Nix +# installations (but works very well for single-user Nix +# installations!) because it requires setting the environment +# variables on the nix-daemon which is either complicated or unsafe +# (i.e: configuring via Nix means the secrets will be persisted +# into the store) +# +# 2. If the DOCKER_CREDENTIALS key with a path to a credentials file +# is added to the NIX_PATH (usually via the '-I ' argument to most +# Nix tools) then an attempt will be made to read credentials from +# it. The semantics are simple, the file should contain two lines +# for the username and password based authentication: +# +# $ cat ./credentials-file.txt +# DOCKER_USER=myusername +# DOCKER_PASS=mypassword +# +# ... and a single line for the token based authentication: +# +# $ cat ./credentials-file.txt +# DOCKER_TOKEN=mytoken +# +# 3. A credential file at /etc/nix-docker-credentials.txt with the +# same format as the file described in #2 can also be used to +# communicate credentials to the builder. This is necessary for +# situations (like Hydra) where you cannot customize the NIX_PATH +# given to the nix-build invocation to provide it with the +# DOCKER_CREDENTIALS path +let + pathParts = + (builtins.filter + ({path, prefix}: "DOCKER_CREDENTIALS" == prefix) + builtins.nixPath); +in + if (pathParts != []) then (builtins.head pathParts).path else "" diff --git a/pkgs/build-support/fetchdocker/default.nix b/pkgs/build-support/fetchdocker/default.nix new file mode 100644 index 00000000000..ae3ae4185e0 --- /dev/null +++ b/pkgs/build-support/fetchdocker/default.nix @@ -0,0 +1,61 @@ +{ stdenv, lib, coreutils, bash, gnutar, jq, writeText }: +let + stripScheme = + builtins.replaceStrings [ "https://" "http://" ] [ "" "" ]; + stripNixStore = + s: lib.removePrefix "/nix/store/" s; +in +{ name +, registry ? "https://registry-1.docker.io/v2/" +, repository ? "library" +, imageName +, tag +, imageLayers +, imageConfig +, image ? "${stripScheme registry}/${repository}/${imageName}:${tag}" +}: + +# Make sure there are *no* slashes in the repository or container +# names since we use these to make the output derivation name for the +# nix-store path. +assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters repository); +assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters imageName); + +let + # Abuse `builtins.toPath` to collapse possible double slashes + repoTag0 = builtins.toString (builtins.toPath "/${stripScheme registry}/${repository}/${imageName}"); + repoTag1 = lib.removePrefix "/" repoTag0; + + layers = builtins.map stripNixStore imageLayers; + + manifest = + writeText "manifest.json" (builtins.toJSON [ + { Config = stripNixStore imageConfig; + Layers = layers; + RepoTags = [ "${repoTag1}:${tag}" ]; + }]); + + repositories = + writeText "repositories" (builtins.toJSON { + "${repoTag1}" = { + "${tag}" = lib.last layers; + }; + }); + + imageFileStorePaths = + writeText "imageFileStorePaths.txt" + (lib.concatStringsSep "\n" ((lib.unique imageLayers) ++ [imageConfig])); +in +stdenv.mkDerivation { + builder = ./fetchdocker-builder.sh; + buildInputs = [ coreutils ]; + preferLocalBuild = true; + + inherit name imageName repository tag; + inherit bash gnutar manifest repositories; + inherit imageFileStorePaths; + + passthru = { + inherit image; + }; +} diff --git a/pkgs/build-support/fetchdocker/fetchDockerConfig.nix b/pkgs/build-support/fetchdocker/fetchDockerConfig.nix new file mode 100644 index 00000000000..9fd813bfa57 --- /dev/null +++ b/pkgs/build-support/fetchdocker/fetchDockerConfig.nix @@ -0,0 +1,13 @@ +pkgargs@{ stdenv, lib, haskellPackages, writeText, gawk }: +let + generic-fetcher = + import ./generic-fetcher.nix pkgargs; +in + +args@{ repository ? "library", imageName, tag, ... }: + +generic-fetcher ({ + fetcher = "hocker-config"; + name = "${repository}_${imageName}_${tag}-config.json"; + tag = "unused"; +} // args) diff --git a/pkgs/build-support/fetchdocker/fetchDockerLayer.nix b/pkgs/build-support/fetchdocker/fetchDockerLayer.nix new file mode 100644 index 00000000000..869ba637429 --- /dev/null +++ b/pkgs/build-support/fetchdocker/fetchDockerLayer.nix @@ -0,0 +1,13 @@ +pkgargs@{ stdenv, lib, haskellPackages, writeText, gawk }: +let + generic-fetcher = + import ./generic-fetcher.nix pkgargs; +in + +args@{ layerDigest, ... }: + +generic-fetcher ({ + fetcher = "hocker-layer"; + name = "docker-layer-${layerDigest}.tar.gz"; + tag = "unused"; +} // args) diff --git a/pkgs/build-support/fetchdocker/fetchdocker-builder.sh b/pkgs/build-support/fetchdocker/fetchdocker-builder.sh new file mode 100644 index 00000000000..7443591e656 --- /dev/null +++ b/pkgs/build-support/fetchdocker/fetchdocker-builder.sh @@ -0,0 +1,28 @@ +source "${stdenv}/setup" +header "exporting ${repository}/${imageName} (tag: ${tag}) into ${out}" +mkdir -p "${out}" + +cat < "${out}/compositeImage.sh" +#! ${bash}/bin/bash +# +# Create a tar archive of a docker image's layers, docker image config +# json, manifest.json, and repositories json; this streams directly to +# stdout and is intended to be used in concert with docker load, i.e: +# +# ${out}/compositeImage.sh | docker load + +# The first character follow the 's' command for sed becomes the +# delimiter sed will use; this makes the transformation regex easy to +# read. We feed tar a file listing the files we want in the archive, +# because the paths are absolute and docker load wants them flattened in +# the archive, we need to transform all of the paths going in by +# stripping everything *including* the last solidus so that we end up +# with the basename of the path. +${gnutar}/bin/tar \ + --transform='s=.*/==' \ + --transform="s=.*-manifest.json=manifest.json=" \ + --transform="s=.*-repositories=repositories=" \ + -c "${manifest}" "${repositories}" -T "${imageFileStorePaths}" +EOF +chmod +x "${out}/compositeImage.sh" +stopNest diff --git a/pkgs/build-support/fetchdocker/generic-fetcher.nix b/pkgs/build-support/fetchdocker/generic-fetcher.nix new file mode 100644 index 00000000000..e051cee0843 --- /dev/null +++ b/pkgs/build-support/fetchdocker/generic-fetcher.nix @@ -0,0 +1,97 @@ +{ stdenv, lib, haskellPackages, writeText, gawk }: +let + awk = "${gawk}/bin/awk"; + dockerCredentialsFile = import ./credentials.nix; + stripScheme = + builtins.replaceStrings [ "https://" "http://" ] [ "" "" ]; +in +{ fetcher +, name + , registry ? "https://registry-1.docker.io/v2/" + , repository ? "library" + , imageName + , sha256 + , tag ? "" + , layerDigest ? "" +}: + +# There must be no slashes in the repository or container names since +# we use these to make the output derivation name for the nix store +# path +assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters repository); +assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters imageName); + +# Only allow hocker-config and hocker-layer as fetchers for now +assert (builtins.elem fetcher ["hocker-config" "hocker-layer"]); + +# If layerDigest is non-empty then it must not have a 'sha256:' prefix! +assert + (if layerDigest != "" + then !lib.hasPrefix "sha256:" layerDigest + else true); + +let + layerDigestFlag = + lib.optionalString (layerDigest != "") "--layer ${layerDigest}"; +in +stdenv.mkDerivation { + inherit name; + builder = writeText "${fetcher}-builder.sh" '' + source "$stdenv/setup" + header "${fetcher} exporting to $out" + + declare -A creds + + # This is a hack for Hydra since we have no way of adding values + # to the NIX_PATH for Hydra jobsets!! + staticCredentialsFile="/etc/nix-docker-credentials.txt" + if [ ! -f "$dockerCredentialsFile" -a -f "$staticCredentialsFile" ]; then + echo "credentials file not set, falling back on static credentials file at: $staticCredentialsFile" + dockerCredentialsFile=$staticCredentialsFile + fi + + if [ -f "$dockerCredentialsFile" ]; then + header "using credentials from $dockerCredentialsFile" + + CREDSFILE=$(cat "$dockerCredentialsFile") + creds[token]=$(${awk} -F'=' '/DOCKER_TOKEN/ {print $2}' <<< "$CREDSFILE" | head -n1) + + # Prefer DOCKER_TOKEN over the username and password + # authentication method + if [ -z "''${creds[token]}" ]; then + creds[user]=$(${awk} -F'=' '/DOCKER_USER/ {print $2}' <<< "$CREDSFILE" | head -n1) + creds[pass]=$(${awk} -F'=' '/DOCKER_PASS/ {print $2}' <<< "$CREDSFILE" | head -n1) + fi + fi + + # These variables will be filled in first by the impureEnvVars, if + # those variables are empty then they will default to the + # credentials that may have been read in from the 'DOCKER_CREDENTIALS' + DOCKER_USER="''${DOCKER_USER:-''${creds[user]}}" + DOCKER_PASS="''${DOCKER_PASS:-''${creds[pass]}}" + DOCKER_TOKEN="''${DOCKER_TOKEN:-''${creds[token]}}" + + ${fetcher} --out="$out" \ + ''${registry:+--registry "$registry"} \ + ''${DOCKER_USER:+--username "$DOCKER_USER"} \ + ''${DOCKER_PASS:+--password "$DOCKER_PASS"} \ + ''${DOCKER_TOKEN:+--token "$DOCKER_TOKEN"} \ + ${layerDigestFlag} \ + "${repository}/${imageName}" \ + "${tag}" + + stopNest + ''; + + buildInputs = [ haskellPackages.hocker ]; + + outputHashAlgo = "sha256"; + outputHashMode = "flat"; + outputHash = sha256; + + preferLocalBuild = true; + + impureEnvVars = [ "DOCKER_USER" "DOCKER_PASS" "DOCKER_TOKEN" ]; + + inherit registry dockerCredentialsFile; +} diff --git a/pkgs/development/haskell-modules/hackage-packages.nix b/pkgs/development/haskell-modules/hackage-packages.nix index 3107d29b090..c4860ac9024 100644 --- a/pkgs/development/haskell-modules/hackage-packages.nix +++ b/pkgs/development/haskell-modules/hackage-packages.nix @@ -103952,8 +103952,8 @@ self: { }: mkDerivation { pname = "hocker"; - version = "1.0.0"; - sha256 = "16indvxpf2zzdkb7hp09zfnn1zkjwc1pcg2560x2vj7x4akh25mv"; + version = "1.0.2"; + sha256 = "1bdzbggvin83m778qq6367mpv2cwgwpbahhlzf290iwikmhmhgr2"; isLibrary = true; isExecutable = true; libraryHaskellDepends = [ diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 2f7cd77bd58..a9566c1df71 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -146,6 +146,12 @@ with pkgs; fetchdarcs = callPackage ../build-support/fetchdarcs { }; + fetchdocker = callPackage ../build-support/fetchdocker { }; + + fetchDockerConfig = callPackage ../build-support/fetchdocker/fetchDockerConfig.nix { }; + + fetchDockerLayer = callPackage ../build-support/fetchdocker/fetchDockerLayer.nix { }; + fetchfossil = callPackage ../build-support/fetchfossil { }; fetchgit = callPackage ../build-support/fetchgit { From c1eb962516abb1ab7900d1f19f423f3a4318b849 Mon Sep 17 00:00:00 2001 From: Parnell Springmeyer Date: Fri, 1 Dec 2017 21:06:16 -0600 Subject: [PATCH 2/3] fetchdocker: Integration test exercising hocker and fetchdocker This change adds a simple integration test exercising the fetchdocker Nix code and hocker utilities for the simple `hello-world` docker container. We exercise: - Fetching the docker image configuration json - Fetching the docker image layers - Building a compositor script - Loading the `hello-world` docker image into docker using the compositor script and `docker load` - Running that loaded container --- nixos/release.nix | 1 + nixos/tests/hocker-fetchdocker/default.nix | 15 +++++++++ .../hello-world-container.nix | 19 ++++++++++++ nixos/tests/hocker-fetchdocker/machine.nix | 31 +++++++++++++++++++ 4 files changed, 66 insertions(+) create mode 100644 nixos/tests/hocker-fetchdocker/default.nix create mode 100644 nixos/tests/hocker-fetchdocker/hello-world-container.nix create mode 100644 nixos/tests/hocker-fetchdocker/machine.nix diff --git a/nixos/release.nix b/nixos/release.nix index ac4dd3d7892..bb6a73eb68e 100644 --- a/nixos/release.nix +++ b/nixos/release.nix @@ -262,6 +262,7 @@ in rec { tests.hardened = callTest tests/hardened.nix { }; tests.hibernate = callTest tests/hibernate.nix {}; tests.hound = callTest tests/hound.nix {}; + tests.hocker-fetchdocker = callTest tests/hocker-fetchdocker {}; tests.i3wm = callTest tests/i3wm.nix {}; tests.initrd-network-ssh = callTest tests/initrd-network-ssh {}; tests.installer = callSubTests tests/installer.nix {}; diff --git a/nixos/tests/hocker-fetchdocker/default.nix b/nixos/tests/hocker-fetchdocker/default.nix new file mode 100644 index 00000000000..4f30f01e403 --- /dev/null +++ b/nixos/tests/hocker-fetchdocker/default.nix @@ -0,0 +1,15 @@ +import ../make-test.nix ({ pkgs, ...} : { + name = "test-hocker-fetchdocker"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ ixmatus ]; + }; + + machine = import ./machine.nix; + + testScript = '' + startAll; + + $machine->waitForUnit("sockets.target"); + $machine->waitUntilSucceeds("docker run registry-1.docker.io/v2/library/hello-world:latest"); + ''; +}) diff --git a/nixos/tests/hocker-fetchdocker/hello-world-container.nix b/nixos/tests/hocker-fetchdocker/hello-world-container.nix new file mode 100644 index 00000000000..a127875264e --- /dev/null +++ b/nixos/tests/hocker-fetchdocker/hello-world-container.nix @@ -0,0 +1,19 @@ +{ fetchDockerConfig, fetchDockerLayer, fetchdocker }: +fetchdocker rec { + name = "hello-world"; + registry = "https://registry-1.docker.io/v2/"; + repository = "library"; + imageName = "hello-world"; + tag = "latest"; + imageConfig = fetchDockerConfig { + inherit tag registry repository imageName; + sha256 = "1ivbd23hyindkahzfw4kahgzi6ibzz2ablmgsz6340vc6qr1gagj"; + }; + imageLayers = let + layer0 = fetchDockerLayer { + inherit registry repository imageName; + layerDigest = "ca4f61b1923c10e9eb81228bd46bee1dfba02b9c7dac1844527a734752688ede"; + sha256 = "1plfd194fwvsa921ib3xkhms1yqxxrmx92r2h7myj41wjaqn2kya"; + }; + in [ layer0 ]; + } diff --git a/nixos/tests/hocker-fetchdocker/machine.nix b/nixos/tests/hocker-fetchdocker/machine.nix new file mode 100644 index 00000000000..0d71aedd054 --- /dev/null +++ b/nixos/tests/hocker-fetchdocker/machine.nix @@ -0,0 +1,31 @@ +{ config, pkgs, ... }: +{ nixpkgs.config.packageOverrides = pkgs': { + hello-world-container = pkgs'.callPackage ./hello-world-container.nix { }; + haskellPackages = pkgs'.haskellPackages.override { + overrides = new: old: { + hocker = pkgs'.haskell.lib.dontCheck old.hocker; + }; + }; + }; + + virtualisation.docker = { + enable = true; + package = pkgs.docker; + }; + + systemd.services.docker-load-fetchdocker-image = { + description = "Docker load hello-world-container"; + wantedBy = [ "multi-user.target" ]; + wants = [ "docker.service" "local-fs.target" ]; + after = [ "docker.service" "local-fs.target" ]; + + script = '' + ${pkgs.hello-world-container}/compositeImage.sh | ${pkgs.docker}/bin/docker load + ''; + + serviceConfig = { + Type = "oneshot"; + }; + }; +} + From 6c9de3f6009fbbebeb92593577622da6d7d05408 Mon Sep 17 00:00:00 2001 From: Parnell Springmeyer Date: Sat, 2 Dec 2017 09:20:22 -0600 Subject: [PATCH 3/3] hocker: Don't check package and wrap the binaries with PATH to nix --- nixos/tests/hocker-fetchdocker/machine.nix | 5 --- .../haskell-modules/configuration-common.nix | 34 +++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/nixos/tests/hocker-fetchdocker/machine.nix b/nixos/tests/hocker-fetchdocker/machine.nix index 0d71aedd054..12c58a01224 100644 --- a/nixos/tests/hocker-fetchdocker/machine.nix +++ b/nixos/tests/hocker-fetchdocker/machine.nix @@ -1,11 +1,6 @@ { config, pkgs, ... }: { nixpkgs.config.packageOverrides = pkgs': { hello-world-container = pkgs'.callPackage ./hello-world-container.nix { }; - haskellPackages = pkgs'.haskellPackages.override { - overrides = new: old: { - hocker = pkgs'.haskell.lib.dontCheck old.hocker; - }; - }; }; virtualisation.docker = { diff --git a/pkgs/development/haskell-modules/configuration-common.nix b/pkgs/development/haskell-modules/configuration-common.nix index 84df4d1f0c4..37b9feb84f0 100644 --- a/pkgs/development/haskell-modules/configuration-common.nix +++ b/pkgs/development/haskell-modules/configuration-common.nix @@ -984,4 +984,38 @@ self: super: { }; }).override { language-c = self.language-c_0_7_0; }; + hocker = + overrideCabal + # Not checking because it's failing on a test that needs a data + # file not included in its source distribution; this will be + # removed when that is fixed + ( dontCheck super.hocker ) + ( oldDerivation: { + testToolDepends = + (oldDerivation.testToolDepends or []) ++[ pkgs.nix ]; + buildDepends = + (oldDerivation.buildDepends or []) ++ [ pkgs.makeWrapper ]; + + postInstall = + (oldDerivation.postInstall or "") + '' + # Globbing for hocker-* fails with: Builder called die: + # makeWrapper doesn't understand the arg /nix/store/rsic1v6y6v63q6lkmpn3xmn7cnzx8irk-hocker-1.0.2/bin/hocker-image + wrapProgram $out/bin/hocker-image \ + --suffix PATH : ${pkgs.nix}/bin + + wrapProgram $out/bin/hocker-layer \ + --suffix PATH : ${pkgs.nix}/bin + + wrapProgram $out/bin/hocker-config \ + --suffix PATH : ${pkgs.nix}/bin + + wrapProgram $out/bin/hocker-manifest \ + --suffix PATH : ${pkgs.nix}/bin + + wrapProgram $out/bin/docker2nix \ + --suffix PATH : ${pkgs.nix}/bin + ''; + } + ); + }