docker: init fetchdocker nix code for docker2nix

This change adds granular, non-docker daemon docker image fetchers and
a docker image layer compositor to be used in conjunction with the
`docker2nix` utility provided by the `haskellPackages.hocker` package.

This change includes a hackage package version bump and updated sha256
for recent fixes released to `hocker` resulting from formulating this
patch.
This commit is contained in:
Parnell Springmeyer 2017-12-01 21:00:52 -06:00
parent fdb8dea0c6
commit 25865688a7
No known key found for this signature in database
GPG key ID: C7FD72B325BC271F
8 changed files with 258 additions and 2 deletions

View file

@ -0,0 +1,38 @@
# We provide three paths to get the credentials into the builder's
# environment:
#
# 1. Via impureEnvVars. This method is difficult for multi-user Nix
# installations (but works very well for single-user Nix
# installations!) because it requires setting the environment
# variables on the nix-daemon which is either complicated or unsafe
# (i.e: configuring via Nix means the secrets will be persisted
# into the store)
#
# 2. If the DOCKER_CREDENTIALS key with a path to a credentials file
# is added to the NIX_PATH (usually via the '-I ' argument to most
# Nix tools) then an attempt will be made to read credentials from
# it. The semantics are simple, the file should contain two lines
# for the username and password based authentication:
#
# $ cat ./credentials-file.txt
# DOCKER_USER=myusername
# DOCKER_PASS=mypassword
#
# ... and a single line for the token based authentication:
#
# $ cat ./credentials-file.txt
# DOCKER_TOKEN=mytoken
#
# 3. A credential file at /etc/nix-docker-credentials.txt with the
# same format as the file described in #2 can also be used to
# communicate credentials to the builder. This is necessary for
# situations (like Hydra) where you cannot customize the NIX_PATH
# given to the nix-build invocation to provide it with the
# DOCKER_CREDENTIALS path
let
pathParts =
(builtins.filter
({path, prefix}: "DOCKER_CREDENTIALS" == prefix)
builtins.nixPath);
in
if (pathParts != []) then (builtins.head pathParts).path else ""

View file

@ -0,0 +1,61 @@
{ stdenv, lib, coreutils, bash, gnutar, jq, writeText }:
let
stripScheme =
builtins.replaceStrings [ "https://" "http://" ] [ "" "" ];
stripNixStore =
s: lib.removePrefix "/nix/store/" s;
in
{ name
, registry ? "https://registry-1.docker.io/v2/"
, repository ? "library"
, imageName
, tag
, imageLayers
, imageConfig
, image ? "${stripScheme registry}/${repository}/${imageName}:${tag}"
}:
# Make sure there are *no* slashes in the repository or container
# names since we use these to make the output derivation name for the
# nix-store path.
assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters repository);
assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters imageName);
let
# Abuse `builtins.toPath` to collapse possible double slashes
repoTag0 = builtins.toString (builtins.toPath "/${stripScheme registry}/${repository}/${imageName}");
repoTag1 = lib.removePrefix "/" repoTag0;
layers = builtins.map stripNixStore imageLayers;
manifest =
writeText "manifest.json" (builtins.toJSON [
{ Config = stripNixStore imageConfig;
Layers = layers;
RepoTags = [ "${repoTag1}:${tag}" ];
}]);
repositories =
writeText "repositories" (builtins.toJSON {
"${repoTag1}" = {
"${tag}" = lib.last layers;
};
});
imageFileStorePaths =
writeText "imageFileStorePaths.txt"
(lib.concatStringsSep "\n" ((lib.unique imageLayers) ++ [imageConfig]));
in
stdenv.mkDerivation {
builder = ./fetchdocker-builder.sh;
buildInputs = [ coreutils ];
preferLocalBuild = true;
inherit name imageName repository tag;
inherit bash gnutar manifest repositories;
inherit imageFileStorePaths;
passthru = {
inherit image;
};
}

View file

@ -0,0 +1,13 @@
pkgargs@{ stdenv, lib, haskellPackages, writeText, gawk }:
let
generic-fetcher =
import ./generic-fetcher.nix pkgargs;
in
args@{ repository ? "library", imageName, tag, ... }:
generic-fetcher ({
fetcher = "hocker-config";
name = "${repository}_${imageName}_${tag}-config.json";
tag = "unused";
} // args)

View file

@ -0,0 +1,13 @@
pkgargs@{ stdenv, lib, haskellPackages, writeText, gawk }:
let
generic-fetcher =
import ./generic-fetcher.nix pkgargs;
in
args@{ layerDigest, ... }:
generic-fetcher ({
fetcher = "hocker-layer";
name = "docker-layer-${layerDigest}.tar.gz";
tag = "unused";
} // args)

View file

@ -0,0 +1,28 @@
source "${stdenv}/setup"
header "exporting ${repository}/${imageName} (tag: ${tag}) into ${out}"
mkdir -p "${out}"
cat <<EOF > "${out}/compositeImage.sh"
#! ${bash}/bin/bash
#
# Create a tar archive of a docker image's layers, docker image config
# json, manifest.json, and repositories json; this streams directly to
# stdout and is intended to be used in concert with docker load, i.e:
#
# ${out}/compositeImage.sh | docker load
# The first character follow the 's' command for sed becomes the
# delimiter sed will use; this makes the transformation regex easy to
# read. We feed tar a file listing the files we want in the archive,
# because the paths are absolute and docker load wants them flattened in
# the archive, we need to transform all of the paths going in by
# stripping everything *including* the last solidus so that we end up
# with the basename of the path.
${gnutar}/bin/tar \
--transform='s=.*/==' \
--transform="s=.*-manifest.json=manifest.json=" \
--transform="s=.*-repositories=repositories=" \
-c "${manifest}" "${repositories}" -T "${imageFileStorePaths}"
EOF
chmod +x "${out}/compositeImage.sh"
stopNest

View file

@ -0,0 +1,97 @@
{ stdenv, lib, haskellPackages, writeText, gawk }:
let
awk = "${gawk}/bin/awk";
dockerCredentialsFile = import ./credentials.nix;
stripScheme =
builtins.replaceStrings [ "https://" "http://" ] [ "" "" ];
in
{ fetcher
, name
, registry ? "https://registry-1.docker.io/v2/"
, repository ? "library"
, imageName
, sha256
, tag ? ""
, layerDigest ? ""
}:
# There must be no slashes in the repository or container names since
# we use these to make the output derivation name for the nix store
# path
assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters repository);
assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters imageName);
# Only allow hocker-config and hocker-layer as fetchers for now
assert (builtins.elem fetcher ["hocker-config" "hocker-layer"]);
# If layerDigest is non-empty then it must not have a 'sha256:' prefix!
assert
(if layerDigest != ""
then !lib.hasPrefix "sha256:" layerDigest
else true);
let
layerDigestFlag =
lib.optionalString (layerDigest != "") "--layer ${layerDigest}";
in
stdenv.mkDerivation {
inherit name;
builder = writeText "${fetcher}-builder.sh" ''
source "$stdenv/setup"
header "${fetcher} exporting to $out"
declare -A creds
# This is a hack for Hydra since we have no way of adding values
# to the NIX_PATH for Hydra jobsets!!
staticCredentialsFile="/etc/nix-docker-credentials.txt"
if [ ! -f "$dockerCredentialsFile" -a -f "$staticCredentialsFile" ]; then
echo "credentials file not set, falling back on static credentials file at: $staticCredentialsFile"
dockerCredentialsFile=$staticCredentialsFile
fi
if [ -f "$dockerCredentialsFile" ]; then
header "using credentials from $dockerCredentialsFile"
CREDSFILE=$(cat "$dockerCredentialsFile")
creds[token]=$(${awk} -F'=' '/DOCKER_TOKEN/ {print $2}' <<< "$CREDSFILE" | head -n1)
# Prefer DOCKER_TOKEN over the username and password
# authentication method
if [ -z "''${creds[token]}" ]; then
creds[user]=$(${awk} -F'=' '/DOCKER_USER/ {print $2}' <<< "$CREDSFILE" | head -n1)
creds[pass]=$(${awk} -F'=' '/DOCKER_PASS/ {print $2}' <<< "$CREDSFILE" | head -n1)
fi
fi
# These variables will be filled in first by the impureEnvVars, if
# those variables are empty then they will default to the
# credentials that may have been read in from the 'DOCKER_CREDENTIALS'
DOCKER_USER="''${DOCKER_USER:-''${creds[user]}}"
DOCKER_PASS="''${DOCKER_PASS:-''${creds[pass]}}"
DOCKER_TOKEN="''${DOCKER_TOKEN:-''${creds[token]}}"
${fetcher} --out="$out" \
''${registry:+--registry "$registry"} \
''${DOCKER_USER:+--username "$DOCKER_USER"} \
''${DOCKER_PASS:+--password "$DOCKER_PASS"} \
''${DOCKER_TOKEN:+--token "$DOCKER_TOKEN"} \
${layerDigestFlag} \
"${repository}/${imageName}" \
"${tag}"
stopNest
'';
buildInputs = [ haskellPackages.hocker ];
outputHashAlgo = "sha256";
outputHashMode = "flat";
outputHash = sha256;
preferLocalBuild = true;
impureEnvVars = [ "DOCKER_USER" "DOCKER_PASS" "DOCKER_TOKEN" ];
inherit registry dockerCredentialsFile;
}

View file

@ -103952,8 +103952,8 @@ self: {
}:
mkDerivation {
pname = "hocker";
version = "1.0.0";
sha256 = "16indvxpf2zzdkb7hp09zfnn1zkjwc1pcg2560x2vj7x4akh25mv";
version = "1.0.2";
sha256 = "1bdzbggvin83m778qq6367mpv2cwgwpbahhlzf290iwikmhmhgr2";
isLibrary = true;
isExecutable = true;
libraryHaskellDepends = [

View file

@ -146,6 +146,12 @@ with pkgs;
fetchdarcs = callPackage ../build-support/fetchdarcs { };
fetchdocker = callPackage ../build-support/fetchdocker { };
fetchDockerConfig = callPackage ../build-support/fetchdocker/fetchDockerConfig.nix { };
fetchDockerLayer = callPackage ../build-support/fetchdocker/fetchDockerLayer.nix { };
fetchfossil = callPackage ../build-support/fetchfossil { };
fetchgit = callPackage ../build-support/fetchgit {