From f12346d49359784617a62731501627dc032419d8 Mon Sep 17 00:00:00 2001 From: Utku Demir Date: Wed, 6 May 2020 21:17:47 +1200 Subject: [PATCH] dockerTools: Calculate tarsum's on the fly Calculating the tarsum after creating a layer is inefficient, since we have to read the tarball we've just written from the disk. This commit simultaneously calculates the tarsum while creating the tarball. --- pkgs/build-support/docker/default.nix | 21 +++----- .../docker/store-path-to-layer.sh | 48 +++++++++---------- 2 files changed, 29 insertions(+), 40 deletions(-) diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index bee6e37cccb..7fd6389aabf 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -392,14 +392,10 @@ rec { (cd layer; eval "$extraCommands") fi - # Tar up the layer and throw it into 'layer.tar'. + # Tar up the layer and throw it into 'layer.tar', while calculating its checksum. echo "Packing layer..." mkdir $out - tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar . - - # Compute a checksum of the tarball. - echo "Computing layer checksum..." - tarhash=$(tarsum < $out/layer.tar) + tarhash=$(tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf - . | tee $out/layer.tar | tarsum) # Add a 'checksum' field to the JSON, with the value set to the # checksum of the tarball. @@ -449,11 +445,7 @@ rec { # Tar up the layer and throw it into 'layer.tar'. echo "Packing layer..." mkdir $out - tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar . - - # Compute a checksum of the tarball. - echo "Computing layer checksum..." - tarhash=$(tarsum < $out/layer.tar) + tarhash=$(tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf - . | tee $out/layer.tar | tarsum) # Add a 'checksum' field to the JSON, with the value set to the # checksum of the tarball. @@ -537,11 +529,10 @@ rec { echo "Packing layer..." mkdir -p $out - tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf $out/layer.tar . + tarhash=$(tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf - . | + tee $out/layer.tar | + ${tarsum}/bin/tarsum) - # Compute the tar checksum and add it to the output json. - echo "Computing checksum..." - tarhash=$(${tarsum}/bin/tarsum < $out/layer.tar) cat ${baseJson} | jshon -s "$tarhash" -i checksum > $out/json # Indicate to docker that we're using schema version 1.0. echo -n "1.0" > $out/VERSION diff --git a/pkgs/build-support/docker/store-path-to-layer.sh b/pkgs/build-support/docker/store-path-to-layer.sh index e9fa22e4b99..d834716e4b2 100755 --- a/pkgs/build-support/docker/store-path-to-layer.sh +++ b/pkgs/build-support/docker/store-path-to-layer.sh @@ -11,37 +11,35 @@ echo "Creating layer #$layerNumber for $@" mkdir -p "$layerPath" # Make sure /nix and /nix/store appear first in the archive. +# # We create the directories here and use them because # when there are other things being added to the # nix store, tar could fail, saying, # "tar: /nix/store: file changed as we read it" mkdir -p nix/store -tar -cf "$layerPath/layer.tar" \ - --mtime="@$SOURCE_DATE_EPOCH" \ - --owner=0 --group=0 \ - --transform='s,nix,/nix,' \ - nix -# We change into the /nix/store in order to avoid a similar -# "file changed as we read it" error as above. Namely, -# if we use the absolute path of /nix/store/123-pkg -# and something new is added to the nix store while tar -# is running, it will detect a change to /nix/store and -# fail. Instead, if we cd into the nix store and copy -# the relative nix store path, tar will ignore changes -# to /nix/store. In order to create the correct structure -# in the tar file, we transform the relative nix store -# path to the absolute store path. -basename -a "$@" | - tar -C /nix/store -rpf "$layerPath/layer.tar" \ - --verbatim-files-from --files-from - \ - --hard-dereference --sort=name \ - --mtime="@$SOURCE_DATE_EPOCH" \ - --owner=0 --group=0 \ - --transform="flags=rS;s,^,/nix/store/," - -# Compute a checksum of the tarball. -tarhash=$(tarsum < $layerPath/layer.tar) +# Then we change into the /nix/store in order to +# avoid a similar "file changed as we read it" error +# as above. Namely, if we use the absolute path of +# /nix/store/123-pkg and something new is added to the nix +# store while tar is running, it will detect a change to +# /nix/store and fail. Instead, if we cd into the nix store +# and copy the relative nix store path, tar will ignore +# changes to /nix/store. In order to create the correct +# structure in the tar file, we transform the relative nix +# store path to the absolute store path. +tarhash=$( + basename -a "$@" | + tar -cp nix \ + -C /nix/store --verbatim-files-from --files-from - \ + --hard-dereference --sort=name \ + --mtime="@$SOURCE_DATE_EPOCH" \ + --owner=0 --group=0 \ + --transform 's,^nix(/|$),/nix/,' \ + --transform 's,^[^/],/nix/store/\0,rS' | + tee "$layerPath/layer.tar" | + tarsum +) # Add a 'checksum' field to the JSON, with the value set to the # checksum of the tarball.