dockerTools: private registry support

* authorization token is optional
* registry url is taken from X-Docker-Endpoints header
* pull.sh correctly resumes partial layer downloads
* detjson.py does not fail on missing keys
This commit is contained in:
Arthur Noel 2016-01-27 19:48:04 +00:00
parent 189693327b
commit 903129f770
4 changed files with 40 additions and 31 deletions

View file

@ -489,7 +489,6 @@ c = lib.makeOverridable f { a = 1; b = 2; }</programlisting>
sha256 = "1bhw5hkz6chrnrih0ymjbmn69hyfriza2lr550xyvpdrnbzr4gk2"; <co xml:id='ex-dockerTools-pullImage-4' /> sha256 = "1bhw5hkz6chrnrih0ymjbmn69hyfriza2lr550xyvpdrnbzr4gk2"; <co xml:id='ex-dockerTools-pullImage-4' />
indexUrl = "https://index.docker.io"; <co xml:id='ex-dockerTools-pullImage-5' /> indexUrl = "https://index.docker.io"; <co xml:id='ex-dockerTools-pullImage-5' />
registryUrl = "https://registry-1.docker.io";
registryVersion = "v1"; registryVersion = "v1";
} }
</programlisting> </programlisting>
@ -534,8 +533,8 @@ c = lib.makeOverridable f { a = 1; b = 2; }</programlisting>
<callout arearefs='ex-dockerTools-pullImage-5'> <callout arearefs='ex-dockerTools-pullImage-5'>
<para> <para>
In the above example the default values are shown for the variables <varname>indexUrl</varname>, In the above example the default values are shown for the variables
<varname>registryUrl</varname> and <varname>registryVersion</varname>. <varname>indexUrl</varname> and <varname>registryVersion</varname>.
Hence by default the Docker.io registry is used to pull the images. Hence by default the Docker.io registry is used to pull the images.
</para> </para>
</callout> </callout>

View file

@ -24,9 +24,11 @@ SAFEDELS["container_config"] = SAFEDELS["config"]
def makedet(j, safedels): def makedet(j, safedels):
for k,v in safedels.items(): for k,v in safedels.items():
if k not in j:
continue
if type(v) == dict: if type(v) == dict:
makedet(j[k], v) makedet(j[k], v)
elif k in j and j[k] == v: elif j[k] == v:
del j[k] del j[k]
def main(): def main():

View file

@ -8,13 +8,14 @@
{ imageName, imageTag ? "latest", imageId ? null { imageName, imageTag ? "latest", imageId ? null
, sha256, name ? "${imageName}-${imageTag}" , sha256, name ? "${imageName}-${imageTag}"
, indexUrl ? "https://index.docker.io" , indexUrl ? "https://index.docker.io"
, registryUrl ? "https://registry-1.docker.io"
, registryVersion ? "v1" , registryVersion ? "v1"
, curlOpts ? "" }: , curlOpts ? "" }:
assert registryVersion == "v1";
let layer = stdenv.mkDerivation { let layer = stdenv.mkDerivation {
inherit name imageName imageTag imageId inherit name imageName imageTag imageId
indexUrl registryUrl registryVersion curlOpts; indexUrl registryVersion curlOpts;
builder = ./pull.sh; builder = ./pull.sh;
detjson = ./detjson.py; detjson = ./detjson.py;
@ -34,10 +35,6 @@ let layer = stdenv.mkDerivation {
# This variable allows the user to pass additional options to curl # This variable allows the user to pass additional options to curl
"NIX_CURL_FLAGS" "NIX_CURL_FLAGS"
# This variable allows overriding the timeout for connecting to
# the hashed mirrors.
"NIX_CONNECT_TIMEOUT"
]; ];
# Doing the download on a remote machine just duplicates network # Doing the download on a remote machine just duplicates network

View file

@ -6,17 +6,20 @@ source $stdenv/setup
# servers to need them during redirects, and work on SSL without a # servers to need them during redirects, and work on SSL without a
# certificate (this isn't a security problem because we check the # certificate (this isn't a security problem because we check the
# cryptographic hash of the output anyway). # cryptographic hash of the output anyway).
curl="curl \ curl=$(command -v curl)
--location --max-redirs 20 \ curl() {
--retry 3 \ [[ -n ${token:-} ]] && set -- -H "Authorization: Token $token" "$@"
--fail \ $curl \
--disable-epsv \ --location --max-redirs 20 \
--cookie-jar cookies \ --retry 3 \
--insecure \ --fail \
$curlOpts \ --disable-epsv \
$NIX_CURL_FLAGS" --cookie-jar cookies \
--insecure \
baseUrl="$registryUrl/$registryVersion" $curlOpts \
$NIX_CURL_FLAGS \
"$@"
}
fetchLayer() { fetchLayer() {
local url="$1" local url="$1"
@ -26,7 +29,7 @@ fetchLayer() {
# if we get error code 18, resume partial download # if we get error code 18, resume partial download
while [ $curlexit -eq 18 ]; do while [ $curlexit -eq 18 ]; do
# keep this inside an if statement, since on failure it doesn't abort the script # keep this inside an if statement, since on failure it doesn't abort the script
if $curl -H "Authorization: Token $token" "$url" --output "$dest"; then if curl -C - "$url" --output "$dest"; then
return 0 return 0
else else
curlexit=$?; curlexit=$?;
@ -36,17 +39,25 @@ fetchLayer() {
return $curlexit return $curlexit
} }
token="$($curl -o /dev/null -D- -H 'X-Docker-Token: true' "$indexUrl/$registryVersion/repositories/$imageName/images" | grep X-Docker-Token | tr -d '\r' | cut -d ' ' -f 2)" headers=$(curl -o /dev/null -D- -H 'X-Docker-Token: true' \
"$indexUrl/$registryVersion/repositories/$imageName/images")
if [ -z "$token" ]; then header() {
echo "error: registry returned no token" grep $1 <<< "$headers" | tr -d '\r' | cut -d ' ' -f 2
exit 1 }
# this only takes the first endpoint, more may be provided
# https://docs.docker.com/v1.6/reference/api/docker-io_api/
if ! registryUrl=$(header X-Docker-Endpoints); then
echo "error: index returned no endpoint"
exit 1
fi fi
baseUrl="https://$registryUrl/$registryVersion"
# token="${token//\"/\\\"}" token="$(header X-Docker-Token || true)";
if [ -z "$imageId" ]; then if [ -z "$imageId" ]; then
imageId="$($curl -H "Authorization: Token $token" "$baseUrl/repositories/$imageName/tags/$imageTag")" imageId="$(curl "$baseUrl/repositories/$imageName/tags/$imageTag")"
imageId="${imageId//\"/}" imageId="${imageId//\"/}"
if [ -z "$imageId" ]; then if [ -z "$imageId" ]; then
echo "error: no image ID found for ${imageName}:${imageTag}" echo "error: no image ID found for ${imageName}:${imageTag}"
@ -62,7 +73,7 @@ jshon -n object \
-n object -s "$imageId" -i "$imageTag" \ -n object -s "$imageId" -i "$imageTag" \
-i "$imageName" > $out/repositories -i "$imageName" > $out/repositories
$curl -H "Authorization: Token $token" "$baseUrl/images/$imageId/ancestry" -o ancestry.json curl "$baseUrl/images/$imageId/ancestry" -o ancestry.json
layerIds=$(jshon -a -u < ancestry.json) layerIds=$(jshon -a -u < ancestry.json)
for layerId in $layerIds; do for layerId in $layerIds; do
@ -70,6 +81,6 @@ for layerId in $layerIds; do
mkdir "$out/$layerId" mkdir "$out/$layerId"
echo '1.0' > "$out/$layerId/VERSION" echo '1.0' > "$out/$layerId/VERSION"
$curl -H "Authorization: Token $token" "$baseUrl/images/$layerId/json" | python $detjson > "$out/$layerId/json" curl "$baseUrl/images/$layerId/json" | python $detjson > "$out/$layerId/json"
fetchLayer "$baseUrl/images/$layerId/layer" "$out/$layerId/layer.tar" fetchLayer "$baseUrl/images/$layerId/layer" "$out/$layerId/layer.tar"
done done