From 35f205a4b624bceca7c53b9c19ddac5f37a5ae4c Mon Sep 17 00:00:00 2001 From: Antoine Eiche Date: Fri, 22 Sep 2017 08:55:24 +0200 Subject: [PATCH] dockerTools.buildImage: Switch to the format image generated by Skopeo We were using 'Combined Image JSON + Filesystem Changeset Format' [1] to unpack and pack image and this patch switches to the format used by the registry. We used the 'repository' file which is not generated by Skopeo when it pulls an image. Moreover, all information of this file are also in the manifest.json file. We then use the manifest.json file instead of 'repository' file. Note also the manifest.json file is required to push an image with Skopeo. Fix #29636 [1] https://github.com/moby/moby/blob/749d90e10f989802638ae542daf54257f3bf71f2/image/spec/v1.1.md#combined-image-json--filesystem-changeset-format --- pkgs/build-support/docker/default.nix | 111 ++++++++------------------ 1 file changed, 34 insertions(+), 77 deletions(-) diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 267a2812a2a..843d6d6b5dd 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -137,7 +137,7 @@ rec { }; inherit fromImage fromImageName fromImageTag; - buildInputs = [ utillinux e2fsprogs jshon rsync ]; + buildInputs = [ utillinux e2fsprogs jshon rsync jq ]; } '' rm -rf $out @@ -146,44 +146,29 @@ rec { mount /dev/${vmTools.hd} disk cd disk + layers="" if [[ -n "$fromImage" ]]; then echo "Unpacking base image..." mkdir image tar -C image -xpf "$fromImage" - - # If the image name isn't set, read it from the image repository json. - if [[ -z "$fromImageName" ]]; then - fromImageName=$(jshon -k < image/repositories | head -n 1) - echo "From-image name wasn't set. Read $fromImageName." - fi - - # If the tag isn't set, use the name as an index into the json - # and read the first key found. - if [[ -z "$fromImageTag" ]]; then - fromImageTag=$(jshon -e $fromImageName -k < image/repositories \ - | head -n1) - echo "From-image tag wasn't set. Read $fromImageTag." - fi - - # Use the name and tag to get the parent ID field. - parentID=$(jshon -e $fromImageName -e $fromImageTag -u \ - < image/repositories) + layers=$(jq -r '.[0].Layers | join(" ")' image/manifest.json) fi - # Unpack all of the parent layers into the image. + # Unpack all of the layers into the image. + # Layer list is ordered starting from the base image lowerdir="" - while [[ -n "$parentID" ]]; do - echo "Unpacking layer $parentID" - mkdir -p image/$parentID/layer - tar -C image/$parentID/layer -xpf image/$parentID/layer.tar - rm image/$parentID/layer.tar + for layer in $layers; do + echo "Unpacking layer $layer" + layerDir=image/$(echo $layer | cut -d':' -f2)"_unpacked" + mkdir -p $layerDir + tar -C $layerDir -xpf image/$layer + chmod a+w image/$layer + rm image/$layer - find image/$parentID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \; + find $layerDir -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \; # Get the next lower directory and continue the loop. - lowerdir=$lowerdir''${lowerdir:+:}image/$parentID/layer - parentID=$(cat image/$parentID/json \ - | (jshon -e parent -u 2>/dev/null || true)) + lowerdir=$lowerdir''${lowerdir:+:}$layerDir done mkdir work @@ -461,26 +446,17 @@ rec { mkdir image touch baseFiles + layers="" if [[ -n "$fromImage" ]]; then echo "Unpacking base image..." tar -C image -xpf "$fromImage" - # Do not import the base image configuration and manifest - chmod a+w image image/*.json - rm -f image/*.json - - if [[ -z "$fromImageName" ]]; then - fromImageName=$(jshon -k < image/repositories|head -n1) - fi - if [[ -z "$fromImageTag" ]]; then - fromImageTag=$(jshon -e $fromImageName -k \ - < image/repositories|head -n1) - fi - parentID=$(jshon -e $fromImageName -e $fromImageTag -u \ - < image/repositories) - - for l in image/*/layer.tar; do - ls_tar $l >> baseFiles + config=$(jq -r '.[0].Config' image/manifest.json) + layers=$(jq -r '.[0].Layers | join(" ")' image/manifest.json) + for l in $layers; do + ls_tar image/$l >> baseFiles done + chmod u+w image image/$config + rm image/$config fi chmod -R ug+rw image @@ -507,47 +483,28 @@ rec { tar -rpf temp/layer.tar --mtime="@$SOURCE_DATE_EPOCH" \ --owner=0 --group=0 --no-recursion --files-from newFiles - echo "Adding meta..." + gzip temp/layer.tar + layerID="sha256:$(sha256sum temp/layer.tar.gz | cut -d ' ' -f 1)" + mv temp/layer.tar.gz image/$layerID - # If we have a parentID, add it to the json metadata. - if [[ -n "$parentID" ]]; then - cat temp/json | jshon -s "$parentID" -i parent > tmpjson - mv tmpjson temp/json - fi - - # Take the sha256 sum of the generated json and use it as the layer ID. - # Compute the size and add it to the json under the 'Size' field. - layerID=$(sha256sum temp/json|cut -d ' ' -f 1) - size=$(stat --printf="%s" temp/layer.tar) - cat temp/json | jshon -s "$layerID" -i id -n $size -i Size > tmpjson - mv tmpjson temp/json - - # Use the temp folder we've been working on to create a new image. - mv temp image/$layerID - - # Create image json and image manifest + echo "Generating image configuration and manifest..." imageJson=$(cat ${baseJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}") manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]") - currentID=$layerID - while [[ -n "$currentID" ]]; do - layerChecksum=$(sha256sum image/$currentID/layer.tar | cut -d ' ' -f1) - imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"${created}\"}] + .") - imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .") - manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$currentID/layer.tar\"] + .") - currentID=$(cat image/$currentID/json | (jshon -e parent -u 2>/dev/null || true)) + # The layer list is ordered starting from the base image + layers=$(echo $layers $layerID) + for i in $(echo $layers); do + imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"${created}\"}] + .") + diffId=$(gzip -dc image/$i | sha256sum | cut -d" " -f1) + imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$diffId\"] + .") + manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$i\"] + .") done imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1) - echo "$imageJson" > "image/$imageJsonChecksum.json" - manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"$imageJsonChecksum.json\"") + echo "$imageJson" > "image/sha256:$imageJsonChecksum" + manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"sha256:$imageJsonChecksum\"") echo "$manifestJson" > image/manifest.json - # Store the json under the name image/repositories. - jshon -n object \ - -n object -s "$layerID" -i "$imageTag" \ - -i "$imageName" > image/repositories - # Make the image read-only. chmod -R a-w image