dockerTools.buildImage: Switch to the format image generated by Skopeo

We were using 'Combined Image JSON + Filesystem Changeset Format' [1] to
unpack and pack image and this patch switches to the format used by the registry.

We used the 'repository' file which is not generated by Skopeo when it
pulls an image. Moreover, all information of this file are also in the
manifest.json file.
We then use the manifest.json file instead of 'repository' file. Note
also the manifest.json file is required to push an image with Skopeo.

Fix #29636

[1] 749d90e10f/image/spec/v1.1.md (combined-image-json--filesystem-changeset-format)

pkgs/build-support/docker/default.nix

@@ -137,7 +137,7 @@ rec {
         };
         inherit fromImage fromImageName fromImageTag;
 
-        buildInputs = [ utillinux e2fsprogs jshon rsync ];
+        buildInputs = [ utillinux e2fsprogs jshon rsync jq ];
       } ''
       rm -rf $out
 
@@ -146,44 +146,29 @@ rec {
       mount /dev/${vmTools.hd} disk
       cd disk
 
+      layers=""
       if [[ -n "$fromImage" ]]; then
         echo "Unpacking base image..."
         mkdir image
         tar -C image -xpf "$fromImage"
-
+        layers=$(jq -r '.[0].Layers | join(" ")' image/manifest.json)
-        # If the image name isn't set, read it from the image repository json.
-        if [[ -z "$fromImageName" ]]; then
-          fromImageName=$(jshon -k < image/repositories | head -n 1)
-          echo "From-image name wasn't set. Read $fromImageName."
-        fi
-
-        # If the tag isn't set, use the name as an index into the json
-        # and read the first key found.
-        if [[ -z "$fromImageTag" ]]; then
-          fromImageTag=$(jshon -e $fromImageName -k < image/repositories \
-                         | head -n1)
-          echo "From-image tag wasn't set. Read $fromImageTag."
-        fi
-
-        # Use the name and tag to get the parent ID field.
-        parentID=$(jshon -e $fromImageName -e $fromImageTag -u \
-                   < image/repositories)
       fi
 
-      # Unpack all of the parent layers into the image.
+      # Unpack all of the layers into the image.
+      # Layer list is ordered starting from the base image
       lowerdir=""
-      while [[ -n "$parentID" ]]; do
+      for layer in $layers; do
-        echo "Unpacking layer $parentID"
+        echo "Unpacking layer $layer"
-        mkdir -p image/$parentID/layer
+        layerDir=image/$(echo $layer | cut -d':' -f2)"_unpacked"
-        tar -C image/$parentID/layer -xpf image/$parentID/layer.tar
+        mkdir -p $layerDir
-        rm image/$parentID/layer.tar
+        tar -C $layerDir -xpf image/$layer
+        chmod a+w image/$layer
+        rm image/$layer
 
-        find image/$parentID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
+        find $layerDir -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
 
         # Get the next lower directory and continue the loop.
-        lowerdir=$lowerdir''${lowerdir:+:}image/$parentID/layer
+        lowerdir=$lowerdir''${lowerdir:+:}$layerDir
-        parentID=$(cat image/$parentID/json \
-                  | (jshon -e parent -u 2>/dev/null || true))
       done
 
       mkdir work
@@ -461,26 +446,17 @@ rec {
 
         mkdir image
         touch baseFiles
+        layers=""
         if [[ -n "$fromImage" ]]; then
           echo "Unpacking base image..."
           tar -C image -xpf "$fromImage"
-          # Do not import the base image configuration and manifest
+          config=$(jq -r '.[0].Config' image/manifest.json)
-          chmod a+w image image/*.json
+          layers=$(jq -r '.[0].Layers | join(" ")' image/manifest.json)
-          rm -f image/*.json
+          for l in $layers; do
-
+            ls_tar image/$l >> baseFiles
-          if [[ -z "$fromImageName" ]]; then
-            fromImageName=$(jshon -k < image/repositories|head -n1)
-          fi
-          if [[ -z "$fromImageTag" ]]; then
-            fromImageTag=$(jshon -e $fromImageName -k \
-                           < image/repositories|head -n1)
-          fi
-          parentID=$(jshon -e $fromImageName -e $fromImageTag -u \
-                     < image/repositories)
-
-          for l in image/*/layer.tar; do
-            ls_tar $l >> baseFiles
           done
+          chmod u+w image image/$config
+          rm image/$config
         fi
 
         chmod -R ug+rw image
@@ -507,47 +483,28 @@ rec {
         tar -rpf temp/layer.tar --mtime="@$SOURCE_DATE_EPOCH" \
           --owner=0 --group=0 --no-recursion --files-from newFiles
 
-        echo "Adding meta..."
+        gzip temp/layer.tar
+        layerID="sha256:$(sha256sum temp/layer.tar.gz | cut -d ' ' -f 1)"
+        mv temp/layer.tar.gz image/$layerID
 
-        # If we have a parentID, add it to the json metadata.
+        echo "Generating image configuration and manifest..."
-        if [[ -n "$parentID" ]]; then
-          cat temp/json | jshon -s "$parentID" -i parent > tmpjson
-          mv tmpjson temp/json
-        fi
-
-        # Take the sha256 sum of the generated json and use it as the layer ID.
-        # Compute the size and add it to the json under the 'Size' field.
-        layerID=$(sha256sum temp/json|cut -d ' ' -f 1)
-        size=$(stat --printf="%s" temp/layer.tar)
-        cat temp/json | jshon -s "$layerID" -i id -n $size -i Size > tmpjson
-        mv tmpjson temp/json
-
-        # Use the temp folder we've been working on to create a new image.
-        mv temp image/$layerID
-
-        # Create image json and image manifest
         imageJson=$(cat ${baseJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}")
         manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]")
-        currentID=$layerID
-        while [[ -n "$currentID" ]]; do
-          layerChecksum=$(sha256sum image/$currentID/layer.tar | cut -d ' ' -f1)
-          imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"${created}\"}] + .")
-          imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .")
-          manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$currentID/layer.tar\"] + .")
 
-          currentID=$(cat image/$currentID/json | (jshon -e parent -u 2>/dev/null || true))
+        # The layer list is ordered starting from the base image
+        layers=$(echo $layers $layerID)
+        for i in $(echo $layers); do
+          imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"${created}\"}] + .")
+          diffId=$(gzip -dc image/$i | sha256sum | cut -d" " -f1)
+          imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$diffId\"] + .")
+          manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$i\"] + .")
         done
 
         imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1)
-        echo "$imageJson" > "image/$imageJsonChecksum.json"
+        echo "$imageJson" > "image/sha256:$imageJsonChecksum"
-        manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"$imageJsonChecksum.json\"")
+        manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"sha256:$imageJsonChecksum\"")
         echo "$manifestJson" > image/manifest.json
 
-        # Store the json under the name image/repositories.
-        jshon -n object \
-          -n object -s "$layerID" -i "$imageTag" \
-          -i "$imageName" > image/repositories
-
         # Make the image read-only.
         chmod -R a-w image