From ecc293ff7aa443f95117294e31e7a670e76cd6c7 Mon Sep 17 00:00:00 2001
From: Viktor Kronvall <viktor.kronvall@gmail.com>
Date: Tue, 30 Mar 2021 16:04:47 +0900
Subject: [PATCH 1/6] dockerTools: Implement merging of image tarballs

The `docker load` command supports loading tarballs that contain
multiple docker images with their respective image names and tags. This
enables distributing these images as a single file which simplifies the
release of software when an application requires multiple services to
run.

However, pkgs.dockerTools only create tarballs with a single docker
image and there exists is no mechanism in nixpkgs to combine the created
tarballs. This commit implements merging of tarballs in a way that is
compatible with `docker load`.
---
 pkgs/build-support/docker/default.nix | 28 +++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix
index b03bfcca87f..aebdb4a6fd8 100644
--- a/pkgs/build-support/docker/default.nix
+++ b/pkgs/build-support/docker/default.nix
@@ -682,6 +682,34 @@ rec {
     in
     result;
 
+  # Merge the tarballs of two images built with buildImage into a single
+  # tarball that contains both images. Running `docker load` on the resulting
+  # tarball with load both images into the docker daemon.
+  mergeImages = a: b: runCommand "merge-docker-images"
+    {
+      nativeBuildInputs = [ pigz jq ];
+    } ''
+    mkdir a b image
+    # Extract images
+    tar -I pigz -xf ${a} -C a
+    tar -I pigz -xf ${b} -C b
+    # Make writable (to enable mv)
+    chmod -R +w a b
+    # Merge repositories objects (image:tag -> hash)
+    jq -s add a/repositories b/repositories > repositories
+    # Merge docker images manifests ([image])
+    jq -s add a/manifest.json b/manifest.json > manifest.json
+    # Move layers to output directory
+    mv --no-clobber a/* image/
+    mv --no-clobber b/* image/
+    # Move merged repositories object and manifest list to output directory
+    mv repositories image/repositories
+    mv manifest.json image/manifest.json
+    # Create tarball and gzip
+    tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | pigz -nT > $out
+  '';
+
+
   # Provide a /etc/passwd and /etc/group that contain root and nobody.
   # Useful when packaging binaries that insist on using nss to look up
   # username/groups (like nginx).

From dcc9aef015ecf43917b36be45c911ca565f4f6a1 Mon Sep 17 00:00:00 2001
From: Viktor Kronvall <viktor.kronvall@gmail.com>
Date: Tue, 30 Mar 2021 18:31:25 +0900
Subject: [PATCH 2/6] dockerTools: take a list of images in mergeImages

---
 pkgs/build-support/docker/default.nix | 33 +++++++++++++--------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix
index aebdb4a6fd8..36e8f3ad226 100644
--- a/pkgs/build-support/docker/default.nix
+++ b/pkgs/build-support/docker/default.nix
@@ -682,27 +682,26 @@ rec {
     in
     result;
 
-  # Merge the tarballs of two images built with buildImage into a single
-  # tarball that contains both images. Running `docker load` on the resulting
-  # tarball with load both images into the docker daemon.
-  mergeImages = a: b: runCommand "merge-docker-images"
+  # Merge the tarballs of images built with buildImage into a single
+  # tarball that contains all images. Running `docker load` on the resulting
+  # tarball will load the images into the docker daemon.
+  mergeImages = images: runCommand "merge-docker-images"
     {
+      inherit images;
       nativeBuildInputs = [ pigz jq ];
     } ''
-    mkdir a b image
+    mkdir image inputs
     # Extract images
-    tar -I pigz -xf ${a} -C a
-    tar -I pigz -xf ${b} -C b
-    # Make writable (to enable mv)
-    chmod -R +w a b
-    # Merge repositories objects (image:tag -> hash)
-    jq -s add a/repositories b/repositories > repositories
-    # Merge docker images manifests ([image])
-    jq -s add a/manifest.json b/manifest.json > manifest.json
-    # Move layers to output directory
-    mv --no-clobber a/* image/
-    mv --no-clobber b/* image/
-    # Move merged repositories object and manifest list to output directory
+    for item in $images; do
+      mkdir inputs/$(basename $item)
+      tar -I pigz -xf $item -C inputs/$(basename $item)
+    done
+    # Copy all layers from input images to output image directory
+    cp -R --no-clobber inputs/*/* image/
+    # Merge repositories objects and manifests
+    jq -s add inputs/*/repositories > repositories
+    jq -s add inputs/*/manifest.json > manifest.json
+    # Replace output image repositories and manifest with merged versions
     mv repositories image/repositories
     mv manifest.json image/manifest.json
     # Create tarball and gzip

From b2aa1f9d7a4f5474ce53cd4ce697a8d1abecc32b Mon Sep 17 00:00:00 2001
From: Viktor Kronvall <viktor.kronvall@gmail.com>
Date: Wed, 31 Mar 2021 02:24:44 +0900
Subject: [PATCH 3/6] dockerTools: preserve order of images in manifest

---
 pkgs/build-support/docker/default.nix | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix
index 36e8f3ad226..0e1aa179cab 100644
--- a/pkgs/build-support/docker/default.nix
+++ b/pkgs/build-support/docker/default.nix
@@ -692,15 +692,24 @@ rec {
     } ''
     mkdir image inputs
     # Extract images
+    repos=()
+    manifests=()
     for item in $images; do
-      mkdir inputs/$(basename $item)
-      tar -I pigz -xf $item -C inputs/$(basename $item)
+      name=$(basename $item)
+      mkdir inputs/$name
+      tar -I pigz -xf $item -C inputs/$name
+      if [ -f inputs/$name/repositories ]; then
+        repos+=(inputs/$name/repositories)
+      fi
+      if [ -f inputs/$name/manifest.json ]; then
+        manifests+=(inputs/$name/manifest.json)
+      fi
     done
     # Copy all layers from input images to output image directory
     cp -R --no-clobber inputs/*/* image/
     # Merge repositories objects and manifests
-    jq -s add inputs/*/repositories > repositories
-    jq -s add inputs/*/manifest.json > manifest.json
+    jq -s add "''${repos[@]}" > repositories
+    jq -s add "''${manifests[@]}" > manifest.json
     # Replace output image repositories and manifest with merged versions
     mv repositories image/repositories
     mv manifest.json image/manifest.json

From 5caed960d36a3f7b442b96e9664d243fd9ab1317 Mon Sep 17 00:00:00 2001
From: Viktor Kronvall <viktor.kronvall@gmail.com>
Date: Tue, 30 Mar 2021 18:37:17 +0900
Subject: [PATCH 4/6] dockerTools: add merged example images

---
 pkgs/build-support/docker/examples.nix | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix
index f6b468942fe..7dbee38feeb 100644
--- a/pkgs/build-support/docker/examples.nix
+++ b/pkgs/build-support/docker/examples.nix
@@ -497,4 +497,23 @@ rec {
       chown 1000 ./home/jane
     '';
   };
+
+  # tarball consisting of both bash and redis images
+  mergedBashAndRedis = pkgs.dockerTools.mergeImages [
+    bash
+    redis
+  ];
+
+  # tarball consisting of bash (without tag) and redis images
+  mergedBashNoTagAndRedis = pkgs.dockerTools.mergeImages [
+    bashNoTag
+    redis
+  ];
+
+  # tarball consisting of bash and layered image with different owner of the
+  # /home/jane directory
+  mergedBashFakeRoot = pkgs.dockerTools.mergeImages [
+    bash
+    layeredImageWithFakeRootCommands
+  ];
 }

From bcc3f8eab5cac2c6ddaeab9d1fdc7afc2c348964 Mon Sep 17 00:00:00 2001
From: Viktor Kronvall <viktor.kronvall@gmail.com>
Date: Tue, 30 Mar 2021 18:53:29 +0900
Subject: [PATCH 5/6] dockerTools: test mergeImages

---
 nixos/tests/docker-tools.nix | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/nixos/tests/docker-tools.nix b/nixos/tests/docker-tools.nix
index 80d527b453f..35b5504ea0f 100644
--- a/nixos/tests/docker-tools.nix
+++ b/nixos/tests/docker-tools.nix
@@ -321,5 +321,40 @@ import ./make-test-python.nix ({ pkgs, ... }: {
         docker.succeed(
             "docker run --rm ${examples.layeredImageWithFakeRootCommands.imageName} sh -c 'stat -c '%u' /home/jane | grep -E ^1000$'"
         )
+
+    with subtest("Ensure docker load on merged images loads all of the constituent images"):
+        docker.succeed(
+            "docker load --input='${examples.mergedBashAndRedis}'"
+        )
+        docker.succeed(
+            "docker images --format '{{.Repository}}-{{.Tag}}' | grep -F '${examples.bash.imageName}-${examples.bash.imageTag}'"
+        )
+        docker.succeed(
+            "docker images --format '{{.Repository}}-{{.Tag}}' | grep -F '${examples.redis.imageName}-${examples.redis.imageTag}'"
+        )
+        docker.succeed("docker run --rm ${examples.bash.imageName} bash --version")
+        docker.succeed("docker run --rm ${examples.redis.imageName} redis-cli --version")
+        docker.succeed("docker rmi ${examples.bash.imageName}")
+        docker.succeed("docker rmi ${examples.redis.imageName}")
+
+    with subtest(
+        "Ensure docker load on merged images loads all of the constituent images (missing tags)"
+    ):
+        docker.succeed(
+            "docker load --input='${examples.mergedBashNoTagAndRedis}'"
+        )
+        docker.succeed(
+            "docker images --format '{{.Repository}}-{{.Tag}}' | grep -F '${examples.bashNoTag.imageName}-${examples.bashNoTag.imageTag}'"
+        )
+        docker.succeed(
+            "docker images --format '{{.Repository}}-{{.Tag}}' | grep -F '${examples.redis.imageName}-${examples.redis.imageTag}'"
+        )
+        # we need to explicitly specify the generated tag here
+        docker.succeed(
+            "docker run --rm ${examples.bashNoTag.imageName}:${examples.bashNoTag.imageTag} bash --version"
+        )
+        docker.succeed("docker run --rm ${examples.redis.imageName} redis-cli --version")
+        docker.succeed("docker rmi ${examples.bashNoTag.imageName}:${examples.bashNoTag.imageTag}")
+        docker.succeed("docker rmi ${examples.redis.imageName}")
   '';
 })

From 10019ab87995ec72e62af95a8b056db9a2e8a341 Mon Sep 17 00:00:00 2001
From: Viktor Kronvall <viktor.kronvall@gmail.com>
Date: Wed, 7 Apr 2021 23:05:36 +0900
Subject: [PATCH 6/6] dockerTools: test that mergeImages preserves owner

---
 nixos/tests/docker-tools.nix | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/nixos/tests/docker-tools.nix b/nixos/tests/docker-tools.nix
index 35b5504ea0f..96662b4540c 100644
--- a/nixos/tests/docker-tools.nix
+++ b/nixos/tests/docker-tools.nix
@@ -356,5 +356,13 @@ import ./make-test-python.nix ({ pkgs, ... }: {
         docker.succeed("docker run --rm ${examples.redis.imageName} redis-cli --version")
         docker.succeed("docker rmi ${examples.bashNoTag.imageName}:${examples.bashNoTag.imageTag}")
         docker.succeed("docker rmi ${examples.redis.imageName}")
+
+    with subtest("mergeImages preserves owners of the original images"):
+        docker.succeed(
+            "docker load --input='${examples.mergedBashFakeRoot}'"
+        )
+        docker.succeed(
+            "docker run --rm ${examples.layeredImageWithFakeRootCommands.imageName} sh -c 'stat -c '%u' /home/jane | grep -E ^1000$'"
+        )
   '';
 })