docker: 19.03.4 -> 20.10.2 (#108960)

This commit refactors the build process to handle Docker engine and
CLI split.

nixos/modules/virtualisation/docker.nix

@@ -155,19 +155,17 @@ in
       users.groups.docker.gid = config.ids.gids.docker;
       systemd.packages = [ cfg.package ];
 
-      # TODO: remove once docker 20.10 is released
-      systemd.enableUnifiedCgroupHierarchy = false;
-
       systemd.services.docker = {
         wantedBy = optional cfg.enableOnBoot "multi-user.target";
         environment = proxy_env;
         serviceConfig = {
+          Type = "notify";
           ExecStart = [
             ""
             ''
               ${cfg.package}/bin/dockerd \
                 --group=docker \
-                --host=fd:// \
+                --host=unix:// \
                 --log-driver=${cfg.logDriver} \
                 ${optionalString (cfg.storageDriver != null) "--storage-driver=${cfg.storageDriver}"} \
                 ${optionalString cfg.liveRestore "--live-restore" } \

pkgs/applications/virtualization/docker/default.nix

@@ -1,5 +1,5 @@
 { stdenv, lib, fetchFromGitHub, fetchpatch, buildGoPackage
-, makeWrapper, installShellFiles, pkgconfig
+, makeWrapper, installShellFiles, pkg-config
 , go-md2man, go, containerd, runc, docker-proxy, tini, libtool
 , sqlite, iproute, lvm2, systemd
 , btrfs-progs, iptables, e2fsprogs, xz, util-linux, xfsprogs, git
@@ -12,6 +12,7 @@ with lib;
 rec {
   dockerGen = {
       version, rev, sha256
+      , mobyRev, mobySha256
       , runcRev, runcSha256
       , containerdRev, containerdSha256
       , tiniRev, tiniSha256
@@ -30,9 +31,7 @@ rec {
       patches = [];
     });
 
-    docker-containerd = let
+    docker-containerd = containerd.overrideAttrs (oldAttrs: {
-      withlibseccomp = lib.versionAtLeast version "19.03";
-    in containerd.overrideAttrs (oldAttrs: {
       name = "docker-containerd-${version}";
       inherit version;
       src = fetchFromGitHub {
@@ -41,11 +40,7 @@ rec {
         rev = containerdRev;
         sha256 = containerdSha256;
       };
-      # disable completion, can be removed when docker uses containerd >= 1.4
+      buildInputs = oldAttrs.buildInputs ++ [ libseccomp ];
-      postInstall = [];
-      # This should be removed once Docker uses containerd >=1.4
-      nativeBuildInputs = oldAttrs.nativeBuildInputs ++ lib.optional withlibseccomp pkgconfig;
-      buildInputs = oldAttrs.buildInputs ++ lib.optional withlibseccomp libseccomp;
     });
 
     docker-tini = tini.overrideAttrs  (oldAttrs: {
@@ -64,16 +59,69 @@ rec {
 
       NIX_CFLAGS_COMPILE = "-DMINIMAL=ON";
     });
+
+    moby = buildGoPackage ((optionalAttrs (stdenv.isLinux)) rec {
+      name = "moby-${version}";
+      inherit version;
+      inherit docker-runc docker-containerd docker-proxy docker-tini;
+
+      src = fetchFromGitHub {
+        owner = "moby";
+        repo = "moby";
+        rev = mobyRev;
+        sha256 = mobySha256;
+      };
+
+      goPackagePath = "github.com/docker/docker";
+
+      nativeBuildInputs = [ makeWrapper pkg-config go-md2man go libtool installShellFiles ];
+      buildInputs = [ sqlite lvm2 btrfs-progs systemd libseccomp ];
+
+      extraPath = optionals (stdenv.isLinux) (makeBinPath [ iproute iptables e2fsprogs xz xfsprogs procps util-linux git ]);
+
+      buildPhase = ''
+        export GOCACHE="$TMPDIR/go-cache"
+        # build engine
+        cd ./go/src/${goPackagePath}
+        export AUTO_GOPATH=1
+        export DOCKER_GITCOMMIT="${rev}"
+        export VERSION="${version}"
+        ./hack/make.sh dynbinary
+        cd -
+      '';
+
+      postPatch = ''
+        patchShebangs .
+        substituteInPlace ./hack/make.sh --replace libsystemd-journal libsystemd
+      '';
+
+      installPhase = ''
+        cd ./go/src/${goPackagePath}
+        install -Dm755 ./bundles/dynbinary-daemon/dockerd $out/libexec/docker/dockerd
+
+        makeWrapper $out/libexec/docker/dockerd $out/bin/dockerd \
+          --prefix PATH : "$out/libexec/docker:$extraPath"
+
+        ln -s ${docker-containerd}/bin/containerd $out/libexec/docker/containerd
+        ln -s ${docker-containerd}/bin/containerd-shim $out/libexec/docker/containerd-shim
+        ln -s ${docker-runc}/bin/runc $out/libexec/docker/runc
+        ln -s ${docker-proxy}/bin/docker-proxy $out/libexec/docker/docker-proxy
+        ln -s ${docker-tini}/bin/tini-static $out/libexec/docker/docker-init
+
+        # systemd
+        install -Dm644 ./contrib/init/systemd/docker.service $out/etc/systemd/system/docker.service
+      '';
+
+      DOCKER_BUILDTAGS = []
+        ++ optional (systemd != null) [ "journald" ]
+        ++ optional (btrfs-progs == null) "exclude_graphdriver_btrfs"
+        ++ optional (lvm2 == null) "exclude_graphdriver_devicemapper"
+        ++ optional (libseccomp != null) "seccomp";
+    });
   in
     buildGoPackage ((optionalAttrs (stdenv.isLinux) {
 
-    inherit docker-runc docker-containerd docker-proxy docker-tini;
+    inherit docker-runc docker-containerd docker-proxy docker-tini moby;
-
-    DOCKER_BUILDTAGS = []
-      ++ optional (systemd != null) [ "journald" ]
-      ++ optional (btrfs-progs == null) "exclude_graphdriver_btrfs"
-      ++ optional (lvm2 == null) "exclude_graphdriver_devicemapper"
-      ++ optional (libseccomp != null) "seccomp";
 
    }) // rec {
     inherit version rev;
@@ -82,97 +130,61 @@ rec {
 
     src = fetchFromGitHub {
       owner = "docker";
-      repo = "docker-ce";
+      repo = "cli";
       rev = "v${version}";
       sha256 = sha256;
     };
 
-    patches = lib.optional (versionAtLeast version "19.03") [
+    goPackagePath = "github.com/docker/cli";
-      # Replace hard-coded cross-compiler with $CC
-      (fetchpatch {
-        url = https://github.com/docker/docker-ce/commit/2fdfb4404ab811cb00227a3de111437b829e55cf.patch;
-        sha256 = "1af20bzakhpfhaixc29qnl9iml9255xdinxdnaqp4an0n1xa686a";
-      })
-    ];
 
-    goPackagePath = "github.com/docker/docker-ce";
+    nativeBuildInputs = [ pkg-config go-md2man go libtool installShellFiles ];
-
-    nativeBuildInputs = [ pkgconfig go-md2man go libtool installShellFiles ];
     buildInputs = [
       makeWrapper
     ] ++ optionals (stdenv.isLinux) [
       sqlite lvm2 btrfs-progs systemd libseccomp
     ];
 
-    dontStrip = true;
+    # Keep eyes on BUILDTIME format - https://github.com/docker/cli/blob/${version}/scripts/build/.variables
-
     buildPhase = ''
       export GOCACHE="$TMPDIR/go-cache"
-    '' + (optionalString (stdenv.isLinux) ''
+
-      # build engine
+      cd ./go/src/${goPackagePath}
-      cd ./go/src/${goPackagePath}/components/engine
-      export AUTO_GOPATH=1
-      export DOCKER_GITCOMMIT="${rev}"
-      export VERSION="${version}"
-      ./hack/make.sh dynbinary
-      cd -
-    '') + ''
-      # build cli
-      cd ./go/src/${goPackagePath}/components/cli
       # Mimic AUTO_GOPATH
       mkdir -p .gopath/src/github.com/docker/
       ln -sf $PWD .gopath/src/github.com/docker/cli
       export GOPATH="$PWD/.gopath:$GOPATH"
       export GITCOMMIT="${rev}"
       export VERSION="${version}"
+      export BUILDTIME="1970-01-01T00:00:00Z"
       source ./scripts/build/.variables
       export CGO_ENABLED=1
       go build -tags pkcs11 --ldflags "$LDFLAGS" github.com/docker/cli/cmd/docker
       cd -
     '';
 
-    # systemd 230 no longer has libsystemd-journal as a separate entity from libsystemd
     postPatch = ''
-      substituteInPlace ./components/cli/scripts/build/.variables --replace "set -eu" ""
-    '' + optionalString (stdenv.isLinux) ''
       patchShebangs .
-      substituteInPlace ./components/engine/hack/make.sh                   --replace libsystemd-journal libsystemd
+      substituteInPlace ./scripts/build/.variables --replace "set -eu" ""
-      substituteInPlace ./components/engine/daemon/logger/journald/read.go --replace libsystemd-journal libsystemd
+      substituteInPlace ./scripts/docs/generate-man.sh --replace "-v md2man" "-v go-md2man"
+      substituteInPlace ./man/md2man-all.sh            --replace md2man go-md2man
     '';
 
     outputs = ["out" "man"];
 
-    extraPath = optionals (stdenv.isLinux) (makeBinPath [ iproute iptables e2fsprogs xz xfsprogs procps util-linux git ]);
-
     installPhase = ''
       cd ./go/src/${goPackagePath}
-      install -Dm755 ./components/cli/docker $out/libexec/docker/docker
+      install -Dm755 ./docker $out/libexec/docker/docker
 
       makeWrapper $out/libexec/docker/docker $out/bin/docker \
         --prefix PATH : "$out/libexec/docker:$extraPath"
     '' + optionalString (stdenv.isLinux) ''
-      install -Dm755 ./components/engine/bundles/dynbinary-daemon/dockerd $out/libexec/docker/dockerd
+      # symlink docker daemon to docker cli derivation
-
+      ln -s ${moby}/bin/dockerd $out/bin/dockerd
-      makeWrapper $out/libexec/docker/dockerd $out/bin/dockerd \
-        --prefix PATH : "$out/libexec/docker:$extraPath"
-
-      # docker uses containerd now
-      ln -s ${docker-containerd}/bin/containerd $out/libexec/docker/containerd
-      ln -s ${docker-containerd}/bin/containerd-shim $out/libexec/docker/containerd-shim
-      ln -s ${docker-runc}/bin/runc $out/libexec/docker/runc
-      ln -s ${docker-proxy}/bin/docker-proxy $out/libexec/docker/docker-proxy
-      ln -s ${docker-tini}/bin/tini-static $out/libexec/docker/docker-init
-
-      # systemd
-      install -Dm644 ./components/engine/contrib/init/systemd/docker.service $out/etc/systemd/system/docker.service
     '' + ''
       # completion (cli)
-      installShellCompletion --bash ./components/cli/contrib/completion/bash/docker
+      installShellCompletion --bash ./contrib/completion/bash/docker
-      installShellCompletion --fish ./components/cli/contrib/completion/fish/docker.fish
+      installShellCompletion --fish ./contrib/completion/fish/docker.fish
-      installShellCompletion --zsh ./components/cli/contrib/completion/zsh/_docker
+      installShellCompletion --zsh  ./contrib/completion/zsh/_docker
-
-      # Include contributed man pages (cli)
-      cd ./components/cli
     '' + lib.optionalString (stdenv.hostPlatform == stdenv.buildPlatform) ''
       # Generate man pages from cobra commands
       echo "Generate man pages from cobra"
@@ -199,30 +211,18 @@ rec {
   });
 
   # Get revisions from
-  # https://github.com/docker/docker-ce/tree/${version}/components/engine/hack/dockerfile/install/*
+  # https://github.com/moby/moby/tree/${version}/hack/dockerfile/install/*
-
+  docker_20_10 = makeOverridable dockerGen rec {
-  docker_18_09 = makeOverridable dockerGen rec {
+    version = "20.10.2";
-    version = "18.09.9";
     rev = "v${version}";
-    sha256 = "0wqhjx9qs96q2jd091wffn3cyv2aslqn2cvpdpgljk8yr9s0yg7h";
+    sha256 = "0z0hpm5hrqh7p8my8lmiwpym2shs48my6p0zv2cc34wym0hcly51";
-    runcRev = "3e425f80a8c931f88e6d94a8c831b9d5aa481657";
+    mobyRev = "v${version}";
-    runcSha256 = "18psc830b2rkwml1x6vxngam5b5wi3pj14mw817rshpzy87prspj";
+    mobySha256 = "0c2zycpnwj4kh8m8xckv1raj3fx07q9bfaj46rr85jihm4p2dp5w";
-    containerdRev = "894b81a4b802e4eb2a91d1ce216b8817763c29fb";
+    runcRev = "ff819c7e9184c13b7c2607fe6c30ae19403a7aff"; # v1.0.0-rc92
-    containerdSha256 = "0sp5mn5wd3xma4svm6hf67hyhiixzkzz6ijhyjkwdrc4alk81357";
+    runcSha256 = "0r4zbxbs03xr639r7848282j1ybhibfdhnxyap9p76j5w8ixms94";
-    tiniRev = "fec3683b971d9c3ef73f284f176672c44b448662";
+    containerdRev = "269548fa27e0089a8b8278fc4fc781d7f65a939b"; # v1.4.3
-    tiniSha256 = "1h20i3wwlbd8x4jr2gz68hgklh0lb0jj7y5xk1wvr8y58fip1rdn";
+    containerdSha256 = "09xvhjg5f8h90w1y94kqqnqzhbhd62dcdd9wb9sdqakisjk6zrl0";
-  };
+    tiniRev = "de40ad007797e0dcd8b7126f27bb87401d224240"; # v0.19.0
-
-  docker_19_03 = makeOverridable dockerGen rec {
-    version = "19.03.14";
-    rev = "v${version}";
-    sha256 = "0szr5dgfrypb5kyj5l1rf7rw4iqj0d0cyx6skdqlbgf4dqwa6g9y";
-    runcRev = "dc9208a3303feef5b3839f4323d9beb36df0a9dd"; # v1.0.0-rc10
-    runcSha256 = "0pi3rvj585997m4z9ljkxz2z9yxf9p2jr0pmqbqrc7bc95f5hagk";
-    # Note: Once all packaged Docker versions use containerd <=1.2 or >=1.4 remove the libseccomp and pkgconfig inputs above
-    containerdRev = "ea765aba0d05254012b0b9e595e995c09186427f"; # v1.3.9
-    containerdSha256 = "1isi1wgq61b4l0lxy1d8n6dnmcb8s5ihn2yqjb6525y3dj5c5i1j";
-    tiniRev = "fec3683b971d9c3ef73f284f176672c44b448662"; # v0.18.0
     tiniSha256 = "1h20i3wwlbd8x4jr2gz68hgklh0lb0jj7y5xk1wvr8y58fip1rdn";
   };
 }

pkgs/top-level/all-packages.nix

@@ -21302,11 +21302,10 @@ in
   };
 
   inherit (callPackage ../applications/virtualization/docker {})
-    docker_18_09
+    docker_20_10;
-    docker_19_03;
 
-  docker = docker_19_03;
+  docker = docker_20_10;
-  docker-edge = docker_19_03;
+  docker-edge = docker_20_10;
 
   docker-proxy = callPackage ../applications/virtualization/docker/proxy.nix { };