Merge pull request #114737 from johanot/kubernetes-1.20-with-containerd

kubernetes: 1.19.5 -> 1.20.4 (dockerd -> containerd)

nixos/doc/manual/release-notes/rl-2105.xml

@@ -788,6 +788,15 @@ self: super:
      and use Maturin as their build tool.
     </para>
    </listitem>
+   <listitem>
+    <para>
+     Kubernetes has <link xlink:href="https://kubernetes.io/blog/2020/12/02/dont-panic-kubernetes-and-docker/">deprecated docker</link> as container runtime.
+     As a consequence, the Kubernetes module now has support for configuration of custom remote container runtimes and enables containerd by default.
+     Note that containerd is more strict regarding container image OCI-compliance.
+     As an example, images with CMD or ENTRYPOINT defined as strings (not lists) will fail on containerd, while working fine on docker.
+     Please test your setup and container images with containerd prior to upgrading.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 </section>

nixos/modules/module-list.nix

@@ -1053,6 +1053,7 @@
   ./testing/service-runner.nix
   ./virtualisation/anbox.nix
   ./virtualisation/container-config.nix
+  ./virtualisation/containerd.nix
   ./virtualisation/containers.nix
   ./virtualisation/nixos-containers.nix
   ./virtualisation/oci-containers.nix

nixos/modules/services/cluster/kubernetes/addons/dns.nix

@@ -3,7 +3,7 @@
 with lib;
 
 let
-  version = "1.6.4";
+  version = "1.7.1";
   cfg = config.services.kubernetes.addons.dns;
   ports = {
     dns = 10053;
@@ -55,9 +55,9 @@ in {
       type = types.attrs;
       default = {
         imageName = "coredns/coredns";
-        imageDigest = "sha256:493ee88e1a92abebac67cbd4b5658b4730e0f33512461442d8d9214ea6734a9b";
+        imageDigest = "sha256:4a6e0769130686518325b21b0c1d0688b54e7c79244d48e1b15634e98e40c6ef";
         finalImageTag = version;
-        sha256 = "0fm9zdjavpf5hni8g7fkdd3csjbhd7n7py7llxjc66sbii087028";
+        sha256 = "02r440xcdsgi137k5lmmvp0z5w5fmk8g9mysq5pnysq1wl8sj6mw";
       };
     };
   };
@@ -156,7 +156,6 @@ in {
             health :${toString ports.health}
             kubernetes ${cfg.clusterDomain} in-addr.arpa ip6.arpa {
               pods insecure
-              upstream
               fallthrough in-addr.arpa ip6.arpa
             }
             prometheus :${toString ports.metrics}

nixos/modules/services/cluster/kubernetes/apiserver.nix

@@ -238,14 +238,40 @@ in
       type = int;
     };
 
+    apiAudiences = mkOption {
+      description = ''
+        Kubernetes apiserver ServiceAccount issuer.
+      '';
+      default = "api,https://kubernetes.default.svc";
+      type = str;
+    };
+
+    serviceAccountIssuer = mkOption {
+      description = ''
+        Kubernetes apiserver ServiceAccount issuer.
+      '';
+      default = "https://kubernetes.default.svc";
+      type = str;
+    };
+
+    serviceAccountSigningKeyFile = mkOption {
+      description = ''
+        Path to the file that contains the current private key of the service
+        account token issuer. The issuer will sign issued ID tokens with this
+        private key.
+      '';
+      type = path;
+    };
+
     serviceAccountKeyFile = mkOption {
       description = ''
-        Kubernetes apiserver PEM-encoded x509 RSA private or public key file,
+        File containing PEM-encoded x509 RSA or ECDSA private or public keys,
-        used to verify ServiceAccount tokens. By default tls private key file
+        used to verify ServiceAccount tokens. The specified file can contain
-        is used.
+        multiple keys, and the flag can be specified multiple times with
+        different files. If unspecified, --tls-private-key-file is used.
+        Must be specified when --service-account-signing-key is provided
       '';
-      default = null;
+      type = path;
-      type = nullOr path;
     };
 
     serviceClusterIpRange = mkOption {
@@ -357,8 +383,10 @@ in
               ${optionalString (cfg.runtimeConfig != "")
                 "--runtime-config=${cfg.runtimeConfig}"} \
               --secure-port=${toString cfg.securePort} \
-              ${optionalString (cfg.serviceAccountKeyFile!=null)
+              --api-audiences=${toString cfg.apiAudiences} \
-                "--service-account-key-file=${cfg.serviceAccountKeyFile}"} \
+              --service-account-issuer=${toString cfg.serviceAccountIssuer} \
+              --service-account-signing-key-file=${cfg.serviceAccountSigningKeyFile} \
+              --service-account-key-file=${cfg.serviceAccountKeyFile} \
               --service-cluster-ip-range=${cfg.serviceClusterIpRange} \
               --storage-backend=${cfg.storageBackend} \
               ${optionalString (cfg.tlsCertFile != null)

nixos/modules/services/cluster/kubernetes/default.nix

@@ -5,6 +5,29 @@ with lib;
 let
   cfg = config.services.kubernetes;
 
+  defaultContainerdConfigFile = pkgs.writeText "containerd.toml" ''
+    version = 2
+    root = "/var/lib/containerd/daemon"
+    state = "/var/run/containerd/daemon"
+    oom_score = 0
+
+    [grpc]
+      address = "/var/run/containerd/containerd.sock"
+
+    [plugins."io.containerd.grpc.v1.cri"]
+      sandbox_image = "pause:latest"
+
+    [plugins."io.containerd.grpc.v1.cri".cni]
+      bin_dir = "/opt/cni/bin"
+      max_conf_num = 0
+
+    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+      runtime_type = "io.containerd.runc.v2"
+
+    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."io.containerd.runc.v2".options]
+      SystemdCgroup = true
+  '';
+
   mkKubeConfig = name: conf: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON {
     apiVersion = "v1";
     kind = "Config";
@@ -222,14 +245,9 @@ in {
     })
 
     (mkIf cfg.kubelet.enable {
-      virtualisation.docker = {
+      virtualisation.containerd = {
         enable = mkDefault true;
-
+        configFile = mkDefault defaultContainerdConfigFile;
-        # kubernetes needs access to logs
-        logDriver = mkDefault "json-file";
-
-        # iptables must be disabled for kubernetes
-        extraOptions = "--iptables=false --ip-masq=false";
       };
     })
 
@@ -269,7 +287,6 @@ in {
       users.users.kubernetes = {
         uid = config.ids.uids.kubernetes;
         description = "Kubernetes user";
-        extraGroups = [ "docker" ];
         group = "kubernetes";
         home = cfg.dataDir;
         createHome = true;

nixos/modules/services/cluster/kubernetes/flannel.nix

@@ -8,16 +8,6 @@ let
 
   # we want flannel to use kubernetes itself as configuration backend, not direct etcd
   storageBackend = "kubernetes";
-
-  # needed for flannel to pass options to docker
-  mkDockerOpts = pkgs.runCommand "mk-docker-opts" {
-    buildInputs = [ pkgs.makeWrapper ];
-  } ''
-    mkdir -p $out
-
-    # bashInteractive needed for `compgen`
-    makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "${pkgs.kubernetes}/bin/mk-docker-opts.sh"
-  '';
 in
 {
   ###### interface
@@ -43,43 +33,17 @@ in
         cniVersion = "0.3.1";
         delegate = {
           isDefaultGateway = true;
-          bridge = "docker0";
+          bridge = "mynet";
         };
       }];
     };
 
-    systemd.services.mk-docker-opts = {
-      description = "Pre-Docker Actions";
-      path = with pkgs; [ gawk gnugrep ];
-      script = ''
-        ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
-        systemctl restart docker
-      '';
-      serviceConfig.Type = "oneshot";
-    };
-
-    systemd.paths.flannel-subnet-env = {
-      wantedBy = [ "flannel.service" ];
-      pathConfig = {
-        PathModified = "/run/flannel/subnet.env";
-        Unit = "mk-docker-opts.service";
-      };
-    };
-
-    systemd.services.docker = {
-      environment.DOCKER_OPTS = "-b none";
-      serviceConfig.EnvironmentFile = "-/run/flannel/docker";
-    };
-
-    # read environment variables generated by mk-docker-opts
-    virtualisation.docker.extraOptions = "$DOCKER_OPTS";
-
     networking = {
       firewall.allowedUDPPorts = [
         8285  # flannel udp
         8472  # flannel vxlan
       ];
-      dhcpcd.denyInterfaces = [ "docker*" "flannel*" ];
+      dhcpcd.denyInterfaces = [ "mynet*" "flannel*" ];
     };
 
     services.kubernetes.pki.certs = {

nixos/modules/services/cluster/kubernetes/kubelet.nix

@@ -23,7 +23,7 @@ let
     name = "pause";
     tag = "latest";
     contents = top.package.pause;
-    config.Cmd = "/bin/pause";
+    config.Cmd = ["/bin/pause"];
   };
 
   kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;
@@ -125,6 +125,18 @@ in
       };
     };
 
+    containerRuntime = mkOption {
+      description = "Which container runtime type to use";
+      type = enum ["docker" "remote"];
+      default = "remote";
+    };
+
+    containerRuntimeEndpoint = mkOption {
+      description = "Endpoint at which to find the container runtime api interface/socket";
+      type = str;
+      default = "unix:///var/run/containerd/containerd.sock";
+    };
+
     enable = mkEnableOption "Kubernetes kubelet.";
 
     extraOpts = mkOption {
@@ -235,16 +247,24 @@ in
   ###### implementation
   config = mkMerge [
     (mkIf cfg.enable {
+
+      environment.etc."cni/net.d".source = cniConfig;
+
       services.kubernetes.kubelet.seedDockerImages = [infraContainer];
 
+      boot.kernel.sysctl = {
+        "net.bridge.bridge-nf-call-iptables"  = 1;
+        "net.ipv4.ip_forward"                 = 1;
+        "net.bridge.bridge-nf-call-ip6tables" = 1;
+      };
+
       systemd.services.kubelet = {
         description = "Kubernetes Kubelet Service";
         wantedBy = [ "kubernetes.target" ];
-        after = [ "network.target" "docker.service" "kube-apiserver.service" ];
+        after = [ "containerd.service" "network.target" "kube-apiserver.service" ];
         path = with pkgs; [
           gitMinimal
           openssh
-          docker
           util-linux
           iproute
           ethtool
@@ -254,8 +274,12 @@ in
         ] ++ lib.optional config.boot.zfs.enabled config.boot.zfs.package ++ top.path;
         preStart = ''
           ${concatMapStrings (img: ''
-            echo "Seeding docker image: ${img}"
+            echo "Seeding container image: ${img}"
-            docker load <${img}
+            ${if (lib.hasSuffix "gz" img) then
+              ''${pkgs.gzip}/bin/zcat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import -''
+            else
+              ''${pkgs.coreutils}/bin/cat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import -''
+            }
           '') cfg.seedDockerImages}
 
           rm /opt/cni/bin/* || true
@@ -306,6 +330,9 @@ in
             ${optionalString (cfg.tlsKeyFile != null)
               "--tls-private-key-file=${cfg.tlsKeyFile}"} \
             ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
+            --container-runtime=${cfg.containerRuntime} \
+            --container-runtime-endpoint=${cfg.containerRuntimeEndpoint} \
+            --cgroup-driver=systemd \
             ${cfg.extraOpts}
           '';
           WorkingDirectory = top.dataDir;
@@ -315,7 +342,7 @@ in
       # Allways include cni plugins
       services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins];
 
-      boot.kernelModules = ["br_netfilter"];
+      boot.kernelModules = ["br_netfilter" "overlay"];
 
       services.kubernetes.kubelet.hostname = with config.networking;
         mkDefault (hostName + optionalString (domain != null) ".${domain}");

nixos/modules/services/cluster/kubernetes/pki.nix

@@ -361,6 +361,7 @@ in
           tlsCertFile = mkDefault cert;
           tlsKeyFile = mkDefault key;
           serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert;
+          serviceAccountSigningKeyFile = mkDefault cfg.certs.serviceAccount.key;
           kubeletClientCaFile = mkDefault caCert;
           kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert;
           kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key;

nixos/modules/services/networking/flannel.nix

@@ -162,10 +162,7 @@ in {
         NODE_NAME = cfg.nodeName;
       };
       path = [ pkgs.iptables ];
-      preStart = ''
+      preStart = optionalString (cfg.storageBackend == "etcd") ''
-        mkdir -p /run/flannel
-        touch /run/flannel/docker
-      '' + optionalString (cfg.storageBackend == "etcd") ''
         echo "setting network configuration"
         until ${pkgs.etcdctl}/bin/etcdctl set /coreos.com/network/config '${builtins.toJSON networkConfig}'
         do
@@ -177,6 +174,7 @@ in {
         ExecStart = "${cfg.package}/bin/flannel";
         Restart = "always";
         RestartSec = "10s";
+        RuntimeDirectory = "flannel";
       };
     };
 

nixos/modules/virtualisation/containerd.nix

@@ -0,0 +1,60 @@
+{ pkgs, lib, config, ... }:
+let
+  cfg = config.virtualisation.containerd;
+  containerdConfigChecked = pkgs.runCommand "containerd-config-checked.toml" { nativeBuildInputs = [pkgs.containerd]; } ''
+    containerd -c ${cfg.configFile} config dump >/dev/null
+    ln -s ${cfg.configFile} $out
+  '';
+in
+{
+
+  options.virtualisation.containerd = with lib.types; {
+    enable = lib.mkEnableOption "containerd container runtime";
+
+    configFile = lib.mkOption {
+      default = null;
+      description = "path to containerd config file";
+      type = nullOr path;
+    };
+
+    args = lib.mkOption {
+      default = {};
+      description = "extra args to append to the containerd cmdline";
+      type = attrsOf str;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    virtualisation.containerd.args.config = lib.mkIf (cfg.configFile != null) (toString containerdConfigChecked);
+
+    environment.systemPackages = [pkgs.containerd];
+
+    systemd.services.containerd = {
+      description = "containerd - container runtime";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      path = with pkgs; [
+        containerd
+        runc
+        iptables
+      ];
+      serviceConfig = {
+        ExecStart = ''${pkgs.containerd}/bin/containerd ${lib.concatStringsSep " " (lib.cli.toGNUCommandLine {} cfg.args)}'';
+        Delegate = "yes";
+        KillMode = "process";
+        Type = "notify";
+        Restart = "always";
+        RestartSec = "5";
+        StartLimitBurst = "8";
+        StartLimitIntervalSec = "120s";
+
+        # "limits" defined below are adopted from upstream: https://github.com/containerd/containerd/blob/master/containerd.service
+        LimitNPROC = "infinity";
+        LimitCORE = "infinity";
+        LimitNOFILE = "infinity";
+        TasksMax = "infinity";
+        OOMScoreAdjust = "-999";
+      };
+    };
+  };
+}

nixos/tests/kubernetes/dns.nix

@@ -34,7 +34,7 @@ let
     name = "redis";
     tag = "latest";
     contents = [ pkgs.redis pkgs.bind.host ];
-    config.Entrypoint = "/bin/redis-server";
+    config.Entrypoint = ["/bin/redis-server"];
   };
 
   probePod = pkgs.writeText "probe-pod.json" (builtins.toJSON {
@@ -55,12 +55,11 @@ let
     name = "probe";
     tag = "latest";
     contents = [ pkgs.bind.host pkgs.busybox ];
-    config.Entrypoint = "/bin/tail";
+    config.Entrypoint = ["/bin/tail"];
   };
 
-  extraConfiguration = { config, pkgs, ... }: {
+  extraConfiguration = { config, pkgs, lib, ... }: {
     environment.systemPackages = [ pkgs.bind.host ];
-    # virtualisation.docker.extraOptions = "--dns=${config.services.kubernetes.addons.dns.clusterIp}";
     services.dnsmasq.enable = true;
     services.dnsmasq.servers = [
       "/cluster.local/${config.services.kubernetes.addons.dns.clusterIp}#53"
@@ -77,7 +76,7 @@ let
       # prepare machine1 for test
       machine1.wait_until_succeeds("kubectl get node machine1.${domain} | grep -w Ready")
       machine1.wait_until_succeeds(
-          "docker load < ${redisImage}"
+          "${pkgs.gzip}/bin/zcat ${redisImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
       )
       machine1.wait_until_succeeds(
           "kubectl create -f ${redisPod}"
@@ -86,7 +85,7 @@ let
           "kubectl create -f ${redisService}"
       )
       machine1.wait_until_succeeds(
-          "docker load < ${probeImage}"
+          "${pkgs.gzip}/bin/zcat ${probeImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
       )
       machine1.wait_until_succeeds(
           "kubectl create -f ${probePod}"
@@ -118,7 +117,7 @@ let
       # prepare machines for test
       machine1.wait_until_succeeds("kubectl get node machine2.${domain} | grep -w Ready")
       machine2.wait_until_succeeds(
-          "docker load < ${redisImage}"
+          "${pkgs.gzip}/bin/zcat ${redisImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
       )
       machine1.wait_until_succeeds(
           "kubectl create -f ${redisPod}"
@@ -127,7 +126,7 @@ let
           "kubectl create -f ${redisService}"
       )
       machine2.wait_until_succeeds(
-          "docker load < ${probeImage}"
+          "${pkgs.gzip}/bin/zcat ${probeImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
       )
       machine1.wait_until_succeeds(
           "kubectl create -f ${probePod}"

nixos/tests/kubernetes/rbac.nix

@@ -85,7 +85,7 @@ let
     name = "kubectl";
     tag = "latest";
     contents = [ kubectl pkgs.busybox kubectlPod2 ];
-    config.Entrypoint = "/bin/sh";
+    config.Entrypoint = ["/bin/sh"];
   };
 
   base = {
@@ -97,7 +97,7 @@ let
       machine1.wait_until_succeeds("kubectl get node machine1.my.zyx | grep -w Ready")
 
       machine1.wait_until_succeeds(
-          "docker load < ${kubectlImage}"
+          "${pkgs.gzip}/bin/zcat ${kubectlImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
       )
 
       machine1.wait_until_succeeds(
@@ -134,7 +134,7 @@ let
       machine1.wait_until_succeeds("kubectl get node machine2.my.zyx | grep -w Ready")
 
       machine2.wait_until_succeeds(
-          "docker load < ${kubectlImage}"
+          "${pkgs.gzip}/bin/zcat ${kubectlImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
       )
 
       machine1.wait_until_succeeds(

pkgs/applications/networking/cluster/kubernetes/default.nix

@@ -20,19 +20,21 @@
 
 stdenv.mkDerivation rec {
   pname = "kubernetes";
-  version = "1.19.5";
+  version = "1.20.4";
 
   src = fetchFromGitHub {
     owner = "kubernetes";
     repo = "kubernetes";
     rev = "v${version}";
-    sha256 = "15bv620fj4x731f2z2a9dcdss18rk379kc40g49bpqsdn42jjx2z";
+    sha256 = "0nni351ya688dphdkpyq94p3wjw2kigg85kmalwdpv5wpz1abl5g";
   };
 
   nativeBuildInputs = [ removeReferencesTo makeWrapper which go rsync installShellFiles ];
 
   outputs = [ "out" "man" "pause" ];
 
+  patches = [ ./fixup-addonmanager-lib-path.patch ];
+
   postPatch = ''
     # go env breaks the sandbox
     substituteInPlace "hack/lib/golang.sh" \
@@ -53,7 +55,7 @@ stdenv.mkDerivation rec {
 
   postBuild = ''
     ./hack/update-generated-docs.sh
-    (cd build/pause && cc pause.c -o pause)
+    (cd build/pause/linux && cc pause.c -o pause)
   '';
 
   installPhase = ''
@@ -61,14 +63,19 @@ stdenv.mkDerivation rec {
       install -D _output/local/go/bin/''${p##*/} -t $out/bin
     done
 
-    install -D build/pause/pause -t $pause/bin
+    install -D build/pause/linux/pause -t $pause/bin
     installManPage docs/man/man1/*.[1-9]
 
-    cp cluster/addons/addon-manager/kube-addons.sh $out/bin/kube-addons
+    # Unfortunately, kube-addons-main.sh only looks for the lib file in either the current working dir
+    # or in /opt. We have to patch this for now.
+    substitute cluster/addons/addon-manager/kube-addons-main.sh $out/bin/kube-addons \
+      --subst-var out
+
+    chmod +x $out/bin/kube-addons
     patchShebangs $out/bin/kube-addons
     wrapProgram $out/bin/kube-addons --set "KUBECTL_BIN" "$out/bin/kubectl"
 
-    cp ${./mk-docker-opts.sh} $out/bin/mk-docker-opts.sh
+    cp cluster/addons/addon-manager/kube-addons.sh $out/bin/kube-addons-lib.sh
 
     for tool in kubeadm kubectl; do
       installShellCompletion --cmd $tool \

pkgs/applications/networking/cluster/kubernetes/fixup-addonmanager-lib-path.patch

@@ -0,0 +1,23 @@
+diff --git a/cluster/addons/addon-manager/kube-addons-main.sh b/cluster/addons/addon-manager/kube-addons-main.sh
+index 849973470d1..e4fef30eaea 100755
+--- a/cluster/addons/addon-manager/kube-addons-main.sh
++++ b/cluster/addons/addon-manager/kube-addons-main.sh
+@@ -17,17 +17,7 @@
+ # Import required functions. The addon manager is installed to /opt in
+ # production use (see the Dockerfile)
+ # Disabling shellcheck following files as the full path would be required.
+-if [ -f "kube-addons.sh" ]; then
+-  # shellcheck disable=SC1091
+-  source "kube-addons.sh"
+-elif [ -f "/opt/kube-addons.sh" ]; then
+-  # shellcheck disable=SC1091
+-  source "/opt/kube-addons.sh"
+-else
+-  # If the required source is missing, we have to fail.
+-  log ERR "== Could not find kube-addons.sh (not in working directory or /opt) at $(date -Is) =="
+-  exit 1
+-fi
++source "@out@/bin/kube-addons-lib.sh"
+ 
+ # The business logic for whether a given object should be created
+ # was already enforced by salt, and /etc/kubernetes/addons is the

pkgs/applications/networking/cluster/kubernetes/mk-docker-opts.sh

@@ -1,113 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2014 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Generate Docker daemon options based on flannel env file.
-
-# exit on any error
-set -e
-
-usage() {
-  echo "$0 [-f FLANNEL-ENV-FILE] [-d DOCKER-ENV-FILE] [-i] [-c] [-m] [-k COMBINED-KEY]
-
-Generate Docker daemon options based on flannel env file
-OPTIONS:
-    -f  Path to flannel env file. Defaults to /run/flannel/subnet.env
-    -d  Path to Docker env file to write to. Defaults to /run/docker_opts.env
-    -i  Output each Docker option as individual var. e.g. DOCKER_OPT_MTU=1500
-    -c  Output combined Docker options into DOCKER_OPTS var
-    -k  Set the combined options key to this value (default DOCKER_OPTS=)
-    -m  Do not output --ip-masq (useful for older Docker version)
-" >/dev/stderr
-  exit 1
-}
-
-flannel_env="/run/flannel/subnet.env"
-docker_env="/run/docker_opts.env"
-combined_opts_key="DOCKER_OPTS"
-indiv_opts=false
-combined_opts=false
-ipmasq=true
-val=""
-
-while getopts "f:d:icmk:" opt; do
-  case $opt in
-    f)
-      flannel_env=$OPTARG
-      ;;
-    d)
-      docker_env=$OPTARG
-      ;;
-    i)
-      indiv_opts=true
-      ;;
-    c)
-      combined_opts=true
-      ;;
-    m)
-      ipmasq=false
-      ;;
-    k)
-      combined_opts_key=$OPTARG
-      ;;
-    \?)
-      usage
-      ;;
-  esac
-done
-
-if [[ $indiv_opts = false ]] && [[ $combined_opts = false ]]; then
-  indiv_opts=true
-  combined_opts=true
-fi
-
-if [[ -f "${flannel_env}" ]]; then
-  source "${flannel_env}"
-fi
-
-if [[ -n "$FLANNEL_SUBNET" ]]; then
-  # shellcheck disable=SC2034  # Variable name referenced in OPT_LOOP below
-  DOCKER_OPT_BIP="--bip=$FLANNEL_SUBNET"
-fi
-
-if [[ -n "$FLANNEL_MTU" ]]; then
-  # shellcheck disable=SC2034  # Variable name referenced in OPT_LOOP below
-  DOCKER_OPT_MTU="--mtu=$FLANNEL_MTU"
-fi
-
-if [[ "$FLANNEL_IPMASQ" = true ]] && [[ $ipmasq = true ]]; then
-  # shellcheck disable=SC2034  # Variable name referenced in OPT_LOOP below
-  DOCKER_OPT_IPMASQ="--ip-masq=false"
-fi
-
-eval docker_opts="\$${combined_opts_key}"
-docker_opts+=" "
-
-echo -n "" >"${docker_env}"
-
-# OPT_LOOP
-for opt in $(compgen -v DOCKER_OPT_); do
-  eval val=\$"${opt}"
-
-  if [[ "$indiv_opts" = true ]]; then
-    echo "$opt=\"$val\"" >>"${docker_env}"
-  fi
-
-  docker_opts+="$val "
-done
-
-if [[ "$combined_opts" = true ]]; then
-  echo "${combined_opts_key}=\"${docker_opts}\"" >>"${docker_env}"
-fi