kubernetes module: flannel support, minor fixes

- add flannel support
- remove deprecated authorizationRBACSuperAdmin option
- rename from deprecated poratalNet to serviceClusterIpRange
- add nodeIp option for kubelet
- kubelet, add br_netfilter to kernelModules
- enable firewall by default
- enable dns by default on node and on master
- disable iptables for docker by default on nodes
- dns, restart on failure
- update tests

and other minor changes

nixos/modules/services/cluster/kubernetes/default.nix

@@ -162,6 +162,17 @@ let
       };
     }
   ]);
+
+  # needed for flannel to pass options to docker
+  mkDockerOpts = pkgs.runCommand "mk-docker-opts" {
+    buildInputs = [ pkgs.makeWrapper ];
+  } ''
+    mkdir -p $out
+    cp ${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh $out/mk-docker-opts.sh
+
+    # bashInteractive needed for `compgen`
+    makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "$out/mk-docker-opts.sh"
+  '';
 in {
 
   ###### interface
@@ -357,20 +368,17 @@ in {
         type = types.listOf types.attrs;
       };
 
-      authorizationRBACSuperAdmin = mkOption {
-        description = "Role based authorization super admin.";
-        default = "admin";
-        type = types.str;
-      };
-
       allowPrivileged = mkOption {
         description = "Whether to allow privileged containers on Kubernetes.";
         default = true;
         type = types.bool;
       };
 
-      portalNet = mkOption {
+      serviceClusterIpRange = mkOption {
-        description = "Kubernetes CIDR notation IP range from which to assign portal IPs.";
+        description = ''
+          A CIDR notation IP range from which to assign service cluster IPs.
+          This must not overlap with any IP ranges assigned to nodes for pods.
+        '';
         default = "10.10.10.10/24";
         type = types.str;
       };
@@ -666,6 +674,12 @@ in {
         type = types.attrsOf (types.submodule [ taintOptions ]);
       };
 
+      nodeIp = mkOption {
+        description = "IP address of the node. If set, kubelet will use this IP address for the node.";
+        default = null;
+        type = types.nullOr types.str;
+      };
+
       extraOpts = mkOption {
         description = "Kubernetes kubelet extra command line options.";
         default = "";
@@ -761,6 +775,12 @@ in {
       type = types.str;
     };
 
+    flannel.enable = mkOption {
+      description = "Whether to enable flannel networking";
+      default = false;
+      type = types.bool;
+    };
+
   };
 
   ###### implementation
@@ -808,6 +828,8 @@ in {
               "--network-plugin=${cfg.kubelet.networkPlugin}"} \
             --cni-conf-dir=${cniConfig} \
             --hairpin-mode=hairpin-veth \
+            ${optionalString (cfg.kubelet.nodeIp != null)
+              "--node-ip=${cfg.kubelet.nodeIp}"} \
             ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
             ${cfg.kubelet.extraOpts}
           '';
@@ -817,6 +839,8 @@ in {
 
       # Allways include cni plugins
       services.kubernetes.kubelet.cni.packages = [pkgs.cni];
+
+      boot.kernelModules = ["br_netfilter"];
     })
 
     (mkIf (cfg.kubelet.applyManifests && cfg.kubelet.enable) {
@@ -879,10 +903,8 @@ in {
                 (concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.apiserver.authorizationPolicy)
               }"
             } \
-            ${optionalString (elem "RBAC" cfg.apiserver.authorizationMode)
-              "--authorization-rbac-super-user=${cfg.apiserver.authorizationRBACSuperAdmin}"} \
             --secure-port=${toString cfg.apiserver.securePort} \
-            --service-cluster-ip-range=${cfg.apiserver.portalNet} \
+            --service-cluster-ip-range=${cfg.apiserver.serviceClusterIpRange} \
             ${optionalString (cfg.apiserver.runtimeConfig != "")
               "--runtime-config=${cfg.apiserver.runtimeConfig}"} \
             --admission_control=${concatStringsSep "," cfg.apiserver.admissionControl} \
@@ -981,10 +1003,9 @@ in {
           WorkingDirectory = cfg.dataDir;
         };
       };
-    })
 
-    (mkIf cfg.kubelet.enable {
+      # kube-proxy needs iptables
-      boot.kernelModules = ["br_netfilter"];
+      networking.firewall.enable = mkDefault true;
     })
 
     (mkIf (any (el: el == "master") cfg.roles) {
@@ -999,15 +1020,25 @@ in {
       services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]);
     })
 
+    # if this node is only a master make it unschedulable by default
     (mkIf (all (el: el == "master") cfg.roles) {
       services.kubernetes.kubelet.unschedulable = mkDefault true;
     })
 
     (mkIf (any (el: el == "node") cfg.roles) {
-      virtualisation.docker.enable = mkDefault true;
+      virtualisation.docker = {
-      virtualisation.docker.logDriver = mkDefault "json-file";
+        enable = mkDefault true;
+
+        # kubernetes needs access to logs
+        logDriver = mkDefault "json-file";
+
+        # iptables must be disabled for kubernetes
+        extraOptions = "--iptables=false --ip-masq=false";
+      };
+
       services.kubernetes.kubelet.enable = mkDefault true;
       services.kubernetes.proxy.enable = mkDefault true;
+      services.kubernetes.dns.enable = mkDefault true;
     })
 
     (mkIf cfg.addonManager.enable {
@@ -1035,8 +1066,17 @@ in {
           Group = "kubernetes";
           AmbientCapabilities = "cap_net_bind_service";
           SendSIGHUP = true;
+          RestartSec = "30s";
+          Restart = "always";
+          StartLimitInterval = "1m";
         };
       };
+
+      networking.firewall.extraCommands = ''
+        # allow container to host communication for DNS traffic
+        ${pkgs.iptables}/bin/iptables -I nixos-fw -p tcp -m tcp -d ${cfg.clusterCidr} --dport 53 -j nixos-fw-accept
+        ${pkgs.iptables}/bin/iptables -I nixos-fw -p udp -m udp -d ${cfg.clusterCidr} --dport 53 -j nixos-fw-accept
+      '';
     })
 
     (mkIf (
@@ -1070,5 +1110,50 @@ in {
       };
       users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes;
     })
+
+    (mkIf cfg.flannel.enable {
+      services.flannel = {
+        enable = mkDefault true;
+        network = mkDefault cfg.clusterCidr;
+        etcd = mkDefault {
+          endpoints = cfg.etcd.servers;
+          inherit (cfg.etcd) caFile certFile keyFile;
+        };
+      };
+
+      services.kubernetes.kubelet = {
+        networkPlugin = mkDefault "cni";
+        cni.config = mkDefault [{
+          name = "mynet";
+          type = "flannel";
+          delegate = {
+            isDefaultGateway = true;
+            bridge = "docker0";
+          };
+        }];
+      };
+
+      systemd.services."mk-docker-opts" = {
+        description = "Pre-Docker Actions";
+        wantedBy = [ "flannel.service" ];
+        before = [ "docker.service" ];
+        after = [ "flannel.service" ];
+        path = [ pkgs.gawk pkgs.gnugrep ];
+        script = ''
+          mkdir -p /run/flannel
+          ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
+        '';
+        serviceConfig.Type = "oneshot";
+      };
+      systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker";
+
+      # read environment variables generated by mk-docker-opts
+      virtualisation.docker.extraOptions = "$DOCKER_OPTS";
+
+      networking.firewall.allowedUDPPorts = [
+        8285  # flannel udp
+        8472  # flannel vxlan
+      ];
+    })
   ];
 }

nixos/tests/kubernetes/kubernetes-common.nix

@@ -1,4 +1,5 @@
 { config, pkgs, certs, servers }:
+
 let
   etcd_key = "${certs}/etcd-key.pem";
   etcd_cert = "${certs}/etcd.pem";
@@ -9,8 +10,6 @@ let
   worker_key = "${certs}/worker-key.pem";
   worker_cert = "${certs}/worker.pem";
 
-  mkDockerOpts = "${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh";
-
   rootCaFile = pkgs.writeScript "rootCaFile.pem" ''
     ${pkgs.lib.readFile "${certs}/ca.pem"}
 
@@ -26,16 +25,9 @@ in
   environment.systemPackages = with pkgs; [ netcat bind etcd.bin ];
 
   networking = {
-    firewall = {
+    firewall.allowedTCPPorts = [
-      enable = true;
+      10250 # kubelet
-      allowedTCPPorts = [
+    ];
-        10250 80 443
-      ];
-      allowedUDPPorts = [
-        8285  # flannel udp
-        8472  # flannel vxlan
-      ];
-    };
     extraHosts = ''
       # register "external" domains
       ${servers.master} etcd.kubernetes.nixos.xyz
@@ -43,42 +35,7 @@ in
       ${mkHosts}
     '';
   };
-  virtualisation.docker.extraOptions = ''
+  services.flannel.iface = "eth1";
-    --iptables=false $DOCKER_OPTS
-  '';
-
-  # lets create environment file for docker startup - network stuff
-  systemd.services."pre-docker" = {
-    description = "Pre-Docker Actions";
-    wantedBy = [ "flannel.service" ];
-    before = [ "docker.service" ];
-    after = [ "flannel.service" ];
-    path = [ pkgs.gawk pkgs.gnugrep ];
-    script = ''
-      mkdir -p /run/flannel
-      # bashInteractive needed for `compgen`
-      ${pkgs.bashInteractive}/bin/bash ${mkDockerOpts} -d /run/flannel/docker
-      cat /run/flannel/docker  # just for debugging
-
-      # allow container to host communication for DNS traffic
-      ${pkgs.iptables}/bin/iptables -I nixos-fw -p tcp -m tcp -i docker0 --dport 53 -j nixos-fw-accept
-      ${pkgs.iptables}/bin/iptables -I nixos-fw -p udp -m udp -i docker0 --dport 53 -j nixos-fw-accept
-    '';
-    serviceConfig.Type = "simple";
-  };
-  systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker";
-
-  services.flannel = {
-    enable = true;
-    network = "10.2.0.0/16";
-    iface = "eth1";
-    etcd = {
-      endpoints = ["https://etcd.kubernetes.nixos.xyz:2379"];
-      keyFile = etcd_client_key;
-      certFile = etcd_client_cert;
-      caFile = ca_pem;
-    };
-  };
   environment.variables = {
     ETCDCTL_CERT_FILE = "${etcd_client_cert}";
     ETCDCTL_KEY_FILE = "${etcd_client_key}";
@@ -88,20 +45,10 @@ in
 
   services.kubernetes = {
     kubelet = {
-      networkPlugin = "cni";
-      cni.config = [{
-        name = "mynet";
-        type = "flannel";
-        delegate = {
-          isDefaultGateway = true;
-          bridge = "docker0";
-        };
-      }];
       tlsKeyFile = worker_key;
       tlsCertFile = worker_cert;
       hostname = "${config.networking.hostName}.nixos.xyz";
-      extraOpts = "--node-ip ${config.networking.primaryIPAddress}";
+      nodeIp = config.networking.primaryIPAddress;
-      clusterDns = config.networking.primaryIPAddress;
     };
     etcd = {
       servers = ["https://etcd.kubernetes.nixos.xyz:2379"];
@@ -110,22 +57,16 @@ in
       caFile = ca_pem;
     };
     kubeconfig = {
-      server = "https://kubernetes.nixos.xyz:4443";
+      server = "https://kubernetes.nixos.xyz";
       caFile = rootCaFile;
       certFile = worker_cert;
       keyFile = worker_key;
     };
+    flannel.enable = true;
 
-    # make sure you cover kubernetes.apiserver.portalNet and flannel networks
-    clusterCidr = "10.0.0.0/8";
-
-    dns.enable = true;
     dns.port = 4453;
   };
 
   services.dnsmasq.enable = true;
   services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
-
-  virtualisation.docker.enable = true;
-  virtualisation.docker.storageDriver = "overlay";
 }

nixos/tests/kubernetes/kubernetes-master.nix

@@ -25,7 +25,7 @@ in
       allowPing = true;
       allowedTCPPorts = [
         2379 2380  # etcd
-        4443  # kubernetes
+        443 # kubernetes apiserver
       ];
     };
   };
@@ -43,14 +43,13 @@ in
     initialCluster = ["master=https://etcd.kubernetes.nixos.xyz:2380"];
     initialAdvertisePeerUrls = ["https://etcd.kubernetes.nixos.xyz:2380"];
   };
+
   services.kubernetes = {
     roles = ["master"];
     scheduler.leaderElect = true;
-    controllerManager.leaderElect = true;
     controllerManager.rootCaFile = rootCaFile;
     controllerManager.serviceAccountKeyFile = apiserver_key;
     apiserver = {
-      securePort = 4443;
       publicAddress = "192.168.1.1";
       advertiseAddress = "192.168.1.1";
       tlsKeyFile = apiserver_key;
@@ -59,9 +58,6 @@ in
       kubeletClientCaFile = rootCaFile;
       kubeletClientKeyFile = worker_key;
       kubeletClientCertFile = worker_cert;
-      portalNet = "10.1.10.0/24";  # --service-cluster-ip-range
-      runtimeConfig = "";
-      /*extraOpts = "--v=2";*/
     };
   };
 }

nixos/tests/kubernetes/multinode-kubectl.nix

@@ -8,7 +8,6 @@ let
   servers.master = "192.168.1.1";
   servers.one = "192.168.1.10";
   servers.two = "192.168.1.20";
-  servers.three = "192.168.1.30";
 
   certs = import ./certs.nix { inherit servers; };
 
@@ -39,7 +38,7 @@ let
     clusters = [{
       name = "local";
       cluster.certificate-authority = "/ca.pem";
-      cluster.server = "https://${servers.master}:4443/";
+      cluster.server = "https://${servers.master}";
     }];
     users = [{
       name = "kubelet";
@@ -61,11 +60,9 @@ let
     $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");
     $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");
     $master->waitUntilSucceeds("kubectl get node two.nixos.xyz | grep Ready");
-    $master->waitUntilSucceeds("kubectl get node three.nixos.xyz | grep Ready");
 
     $one->execute("docker load < ${kubectlImage}");
     $two->execute("docker load < ${kubectlImage}");
-    $three->execute("docker load < ${kubectlImage}");
 
     $master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}");
 
@@ -116,19 +113,6 @@ in makeTest {
           }
           (import ./kubernetes-common.nix { inherit pkgs config certs servers; })
         ];
-
-    three =
-      { config, pkgs, lib, nodes, ... }:
-        mkMerge [
-          {
-            virtualisation.memorySize = 768;
-            virtualisation.diskSize = 4096;
-            networking.interfaces.eth1.ip4 = mkForce [{address = servers.three; prefixLength = 24;}];
-            networking.primaryIPAddress = mkForce servers.three;
-            services.kubernetes.roles = ["node"];
-          }
-          (import ./kubernetes-common.nix { inherit pkgs config certs servers; })
-        ];
   };
 
   testScript = ''

nixos/tests/kubernetes/singlenode.nix

@@ -46,37 +46,30 @@ let
     $kubernetes->waitUntilSucceeds("kubectl get pod redis | grep Running");
     $kubernetes->succeed("nc -z \$\(dig redis.default.svc.cluster.local +short\) 6379");
   '';
-in {
+in makeTest {
-  # This test runs kubernetes on a single node
+  name = "kubernetes-singlenode";
-  singlenode = makeTest {
-    name = "kubernetes-singlenode";
 
-    nodes = {
+  nodes = {
-      kubernetes =
+    kubernetes =
-        { config, pkgs, lib, nodes, ... }:
+      { config, pkgs, lib, nodes, ... }:
-          {
+        {
-            virtualisation.memorySize = 768;
+          virtualisation.memorySize = 768;
-            virtualisation.diskSize = 2048;
+          virtualisation.diskSize = 2048;
 
-            programs.bash.enableCompletion = true;
+          programs.bash.enableCompletion = true;
-            environment.systemPackages = with pkgs; [ netcat bind ];
+          environment.systemPackages = with pkgs; [ netcat bind ];
 
-            services.kubernetes.roles = ["master" "node"];
+          services.kubernetes.roles = ["master" "node"];
-            services.kubernetes.dns.port = 4453;
+          services.kubernetes.dns.port = 4453;
-            virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0";
 
-            networking.bridges.cbr0.interfaces = [];
+          services.dnsmasq.enable = true;
-            networking.interfaces.cbr0 = {};
+          services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
-
+        };
-            services.dnsmasq.enable = true;
-            services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
-          };
-    };
-
-    testScript = ''
-      startAll;
-
-      ${testSimplePod}
-    '';
   };
+
+  testScript = ''
+    startAll;
+
+    ${testSimplePod}
+  '';
 }