Cleanup pki: flannel

nixos/modules/services/cluster/kubernetes/flannel.nix

@@ -24,16 +24,26 @@ in
   ###### interface
   options.services.kubernetes.flannel = {
     enable = mkEnableOption "enable flannel networking";
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes flannel";
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
+  config = let
+
+    flannelPaths = filter (a: a != null) [
+      cfg.kubeconfig.caFile
+      cfg.kubeconfig.certFile
+      cfg.kubeconfig.keyFile
+    ];
+    kubeconfig = top.lib.mkKubeConfig "flannel" cfg.kubeconfig;
+
+  in mkIf cfg.enable {
     services.flannel = {
 
       enable = mkDefault true;
       network = mkDefault top.clusterCidr;
-      inherit storageBackend;
+      inherit storageBackend kubeconfig;
-      nodeName = config.services.kubernetes.kubelet.hostname;
+      nodeName = top.kubelet.hostname;
     };
 
     services.kubernetes.kubelet = {
@@ -79,16 +89,35 @@ in
       wantedBy = [ "flannel.target" ];
       after = [ "kubelet.target" ];
       before = [ "flannel.target" ];
-      path = [ pkgs.iptables ];
+      path = with pkgs; [ iptables kubectl ];
-      preStart = ''
+      environment.KUBECONFIG = kubeconfig;
-        ${top.lib.mkWaitCurl ( with config.systemd.services.flannel; {
+      preStart = let
-          path = "/api/v1/nodes";
+        args = [
-          cacert = top.caFile;
+          "--selector=kubernetes.io/hostname=${top.kubelet.hostname}"
-          args = "-o - | grep podCIDR >/dev/null";
+          # flannel exits if node is not registered yet, before that there is no podCIDR
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+          "--output=jsonpath={.items[0].spec.podCIDR}"
+          # if jsonpath cannot be resolved exit with status 1
+          "--allow-missing-template-keys=false"
+        ];
+      in ''
+        until kubectl get nodes ${concatStringsSep " " args} 2>/dev/null; do
+          echo Waiting for ${top.kubelet.hostname} to be RegisteredNode
+          sleep 1
+        done
       '';
+      unitConfig.ConditionPathExists = flannelPaths;
     };
 
+    systemd.paths.flannel = {
+      wantedBy = [ "flannel.service" ];
+      pathConfig = {
+        PathExists = flannelPaths;
+        PathChanged = flannelPaths;
+      };
+    };
+
+    services.kubernetes.flannel.kubeconfig.server = mkDefault top.apiserverAddress;
+
     systemd.services.docker = {
       environment.DOCKER_OPTS = "-b none";
       serviceConfig.EnvironmentFile = "-/run/flannel/docker";

nixos/modules/services/cluster/kubernetes/pki.nix

@@ -124,10 +124,6 @@ in
       top.caFile
       certmgrAPITokenPath
     ];
-    flannelPaths = [
-      cfg.certs.flannelClient.cert
-      cfg.certs.flannelClient.key
-    ];
     proxyPaths = mkIf top.proxy.enable [
       cfg.certs.kubeProxyClient.cert
       cfg.certs.kubeProxyClient.key
@@ -375,27 +371,6 @@ in
         127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
       '';
 
-      services.flannel = with cfg.certs.flannelClient; {
-        kubeconfig = top.lib.mkKubeConfig "flannel" {
-          server = top.apiserverAddress;
-          certFile = cert;
-          keyFile = key;
-        };
-      };
-
-      systemd.services.flannel = mkIf top.flannel.enable {
-        environment = { inherit (top.pki.certs.flannelClient) cert key; };
-        unitConfig.ConditionPathExists = flannelPaths;
-      };
-
-      systemd.paths.flannel = mkIf top.flannel.enable {
-        wantedBy = [ "flannel.service" ];
-        pathConfig = {
-          PathExists = flannelPaths;
-          PathChanged = flannelPaths;
-        };
-      };
-
       systemd.services.kube-proxy = mkIf top.proxy.enable {
         environment = { inherit (top.pki.certs.kubeProxyClient) cert key; };
         unitConfig.ConditionPathExists = proxyPaths;
@@ -453,6 +428,12 @@ in
             keyFile = mkDefault key;
           };
         };
+        flannel = mkIf top.flannel.enable {
+          kubeconfig = with cfg.certs.flannelClient; {
+            certFile = cert;
+            keyFile = key;
+          };
+        };
         scheduler = mkIf top.scheduler.enable {
           kubeconfig = with cfg.certs.schedulerClient; {
             certFile = mkDefault cert;