nixos/kubernetes: Address review: Move bootstrapping addons into own service

nixos/modules/services/cluster/kubernetes/addon-manager.nix

@@ -72,9 +72,16 @@ in
     systemd.services.kube-addon-manager = {
       description = "Kubernetes addon manager";
       wantedBy = [ "kube-control-plane-online.target" ];
+      after = [ "kube-addon-manager-bootstrap.service" ];
       before = [ "kube-control-plane-online.target" ];
       environment.ADDON_PATH = "/etc/kubernetes/addons/";
       path = [ pkgs.gawk ];
+      preStart = ''
+        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager; {
+          path = "/api/v1/namespaces/kube-system/serviceaccounts/default";
+          cacert = top.caFile;
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+      '';
       serviceConfig = {
         Slice = "kubernetes.slice";
         ExecStart = "${top.package}/bin/kube-addons";
@@ -86,6 +93,25 @@ in
       };
     };
 
+    systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
+      wantedBy = [ "kube-control-plane-online.target" ];
+      after = [ "kube-apiserver.service" ];
+      before = [ "kube-control-plane-online.target" ];
+      path = [ pkgs.kubectl ];
+      preStart = with pkgs; let
+        files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
+          cfg.bootstrapAddons;
+      in ''
+        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager-bootstrap; {
+          path = "/api";
+          cacert = top.caFile;
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+
+        kubectl apply -f ${concatStringsSep " \\\n -f " files}
+      '';
+      script = "echo Ok";
+    };
+
     services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled
     (let
       name = system:kube-addon-manager;

nixos/modules/services/cluster/kubernetes/flannel.nix

@@ -27,12 +27,7 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable (let
+  config = mkIf cfg.enable {
-    flannelBootstrapPaths = mkIf top.apiserver.enable [
-      top.pki.certs.clusterAdmin.cert
-      top.pki.certs.clusterAdmin.key
-    ];
-  in {
     services.flannel = {
 
       enable = mkDefault true;
@@ -112,69 +107,42 @@ in
     };
 
     # give flannel som kubernetes rbac permissions if applicable
-    systemd.services.flannel-rbac-bootstrap = mkIf (top.apiserver.enable && (elem "RBAC" top.apiserver.authorizationMode)) {
+    services.kubernetes.addonManager.bootstrapAddons = mkIf ((storageBackend == "kubernetes") && (elem "RBAC" top.apiserver.authorizationMode)) {
+      flannel-cr = {
+        apiVersion = "rbac.authorization.k8s.io/v1beta1";
+        kind = "ClusterRole";
+        metadata = { name = "flannel"; };
+        rules = [{
+          apiGroups = [ "" ];
+          resources = [ "pods" ];
+          verbs = [ "get" ];
+        }
+        {
+          apiGroups = [ "" ];
+          resources = [ "nodes" ];
+          verbs = [ "list" "watch" ];
+        }
+        {
+          apiGroups = [ "" ];
+          resources = [ "nodes/status" ];
+          verbs = [ "patch" ];
+        }];
+      };
 
-      wantedBy = [ "kube-apiserver-online.target" ];
+      flannel-crb = {
-      after = [ "kube-apiserver-online.target" ];
+        apiVersion = "rbac.authorization.k8s.io/v1beta1";
-      before = [ "flannel.service" ];
+        kind = "ClusterRoleBinding";
-      path = with pkgs; [ kubectl ];
+        metadata = { name = "flannel"; };
-      preStart = let
+        roleRef = {
-        files = mapAttrsToList (n: v: pkgs.writeText "${n}.json" (builtins.toJSON v)) {
+          apiGroup = "rbac.authorization.k8s.io";
-          flannel-cr = {
+          kind = "ClusterRole";
-            apiVersion = "rbac.authorization.k8s.io/v1beta1";
+          name = "flannel";
-            kind = "ClusterRole";
-            metadata = { name = "flannel"; };
-            rules = [{
-              apiGroups = [ "" ];
-              resources = [ "pods" ];
-              verbs = [ "get" ];
-            }
-            {
-              apiGroups = [ "" ];
-              resources = [ "nodes" ];
-              verbs = [ "list" "watch" ];
-            }
-            {
-              apiGroups = [ "" ];
-              resources = [ "nodes/status" ];
-              verbs = [ "patch" ];
-            }];
-          };
-
-          flannel-crb = {
-            apiVersion = "rbac.authorization.k8s.io/v1beta1";
-            kind = "ClusterRoleBinding";
-            metadata = { name = "flannel"; };
-            roleRef = {
-              apiGroup = "rbac.authorization.k8s.io";
-              kind = "ClusterRole";
-              name = "flannel";
-            };
-            subjects = [{
-              kind = "User";
-              name = "flannel-client";
-            }];
-          };
         };
-      in ''
+        subjects = [{
-        ${top.lib.mkWaitCurl (with top.pki.certs.clusterAdmin; {
+          kind = "User";
-          path = "/";
+          name = "flannel-client";
-          cacert = top.caFile;
+        }];
-          inherit cert key;
-        })}
-
-        kubectl -s ${top.apiserverAddress} --certificate-authority=${top.caFile} --client-certificate=${top.pki.certs.clusterAdmin.cert} --client-key=${top.pki.certs.clusterAdmin.key} apply -f ${concatStringsSep " \\\n -f " files}
-      '';
-      script = "echo Ok";
-      unitConfig.ConditionPathExists = flannelBootstrapPaths;
-    };
-
-    systemd.paths.flannel-rbac-bootstrap = mkIf top.apiserver.enable {
-      wantedBy = [ "flannel-rbac-bootstrap.service" ];
-      pathConfig = {
-        PathExists = flannelBootstrapPaths;
-        PathChanged = flannelBootstrapPaths;
       };
     };
-  });
+  };
 }

nixos/modules/services/cluster/kubernetes/pki.nix

@@ -304,41 +304,29 @@ in
         };
       };
 
+      systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
+        environment = {
+          KUBECONFIG = clusterAdminKubeconfig;
+          inherit (cfg.certs.clusterAdmin) cert key;
+        };
+      };
+
       #TODO: Get rid of kube-addon-manager in the future for the following reasons
       # - it is basically just a shell script wrapped around kubectl
       # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
       # - it is designed to be used with k8s system components only
       # - it would be better with a more Nix-oriented way of managing addons
-      systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{
+      systemd.services.kube-addon-manager = mkIf top.addonManager.enable {
-        environment.KUBECONFIG = with cfg.certs.addonManager;
+        environment = with cfg.certs.addonManager; {
-          top.lib.mkKubeConfig "addon-manager" {
+          KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager" {
-            server = top.apiserverAddress;
+              server = top.apiserverAddress;
-            certFile = cert;
+              certFile = cert;
-            keyFile = key;
+              keyFile = key;
           };
-        }
+          inherit cert key;
-
+        };
-        (optionalAttrs (top.addonManager.bootstrapAddons != {}) {
+        unitConfig.ConditionPathExists = addonManagerPaths;
-          serviceConfig.PermissionsStartOnly = true;
+      };
-          preStart = with pkgs;
-          let
-            files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
-              top.addonManager.bootstrapAddons;
-          in
-          ''
-            export KUBECONFIG=${clusterAdminKubeconfig}
-            ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
-
-            ${top.lib.mkWaitCurl (with top.pki.certs.addonManager; {
-              path = "/api/v1/namespaces/kube-system/serviceaccounts/default";
-              cacert = top.caFile;
-              inherit cert key;
-            })}
-          '';
-        })
-        {
-          unitConfig.ConditionPathExists = addonManagerPaths;
-        }]);
 
       systemd.paths.kube-addon-manager = mkIf top.addonManager.enable {
         wantedBy = [ "kube-addon-manager.service" ];