nixos/kubernetes: Address review: Move bootstrapping addons into own service

nixos/modules/services/cluster/kubernetes/addon-manager.nix

@@ -72,9 +72,16 @@ in
     systemd.services.kube-addon-manager = {
       description = "Kubernetes addon manager";
       wantedBy = [ "kube-control-plane-online.target" ];
+      after = [ "kube-addon-manager-bootstrap.service" ];
       before = [ "kube-control-plane-online.target" ];
       environment.ADDON_PATH = "/etc/kubernetes/addons/";
       path = [ pkgs.gawk ];
+      preStart = ''
+        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager; {
+          path = "/api/v1/namespaces/kube-system/serviceaccounts/default";
+          cacert = top.caFile;
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+      '';
       serviceConfig = {
         Slice = "kubernetes.slice";
         ExecStart = "${top.package}/bin/kube-addons";
@@ -86,6 +93,25 @@ in
       };
     };
 
+    systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
+      wantedBy = [ "kube-control-plane-online.target" ];
+      after = [ "kube-apiserver.service" ];
+      before = [ "kube-control-plane-online.target" ];
+      path = [ pkgs.kubectl ];
+      preStart = with pkgs; let
+        files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
+          cfg.bootstrapAddons;
+      in ''
+        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager-bootstrap; {
+          path = "/api";
+          cacert = top.caFile;
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+
+        kubectl apply -f ${concatStringsSep " \\\n -f " files}
+      '';
+      script = "echo Ok";
+    };
+
     services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled
     (let
       name = system:kube-addon-manager;

nixos/modules/services/cluster/kubernetes/flannel.nix

@@ -27,12 +27,7 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable (let
+  config = mkIf cfg.enable {
-    flannelBootstrapPaths = mkIf top.apiserver.enable [
-      top.pki.certs.clusterAdmin.cert
-      top.pki.certs.clusterAdmin.key
-    ];
-  in {
     services.flannel = {
 
       enable = mkDefault true;
@@ -112,14 +107,7 @@ in
     };
 
     # give flannel som kubernetes rbac permissions if applicable
-    systemd.services.flannel-rbac-bootstrap = mkIf (top.apiserver.enable && (elem "RBAC" top.apiserver.authorizationMode)) {
+    services.kubernetes.addonManager.bootstrapAddons = mkIf ((storageBackend == "kubernetes") && (elem "RBAC" top.apiserver.authorizationMode)) {
-
-      wantedBy = [ "kube-apiserver-online.target" ];
-      after = [ "kube-apiserver-online.target" ];
-      before = [ "flannel.service" ];
-      path = with pkgs; [ kubectl ];
-      preStart = let
-        files = mapAttrsToList (n: v: pkgs.writeText "${n}.json" (builtins.toJSON v)) {
       flannel-cr = {
         apiVersion = "rbac.authorization.k8s.io/v1beta1";
         kind = "ClusterRole";
@@ -156,25 +144,5 @@ in
         }];
       };
     };
-      in ''
-        ${top.lib.mkWaitCurl (with top.pki.certs.clusterAdmin; {
-          path = "/";
-          cacert = top.caFile;
-          inherit cert key;
-        })}
-
-        kubectl -s ${top.apiserverAddress} --certificate-authority=${top.caFile} --client-certificate=${top.pki.certs.clusterAdmin.cert} --client-key=${top.pki.certs.clusterAdmin.key} apply -f ${concatStringsSep " \\\n -f " files}
-      '';
-      script = "echo Ok";
-      unitConfig.ConditionPathExists = flannelBootstrapPaths;
   };
-
-    systemd.paths.flannel-rbac-bootstrap = mkIf top.apiserver.enable {
-      wantedBy = [ "flannel-rbac-bootstrap.service" ];
-      pathConfig = {
-        PathExists = flannelBootstrapPaths;
-        PathChanged = flannelBootstrapPaths;
-      };
-    };
-  });
 }

nixos/modules/services/cluster/kubernetes/pki.nix

@@ -304,41 +304,29 @@ in
         };
       };
 
+      systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
+        environment = {
+          KUBECONFIG = clusterAdminKubeconfig;
+          inherit (cfg.certs.clusterAdmin) cert key;
+        };
+      };
+
       #TODO: Get rid of kube-addon-manager in the future for the following reasons
       # - it is basically just a shell script wrapped around kubectl
       # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
       # - it is designed to be used with k8s system components only
       # - it would be better with a more Nix-oriented way of managing addons
-      systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{
+      systemd.services.kube-addon-manager = mkIf top.addonManager.enable {
-        environment.KUBECONFIG = with cfg.certs.addonManager;
+        environment = with cfg.certs.addonManager; {
-          top.lib.mkKubeConfig "addon-manager" {
+          KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager" {
               server = top.apiserverAddress;
               certFile = cert;
               keyFile = key;
           };
-        }
-
-        (optionalAttrs (top.addonManager.bootstrapAddons != {}) {
-          serviceConfig.PermissionsStartOnly = true;
-          preStart = with pkgs;
-          let
-            files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
-              top.addonManager.bootstrapAddons;
-          in
-          ''
-            export KUBECONFIG=${clusterAdminKubeconfig}
-            ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
-
-            ${top.lib.mkWaitCurl (with top.pki.certs.addonManager; {
-              path = "/api/v1/namespaces/kube-system/serviceaccounts/default";
-              cacert = top.caFile;
           inherit cert key;
-            })}
+        };
-          '';
-        })
-        {
         unitConfig.ConditionPathExists = addonManagerPaths;
-        }]);
+      };
 
       systemd.paths.kube-addon-manager = mkIf top.addonManager.enable {
         wantedBy = [ "kube-addon-manager.service" ];