diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix index bd37c49486e..a37f4fd1d75 100644 --- a/nixos/modules/services/cluster/kubernetes/default.nix +++ b/nixos/modules/services/cluster/kubernetes/default.nix @@ -15,19 +15,19 @@ let config.Cmd = "/bin/pause"; }; - kubeconfig = pkgs.writeText "kubeconfig" (builtins.toJSON { + mkKubeConfig = name: cfg: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON { apiVersion = "v1"; kind = "Config"; clusters = [{ name = "local"; - cluster.certificate-authority = cfg.kubeconfig.caFile; - cluster.server = cfg.kubeconfig.server; + cluster.certificate-authority = cfg.caFile; + cluster.server = cfg.server; }]; users = [{ name = "kubelet"; user = { - client-certificate = cfg.kubeconfig.certFile; - client-key = cfg.kubeconfig.keyFile; + client-certificate = cfg.certFile; + client-key = cfg.keyFile; }; }]; contexts = [{ @@ -39,6 +39,39 @@ let }]; }); + mkKubeConfigOptions = prefix: { + server = mkOption { + description = "${prefix} kube-apiserver server address."; + default = "http://${cfg.apiserver.address}:${toString cfg.apiserver.port}"; + type = types.str; + }; + + caFile = mkOption { + description = "${prefix} certificate authrority file used to connect to kube-apiserver."; + type = types.nullOr types.path; + default = null; + }; + + certFile = mkOption { + description = "${prefix} client certificate file used to connect to kube-apiserver."; + type = types.nullOr types.path; + default = null; + }; + + keyFile = mkOption { + description = "${prefix} client key file used to connect to kube-apiserver."; + type = types.nullOr types.path; + default = null; + }; + }; + + kubeConfigDefaults = { + server = mkDefault cfg.kubeconfig.server; + caFile = mkDefault cfg.kubeconfig.caFile; + certFile = mkDefault cfg.kubeconfig.certFile; + keyFile = mkDefault cfg.kubeconfig.keyFile; + }; + cniConfig = pkgs.buildEnv { name = "kubernetes-cni-config"; paths = imap (i: entry: @@ -228,31 +261,7 @@ in { }; }; - kubeconfig = { - server = mkOption { - description = "Kubernetes apiserver server address."; - default = "http://${cfg.apiserver.address}:${toString cfg.apiserver.port}"; - type = types.str; - }; - - caFile = mkOption { - description = "Certificate authrority file to use to connect to Kubernetes apiserver."; - type = types.nullOr types.path; - default = null; - }; - - certFile = mkOption { - description = "Client certificate file to use to connect to Kubernetes."; - type = types.nullOr types.path; - default = null; - }; - - keyFile = mkOption { - description = "Client key file to use to connect to Kubernetes."; - type = types.nullOr types.path; - default = null; - }; - }; + kubeconfig = mkKubeConfigOptions "Default kubeconfig"; dataDir = mkOption { description = "Kubernetes root directory for managing kubelet files."; @@ -472,6 +481,8 @@ in { default = false; }; + kubeconfig = mkKubeConfigOptions "Kubernetes scheduler"; + extraOpts = mkOption { description = "Kubernetes scheduler extra command line options."; default = ""; @@ -522,6 +533,8 @@ in { type = types.nullOr types.path; }; + kubeconfig = mkKubeConfigOptions "Kubernetes controller manager"; + extraOpts = mkOption { description = "Kubernetes controller manager extra command line options."; default = ""; @@ -680,6 +693,8 @@ in { type = types.nullOr types.str; }; + kubeconfig = mkKubeConfigOptions "Kubelet"; + extraOpts = mkOption { description = "Kubernetes kubelet extra command line options."; default = ""; @@ -700,6 +715,8 @@ in { type = types.str; }; + kubeconfig = mkKubeConfigOptions "Kubernetes proxy"; + extraOpts = mkOption { description = "Kubernetes proxy extra command line options."; default = ""; @@ -756,6 +773,8 @@ in { type = types.str; }; + kubeconfig = mkKubeConfigOptions "Kubernetes dns"; + extraOpts = mkOption { description = "Kubernetes DNS extra command line options."; default = ""; @@ -804,7 +823,7 @@ in { "--pod-manifest-path=${manifests}"} \ ${optionalString (taints != "") "--register-with-taints=${taints}"} \ - --kubeconfig=${kubeconfig} \ + --kubeconfig=${mkKubeConfig "kubelet" cfg.kubelet.kubeconfig} \ --require-kubeconfig \ --address=${cfg.kubelet.address} \ --port=${toString cfg.kubelet.port} \ @@ -841,6 +860,8 @@ in { services.kubernetes.kubelet.cni.packages = [pkgs.cni]; boot.kernelModules = ["br_netfilter"]; + + services.kubernetes.kubelet.kubeconfig = kubeConfigDefaults; }) (mkIf (cfg.kubelet.applyManifests && cfg.kubelet.enable) { @@ -936,7 +957,7 @@ in { --address=${cfg.scheduler.address} \ --port=${toString cfg.scheduler.port} \ --leader-elect=${boolToString cfg.scheduler.leaderElect} \ - --kubeconfig=${kubeconfig} \ + --kubeconfig=${mkKubeConfig "kube-scheduler" cfg.scheduler.kubeconfig} \ ${optionalString cfg.verbose "--v=6"} \ ${optionalString cfg.verbose "--log-flush-frequency=1s"} \ ${cfg.scheduler.extraOpts} @@ -946,6 +967,8 @@ in { Group = "kubernetes"; }; }; + + services.kubernetes.scheduler.kubeconfig = kubeConfigDefaults; }) (mkIf cfg.controllerManager.enable { @@ -960,7 +983,7 @@ in { ExecStart = ''${cfg.package}/bin/kube-controller-manager \ --address=${cfg.controllerManager.address} \ --port=${toString cfg.controllerManager.port} \ - --kubeconfig=${kubeconfig} \ + --kubeconfig=${mkKubeConfig "kube-controller-manager" cfg.controllerManager.kubeconfig} \ --leader-elect=${boolToString cfg.controllerManager.leaderElect} \ ${if (cfg.controllerManager.serviceAccountKeyFile!=null) then "--service-account-private-key-file=${cfg.controllerManager.serviceAccountKeyFile}" @@ -981,6 +1004,8 @@ in { }; path = cfg.path; }; + + services.kubernetes.controllerManager.kubeconfig = kubeConfigDefaults; }) (mkIf cfg.proxy.enable { @@ -992,7 +1017,7 @@ in { serviceConfig = { Slice = "kubernetes.slice"; ExecStart = ''${cfg.package}/bin/kube-proxy \ - --kubeconfig=${kubeconfig} \ + --kubeconfig=${mkKubeConfig "kube-proxy" cfg.proxy.kubeconfig} \ --bind-address=${cfg.proxy.address} \ ${optionalString cfg.verbose "--v=6"} \ ${optionalString cfg.verbose "--log-flush-frequency=1s"} \ @@ -1006,6 +1031,8 @@ in { # kube-proxy needs iptables networking.firewall.enable = mkDefault true; + + services.kubernetes.proxy.kubeconfig = kubeConfigDefaults; }) (mkIf (any (el: el == "master") cfg.roles) { @@ -1054,7 +1081,7 @@ in { serviceConfig = { Slice = "kubernetes.slice"; ExecStart = ''${pkgs.kube-dns}/bin/kube-dns \ - --kubecfg-file=${kubeconfig} \ + --kubecfg-file=${mkKubeConfig "kube-dns" cfg.dns.kubeconfig} \ --dns-port=${toString cfg.dns.port} \ --domain=${cfg.dns.domain} \ ${optionalString cfg.verbose "--v=6"} \ @@ -1077,6 +1104,8 @@ in { ${pkgs.iptables}/bin/iptables -I nixos-fw -p tcp -m tcp -d ${cfg.clusterCidr} --dport 53 -j nixos-fw-accept ${pkgs.iptables}/bin/iptables -I nixos-fw -p udp -m udp -d ${cfg.clusterCidr} --dport 53 -j nixos-fw-accept ''; + + services.kubernetes.dns.kubeconfig = kubeConfigDefaults; }) (mkIf (