From 466beb02143f99815eef90ef8a69c91cd898a998 Mon Sep 17 00:00:00 2001 From: Johan Thomsen Date: Tue, 12 Feb 2019 16:48:23 +0100 Subject: [PATCH] nixos/kubernetes: let flannel use kubernetes as storage backend + isolate etcd on the master node by letting it listen only on loopback + enabling kubelet on master and taint master with NoSchedule The reason for the latter is that flannel requires all nodes to be "registered" in the cluster in order to setup the cluster network. This means that the kubelet is needed even at nodes on which we don't plan to schedule anything. --- .../services/cluster/kubernetes/apiserver.nix | 1 + .../services/cluster/kubernetes/default.nix | 10 ++++ .../services/cluster/kubernetes/flannel.nix | 52 +++++++++++++++++-- .../services/cluster/kubernetes/pki.nix | 23 ++++++-- nixos/tests/kubernetes/base.nix | 1 - 5 files changed, 78 insertions(+), 9 deletions(-) diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix index 465d74d83c8..81e45b417de 100644 --- a/nixos/modules/services/cluster/kubernetes/apiserver.nix +++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix @@ -411,6 +411,7 @@ in name = "etcd"; CN = top.masterAddress; hosts = [ + "etcd.local" "etcd.${top.addons.dns.clusterDomain}" top.masterAddress cfg.advertiseAddress diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix index a5a59f4a5fc..375e33e91b5 100644 --- a/nixos/modules/services/cluster/kubernetes/default.nix +++ b/nixos/modules/services/cluster/kubernetes/default.nix @@ -189,6 +189,16 @@ in { services.kubernetes.addonManager.enable = mkDefault true; services.kubernetes.proxy.enable = mkDefault true; services.etcd.enable = true; # Cannot mkDefault because of flannel default options + services.kubernetes.kubelet = { + enable = mkDefault true; + taints = mkIf (!(elem "node" cfg.roles)) { + master = { + key = "node-role.kubernetes.io/master"; + value = "true"; + effect = "NoSchedule"; + }; + }; + }; }) diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix index 38dc1e2b47d..fb70e6513d3 100644 --- a/nixos/modules/services/cluster/kubernetes/flannel.nix +++ b/nixos/modules/services/cluster/kubernetes/flannel.nix @@ -6,6 +6,9 @@ let top = config.services.kubernetes; cfg = top.flannel; + # we want flannel to use kubernetes itself as configuration backend, not direct etcd + storageBackend = "kubernetes"; + # needed for flannel to pass options to docker mkDockerOpts = pkgs.runCommand "mk-docker-opts" { buildInputs = [ pkgs.makeWrapper ]; @@ -29,6 +32,8 @@ in enable = mkDefault true; network = mkDefault top.clusterCidr; + inherit storageBackend; + nodeName = config.services.kubernetes.kubelet.hostname; }; services.kubernetes.kubelet = { @@ -69,11 +74,52 @@ in }; services.kubernetes.pki.certs = { - flannelEtcdClient = top.lib.mkCert { - name = "flannel-etcd-client"; - CN = "flannel-etcd-client"; + flannelClient = top.lib.mkCert { + name = "flannel-client"; + CN = "flannel-client"; action = "systemctl restart flannel.service"; }; }; + + # give flannel som kubernetes rbac permissions if applicable + services.kubernetes.addonManager.bootstrapAddons = mkIf ((storageBackend == "kubernetes") && (elem "RBAC" top.apiserver.authorizationMode)) { + + flannel-cr = { + apiVersion = "rbac.authorization.k8s.io/v1beta1"; + kind = "ClusterRole"; + metadata = { name = "flannel"; }; + rules = [{ + apiGroups = [ "" ]; + resources = [ "pods" ]; + verbs = [ "get" ]; + } + { + apiGroups = [ "" ]; + resources = [ "nodes" ]; + verbs = [ "list" "watch" ]; + } + { + apiGroups = [ "" ]; + resources = [ "nodes/status" ]; + verbs = [ "patch" ]; + }]; + }; + + flannel-crb = { + apiVersion = "rbac.authorization.k8s.io/v1beta1"; + kind = "ClusterRoleBinding"; + metadata = { name = "flannel"; }; + roleRef = { + apiGroup = "rbac.authorization.k8s.io"; + kind = "ClusterRole"; + name = "flannel"; + }; + subjects = [{ + kind = "User"; + name = "flannel-client"; + }]; + }; + + }; }; } diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix index 4587373d519..38deca23a99 100644 --- a/nixos/modules/services/cluster/kubernetes/pki.nix +++ b/nixos/modules/services/cluster/kubernetes/pki.nix @@ -305,7 +305,7 @@ in ''} ${optionalString top.flannel.enable '' - while [ ! -f ${cfg.certs.flannelEtcdClient.cert} ]; do sleep 1; done + while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done echo "Restarting flannel..." >&1 systemctl restart flannel ''} @@ -313,22 +313,35 @@ in echo "Node joined succesfully" '')]; + # isolate etcd on loopback at the master node + # easyCerts doesn't support multimaster clusters anyway atm. services.etcd = with cfg.certs.etcd; { + listenClientUrls = ["https://127.0.0.1:2379"]; + listenPeerUrls = ["https://127.0.0.1:2380"]; + advertiseClientUrls = ["https://etcd.local:2379"]; + initialCluster = ["${top.masterAddress}=https://etcd.local:2380"]; + initialAdvertisePeerUrls = ["https://etcd.local:2380"]; certFile = mkDefault cert; keyFile = mkDefault key; trustedCaFile = mkDefault caCert; }; + networking.extraHosts = mkIf (config.services.etcd.enable) '' + 127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local + ''; - services.flannel.etcd = with cfg.certs.flannelEtcdClient; { - certFile = mkDefault cert; - keyFile = mkDefault key; - caFile = mkDefault caCert; + services.flannel = with cfg.certs.flannelClient; { + kubeconfig = top.lib.mkKubeConfig "flannel" { + server = top.apiserverAddress; + certFile = cert; + keyFile = key; + }; }; services.kubernetes = { apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; { etcd = with cfg.certs.apiserverEtcdClient; { + servers = ["https://etcd.local:2379"]; certFile = mkDefault cert; keyFile = mkDefault key; caFile = mkDefault caCert; diff --git a/nixos/tests/kubernetes/base.nix b/nixos/tests/kubernetes/base.nix index 3529f35f60e..ec1a75e74c4 100644 --- a/nixos/tests/kubernetes/base.nix +++ b/nixos/tests/kubernetes/base.nix @@ -65,7 +65,6 @@ let } (optionalAttrs (any (role: role == "master") machine.roles) { networking.firewall.allowedTCPPorts = [ - 2379 2380 # etcd 443 # kubernetes apiserver ]; })