From 9522ca5ce98af6a5b227adaa5164697385150366 Mon Sep 17 00:00:00 2001 From: Johan Thomsen Date: Mon, 11 Feb 2019 13:47:45 +0100 Subject: [PATCH 1/3] nixos/flannel: add options to configure kubernetes as config backend for flannel --- nixos/modules/services/networking/flannel.nix | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/nixos/modules/services/networking/flannel.nix b/nixos/modules/services/networking/flannel.nix index b93e28e34ef..cb39a53b5f9 100644 --- a/nixos/modules/services/networking/flannel.nix +++ b/nixos/modules/services/networking/flannel.nix @@ -73,11 +73,26 @@ in { }; }; + kubeconfig = mkOption { + description = '' + Path to kubeconfig to use for storing flannel config using the + Kubernetes API + ''; + type = types.nullOr types.path; + default = null; + }; + network = mkOption { description = " IPv4 network in CIDR format to use for the entire flannel network."; type = types.str; }; + storageBackend = mkOption { + description = "Determines where flannel stores its configuration at runtime"; + type = types.enum ["etcd" "kubernetes"]; + default = "etcd"; + }; + subnetLen = mkOption { description = '' The size of the subnet allocated to each host. Defaults to 24 (i.e. /24) @@ -122,17 +137,21 @@ in { after = [ "network.target" ]; environment = { FLANNELD_PUBLIC_IP = cfg.publicIp; + FLANNELD_IFACE = cfg.iface; + } // optionalAttrs (cfg.storageBackend == "etcd") { FLANNELD_ETCD_ENDPOINTS = concatStringsSep "," cfg.etcd.endpoints; FLANNELD_ETCD_KEYFILE = cfg.etcd.keyFile; FLANNELD_ETCD_CERTFILE = cfg.etcd.certFile; FLANNELD_ETCD_CAFILE = cfg.etcd.caFile; - FLANNELD_IFACE = cfg.iface; ETCDCTL_CERT_FILE = cfg.etcd.certFile; ETCDCTL_KEY_FILE = cfg.etcd.keyFile; ETCDCTL_CA_FILE = cfg.etcd.caFile; ETCDCTL_PEERS = concatStringsSep "," cfg.etcd.endpoints; + } // optionalAttrs (cfg.storageBackend == "kubernetes") { + FLANNELD_KUBE_SUBNET_MGR = "true"; + FLANNELD_KUBECONFIG_FILE = cfg.kubeconfig; }; - preStart = '' + preStart = mkIf (cfg.storageBackend == "etcd") '' echo "setting network configuration" until ${pkgs.etcdctl.bin}/bin/etcdctl set /coreos.com/network/config '${builtins.toJSON networkConfig}' do @@ -149,6 +168,12 @@ in { serviceConfig.ExecStart = "${cfg.package}/bin/flannel"; }; - services.etcd.enable = mkDefault (cfg.etcd.endpoints == ["http://127.0.0.1:2379"]); + services.etcd.enable = mkDefault (cfg.storageBackend == "etcd" && cfg.etcd.endpoints == ["http://127.0.0.1:2379"]); + + # for some reason, flannel doesn't let you configure this path + # see: https://github.com/coreos/flannel/blob/master/Documentation/configuration.md#configuration + environment.etc."kube-flannel/net-conf.json" = mkIf (cfg.storageBackend == "kubernetes") { + source = pkgs.writeText "net-conf.json" (builtins.toJSON networkConfig); + }; }; } From adc9da617884b240f3799875b8e3b1ae3ae3185e Mon Sep 17 00:00:00 2001 From: Johan Thomsen Date: Tue, 12 Feb 2019 18:26:08 +0100 Subject: [PATCH 2/3] nixos/flannel: fix flannel nixos test, add test to all-tests.nix --- nixos/tests/all-tests.nix | 1 + nixos/tests/flannel.nix | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 7e207fa419f..229f2c3abf7 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -73,6 +73,7 @@ in ferm = handleTest ./ferm.nix {}; firefox = handleTest ./firefox.nix {}; firewall = handleTest ./firewall.nix {}; + flannel = handleTestOn ["x86_64-linux"] ./flannel.nix {}; flatpak = handleTest ./flatpak.nix {}; fsck = handleTest ./fsck.nix {}; fwupd = handleTestOn ["x86_64-linux"] ./fwupd.nix {}; # libsmbios is unsupported on aarch64 diff --git a/nixos/tests/flannel.nix b/nixos/tests/flannel.nix index fb66fe28209..0b261a68477 100644 --- a/nixos/tests/flannel.nix +++ b/nixos/tests/flannel.nix @@ -21,8 +21,9 @@ import ./make-test.nix ({ pkgs, ...} : rec { services = { etcd = { enable = true; - listenClientUrls = ["http://etcd:2379"]; - listenPeerUrls = ["http://etcd:2380"]; + listenClientUrls = ["http://0.0.0.0:2379"]; # requires ip-address for binding + listenPeerUrls = ["http://0.0.0.0:2380"]; # requires ip-address for binding + advertiseClientUrls = ["http://etcd:2379"]; initialAdvertisePeerUrls = ["http://etcd:2379"]; initialCluster = ["etcd=http://etcd:2379"]; }; From 94136fdc1b6c0eb71d10b27a9a2cb597d73ca33e Mon Sep 17 00:00:00 2001 From: Johan Thomsen Date: Wed, 13 Feb 2019 17:17:52 +0100 Subject: [PATCH 3/3] nixos/flannel: node name needs to be configured for flannel to work with kubernetes storage backend --- nixos/modules/services/networking/flannel.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nixos/modules/services/networking/flannel.nix b/nixos/modules/services/networking/flannel.nix index cb39a53b5f9..6c43573851b 100644 --- a/nixos/modules/services/networking/flannel.nix +++ b/nixos/modules/services/networking/flannel.nix @@ -87,6 +87,15 @@ in { type = types.str; }; + nodeName = mkOption { + description = '' + Needed when running with Kubernetes as backend as this cannot be auto-detected"; + ''; + type = types.nullOr types.str; + default = with config.networking; (hostName + optionalString (!isNull domain) ".${domain}"); + example = "node1.example.com"; + }; + storageBackend = mkOption { description = "Determines where flannel stores its configuration at runtime"; type = types.enum ["etcd" "kubernetes"]; @@ -150,6 +159,7 @@ in { } // optionalAttrs (cfg.storageBackend == "kubernetes") { FLANNELD_KUBE_SUBNET_MGR = "true"; FLANNELD_KUBECONFIG_FILE = cfg.kubeconfig; + NODE_NAME = cfg.nodeName; }; preStart = mkIf (cfg.storageBackend == "etcd") '' echo "setting network configuration"