nixpkgs/nixos/modules/services/cluster/kubernetes.nix

{ config, lib, pkgs, ... }:

with lib;

let
  cfg = config.services.kubernetes;

in {

  ###### interface

  options.services.kubernetes = {
    package = mkOption {
      description = "Kubernetes package to use.";
      type = types.package;
    };

    verbose = mkOption {
      description = "Kubernetes enable verbose mode for debugging";
      default = false;
      type = types.bool;
    };

    etcdServers = mkOption {
      description = "Kubernetes list of etcd servers to watch.";
      default = [ "127.0.0.1:4001" ];
      type = types.listOf types.str;
    };

    roles = mkOption {
      description = ''
        Kubernetes role that this machine should take.

        Master role will enable etcd, apiserver, scheduler and controller manager
        services. Node role will enable etcd, docker, kubelet and proxy services.
      '';
      default = [];
      type = types.listOf (types.enum ["master" "node"]);
    };

    dataDir = mkOption {
      description = "Kubernetes root directory for managing kubelet files.";
      default = "/var/lib/kubernetes";
      type = types.path;
    };

    dockerCfg = mkOption {
      description = "Kubernetes contents of dockercfg file.";
      default = "";
      type = types.lines;
    };

    apiserver = {
      enable = mkOption {
        description = "Whether to enable kubernetes apiserver.";
        default = false;
        type = types.bool;
      };

      address = mkOption {
        description = "Kubernetes apiserver listening address.";
        default = "127.0.0.1";
        type = types.str;
      };

      publicAddress = mkOption {
        description = ''
          Kubernetes apiserver public listening address used for read only and
          secure port.
        '';
        default = cfg.apiserver.address;
        type = types.str;
      };

      port = mkOption {
        description = "Kubernetes apiserver listening port.";
        default = 8080;
        type = types.int;
      };

      securePort = mkOption {
        description = "Kubernetes apiserver secure port.";
        default = 6443;
        type = types.int;
      };

      tlsCertFile = mkOption {
        description = "Kubernetes apiserver certificate file.";
        default = "";
        type = types.str;
      };

      tlsPrivateKeyFile = mkOption {
        description = "Kubernetes apiserver private key file.";
        default = "";
        type = types.str;
      };

      clientCaFile = mkOption {
        description = "Kubernetes apiserver CA file for client auth.";
        default = "";
        type = types.str;
      };

      tokenAuth = mkOption {
        description = ''
          Kubernetes apiserver token authentication file. See
          <link xlink:href="http://kubernetes.io/v1.0/docs/admin/authentication.html"/>
        '';
        default = {};
        example = literalExample ''
          {
            alice = "abc123";
            bob = "xyz987";
          }
        '';
        type = types.attrsOf types.str;
      };

      authorizationMode = mkOption {
        description = ''
          Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC). See
          <link xlink:href="http://kubernetes.io/v1.0/docs/admin/authorization.html"/>
        '';
        default = "AlwaysAllow";
        type = types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC"];
      };

      authorizationPolicy = mkOption {
        description = ''
          Kubernetes apiserver authorization policy file. See
          <link xlink:href="http://kubernetes.io/v1.0/docs/admin/authorization.html"/>
        '';
        default = [];
        example = literalExample ''
          [
            {user = "admin";}
            {user = "scheduler"; readonly = true; kind= "pods";}
            {user = "scheduler"; kind = "bindings";}
            {user = "kubelet";  readonly = true; kind = "bindings";}
            {user = "kubelet"; kind = "events";}
            {user= "alice"; ns = "projectCaribou";}
            {user = "bob"; readonly = true; ns = "projectCaribou";}
          ]
        '';
        type = types.listOf types.attrs;
      };

      allowPrivileged = mkOption {
        description = "Whether to allow privileged containers on kubernetes.";
        default = false;
        type = types.bool;
      };

      portalNet = mkOption {
        description = "Kubernetes CIDR notation IP range from which to assign portal IPs";
        default = "10.10.10.10/16";
        type = types.str;
      };

      runtimeConfig = mkOption {
        description = ''
          Api runtime configuration. See
          <link xlink:href="http://kubernetes.io/v1.0/docs/admin/cluster-management.html"/>
        '';
        default = "";
        example = "api/all=false,api/v1=true";
        type = types.str;
      };

      admissionControl = mkOption {
        description = ''
          Kubernetes admission control plugins to use. See
          <link xlink:href="http://kubernetes.io/v1.0/docs/admin/admission-controllers.html"/>
        '';
        default = ["AlwaysAdmit"];
        example = [
          "NamespaceLifecycle" "NamespaceExists" "LimitRanger"
          "SecurityContextDeny" "ServiceAccount" "ResourceQuota"
        ];
        type = types.listOf types.str;
      };

      serviceAccountKey = mkOption {
        description = ''
          Kubernetes apiserver PEM-encoded x509 RSA private or public key file,
          used to verify ServiceAccount tokens.
        '';
        default = null;
        type = types.nullOr types.path;
      };

      extraOpts = mkOption {
        description = "Kubernetes apiserver extra command line options.";
        default = "";
        type = types.str;
      };
    };

    scheduler = {
      enable = mkOption {
        description = "Whether to enable kubernetes scheduler.";
        default = false;
        type = types.bool;
      };

      address = mkOption {
        description = "Kubernetes scheduler listening address.";
        default = "127.0.0.1";
        type = types.str;
      };

      port = mkOption {
        description = "Kubernetes scheduler listening port.";
        default = 10251;
        type = types.int;
      };

      master = mkOption {
        description = "Kubernetes apiserver address";
        default = "${cfg.apiserver.address}:${toString cfg.apiserver.port}";
        type = types.str;
      };

      extraOpts = mkOption {
        description = "Kubernetes scheduler extra command line options.";
        default = "";
        type = types.str;
      };
    };

    controllerManager = {
      enable = mkOption {
        description = "Whether to enable kubernetes controller manager.";
        default = false;
        type = types.bool;
      };

      address = mkOption {
        description = "Kubernetes controller manager listening address.";
        default = "127.0.0.1";
        type = types.str;
      };

      port = mkOption {
        description = "Kubernetes controller manager listening port.";
        default = 10252;
        type = types.int;
      };

      master = mkOption {
        description = "Kubernetes apiserver address";
        default = "${cfg.apiserver.address}:${toString cfg.apiserver.port}";
        type = types.str;
      };

      serviceAccountPrivateKey = mkOption {
        description = ''
          Kubernetes controller manager PEM-encoded private RSA key file used to
          sign service account tokens
        '';
        default = null;
        type = types.nullOr types.path;
      };

      rootCaFile = mkOption {
        description = ''
          Kubernetes controller manager certificate authority file included in
          service account's token secret.
        '';
        default = null;
        type = types.nullOr types.path;
      };

      extraOpts = mkOption {
        description = "Kubernetes controller manager extra command line options.";
        default = "";
        type = types.str;
      };
    };

    kubelet = {
      enable = mkOption {
        description = "Whether to enable kubernetes kubelet.";
        default = false;
        type = types.bool;
      };

      registerNode = mkOption {
        description = "Whether to auto register kubelet with API server.";
        default = true;
        type = types.bool;
      };

      address = mkOption {
        description = "Kubernetes kubelet info server listening address.";
        default = "0.0.0.0";
        type = types.str;
      };

      port = mkOption {
        description = "Kubernetes kubelet info server listening port.";
        default = 10250;
        type = types.int;
      };

      healthz = {
        bind = mkOption {
          description = "Kubernetes kubelet healthz listening address.";
          default = "127.0.0.1";
          type = types.str;
        };

        port = mkOption {
          description = "Kubernetes kubelet healthz port.";
          default = 10248;
          type = types.int;
        };
      };

      hostname = mkOption {
        description = "Kubernetes kubelet hostname override";
        default = config.networking.hostName;
        type = types.str;
      };

      allowPrivileged = mkOption {
        description = "Whether to allow kubernetes containers to request privileged mode.";
        default = false;
        type = types.bool;
      };

      apiServers = mkOption {
        description = ''
          Kubernetes kubelet list of Kubernetes API servers for publishing events,
          and reading pods and services.
        '';
        default = ["${cfg.apiserver.address}:${toString cfg.apiserver.port}"];
        type = types.listOf types.str;
      };

      cadvisorPort = mkOption {
        description = "Kubernetes kubelet local cadvisor port.";
        default = 4194;
        type = types.int;
      };

      clusterDns = mkOption {
        description = "Use alternative dns.";
        default = "";
        type = types.str;
      };

      clusterDomain = mkOption {
        description = "Use alternative domain.";
        default = "kubernetes.io";
        type = types.str;
      };

      extraOpts = mkOption {
        description = "Kubernetes kubelet extra command line options.";
        default = "";
        type = types.str;
      };
    };

    proxy = {
      enable = mkOption {
        description = "Whether to enable kubernetes proxy.";
        default = false;
        type = types.bool;
      };

      address = mkOption {
        description = "Kubernetes proxy listening address.";
        default = "0.0.0.0";
        type = types.str;
      };

      master = mkOption {
        description = "Kubernetes apiserver address";
        default = "${cfg.apiserver.address}:${toString cfg.apiserver.port}";
        type = types.str;
      };

      extraOpts = mkOption {
        description = "Kubernetes proxy extra command line options.";
        default = "";
        type = types.str;
      };
    };

    kube2sky = {
      enable = mkEnableOption "Whether to enable kube2sky dns service.";

      domain = mkOption  {
        description = "Kuberntes kube2sky domain under which all DNS names will be hosted.";
        default = cfg.kubelet.clusterDomain;
        type = types.str;
      };

      master = mkOption {
        description = "Kubernetes apiserver address";
        default = "${cfg.apiserver.address}:${toString cfg.apiserver.port}";
        type = types.str;
      };

      extraOpts = mkOption {
        description = "Kubernetes kube2sky extra command line options.";
        default = "";
        type = types.str;
      };
    };
  };

  ###### implementation

  config = mkMerge [
    (mkIf cfg.apiserver.enable {
      systemd.services.kube-apiserver = {
        description = "Kubernetes Api Server";
        wantedBy = [ "multi-user.target" ];
        requires = ["kubernetes-setup.service"];
        after = [ "network-interfaces.target" "etcd.service" ];
        serviceConfig = {
          ExecStart = let
            authorizationPolicyFile =
              pkgs.writeText "kubernetes-policy"
                (builtins.toJSON cfg.apiserver.authorizationPolicy);
            tokenAuthFile =
              pkgs.writeText "kubernetes-auth"
                (concatImapStringsSep "\n" (i: v: v + "," + (toString i))
                    (mapAttrsToList (name: token: token + "," + name) cfg.apiserver.tokenAuth));
          in ''${cfg.package}/bin/kube-apiserver \
            --etcd-servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \
            --insecure-bind-address=${cfg.apiserver.address} \
            --insecure-port=${toString cfg.apiserver.port} \
            --bind-address=${cfg.apiserver.publicAddress} \
            --allow-privileged=${if cfg.apiserver.allowPrivileged then "true" else "false"} \
            ${optionalString (cfg.apiserver.tlsCertFile!="")
              "--tls-cert-file=${cfg.apiserver.tlsCertFile}"} \
            ${optionalString (cfg.apiserver.tlsPrivateKeyFile!="")
              "--tls-private-key-file=${cfg.apiserver.tlsPrivateKeyFile}"} \
            ${optionalString (cfg.apiserver.tokenAuth!=[])
              "--token-auth-file=${tokenAuthFile}"} \
            ${optionalString (cfg.apiserver.clientCaFile!="")
              "--client-ca-file=${cfg.apiserver.clientCaFile}"} \
            --authorization-mode=${cfg.apiserver.authorizationMode} \
            ${optionalString (cfg.apiserver.authorizationMode == "ABAC")
              "--authorization-policy-file=${authorizationPolicyFile}"} \
            --secure-port=${toString cfg.apiserver.securePort} \
            --service-cluster-ip-range=${cfg.apiserver.portalNet} \
            ${optionalString (cfg.apiserver.runtimeConfig!="")
              "--runtime-config=${cfg.apiserver.runtimeConfig}"} \
            --admission_control=${concatStringsSep "," cfg.apiserver.admissionControl} \
            ${optionalString (cfg.apiserver.serviceAccountKey!=null)
              "--service-account-key-file=${cfg.apiserver.serviceAccountKey}"} \
            --logtostderr=true \
            ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
            ${cfg.apiserver.extraOpts}
          '';
          User = "kubernetes";
        };
      };
    })

    (mkIf cfg.scheduler.enable {
      systemd.services.kube-scheduler = {
        description = "Kubernetes Scheduler Service";
        wantedBy = [ "multi-user.target" ];
        after = [ "network-interfaces.target" "kubernetes-apiserver.service" ];
        serviceConfig = {
          ExecStart = ''${cfg.package}/bin/kube-scheduler \
            --address=${cfg.scheduler.address} \
            --port=${toString cfg.scheduler.port} \
            --master=${cfg.scheduler.master} \
            --logtostderr=true \
            ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
            ${cfg.scheduler.extraOpts}
          '';
          User = "kubernetes";
        };
      };
    })

    (mkIf cfg.controllerManager.enable {
      systemd.services.kube-controller-manager = {
        description = "Kubernetes Controller Manager Service";
        wantedBy = [ "multi-user.target" ];
        after = [ "network-interfaces.target" "kubernetes-apiserver.service" ];
        serviceConfig = {
          ExecStart = ''${cfg.package}/bin/kube-controller-manager \
            --address=${cfg.controllerManager.address} \
            --port=${toString cfg.controllerManager.port} \
            --master=${cfg.controllerManager.master} \
            ${optionalString (cfg.controllerManager.serviceAccountPrivateKey!=null)
              "--service-account-private-key-file=${cfg.controllerManager.serviceAccountPrivateKey}"} \
            ${optionalString (cfg.controllerManager.rootCaFile!=null)
              "--root-ca-file=${cfg.controllerManager.rootCaFile}"} \
            --logtostderr=true \
            ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
            ${cfg.controllerManager.extraOpts}
          '';
          User = "kubernetes";
        };
      };
    })

    (mkIf cfg.kubelet.enable {
      systemd.services.kubelet = {
        description = "Kubernetes Kubelet Service";
        wantedBy = [ "multi-user.target" ];
        requires = ["kubernetes-setup.service"];
        after = [ "network-interfaces.target" "etcd.service" "docker.service" ];
        path = [ pkgs.gitMinimal pkgs.openssh ];
        script = ''
          export PATH="/bin:/sbin:/usr/bin:/usr/sbin:$PATH"
          exec ${cfg.package}/bin/kubelet \
            --api-servers=${concatMapStringsSep "," (f: "http://${f}") cfg.kubelet.apiServers}  \
            --register-node=${if cfg.kubelet.registerNode then "true" else "false"} \
            --address=${cfg.kubelet.address} \
            --port=${toString cfg.kubelet.port} \
            --healthz-bind-address=${cfg.kubelet.healthz.bind} \
            --healthz-port=${toString cfg.kubelet.healthz.port} \
            --hostname-override=${cfg.kubelet.hostname} \
            --allow-privileged=${if cfg.kubelet.allowPrivileged then "true" else "false"} \
            --root-dir=${cfg.dataDir} \
            --cadvisor_port=${toString cfg.kubelet.cadvisorPort} \
            ${optionalString (cfg.kubelet.clusterDns != "")
                ''--cluster-dns=${cfg.kubelet.clusterDns}''} \
            ${optionalString (cfg.kubelet.clusterDomain != "")
                ''--cluster-domain=${cfg.kubelet.clusterDomain}''} \
            --logtostderr=true \
            ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
            ${cfg.kubelet.extraOpts}
          '';
        serviceConfig.WorkingDirectory = cfg.dataDir;
      };
    })

    (mkIf cfg.proxy.enable {
      systemd.services.kube-proxy = {
        description = "Kubernetes Proxy Service";
        wantedBy = [ "multi-user.target" ];
        after = [ "network-interfaces.target" "etcd.service" ];
        serviceConfig = {
          ExecStart = ''${cfg.package}/bin/kube-proxy \
            --master=${cfg.proxy.master} \
            --bind-address=${cfg.proxy.address} \
            --logtostderr=true \
            ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
            ${cfg.proxy.extraOpts}
          '';
          Restart = "always"; # Retry connection
          RestartSec = "5s";
        };
      };
    })

    (mkIf cfg.kube2sky.enable {
      systemd.services.kube2sky = {
        description = "Kubernetes Dns Bridge Service";
        wantedBy = [ "multi-user.target" ];
        after = [ "network.target" "skydns.service" "etcd.service" "kubernetes-apiserver.service" ];
        serviceConfig = {
          ExecStart = ''${cfg.package}/bin/kube2sky \
            -etcd-server=http://${head cfg.etcdServers} \
            -domain=${cfg.kube2sky.domain} \
            -kube_master_url=http://${cfg.kube2sky.master} \
            -logtostderr=true \
            ${optionalString cfg.verbose "--v=6 --log-flush-frequency=1s"} \
            ${cfg.kube2sky.extraOpts}
          '';
          User = "kubernetes";
        };
      };
    })

    (mkIf (any (el: el == "master") cfg.roles) {
      services.kubernetes.apiserver.enable = mkDefault true;
      services.kubernetes.scheduler.enable = mkDefault true;
      services.kubernetes.controllerManager.enable = mkDefault true;
      services.kubernetes.kube2sky.enable = mkDefault true;
    })

    (mkIf (any (el: el == "node") cfg.roles) {
      virtualisation.docker.enable = mkDefault true;
      services.kubernetes.kubelet.enable = mkDefault true;
      services.kubernetes.proxy.enable = mkDefault true;
    })

    (mkIf (any (el: el == "node" || el == "master") cfg.roles) {
      services.etcd.enable = mkDefault true;

      services.skydns.enable = mkDefault true;
      services.skydns.domain = mkDefault cfg.kubelet.clusterDomain;
    })

    (mkIf (
        cfg.apiserver.enable ||
        cfg.scheduler.enable ||
        cfg.controllerManager.enable ||
        cfg.kubelet.enable ||
        cfg.proxy.enable
    ) {
      systemd.services.kubernetes-setup = {
        description = "Kubernetes setup.";
        serviceConfig.Type = "oneshot";
        script = ''
          mkdir -p /var/run/kubernetes
          chown kubernetes /var/lib/kubernetes

          rm ${cfg.dataDir}/.dockercfg || true
          ln -fs ${pkgs.writeText "kubernetes-dockercfg" cfg.dockerCfg} ${cfg.dataDir}/.dockercfg
        '';
      };

      services.kubernetes.package = mkDefault pkgs.kubernetes;

      environment.systemPackages = [ cfg.package ];

      users.extraUsers = singleton {
        name = "kubernetes";
        uid = config.ids.uids.kubernetes;
        description = "Kubernetes user";
        extraGroups = [ "docker" ];
        group = "kubernetes";
        home = cfg.dataDir;
        createHome = true;
      };
      users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes;
    })

  ];
}