nixos-config/config/service/metrics.nix

{ config, lib, pkgs, ... }@toplevel:

with lib;
let
  hostname = config.instance.hostname;
  domainName = config.fudo.hosts."${hostname}".domain;
  domain = config.fudo.domains."${domainName}";

  inherit (pkgs.lib) getHostIpv4 getHostIpv6 getHostFqdn;

  hostSecrets = config.fudo.secrets.host-secrets."${hostname}";

  metricsEnabled = !isNull domain.metrics;
  isPrometheus = hostname == domain.metrics.prometheus-host;
  isGrafana = hostname == domain.metrics.grafana-host;

  grafanaHost = domain.metrics.grafana-host;
  prometheusHost = domain.metrics.prometheus-host;

  prometheusCfg = config.fudo.services.metrics.prometheus;
  grafanaCfg = config.fudo.services.metrics.grafana;
  privateNetwork = config.fudo.services.metrics.private-network;

  grafana-smtp-password-file =
    pkgs.lib.passwd.stablerandom-passwd-file "grafana-smtp-passwd"
    config.instance.build-seed;

  grafana-admin-password-file =
    pkgs.lib.passwd.stablerandom-passwd-file "grafana-admin-passwd"
    config.instance.build-seed;

  grafana-secret-key-file =
    pkgs.lib.passwd.stablerandom-passwd-file "grafana-secret-key"
    config.instance.build-seed;

in {
  options.fudo.services.metrics = with types; {
    private-network = mkOption {
      type = bool;
      description = "Network is private, encryption not required.";
      default = false;
    };

    prometheus = {
      static-targets = mkOption {
        type = attrsOf (listOf str);
        description =
          "A map of exporter type to a list of host:ports from which to collect metrics.";
        example = { dovecot = [ "my.host.name:1111" ]; };
        default = { };
      };

      state-directory = mkOption {
        type = str;
        description = "Path at which to store Prometheus state.";
        default = "/var/lib/prometheus";
      };
    };
    grafana = {
      smtp = {
        username = mkOption {
          type = str;
          description = "Username from which to send Grafana alerts.";
          default = "monitor";
        };

        hostname = mkOption {
          type = str;
          description = "Hostname of the SMTP host.";
          default = "mail.${toplevel.config.instance.local-domain}";
        };
      };

      oauth = let
        oauthOpts.options = {
          hostname = mkOption {
            type = str;
            description = "Host of the OAuth server.";
          };

          client-id = mkOption {
            type = str;
            description =
              "Path to file containing the Grafana OAuth client ID.";
          };

          client-secret = mkOption {
            type = str;
            description =
              "Path to file containing the Grafana OAuth client secret.";
          };

          slug = mkOption {
            type = str;
            description = "The application slug on the OAuth server.";
          };
        };
      in mkOption {
        type = nullOr (submodule oauthOpts);
        default = null;
      };

      state-directory = mkOption {
        type = str;
        description = "Path at which to store Grafana state.";
        default = "/var/lib/grafana";
      };
    };
  };

  config = mkIf metricsEnabled {

    fudo = {
      secrets.host-secrets =
        let grafana-user = config.systemd.services.grafana.serviceConfig.User;
        in {
          "${hostname}" = {
            grafana-admin-password = mkIf isGrafana {
              source-file = grafana-admin-password-file;
              target-file = "/run/metrics/grafana/admin.passwd";
              user = grafana-user;
            };

            grafana-secret-key = mkIf isGrafana {
              source-file = grafana-secret-key-file;
              target-file = "/run/metrics/grafana/secret.key";
              user = grafana-user;
            };

            grafana-client-id = mkIf (isGrafana && !isNull grafanaCfg.oauth) {
              source-file = grafanaCfg.oauth.client-id;
              target-file = "/run/metrics/grafana/oauth-client-id";
              user = grafana-user;
            };

            grafana-client-secret =
              mkIf (isGrafana && !isNull grafanaCfg.oauth) {
                source-file = grafanaCfg.oauth.client-secret;
                target-file = "/run/metrics/grafana/oauth-client-secret";
                user = grafana-user;
              };
          };
        };

      zones."${domain.zone}" = {
        hosts = {
          grafana = {
            ipv4-address = getHostIpv4 grafanaHost;
            ipv6-address = getHostIpv6 grafanaHost;
            description = "Grafana Metrics Analysis on ${grafanaHost}.";
          };
          prometheus = {
            ipv4-address = getHostIpv4 prometheusHost;
            ipv6-address = getHostIpv6 prometheusHost;
            description = "Prometheus Metrics Aggregator on ${prometheusHost}.";
          };
        };

        aliases = {
          metrics = "prometheus.${domainName}";
          monitor = "grafana.${domainName}";
        };

        metric-records = let
          domainHosts = filterAttrs (hostname: hostOpts:
            hostOpts.domain == domainName && hostOpts.nixos-system)
            config.fudo.hosts;
        in {
          node = map (hostname: {
            host = getHostFqdn hostname;
            port = if privateNetwork then 80 else 443;
          }) (attrNames domainHosts);
        };
      };

      metrics = {
        node-exporter = {
          enable = true;
          hostname = getHostFqdn hostname;
          private-network = privateNetwork;
        };

        prometheus = mkIf isPrometheus {
          enable = true;
          service-discovery-dns = {
            node = [ "node._metrics._tcp.${domainName}" ];
          };
          static-targets = prometheusCfg.static-targets;
          hostname = "prometheus.${domainName}";
          state-directory = prometheusCfg.state-directory;
          private-network = privateNetwork;
        };
      };

      services.grafana = mkIf isGrafana {
        enable = true;
        state-directory = grafanaCfg.state-directory;
        base-url = let scheme = if privateNetwork then "http" else "https";
        in "${scheme}://grafana.${domainName}";
        admin-password-file = hostSecrets.grafana-admin-password.target-file;
        secret-key-file = hostSecrets.grafana-secret-key.target-file;
        datasources = {
          "${domainName}" = {
            url = let scheme = if privateNetwork then "http" else "https";
            in "${scheme}://prometheus.${domainName}";
            type = "prometheus";
            default = true;
          };
        };
        oauth = mkIf (!isNull grafanaCfg.oauth) {
          inherit (grafanaCfg.oauth) hostname slug;
          client-id = hostSecrets.grafana-client-id.target-file;
          client-secret = hostSecrets.grafana-client-secret.target-file;
        };
      };
    };

    services.nginx = mkIf (isPrometheus || isGrafana) {
      enable = true;
      recommendedOptimisation = true;
      recommendedProxySettings = true;

      virtualHosts = let scheme = if privateNetwork then "http" else "https";
      in {
        "metrics.${domainName}".locations."/".return =
          "302 http://prometheus.${domainName}";
        "monitor.${domainName}".locations."/".return =
          "302 http://grafana.${domainName}";

        "grafana.${domainName}" = {
          enableACME = !privateNetwork;
          forceSSL = !privateNetwork;
          locations."/".proxyPass =
            "http://localhost:${toString config.fudo.services.grafana.port}";
        };
      };
    };
  };
}