{ config, lib, pkgs, ... }@toplevel: with lib; let hostname = config.instance.hostname; domainName = config.fudo.hosts."${hostname}".domain; domain = config.fudo.domains."${domainName}"; inherit (pkgs.lib) getHostIpv4 getHostIpv6 getHostFqdn; hostSecrets = config.fudo.secrets.host-secrets."${hostname}"; metricsEnabled = !isNull domain.metrics; isPrometheus = hostname == domain.metrics.prometheus-host; isGrafana = hostname == domain.metrics.grafana-host; grafanaHost = domain.metrics.grafana-host; prometheusHost = domain.metrics.prometheus-host; prometheusCfg = config.fudo.services.metrics.prometheus; grafanaCfg = config.fudo.services.metrics.grafana; privateNetwork = config.fudo.services.metrics.private-network; grafana-smtp-password-file = pkgs.lib.passwd.stablerandom-passwd-file "grafana-smtp-passwd" config.instance.build-seed; grafana-admin-password-file = pkgs.lib.passwd.stablerandom-passwd-file "grafana-admin-passwd" config.instance.build-seed; grafana-secret-key-file = pkgs.lib.passwd.stablerandom-passwd-file "grafana-secret-key" config.instance.build-seed; in { options.fudo.services.metrics = with types; { private-network = mkOption { type = bool; description = "Network is private, encryption not required."; default = false; }; prometheus = { static-targets = mkOption { type = attrsOf (listOf str); description = "A map of exporter type to a list of host:ports from which to collect metrics."; example = { dovecot = [ "my.host.name:1111" ]; }; default = { }; }; state-directory = mkOption { type = str; description = "Path at which to store Prometheus state."; default = "/var/lib/prometheus"; }; }; grafana = { smtp = { username = mkOption { type = str; description = "Username from which to send Grafana alerts."; default = "monitor"; }; hostname = mkOption { type = str; description = "Hostname of the SMTP host."; default = "mail.${toplevel.config.instance.local-domain}"; }; }; oauth = let oauthOpts.options = { hostname = mkOption { type = str; description = "Host of the OAuth server."; }; client-id = mkOption { type = str; description = "Path to file containing the Grafana OAuth client ID."; }; client-secret = mkOption { type = str; description = "Path to file containing the Grafana OAuth client secret."; }; slug = mkOption { type = str; description = "The application slug on the OAuth server."; }; }; in mkOption { type = nullOr (submodule oauthOpts); default = null; }; state-directory = mkOption { type = str; description = "Path at which to store Grafana state."; default = "/var/lib/grafana"; }; }; }; config = mkIf metricsEnabled { fudo = { secrets.host-secrets = let grafana-user = config.systemd.services.grafana.serviceConfig.User; in { "${hostname}" = { grafana-admin-password = mkIf isGrafana { source-file = grafana-admin-password-file; target-file = "/run/metrics/grafana/admin.passwd"; user = grafana-user; }; grafana-secret-key = mkIf isGrafana { source-file = grafana-secret-key-file; target-file = "/run/metrics/grafana/secret.key"; user = grafana-user; }; grafana-client-id = mkIf (isGrafana && !isNull grafanaCfg.oauth) { source-file = grafanaCfg.oauth.client-id; target-file = "/run/metrics/grafana/oauth-client-id"; user = grafana-user; }; grafana-client-secret = mkIf (isGrafana && !isNull grafanaCfg.oauth) { source-file = grafanaCfg.oauth.client-secret; target-file = "/run/metrics/grafana/oauth-client-secret"; user = grafana-user; }; }; }; zones."${domain.zone}" = { hosts = { grafana = { ipv4-address = getHostIpv4 grafanaHost; ipv6-address = getHostIpv6 grafanaHost; description = "Grafana Metrics Analysis on ${grafanaHost}."; }; prometheus = { ipv4-address = getHostIpv4 prometheusHost; ipv6-address = getHostIpv6 prometheusHost; description = "Prometheus Metrics Aggregator on ${prometheusHost}."; }; }; aliases = { metrics = "prometheus.${domainName}"; monitor = "grafana.${domainName}"; }; metric-records = let domainHosts = filterAttrs (hostname: hostOpts: hostOpts.domain == domainName && hostOpts.nixos-system) config.fudo.hosts; in { node = map (hostname: { host = getHostFqdn hostname; port = if privateNetwork then 80 else 443; }) (attrNames domainHosts); }; }; metrics = { node-exporter = { enable = true; hostname = getHostFqdn hostname; private-network = privateNetwork; }; prometheus = mkIf isPrometheus { enable = true; service-discovery-dns = { node = [ "node._metrics._tcp.${domainName}" ]; }; static-targets = prometheusCfg.static-targets; hostname = "prometheus.${domainName}"; state-directory = prometheusCfg.state-directory; private-network = privateNetwork; }; }; services.grafana = mkIf isGrafana { enable = true; state-directory = grafanaCfg.state-directory; base-url = let scheme = if privateNetwork then "http" else "https"; in "${scheme}://grafana.${domainName}"; admin-password-file = hostSecrets.grafana-admin-password.target-file; secret-key-file = hostSecrets.grafana-secret-key.target-file; datasources = { "${domainName}" = { url = let scheme = if privateNetwork then "http" else "https"; in "${scheme}://prometheus.${domainName}"; type = "prometheus"; default = true; }; }; oauth = mkIf (!isNull grafanaCfg.oauth) { inherit (grafanaCfg.oauth) hostname slug; client-id = hostSecrets.grafana-client-id.target-file; client-secret = hostSecrets.grafana-client-secret.target-file; }; }; }; services.nginx = mkIf (isPrometheus || isGrafana) { enable = true; recommendedOptimisation = true; recommendedProxySettings = true; virtualHosts = let scheme = if privateNetwork then "http" else "https"; in { "metrics.${domainName}".locations."/".return = "302 http://prometheus.${domainName}"; "monitor.${domainName}".locations."/".return = "302 http://grafana.${domainName}"; "grafana.${domainName}" = { enableACME = !privateNetwork; forceSSL = !privateNetwork; locations."/".proxyPass = "http://localhost:${toString config.fudo.services.grafana.port}"; }; }; }; }; }