{ config, lib, pkgs, ... }@toplevel: with lib; let hostname = config.instance.hostname; domain-name = config.fudo.hosts."${hostname}".domain; domain = config.fudo.domains."${domain-name}"; host-secrets = config.fudo.secrets.host-secrets."${hostname}"; notEmpty = lst: (length lst) > 0; metricsEnabled = notEmpty domain.prometheus-hosts; metricsScraper = elem hostname domain.prometheus-hosts; metricsMonitor = elem hostname domain.grafana-hosts; prometheus-cfg = config.fudo.services.metrics.prometheus; grafana-cfg = config.fudo.services.metrics.grafana; host-fqdn = hostname: let host-domain = config.fudo.hosts.${hostname}.domain; in "${hostname}.${host-domain}"; host-auth-fqdn = hostname: "${host-fqdn hostname}."; make-alias-map = type: hosts: listToAttrs (imap0 (i: hostname: nameValuePair hostname "${type}-${toString i}") hosts); headOrNull = lst: if notEmpty lst then head lst else null; metrics-master = headOrNull domain.prometheus-hosts; monitor-master = headOrNull domain.grafana-hosts; metrics-alias-map = make-alias-map "metrics" domain.prometheus-hosts; monitor-alias-map = make-alias-map "monitor" domain.grafana-hosts; alias-map-to-cnames = mapAttrs' (hostname: alias: nameValuePair alias (host-auth-fqdn hostname)); alias-map-to-hostnames = mapAttrsToList (hostname: alias: "${alias}.${domain-name}"); grafana-smtp-password-file = pkgs.lib.passwd.stablerandom-passwd-file "grafana-smtp-passwd" config.instance.build-seed; grafana-auth-password-file = pkgs.lib.passwd.stablerandom-passwd-file "grafana-auth-passwd" config.instance.build-seed; grafana-admin-password-file = pkgs.lib.passwd.stablerandom-passwd-file "grafana-admin-passwd" config.instance.build-seed; grafana-secret-key-file = pkgs.lib.passwd.stablerandom-passwd-file "grafana-secret-key" config.instance.build-seed; grafana-database-password-file = pkgs.lib.passwd.stablerandom-passwd-file "grafana-database-postgres" config.instance.build-seed; site = let site-name = config.fudo.hosts."${hostname}".site; in config.fudo.sites."${site-name}"; is-private-network = site.local-gateway != null; domainToBaseDn = domain: concatStringsSep "," (map (el: "dc=${el}") (splitString "." domain)); ldapEnabled = domain.ldap-servers != [ ]; isPostgresServer = config.instance.hostname == domain.postgresql-server; postgresServer = pkgs.lib.getHostFqdn domain.postgresql-server; in { options.fudo.services.metrics = with types; { prometheus = { static-targets = mkOption { type = attrsOf (listOf str); description = "A map of exporter type to a list of host:ports from which to collect metrics."; example = { dovecot = [ "my.host.name:1111" ]; }; default = { }; }; state-directory = mkOption { type = str; description = "Path at which to store Prometheus state."; default = "/var/lib/prometheus"; }; }; grafana = { smtp = { username = mkOption { type = str; description = "Username from which to send Grafana alerts."; default = "monitor"; }; hostname = mkOption { type = str; description = "Hostname of the SMTP host."; default = "mail.${toplevel.config.instance.local-domain}"; }; }; ldap = let base-dn = domainToBaseDn config.instance.local-domain; in { base-dn = mkOption { type = str; description = "DN under which to search for users."; default = base-dn; }; bind-user = mkOption { type = str; description = "DN as which to bind to the LDAP server."; default = "grafana_reader"; }; bind-passwd = mkOption { type = nullOr str; description = "Path to file with bind password. Generated if null."; default = null; }; }; # database = { # hostname = mkOption { # type = str; # description = "Hostname of the postgresql database."; # default = "localhost"; # }; # user = mkOption { # type = str; # description = # "Username as which to authenticate to the postgresql database."; # }; # password-file = mkOption { # type = str; # description = # "Password file (on the target host) which to authenticate to the postgresql database."; # }; # name = mkOption { # type = str; # description = "Database name."; # default = "grafana"; # }; # }; state-directory = mkOption { type = str; description = "Path at which to store Grafana state."; default = "/var/lib/grafana"; }; }; }; config = mkIf metricsEnabled { fudo = { system-users = { "${grafana-cfg.smtp.username}" = { description = "Grafana Alerts"; ldap-hashed-password = pkgs.lib.passwd.hash-ldap-passwd "grafana-smtp-passwd" grafana-smtp-password-file; }; "${grafana-cfg.ldap.bind-user}" = mkIf ((domain.ldap-servers != [ ]) && (grafana-cfg.ldap.bind-passwd == null)) { description = "Grafana Authentication Reader"; ldap-hashed-password = pkgs.lib.passwd.hash-ldap-passwd "grafana-auth-passwd" grafana-auth-password-file; }; }; secrets.host-secrets = let grafana-user = config.systemd.services.grafana.serviceConfig.User; in { "${hostname}" = { grafana-smtp-password = mkIf metricsMonitor { source-file = grafana-smtp-password-file; target-file = "/run/metrics/grafana/smtp.passwd"; user = grafana-user; }; grafana-admin-password = mkIf metricsMonitor { source-file = grafana-admin-password-file; target-file = "/run/metrics/grafana/admin.passwd"; user = grafana-user; }; grafana-secret-key = mkIf metricsMonitor { source-file = grafana-secret-key-file; target-file = "/run/metrics/grafana/secret.key"; user = grafana-user; }; grafana-postgresql-password = mkIf metricsMonitor { source-file = grafana-database-password-file; target-file = "/run/metrics/grafana/postgres.passwd"; user = grafana-user; }; postgresql-grafana-password = mkIf isPostgresServer { source-file = grafana-database-password-file; target-file = "/run/postgres-users/grafana.passwd"; user = config.systemd.services.postgresql.serviceConfig.User; }; }; }; zones."${domain.zone}" = { aliases = let metrics-aliases = alias-map-to-cnames metrics-alias-map; monitor-aliases = alias-map-to-cnames monitor-alias-map; metrics-master-cname = optionalAttrs (metrics-master != null) { metrics = "${metrics-master}.${domain-name}."; }; monitor-master-cname = optionalAttrs (monitor-master != null) { monitor = "${monitor-master}.${domain-name}."; }; in metrics-aliases // monitor-aliases // metrics-master-cname // monitor-master-cname; metric-records = let domain-hosts = filterAttrs (hostname: hostOpts: hostOpts.domain == domain-name && hostOpts.nixos-system) config.fudo.hosts; in { node = map (hostname: { host = "${hostname}.${domain-name}"; port = if is-private-network then 80 else 443; }) (attrNames domain-hosts); }; }; postgresql = mkIf isPostgresServer { users.grafana = { password-file = host-secrets.postgresql-grafana-password.target-file; databases.grafana = { access = "CONNECT"; entity-access = { "ALL TABLES IN SCHEMA public" = "ALL PRIVILEGES"; # "SELECT,INSERT,UPDATE,DELETE"; "ALL SEQUENCES IN SCHEMA public" = "ALL PRIVILEGES"; # "SELECT, UPDATE"; }; }; }; databases.grafana.users = config.instance.local-admins; }; metrics = { node-exporter = { enable = true; hostname = host-fqdn hostname; private-network = is-private-network; }; prometheus = mkIf metricsScraper { enable = true; service-discovery-dns = { node = [ "node._metrics._tcp.${domain-name}" ]; }; static-targets = prometheus-cfg.static-targets; hostname = let alias = metrics-alias-map.${hostname}; in "${alias}.${domain-name}"; state-directory = prometheus-cfg.state-directory; private-network = is-private-network; }; grafana = mkIf metricsMonitor { enable = true; hostname = let alias = monitor-alias-map.${hostname}; in "${alias}.${domain-name}"; smtp = let cfg = grafana-cfg.smtp; in { username = cfg.username; password-file = host-secrets.grafana-smtp-password.target-file; hostname = cfg.hostname; email = "${cfg.username}@${domain-name}"; }; database = let cfg = grafana-cfg.database; in { name = "grafana"; user = "grafana"; password-file = host-secrets.grafana-postgresql-password.target-file; hostname = postgresServer; }; ldap = mkIf (domain.ldap-servers != [ ]) { hosts = map host-fqdn domain.ldap-servers; base-dn = grafana-cfg.ldap.base-dn; bind-dn = "cn=${grafana-cfg.ldap.bind-user},${grafana-cfg.ldap.base-dn}"; bind-passwd = if (grafana-cfg.ldap.bind-passwd != null) then grafana-cfg.ldap.bind-passwd else (readFile grafana-auth-password-file); }; admin-password-file = host-secrets.grafana-admin-password.target-file; secret-key-file = host-secrets.grafana-secret-key.target-file; datasources = let scheme = if is-private-network then "http" else "https"; host-config = hostname: { url = "${scheme}://${hostname}.${domain-name}"; type = "prometheus"; default = hostname == "metrics-0"; }; in listToAttrs (map (host: nameValuePair "prometheus-${host}" (host-config host)) (attrValues metrics-alias-map)); state-directory = grafana-cfg.state-directory; private-network = is-private-network; }; }; }; services.nginx = mkIf (hostname == metrics-master || hostname == monitor-master) { enable = true; recommendedOptimisation = true; recommendedProxySettings = true; virtualHosts = let scheme = if is-private-network then "http" else "https"; in { "metrics.${domain-name}" = mkIf (hostname == metrics-master) { enableACME = !is-private-network; forceSSL = !is-private-network; locations."/".return = let alias = metrics-alias-map.${hostname}; in "301 ${scheme}://${alias}.${domain-name}$request_uri"; }; "monitor.${domain-name}" = mkIf (hostname == monitor-master) { enableACME = !is-private-network; forceSSL = !is-private-network; locations."/".return = let alias = monitor-alias-map.${hostname}; in "301 ${scheme}://${alias}.${domain-name}$request_uri"; }; }; }; }; }