{ config, lib, pkgs, ... }: with lib; let hostname = config.instance.hostname; domain-name = config.fudo.hosts.${hostname}.domain; domain = config.fudo.domains.${domain-name}; host-fqdn = hostname: let host-domain = config.fudo.hosts.${hostname}.domain; in "${hostname}.${host-domain}"; logAggregationEnabled = domain.log-aggregator != null; isLogAggregator = hostname == domain.log-aggregator; is-private-network = let site-name = config.fudo.hosts.${hostname}.site; in config.fudo.sites.${site-name}.local-gateway != null; cfg = config.fudo.services.logging; in { options.fudo.services.logging = with types; { loki = { port = mkOption { type = port; description = "Port on which to listen on localhost."; default = 3021; }; state-directory = mkOption { type = str; description = "Path at which to store state for Loki."; }; }; promtail = { http-listen-port = mkOption { type = port; default = 6041; }; grpc-listen-port = mkOption { type = port; default = 6042; }; user = mkOption { type = str; description = "User as which to run promtail job."; default = "promtail"; }; }; }; config = mkIf logAggregationEnabled { users = { users.${cfg.promtail.user} = { isSystemUser = true; uid = 441; group = cfg.promtail.user; }; groups = { ${cfg.promtail.user}.members = [ cfg.promtail.user ]; systemd-journal = { members = [ cfg.promtail.user ]; }; }; }; fudo = let aggregator-domain = config.fudo.hosts.${domain.log-aggregator}.domain; log-aggregator-fqdn = "${domain.log-aggregator}.${aggregator-domain}"; in { zones.${domain.zone} = { aliases.log-aggregator = "${log-aggregator-fqdn}."; }; metrics.grafana.datasources = let scheme = if is-private-network then "http" else "https"; in { "loki-${domain.log-aggregator}" = { url = "${scheme}://log-aggregator.${aggregator-domain}"; type = "loki"; }; }; }; services = { nginx = mkIf isLogAggregator { enable = true; virtualHosts."log-aggregator.${domain-name}" = { enableACME = !is-private-network; forceSSL = !is-private-network; locations."/" = { proxyPass = "http://127.0.0.1:${toString cfg.loki.port}"; extraConfig = let local-networks = config.instance.local-networks; in "${optionalString ((length local-networks) > 0) (concatStringsSep "\n" (map (network: "allow ${network};") local-networks)) + '' deny all;''}"; }; }; }; loki = mkIf isLogAggregator { enable = true; dataDir = cfg.loki.state-directory; # cargo-culted from https://gist.github.com/Xe/c3c786b41ec2820725ee77a7af551225 configuration = { auth_enabled = false; server.http_listen_port = cfg.loki.port; ingester = { lifecycler = { address = "127.0.0.1"; ring = { kvstore.store = "inmemory"; replication_factor = 1; }; final_sleep = "0s"; }; chunk_idle_period = "1h"; max_chunk_age = "1h"; chunk_target_size = 10485576; chunk_retain_period = "30s"; max_transfer_retries = 0; }; compactor = { working_directory = "${cfg.loki.state-directory}/compactor"; }; schema_config.configs = [{ from = "2022-01-01"; store = "boltdb-shipper"; object_store = "filesystem"; schema = "v11"; index = { prefix = "index_"; period = "24h"; }; }]; storage_config = { boltdb_shipper = { active_index_directory = "${cfg.loki.state-directory}/boltdb-shipper/active"; cache_location = "${cfg.loki.state-directory}/boltdb-shipper/cache"; cache_ttl = "24h"; shared_store = "filesystem"; }; filesystem.directory = "${cfg.loki.state-directory}/chunks"; }; limits_config = { reject_old_samples = true; reject_old_samples_max_age = "168h"; }; chunk_store_config.max_look_back_period = "0s"; table_manager = { retention_deletes_enabled = false; retention_period = "0s"; }; }; }; }; systemd = let lokiUser = config.services.loki.user; statedir = cfg.loki.state-directory; in { tmpfiles.rules = optionals isLogAggregator [ "d ${statedir} 0700 ${lokiUser} - - -" "d ${statedir}/boltdb-shipper 0700 ${lokiUser} - - -" "d ${statedir}/boltdb-shipper/active 0700 ${lokiUser} - - -" "d ${statedir}/boltdb-shipper/cache 0700 ${lokiUser} - - -" "d ${statedir}/chunks 0700 ${lokiUser} - - -" "d ${statedir}/compactor 0700 ${lokiUser} - - -" ]; services = { loki.serviceConfig = mkIf isLogAggregator { ExecStartPre = "+${pkgs.coreutils}/bin/chown -R ${lokiUser}:root ${statedir}"; }; promtail = let scheme = if is-private-network then "http" else "https"; loki-host = host-fqdn domain.log-aggregator; config = builtins.toJSON { server = { http_listen_port = cfg.promtail.http-listen-port; grpc_listen_port = cfg.promtail.grpc-listen-port; }; positions.filename = "/tmp/positions.yml"; clients = [{ url = "${scheme}://log-aggregator.${domain-name}/loki/api/v1/push"; }]; scrape_configs = [{ job_name = "journal"; journal = { max_age = "12h"; labels = { job = "systemd-journal"; host = hostname; }; }; relabel_configs = [{ source_labels = [ "__journal__systemd_unit" ]; target_label = "unit"; }]; }]; }; config-file = pkgs.writeText "promtail-config.json" config; in { description = "PromTail log aggregator client for Loki."; wantedBy = [ "multi-user.target" ]; serviceConfig = { ExecStart = '' ${pkgs.grafana-loki}/bin/promtail --config.file ${config-file} ''; User = cfg.promtail.user; PrivateTmp = true; }; }; }; }; }; }