nixos-config/config/service/logging.nix

{ config, lib, pkgs, ... }:

with lib;
let
  hostname = config.instance.hostname;
  domain-name = config.fudo.hosts.${hostname}.domain;
  domain = config.fudo.domains.${domain-name};

  host-fqdn = hostname:
    let host-domain = config.fudo.hosts.${hostname}.domain;
    in "${hostname}.${host-domain}";

  logAggregationEnabled = domain.log-aggregator != null;
  isLogAggregator = hostname == domain.log-aggregator;

  is-private-network = let site-name = config.fudo.hosts.${hostname}.site;
  in config.fudo.sites.${site-name}.local-gateway != null;

  cfg = config.fudo.services.logging;

in {
  options.fudo.services.logging = with types; {
    loki = {
      port = mkOption {
        type = port;
        description = "Port on which to listen on localhost.";
        default = 3021;
      };

      state-directory = mkOption {
        type = str;
        description = "Path at which to store state for Loki.";
      };
    };

    promtail = {
      http-listen-port = mkOption {
        type = port;
        default = 6041;
      };
      grpc-listen-port = mkOption {
        type = port;
        default = 6042;
      };
      user = mkOption {
        type = str;
        description = "User as which to run promtail job.";
        default = "promtail";
      };
    };
  };

  config = mkIf logAggregationEnabled {
    users = {
      users.${cfg.promtail.user} = {
        isSystemUser = true;
        uid = 441;
        group = cfg.promtail.user;
      };
      groups = {
        ${cfg.promtail.user}.members = [ cfg.promtail.user ];
        systemd-journal = { members = [ cfg.promtail.user ]; };
      };
    };

    fudo = let
      aggregator-domain = config.fudo.hosts.${domain.log-aggregator}.domain;
      log-aggregator-fqdn = "${domain.log-aggregator}.${aggregator-domain}";
    in {
      zones.${domain.zone} = {
        aliases.log-aggregator = "${log-aggregator-fqdn}.";
      };

      metrics.grafana.datasources =
        let scheme = if is-private-network then "http" else "https";
        in {
          "loki-${domain.log-aggregator}" = {
            url = "${scheme}://log-aggregator.${aggregator-domain}";
            type = "loki";
          };
        };
    };

    services = {
      nginx = mkIf isLogAggregator {
        enable = true;

        virtualHosts."log-aggregator.${domain-name}" = {
          enableACME = !is-private-network;
          forceSSL = !is-private-network;
          locations."/" = {
            proxyPass = "http://127.0.0.1:${toString cfg.loki.port}";
            extraConfig = let local-networks = config.instance.local-networks;
            in "${optionalString ((length local-networks) > 0)
            (concatStringsSep "\n"
              (map (network: "allow ${network};") local-networks)) + ''

                deny all;''}";
          };
        };
      };

      loki = mkIf isLogAggregator {
        enable = true;
        dataDir = cfg.loki.state-directory;
        # cargo-culted from https://gist.github.com/Xe/c3c786b41ec2820725ee77a7af551225
        configuration = {
          auth_enabled = false;

          server.http_listen_port = cfg.loki.port;

          ingester = {
            lifecycler = {
              address = "127.0.0.1";
              ring = {
                kvstore.store = "inmemory";
                replication_factor = 1;
              };
              final_sleep = "0s";
            };
            chunk_idle_period = "1h";
            max_chunk_age = "1h";
            chunk_target_size = 10485576;
            chunk_retain_period = "30s";
            max_transfer_retries = 0;
          };

          compactor = {
            working_directory = "${cfg.loki.state-directory}/compactor";
          };

          schema_config.configs = [{
            from = "2022-01-01";
            store = "boltdb-shipper";
            object_store = "filesystem";
            schema = "v11";
            index = {
              prefix = "index_";
              period = "24h";
            };
          }];

          storage_config = {
            boltdb_shipper = {
              active_index_directory =
                "${cfg.loki.state-directory}/boltdb-shipper/active";
              cache_location =
                "${cfg.loki.state-directory}/boltdb-shipper/cache";
              cache_ttl = "24h";
              shared_store = "filesystem";
            };
            filesystem.directory = "${cfg.loki.state-directory}/chunks";
          };

          limits_config = {
            reject_old_samples = true;
            reject_old_samples_max_age = "168h";
          };

          chunk_store_config.max_look_back_period = "0s";

          table_manager = {
            retention_deletes_enabled = false;
            retention_period = "0s";
          };
        };
      };
    };

    systemd = let
      lokiUser = config.services.loki.user;
      statedir = cfg.loki.state-directory;
    in {
      tmpfiles.rules = optionals isLogAggregator [
        "d ${statedir} 0700 ${lokiUser} - - -"
        "d ${statedir}/boltdb-shipper 0700 ${lokiUser} - - -"
        "d ${statedir}/boltdb-shipper/active 0700 ${lokiUser} - - -"
        "d ${statedir}/boltdb-shipper/cache 0700 ${lokiUser} - - -"
        "d ${statedir}/chunks 0700 ${lokiUser} - - -"
        "d ${statedir}/compactor 0700 ${lokiUser} - - -"
      ];

      services = {
        loki.serviceConfig = mkIf isLogAggregator {
          ExecStartPre =
            "+${pkgs.coreutils}/bin/chown -R ${lokiUser}:root ${statedir}";
        };

        promtail = let
          scheme = if is-private-network then "http" else "https";

          loki-host = host-fqdn domain.log-aggregator;

          config = builtins.toJSON {
            server = {
              http_listen_port = cfg.promtail.http-listen-port;
              grpc_listen_port = cfg.promtail.grpc-listen-port;
            };
            positions.filename = "/tmp/positions.yml";
            clients = [{
              url =
                "${scheme}://log-aggregator.${domain-name}/loki/api/v1/push";
            }];
            scrape_configs = [{
              job_name = "journal";
              journal = {
                max_age = "12h";
                labels = {
                  job = "systemd-journal";
                  host = hostname;
                };
              };
              relabel_configs = [{
                source_labels = [ "__journal__systemd_unit" ];
                target_label = "unit";
              }];
            }];
          };

          config-file = pkgs.writeText "promtail-config.json" config;
        in {
          description = "PromTail log aggregator client for Loki.";
          wantedBy = [ "multi-user.target" ];
          serviceConfig = {
            ExecStart = ''
              ${pkgs.grafana-loki}/bin/promtail --config.file ${config-file}
            '';
            User = cfg.promtail.user;
            PrivateTmp = true;
          };
        };
      };
    };
  };
}