nixos-config/config/service/logging.nix

{ config, lib, pkgs, ... }:

with lib;
let
  hostname = config.instance.hostname;
  domain-name = config.fudo.hosts.${hostname}.domain;
  domain = config.fudo.domains.${domain-name};

  host-fqdn = hostname: let
    host-domain = config.fudo.hosts.${hostname}.domain;
  in "${hostname}.${host-domain}";

  logAggregationEnabled = domain.log-aggregator != null;
  isLogAggregator = hostname == domain.log-aggregator;

  is-private-network = let
    site-name = config.fudo.hosts.${hostname}.site;
  in config.fudo.sites.${site-name}.local-gateway != null;

  cfg = config.fudo.services.logging;

in {
  options.fudo.services.logging = with types; {
    loki = {
      port = mkOption {
        type = port;
        description = "Port on which to listen on localhost.";
        default = 3021;
      };

      state-directory = mkOption {
        type = str;
        description = "Path at which to store state for Loki.";
      };
    };

    promtail = {
      http-listen-port = mkOption {
        type = port;
        default = 6041;
      };
      grpc-listen-port = mkOption {
        type = port;
        default = 6042;
      };
      user = mkOption {
        type = str;
        description = "User as which to run promtail job.";
        default = "promtail";
      };
    };
  };

  config = mkIf logAggregationEnabled {
    users = {
      users.${cfg.promtail.user} = {
        isSystemUser = true;
        uid = 441;
        group = cfg.promtail.user;
      };
      groups = {
        ${cfg.promtail.user}.members = [ cfg.promtail.user ];
        systemd-journal = {
          members = [ cfg.promtail.user ];
        };
      };
    };

    fudo = let
      aggregator-domain = config.fudo.hosts.${domain.log-aggregator}.domain;
      log-aggregator-fqdn = "${domain.log-aggregator}.${aggregator-domain}";
    in {
      zones.${domain.zone} = {
        aliases.log-aggregator = "${log-aggregator-fqdn}.";
      };

      metrics.grafana.datasources = let
        scheme = if is-private-network then "http" else "https";
      in {
        "loki-${domain.log-aggregator}" = {
          url = "${scheme}://log-aggregator.${aggregator-domain}";
          type = "loki";
        };
      };
    };

    services = {
      nginx = mkIf isLogAggregator {
        enable = true;

        virtualHosts."log-aggregator.${domain-name}" = {
          enableACME = ! is-private-network;
          forceSSL = ! is-private-network;
          locations."/" = {
            proxyPass = "http://127.0.0.1:${toString cfg.loki.port}";
            extraConfig = let
              local-networks = config.instance.local-networks;
            in "${optionalString ((length local-networks) > 0)
              (concatStringsSep "\n"
                (map (network: "allow ${network};") local-networks)) + "\ndeny all;"}";
          };
        };
      };

      loki = mkIf isLogAggregator {
        enable = true;
        dataDir = cfg.loki.state-directory;
        # cargo-culted from https://gist.github.com/Xe/c3c786b41ec2820725ee77a7af551225
        configuration = {
          auth_enabled = false;

          server.http_listen_port = cfg.loki.port;

          ingester = {
            lifecycler = {
              address = "127.0.0.1";
              ring = {
                kvstore.store = "inmemory";
                replication_factor = 1;
              };
              final_sleep = "0s";
            };
            chunk_idle_period = "1h";
            max_chunk_age = "1h";
            chunk_target_size = 10485576;
            chunk_retain_period = "30s";
            max_transfer_retries = 0;
          };

          compactor = {
            working_directory = "${cfg.loki.state-directory}/compactor";
          };

          schema_config.configs = [
            {
              from = "2022-01-01";
              store = "boltdb-shipper";
              object_store = "filesystem";
              schema = "v11";
              index = {
                prefix = "index_";
                period = "24h";
              };
            }
          ];

          storage_config = {
            boltdb_shipper = {
              active_index_directory = "${cfg.loki.state-directory}/boltdb-shipper/active";
              cache_location = "${cfg.loki.state-directory}/boltdb-shipper/cache";
              cache_ttl = "24h";
              shared_store = "filesystem";
            };
            filesystem.directory = "${cfg.loki.state-directory}/chunks";
          };

          limits_config = {
            reject_old_samples = true;
            reject_old_samples_max_age = "168h";
          };

          chunk_store_config.max_look_back_period = "0s";

          table_manager = {
            retention_deletes_enabled = false;
            retention_period = "0s";
          };
        };
      };
    };

    systemd = {
      tmpfiles.rules = mkIf isLogAggregator (let
        user = config.services.loki.user;
        statedir = cfg.loki.state-directory;
      in [
        "d ${statedir} 0700 ${user} - - -"
        "d ${statedir}/boltdb-shipper 0700 ${user} - - -"
        "d ${statedir}/boltdb-shipper/active 0700 ${user} - - -"
        "d ${statedir}/boltdb-shipper/cache 0700 ${user} - - -"
        "d ${statedir}/chunks 0700 ${user} - - -"
        "d ${statedir}/compactor 0700 ${user} - - -"
      ]);

      services.promtail = let
        scheme = if is-private-network then "http" else "https";

        loki-host = host-fqdn domain.log-aggregator;

        config = builtins.toJSON {
          server = {
            http_listen_port = cfg.promtail.http-listen-port;
            grpc_listen_port = cfg.promtail.grpc-listen-port;
          };
          positions.filename = "/tmp/positions.yml";
          clients = [
            { url = "${scheme}://log-aggregator.${domain-name}/loki/api/v1/push"; }
          ];
          scrape_configs = [
            {
              job_name = "journal";
              journal = {
                max_age = "12h";
                labels = {
                  job = "systemd-journal";
                  host = hostname;
                };
              };
              relabel_configs = [{
                source_labels = [ "__journal__systemd_unit" ];
                target_label = "unit";
              }];
            }
          ];
        };

        config-file = pkgs.writeText "promtail-config.json" config;
      in {
        description = "PromTail log aggregator client for Loki.";
        wantedBy = [ "multi-user.target" ];
        serviceConfig = {
          ExecStart = ''
          ${pkgs.grafana-loki}/bin/promtail --config.file ${config-file}
        '';
          User = cfg.promtail.user;
          PrivateTmp = true;
        };
      };
    };
  };
}