Add prometheus2 configuration to the prometheus modules

As the configuration for the exporters and alertmanager is unchanged
between the two major versions this patch tries to minimize
duplication while at the same time as there's no upgrade path from 1.x
to 2.x, it allows running the two services in parallel. See also #56037

nixos/modules/misc/ids.nix

@@ -339,6 +339,7 @@
       rss2email = 312;
       cockroachdb = 313;
       zoneminder = 314;
+      prometheus2 = 315;
 
       # When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399!
 
@@ -638,6 +639,7 @@
       rss2email = 312;
       cockroachdb = 313;
       zoneminder = 314;
+      prometheus2 = 315;
 
       # When adding a gid, make sure it doesn't match an existing
       # uid. Users and groups with the same name should have equal

nixos/modules/services/monitoring/prometheus/alertmanager.nix

@@ -4,31 +4,33 @@ with lib;
 
 let
   cfg = config.services.prometheus.alertmanager;
-  mkConfigFile = pkgs.writeText "alertmanager.yml" (builtins.toJSON cfg.configuration);
+  cfg2 = config.services.prometheus2.alertmanager;
+  mkConfigFile = amCfg:
+    pkgs.writeText "alertmanager.yml" (builtins.toJSON amCfg.configuration);
 
-  checkedConfig = file: pkgs.runCommand "checked-config" { buildInputs = [ cfg.package ]; } ''
+  mkAlertmanagerYml = amCfg: let
+      checkedConfig = file:
+        pkgs.runCommand "checked-config" { buildInputs = [ amCfg.package ]; } ''
         ln -s ${file} $out
         amtool check-config $out
       '';
+      yml = if amCfg.configText != null then
+        pkgs.writeText "alertmanager.yml" amCfg.configText
+        else mkConfigFile amCfg;
+    in
+      checkedConfig yml;
 
-  alertmanagerYml = let
+  mkCmdlineArgs = amCfg:
-    yml = if cfg.configText != null then
+    amCfg.extraFlags ++ [
-        pkgs.writeText "alertmanager.yml" cfg.configText
+    "--config.file ${mkAlertmanagerYml amCfg}"
-        else mkConfigFile;
+    "--web.listen-address ${amCfg.listenAddress}:${toString amCfg.port}"
-    in checkedConfig yml;
+    "--log.level ${amCfg.logLevel}"
-
+    ] ++ (optional (amCfg.webExternalUrl != null)
-  cmdlineArgs = cfg.extraFlags ++ [
+      "--web.external-url ${amCfg.webExternalUrl}"
-    "--config.file ${alertmanagerYml}"
+    ) ++ (optional (amCfg.logFormat != null)
-    "--web.listen-address ${cfg.listenAddress}:${toString cfg.port}"
+      "--log.format ${amCfg.logFormat}"
-    "--log.level ${cfg.logLevel}"
-    ] ++ (optional (cfg.webExternalUrl != null)
-      "--web.external-url ${cfg.webExternalUrl}"
-    ) ++ (optional (cfg.logFormat != null)
-      "--log.format ${cfg.logFormat}"
     );
-in {
+    amOptions = {
-  options = {
-    services.prometheus.alertmanager = {
       enable = mkEnableOption "Prometheus Alertmanager";
 
       package = mkOption {
@@ -135,30 +137,28 @@ in {
         '';
       };
     };
-  };
+    mkAMConfig = amCfg: amVersion:
-
       config = mkMerge [
-    (mkIf cfg.enable {
+        (mkIf amCfg.enable {
           assertions = singleton {
-        assertion = cfg.configuration != null || cfg.configText != null;
+            assertion = amCfg.configuration != null || amCfg.configText != null;
             message = "Can not enable alertmanager without a configuration. "
              + "Set either the `configuration` or `configText` attribute.";
           };
         })
-    (mkIf cfg.enable {
+        (mkIf amCfg.enable {
-      networking.firewall.allowedTCPPorts = optional cfg.openFirewall cfg.port;
+          networking.firewall.allowedTCPPorts = optional amCfg.openFirewall amCfg.port;
 
-      systemd.services.alertmanager = {
+          systemd.services."alertmanager${amVersion}" = {
             wantedBy = [ "multi-user.target" ];
             after    = [ "network.target" ];
             script = ''
-          ${cfg.package}/bin/alertmanager \
+              ${amCfg.package}/bin/alertmanager \
                 ${concatStringsSep " \\\n  " cmdlineArgs}
             '';
-
             serviceConfig = {
-          User = cfg.user;
+              User = amCfg.user;
-          Group = cfg.group;
+              Group = amCfg.group;
               Restart  = "always";
               PrivateTmp = true;
               WorkingDirectory = "/tmp";
@@ -167,4 +167,14 @@ in {
           };
         })
       ];
+in {
+  options = {
+    services.prometheus.alertmanager = amOptions;
+    services.prometheus2.alertmanager = amOptions;
+  };
+
+  config = mkMerge [
+    (mkAMConfig cfg "")
+    (mkAMConfig cfg2 "2")
+  ];
 }

nixos/modules/services/monitoring/prometheus/default.nix

@@ -4,8 +4,11 @@ with lib;
 
 let
   cfg = config.services.prometheus;
+  cfg2 = config.services.prometheus2;
   promUser = "prometheus";
   promGroup = "prometheus";
+  prom2User = "prometheus2";
+  prom2Group = "prometheus2";
 
   # Get a submodule without any embedded metadata:
   _filter = x: filterAttrs (k: v: k != "_module") x;
@@ -17,13 +20,21 @@ let
     promtool ${what} $out
   '';
 
+  # a wrapper that verifies that the configuration is valid for
+  # prometheus 2
+  prom2toolCheck = what: name: file: pkgs.runCommand "${name}-${what}-checked"
+    { buildInputs = [ cfg2.package ]; } ''
+    ln -s ${file} $out
+    promtool ${what} $out
+  '';
+
   # Pretty-print JSON to a file
   writePrettyJSON = name: x:
     pkgs.runCommand name { preferLocalBuild = true; } ''
       echo '${builtins.toJSON x}' | ${pkgs.jq}/bin/jq . > $out
     '';
 
-  # This becomes the main config file
+  # This becomes the main config file for Prometheus 1
   promConfig = {
     global = cfg.globalConfig;
     rule_files = map (promtoolCheck "check-rules" "rules") (cfg.ruleFiles ++ [
@@ -50,6 +61,39 @@ let
     (optionalString (cfg.webExternalUrl != null) "-web.external-url=${cfg.webExternalUrl}")
   ];
 
+  # This becomes the main config file for Prometheus 2
+  promConfig2 = {
+    global = cfg2.globalConfig;
+    rule_files = map (prom2toolCheck "check-rules" "rules") (cfg2.ruleFiles ++ [
+      (pkgs.writeText "prometheus.rules" (concatStringsSep "\n" cfg2.rules))
+    ]);
+    scrape_configs = cfg2.scrapeConfigs;
+    alerting = optionalAttrs (cfg2.alertmanagerURL != []) {
+      alertmanagers = [{
+        static_configs = [{
+          targets = cfg2.alertmanagerURL;
+        }];
+      }];
+    };
+  };
+
+  generatedPrometheus2Yml = writePrettyJSON "prometheus.yml" promConfig2;
+
+  prometheus2Yml = let
+    yml = if cfg2.configText != null then
+      pkgs.writeText "prometheus.yml" cfg2.configText
+      else generatedPrometheus2Yml;
+    in promtoo2lCheck "check-config" "prometheus.yml" yml;
+
+  cmdlineArgs2 = cfg2.extraFlags ++ [
+    "--storage.tsdb.path=${cfg2.dataDir}/data/"
+    "--config.file=${prometheus2Yml}"
+    "--web.listen-address=${cfg2.listenAddress}"
+    "--alertmanager.notification-queue-capacity=${toString cfg2.alertmanagerNotificationQueueCapacity}"
+    "--alertmanager.timeout=${toString cfg2.alertmanagerTimeout}s"
+    (optionalString (cfg2.webExternalUrl != null) "-web.external-url=${cfg2.webExternalUrl}")
+  ];
+
   promTypes.globalConfig = types.submodule {
     options = {
       scrape_interval = mkOption {
@@ -497,9 +541,132 @@ in {
         '';
       };
     };
+    services.prometheus2 = {
+
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Enable the Prometheus 2 monitoring daemon.
+        '';
       };
 
-  config = mkIf cfg.enable {
+      package = mkOption {
+        type = types.package;
+        default = pkgs.prometheus_2;
+        defaultText = "pkgs.prometheus_2";
+        description = ''
+          The prometheus2 package that should be used.
+        '';
+      };
+
+      listenAddress = mkOption {
+        type = types.str;
+        default = "0.0.0.0:9090";
+        description = ''
+          Address to listen on for the web interface, API, and telemetry.
+        '';
+      };
+
+      dataDir = mkOption {
+        type = types.path;
+        default = "/var/lib/prometheus2";
+        description = ''
+          Directory to store Prometheus 2 metrics data.
+        '';
+      };
+
+      extraFlags = mkOption {
+        type = types.listOf types.str;
+        default = [];
+        description = ''
+          Extra commandline options when launching Prometheus 2.
+        '';
+      };
+
+      configText = mkOption {
+        type = types.nullOr types.lines;
+        default = null;
+        description = ''
+          If non-null, this option defines the text that is written to
+          prometheus.yml. If null, the contents of prometheus.yml is generated
+          from the structured config options.
+        '';
+      };
+
+      globalConfig = mkOption {
+        type = promTypes.globalConfig;
+        default = {};
+        apply = _filter;
+        description = ''
+          Parameters that are valid in all  configuration contexts. They
+          also serve as defaults for other configuration sections
+        '';
+      };
+
+      rules = mkOption {
+        type = types.listOf types.str;
+        default = [];
+        description = ''
+          Alerting and/or Recording rules to evaluate at runtime.
+        '';
+      };
+
+      ruleFiles = mkOption {
+        type = types.listOf types.path;
+        default = [];
+        description = ''
+          Any additional rules files to include in this configuration.
+        '';
+      };
+
+      scrapeConfigs = mkOption {
+        type = types.listOf promTypes.scrape_config;
+        default = [];
+        apply = x: map _filter x;
+        description = ''
+          A list of scrape configurations.
+        '';
+      };
+
+      alertmanagerURL = mkOption {
+        type = types.listOf types.str;
+        default = [];
+        description = ''
+          List of Alertmanager URLs to send notifications to.
+        '';
+      };
+
+      alertmanagerNotificationQueueCapacity = mkOption {
+        type = types.int;
+        default = 10000;
+        description = ''
+          The capacity of the queue for pending alert manager notifications.
+        '';
+      };
+
+      alertmanagerTimeout = mkOption {
+        type = types.int;
+        default = 10;
+        description = ''
+          Alert manager HTTP API timeout (in seconds).
+        '';
+      };
+
+      webExternalUrl = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        example = "https://example.com/";
+        description = ''
+          The URL under which Prometheus is externally reachable (for example,
+          if Prometheus is served via a reverse proxy).
+        '';
+      };
+    };
+   };
+
+  config = mkMerge [
+    (mkIf cfg.enable {
       users.groups.${promGroup}.gid = config.ids.gids.prometheus;
       users.users.${promUser} = {
         description = "Prometheus daemon user";
@@ -522,5 +689,30 @@ in {
           WorkingDirectory = cfg.dataDir;
         };
       };
+    })
+    (mkIf cfg2.enable {
+      users.groups.${prom2Group}.gid = config.ids.gids.prometheus2;
+      users.users.${prom2User} = {
+        description = "Prometheus2 daemon user";
+        uid = config.ids.uids.prometheus2;
+        group = prom2Group;
+        home = cfg2.dataDir;
+        createHome = true;
       };
+      systemd.services.prometheus2 = {
+        wantedBy = [ "multi-user.target" ];
+        after    = [ "network.target" ];
+        script = ''
+          #!/bin/sh
+          exec ${cfg.package}/bin/prometheus \
+            ${concatStringsSep " \\\n  " cmdlineArgs2}
+        '';
+        serviceConfig = {
+          User = prom2User;
+          Restart  = "always";
+          WorkingDirectory = cfg2.dataDir;
+        };
+      };
+    })
+  ];
 }

nixos/modules/services/monitoring/prometheus/exporters.nix

@@ -4,8 +4,10 @@ with lib;
 
 let
   cfg = config.services.prometheus.exporters;
+  cfg2 = config.services.prometheus2.exporters;
 
-  # each attribute in `exporterOpts` is expected to have specified:
+  # each attribute in `exporterOpts` is a function that when executed
+  # with `cfg` or `cfg2` as parameter is expected to have specified:
   #   - port        (types.int):   port on which the exporter listens
   #   - serviceOpts (types.attrs): config that is merged with the
   #                                default definition of the exporter's
@@ -108,11 +110,16 @@ let
     };
   };
 
-  mkSubModules = (foldl' (a: b: a//b) {}
+  mkSubModules = exCfg:
-    (mapAttrsToList (name: opts: mkSubModule {
+    (foldl' (a: b: a//b) {}
+      (mapAttrsToList (name: confGen:
+        let
+          conf = (confGen exCfg);
+        in
+          mkSubModule {
             inherit name;
-      inherit (opts) port serviceOpts;
+            inherit (conf) port serviceOpts;
-      extraOpts = opts.extraOpts or {};
+            extraOpts = conf.extraOpts or {};
           }) exporterOpts)
     );
 
@@ -133,11 +140,36 @@ let
         serviceConfig.Group = conf.group;
       });
   };
+  mkExportersConfig = exCfg: promVersion:
+    ([{
+      assertions = [{
+        assertion = (exCfg.snmp.configurationPath == null) != (exCfg.snmp.configuration == null);
+        message = ''
+          Please ensure you have either `services.prometheus.exporters.snmp.configuration'
+          or `services.prometheus${promVersion}.exporters.snmp.configurationPath' set!
+        '';
+      }];
+    }] ++ [(mkIf config.services.minio.enable {
+      services."prometheus${promVersion}".exporters.minio = {
+        minioAddress  = mkDefault "http://localhost:9000";
+        minioAccessKey = mkDefault config.services.minio.accessKey;
+        minioAccessSecret = mkDefault config.services.minio.secretKey;
+      };
+    })] ++ (mapAttrsToList (name: confGen:
+      let
+        conf = (confGen exCfg);
+      in
+      mkExporterConf {
+        inherit name;
+        inherit (conf) serviceOpts;
+        conf = exCfg.${name};
+      }) exporterOpts)
+    );
 in
 {
   options.services.prometheus.exporters = mkOption {
     type = types.submodule {
-      options = (mkSubModules);
+      options = (mkSubModules cfg);
     };
     description = "Prometheus exporter configuration";
     default = {};
@@ -152,25 +184,24 @@ in
     '';
   };
 
-  config = mkMerge ([{
+  options.services.prometheus2.exporters = mkOption {
-    assertions = [{
+    type = types.submodule {
-      assertion = (cfg.snmp.configurationPath == null) != (cfg.snmp.configuration == null);
+      options = (mkSubModules cfg2);
-      message = ''
+    };
-        Please ensure you have either `services.prometheus.exporters.snmp.configuration'
+    description = "Prometheus 2 exporter configuration";
-          or `services.prometheus.exporters.snmp.configurationPath' set!
+    default = {};
+    example = literalExample ''
+      {
+        node = {
+          enable = true;
+          enabledCollectors = [ "systemd" ];
+        };
+        varnish.enable = true;
+      }
     '';
-    }];
+  };
-  }] ++ [(mkIf config.services.minio.enable {
+
-    services.prometheus.exporters.minio.minioAddress  = mkDefault "http://localhost:9000";
+  config = mkMerge ((mkExportersConfig cfg "") ++ (mkExportersConfig cfg2 "2"));
-    services.prometheus.exporters.minio.minioAccessKey = mkDefault config.services.minio.accessKey;
-    services.prometheus.exporters.minio.minioAccessSecret = mkDefault config.services.minio.secretKey;
-  })] ++ (mapAttrsToList (name: conf:
-    mkExporterConf {
-      inherit name;
-      inherit (conf) serviceOpts;
-      conf = cfg.${name};
-    }) exporterOpts)
-  );
 
   meta = {
     doc = ./exporters.xml;

nixos/modules/services/monitoring/prometheus/exporters/bind.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.bind;
+    cfg = baseCfg.bind;
   in
   {
     port = 9119;

nixos/modules/services/monitoring/prometheus/exporters/blackbox.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.blackbox;
+    cfg = baseCfg.blackbox;
   in
   {
     port = 9115;

nixos/modules/services/monitoring/prometheus/exporters/collectd.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.collectd;
+    cfg = baseCfg.collectd;
   in
   {
     port = 9103;

nixos/modules/services/monitoring/prometheus/exporters/dnsmasq.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.dnsmasq;
+    cfg = baseCfg.dnsmasq;
   in
   {
     port = 9153;

nixos/modules/services/monitoring/prometheus/exporters/dovecot.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.dovecot;
+    cfg = baseCfg.dovecot;
   in
   {
     port = 9166;

nixos/modules/services/monitoring/prometheus/exporters/fritzbox.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.fritzbox;
+    cfg = baseCfg.fritzbox;
   in
   {
     port = 9133;

nixos/modules/services/monitoring/prometheus/exporters/json.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.json;
+    cfg = baseCfg.json;
   in
   {
     port = 7979;

nixos/modules/services/monitoring/prometheus/exporters/minio.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.minio;
+    cfg = baseCfg.minio;
   in
   {
     port = 9290;

nixos/modules/services/monitoring/prometheus/exporters/nginx.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.nginx;
+    cfg = baseCfg.nginx;
   in
   {
     port = 9113;

nixos/modules/services/monitoring/prometheus/exporters/node.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.node;
+    cfg = baseCfg.node;
   in
   {
     port = 9100;

nixos/modules/services/monitoring/prometheus/exporters/postfix.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.postfix;
+    cfg = baseCfg.postfix;
   in
   {
     port = 9154;

nixos/modules/services/monitoring/prometheus/exporters/snmp.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.snmp;
+    cfg = baseCfg.snmp;
   in
   {
     port = 9116;
@@ -60,10 +61,10 @@ in
         DynamicUser = true;
         ExecStart = ''
           ${pkgs.prometheus-snmp-exporter.bin}/bin/snmp_exporter \
-          --config.file=${configFile} \
+            --config.file ${configFile} \
-          --log.format=${cfg.logFormat} \
+            --log.format ${cfg.logFormat} \
-          --log.level=${cfg.logLevel} \
+            --log.level ${cfg.logLevel} \
-          --web.listen-address=${cfg.listenAddress}:${toString cfg.port} \
+            --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
             ${concatStringsSep " \\\n  " cfg.extraFlags}
         '';
       };

nixos/modules/services/monitoring/prometheus/exporters/surfboard.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.surfboard;
+    cfg = baseCfg.surfboard;
   in
   {
     port = 9239;

nixos/modules/services/monitoring/prometheus/exporters/tor.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.tor;
+    cfg = baseCfg.tor;
   in
   {
     port = 9130;

nixos/modules/services/monitoring/prometheus/exporters/unifi.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.unifi;
+    cfg = baseCfg.unifi;
   in
   {
     port = 9130;

nixos/modules/services/monitoring/prometheus/exporters/varnish.nix

@@ -2,8 +2,9 @@
 
 with lib;
 
+baseCfg:
   let
-  cfg = config.services.prometheus.exporters.varnish;
+    cfg = baseCfg.varnish;
   in
   {
     port = 9131;