Merge pull request #58255 from jbgi/prometheus2

Add Prometheus 2 service in parallel with 1.x version (continuation)

nixos/doc/manual/release-notes/rl-1909.xml

@@ -37,7 +37,52 @@
 
   <itemizedlist>
    <listitem>
-    <para />
+    <para>
+       Besides the existing <option>services.prometheus</option> module which
+       targets Prometheus-1 a new <option>services.prometheus2</option> module
+       has been added which targets Prometheus-2.
+    </para>
+    <para>
+       Both modules can be enabled at the same time. In fact
+       <link xlink:href="https://prometheus.io/docs/prometheus/latest/migration/#storage">
+       this is needed for upgrading existing Prometheus-1 data to Prometheus-2
+       </link>.
+    </para>
+   </listitem>
+  </itemizedlist>
+ </section>
+
+ <section xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/XInclude"
+         version="5.0"
+         xml:id="sec-release-19.09-incompatibilities">
+  <title>Backward Incompatibilities</title>
+
+  <para>
+   When upgrading from a previous release, please be aware of the following
+   incompatible changes:
+  </para>
+
+  <itemizedlist>
+   <listitem>
+    <para>
+      The directory where Prometheus will store its metric data is now
+      managed by systemd's StateDirectory mechanism. It still defaults
+      to <literal>/var/lib/prometheus</literal>.
+    </para>
+    <para>
+      Its location can be specified by the new
+      <option>services.prometheus.stateDir</option> option which
+      defaults to <literal>prometheus</literal>. Note that this should
+      be a directory relative to <literal>/var/lib/</literal>.
+    </para>
+    <para>
+      The option <option>services.prometheus.dataDir</option> has been
+      deprecated. You can still set it but it's now required to have
+      <literal>/var/lib/</literal> as a prefix and you can't set
+      <option>services.prometheus.stateDir</option> at the same time.
+    </para>
    </listitem>
   </itemizedlist>
  </section>

nixos/modules/services/monitoring/prometheus/default.nix

@@ -4,9 +4,24 @@ with lib;
 
 let
   cfg = config.services.prometheus;
+  cfg2 = config.services.prometheus2;
   promUser = "prometheus";
   promGroup = "prometheus";
 
+  stateDir =
+    if cfg.stateDir != null
+    then cfg.stateDir
+    else
+      if cfg.dataDir != null
+      then
+        # This assumes /var/lib/ is a prefix of cfg.dataDir.
+        # This is checked as an assertion below.
+        removePrefix stateDirBase cfg.dataDir
+      else "prometheus";
+  stateDirBase = "/var/lib/";
+  workingDir  = stateDirBase + stateDir;
+  workingDir2 = stateDirBase + cfg2.stateDir;
+
   # Get a submodule without any embedded metadata:
   _filter = x: filterAttrs (k: v: k != "_module") x;
 
@@ -17,13 +32,23 @@ let
     promtool ${what} $out
   '';
 
+  # a wrapper that verifies that the configuration is valid for
+  # prometheus 2
+  prom2toolCheck = what: name: file:
+    pkgs.runCommand
+      "${name}-${replaceStrings [" "] [""] what}-checked"
+      { buildInputs = [ cfg2.package ]; } ''
+    ln -s ${file} $out
+    promtool ${what} $out
+  '';
+
   # Pretty-print JSON to a file
   writePrettyJSON = name: x:
     pkgs.runCommand name { preferLocalBuild = true; } ''
       echo '${builtins.toJSON x}' | ${pkgs.jq}/bin/jq . > $out
     '';
 
-  # This becomes the main config file
+  # This becomes the main config file for Prometheus 1
   promConfig = {
     global = cfg.globalConfig;
     rule_files = map (promtoolCheck "check-rules" "rules") (cfg.ruleFiles ++ [
@@ -35,20 +60,53 @@ let
   generatedPrometheusYml = writePrettyJSON "prometheus.yml" promConfig;
 
   prometheusYml = let
-    yml =  if cfg.configText != null then
+    yml = if cfg.configText != null then
       pkgs.writeText "prometheus.yml" cfg.configText
       else generatedPrometheusYml;
     in promtoolCheck "check-config" "prometheus.yml" yml;
 
   cmdlineArgs = cfg.extraFlags ++ [
-    "-storage.local.path=${cfg.dataDir}/metrics"
+    "-storage.local.path=${workingDir}/metrics"
     "-config.file=${prometheusYml}"
     "-web.listen-address=${cfg.listenAddress}"
     "-alertmanager.notification-queue-capacity=${toString cfg.alertmanagerNotificationQueueCapacity}"
     "-alertmanager.timeout=${toString cfg.alertmanagerTimeout}s"
-    (optionalString (cfg.alertmanagerURL != []) "-alertmanager.url=${concatStringsSep "," cfg.alertmanagerURL}")
+  ] ++
-    (optionalString (cfg.webExternalUrl != null) "-web.external-url=${cfg.webExternalUrl}")
+  optional (cfg.alertmanagerURL != []) "-alertmanager.url=${concatStringsSep "," cfg.alertmanagerURL}" ++
-  ];
+  optional (cfg.webExternalUrl != null) "-web.external-url=${cfg.webExternalUrl}";
+
+  # This becomes the main config file for Prometheus 2
+  promConfig2 = {
+    global = cfg2.globalConfig;
+    rule_files = map (prom2toolCheck "check rules" "rules") (cfg2.ruleFiles ++ [
+      (pkgs.writeText "prometheus.rules" (concatStringsSep "\n" cfg2.rules))
+    ]);
+    scrape_configs = cfg2.scrapeConfigs;
+    alerting = optionalAttrs (cfg2.alertmanagerURL != []) {
+      alertmanagers = [{
+        static_configs = [{
+          targets = cfg2.alertmanagerURL;
+        }];
+      }];
+    };
+  };
+
+  generatedPrometheus2Yml = writePrettyJSON "prometheus.yml" promConfig2;
+
+  prometheus2Yml = let
+    yml = if cfg2.configText != null then
+      pkgs.writeText "prometheus.yml" cfg2.configText
+      else generatedPrometheus2Yml;
+    in prom2toolCheck "check config" "prometheus.yml" yml;
+
+  cmdlineArgs2 = cfg2.extraFlags ++ [
+    "--storage.tsdb.path=${workingDir2}/data/"
+    "--config.file=${prometheus2Yml}"
+    "--web.listen-address=${cfg2.listenAddress}"
+    "--alertmanager.notification-queue-capacity=${toString cfg2.alertmanagerNotificationQueueCapacity}"
+    "--alertmanager.timeout=${toString cfg2.alertmanagerTimeout}s"
+  ] ++
+  optional (cfg2.webExternalUrl != null) "--web.external-url=${cfg2.webExternalUrl}";
 
   promTypes.globalConfig = types.submodule {
     options = {
@@ -403,10 +461,21 @@ in {
       };
 
       dataDir = mkOption {
-        type = types.path;
+        type = types.nullOr types.path;
-        default = "/var/lib/prometheus";
+        default = null;
         description = ''
           Directory to store Prometheus metrics data.
+          This option is deprecated, please use <option>services.prometheus.stateDir</option>.
+        '';
+      };
+
+      stateDir = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = ''
+          Directory below <literal>${stateDirBase}</literal> to store Prometheus metrics data.
+          This directory will be created automatically using systemd's StateDirectory mechanism.
+          Defaults to <literal>prometheus</literal>.
         '';
       };
 
@@ -497,30 +566,201 @@ in {
         '';
       };
     };
-  };
+    services.prometheus2 = {
 
-  config = mkIf cfg.enable {
+      enable = mkOption {
-    users.groups.${promGroup}.gid = config.ids.gids.prometheus;
+        type = types.bool;
-    users.users.${promUser} = {
+        default = false;
-      description = "Prometheus daemon user";
+        description = ''
-      uid = config.ids.uids.prometheus;
+          Enable the Prometheus 2 monitoring daemon.
-      group = promGroup;
+        '';
-      home = cfg.dataDir;
+      };
-      createHome = true;
+
-    };
+      package = mkOption {
-    systemd.services.prometheus = {
+        type = types.package;
-      wantedBy = [ "multi-user.target" ];
+        default = pkgs.prometheus_2;
-      after    = [ "network.target" ];
+        defaultText = "pkgs.prometheus_2";
-      script = ''
+        description = ''
-        #!/bin/sh
+          The prometheus2 package that should be used.
-        exec ${cfg.package}/bin/prometheus \
+        '';
-          ${concatStringsSep " \\\n  " cmdlineArgs}
+      };
-      '';
+
-      serviceConfig = {
+      listenAddress = mkOption {
-        User = promUser;
+        type = types.str;
-        Restart  = "always";
+        default = "0.0.0.0:9090";
-        WorkingDirectory = cfg.dataDir;
+        description = ''
+          Address to listen on for the web interface, API, and telemetry.
+        '';
+      };
+
+      stateDir = mkOption {
+        type = types.str;
+        default = "prometheus2";
+        description = ''
+          Directory below <literal>${stateDirBase}</literal> to store Prometheus metrics data.
+          This directory will be created automatically using systemd's StateDirectory mechanism.
+          Defaults to <literal>prometheus2</literal>.
+        '';
+      };
+
+      extraFlags = mkOption {
+        type = types.listOf types.str;
+        default = [];
+        description = ''
+          Extra commandline options when launching Prometheus 2.
+        '';
+      };
+
+      configText = mkOption {
+        type = types.nullOr types.lines;
+        default = null;
+        description = ''
+          If non-null, this option defines the text that is written to
+          prometheus.yml. If null, the contents of prometheus.yml is generated
+          from the structured config options.
+        '';
+      };
+
+      globalConfig = mkOption {
+        type = promTypes.globalConfig;
+        default = {};
+        apply = _filter;
+        description = ''
+          Parameters that are valid in all  configuration contexts. They
+          also serve as defaults for other configuration sections
+        '';
+      };
+
+      rules = mkOption {
+        type = types.listOf types.str;
+        default = [];
+        description = ''
+          Alerting and/or Recording rules to evaluate at runtime.
+        '';
+      };
+
+      ruleFiles = mkOption {
+        type = types.listOf types.path;
+        default = [];
+        description = ''
+          Any additional rules files to include in this configuration.
+        '';
+      };
+
+      scrapeConfigs = mkOption {
+        type = types.listOf promTypes.scrape_config;
+        default = [];
+        apply = x: map _filter x;
+        description = ''
+          A list of scrape configurations.
+        '';
+      };
+
+      alertmanagerURL = mkOption {
+        type = types.listOf types.str;
+        default = [];
+        description = ''
+          List of Alertmanager URLs to send notifications to.
+        '';
+      };
+
+      alertmanagerNotificationQueueCapacity = mkOption {
+        type = types.int;
+        default = 10000;
+        description = ''
+          The capacity of the queue for pending alert manager notifications.
+        '';
+      };
+
+      alertmanagerTimeout = mkOption {
+        type = types.int;
+        default = 10;
+        description = ''
+          Alert manager HTTP API timeout (in seconds).
+        '';
+      };
+
+      webExternalUrl = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        example = "https://example.com/";
+        description = ''
+          The URL under which Prometheus is externally reachable (for example,
+          if Prometheus is served via a reverse proxy).
+        '';
       };
     };
-  };
+   };
+
+  config = mkMerge [
+    (mkIf (cfg.enable || cfg2.enable) {
+      users.groups.${promGroup}.gid = config.ids.gids.prometheus;
+      users.users.${promUser} = {
+        description = "Prometheus daemon user";
+        uid = config.ids.uids.prometheus;
+        group = promGroup;
+      };
+    })
+    (mkIf cfg.enable {
+      warnings =
+        optional (cfg.dataDir != null) ''
+          The option services.prometheus.dataDir is deprecated, please use
+          services.prometheus.stateDir.
+        '';
+      assertions = [
+        {
+          assertion = !(cfg.dataDir != null && cfg.stateDir != null);
+          message =
+            "The options services.prometheus.dataDir and services.prometheus.stateDir" +
+            " can't both be set at the same time! It's recommended to only set the latter" +
+            " since the former is deprecated.";
+        }
+        {
+          assertion = cfg.dataDir != null -> hasPrefix stateDirBase cfg.dataDir;
+          message =
+            "The option services.prometheus.dataDir should have ${stateDirBase} as a prefix!";
+        }
+        {
+          assertion = cfg.stateDir != null -> !hasPrefix "/" cfg.stateDir;
+          message =
+            "The option services.prometheus.stateDir shouldn't be an absolute directory." +
+            " It should be a directory relative to ${stateDirBase}.";
+        }
+        {
+          assertion = cfg2.stateDir != null -> !hasPrefix "/" cfg2.stateDir;
+          message =
+            "The option services.prometheus2.stateDir shouldn't be an absolute directory." +
+            " It should be a directory relative to ${stateDirBase}.";
+        }
+      ];
+      systemd.services.prometheus = {
+        wantedBy = [ "multi-user.target" ];
+        after    = [ "network.target" ];
+        serviceConfig = {
+          ExecStart = "${cfg.package}/bin/prometheus" +
+            optionalString (length cmdlineArgs != 0) (" \\\n  " +
+              concatStringsSep " \\\n  " cmdlineArgs);
+          User = promUser;
+          Restart  = "always";
+          WorkingDirectory = workingDir;
+          StateDirectory = stateDir;
+        };
+      };
+    })
+    (mkIf cfg2.enable {
+      systemd.services.prometheus2 = {
+        wantedBy = [ "multi-user.target" ];
+        after    = [ "network.target" ];
+        serviceConfig = {
+          ExecStart = "${cfg2.package}/bin/prometheus" +
+            optionalString (length cmdlineArgs2 != 0) (" \\\n  " +
+              concatStringsSep " \\\n  " cmdlineArgs2);
+          User = promUser;
+          Restart  = "always";
+          WorkingDirectory = workingDir2;
+          StateDirectory = cfg2.stateDir;
+        };
+      };
+    })
+  ];
 }

nixos/tests/all-tests.nix

@@ -194,6 +194,7 @@ in
   predictable-interface-names = handleTest ./predictable-interface-names.nix {};
   printing = handleTest ./printing.nix {};
   prometheus = handleTest ./prometheus.nix {};
+  prometheus2 = handleTest ./prometheus-2.nix {};
   prometheus-exporters = handleTest ./prometheus-exporters.nix {};
   prosody = handleTest ./prosody.nix {};
   proxy = handleTest ./proxy.nix {};

nixos/tests/prometheus-2.nix

@@ -0,0 +1,34 @@
+import ./make-test.nix {
+  name = "prometheus-2";
+
+  nodes = {
+    one = { pkgs, ... }: {
+      services.prometheus2 = {
+        enable = true;
+        scrapeConfigs = [{
+          job_name = "prometheus";
+          static_configs = [{
+            targets = [ "127.0.0.1:9090" ];
+            labels = { instance = "localhost"; };
+          }];
+        }];
+        rules = [
+          ''
+            groups:
+              - name: test
+                rules:
+                  - record: testrule
+                    expr: count(up{job="prometheus"})
+          ''
+        ];
+      };
+    };
+  };
+
+  testScript = ''
+    startAll;
+    $one->waitForUnit("prometheus2.service");
+    $one->waitForOpenPort(9090);
+    $one->succeed("curl -s http://127.0.0.1:9090/metrics");
+  '';
+}