diff --git a/nixos/doc/manual/release-notes/rl-1909.xml b/nixos/doc/manual/release-notes/rl-1909.xml
index 2cfaec49c02..2d391b75053 100644
--- a/nixos/doc/manual/release-notes/rl-1909.xml
+++ b/nixos/doc/manual/release-notes/rl-1909.xml
@@ -37,7 +37,52 @@
-
+
+ Besides the existing module which
+ targets Prometheus-1 a new module
+ has been added which targets Prometheus-2.
+
+
+ Both modules can be enabled at the same time. In fact
+
+ this is needed for upgrading existing Prometheus-1 data to Prometheus-2
+ .
+
+
+
+
+
+
+ Backward Incompatibilities
+
+
+ When upgrading from a previous release, please be aware of the following
+ incompatible changes:
+
+
+
+
+
+ The directory where Prometheus will store its metric data is now
+ managed by systemd's StateDirectory mechanism. It still defaults
+ to /var/lib/prometheus.
+
+
+ Its location can be specified by the new
+ option which
+ defaults to prometheus. Note that this should
+ be a directory relative to /var/lib/.
+
+
+ The option has been
+ deprecated. You can still set it but it's now required to have
+ /var/lib/ as a prefix and you can't set
+ at the same time.
+
diff --git a/nixos/modules/services/monitoring/prometheus/default.nix b/nixos/modules/services/monitoring/prometheus/default.nix
index cc703573d8c..25385be9704 100644
--- a/nixos/modules/services/monitoring/prometheus/default.nix
+++ b/nixos/modules/services/monitoring/prometheus/default.nix
@@ -4,9 +4,24 @@ with lib;
let
cfg = config.services.prometheus;
+ cfg2 = config.services.prometheus2;
promUser = "prometheus";
promGroup = "prometheus";
+ stateDir =
+ if cfg.stateDir != null
+ then cfg.stateDir
+ else
+ if cfg.dataDir != null
+ then
+ # This assumes /var/lib/ is a prefix of cfg.dataDir.
+ # This is checked as an assertion below.
+ removePrefix stateDirBase cfg.dataDir
+ else "prometheus";
+ stateDirBase = "/var/lib/";
+ workingDir = stateDirBase + stateDir;
+ workingDir2 = stateDirBase + cfg2.stateDir;
+
# Get a submodule without any embedded metadata:
_filter = x: filterAttrs (k: v: k != "_module") x;
@@ -17,13 +32,23 @@ let
promtool ${what} $out
'';
+ # a wrapper that verifies that the configuration is valid for
+ # prometheus 2
+ prom2toolCheck = what: name: file:
+ pkgs.runCommand
+ "${name}-${replaceStrings [" "] [""] what}-checked"
+ { buildInputs = [ cfg2.package ]; } ''
+ ln -s ${file} $out
+ promtool ${what} $out
+ '';
+
# Pretty-print JSON to a file
writePrettyJSON = name: x:
pkgs.runCommand name { preferLocalBuild = true; } ''
echo '${builtins.toJSON x}' | ${pkgs.jq}/bin/jq . > $out
'';
- # This becomes the main config file
+ # This becomes the main config file for Prometheus 1
promConfig = {
global = cfg.globalConfig;
rule_files = map (promtoolCheck "check-rules" "rules") (cfg.ruleFiles ++ [
@@ -35,20 +60,53 @@ let
generatedPrometheusYml = writePrettyJSON "prometheus.yml" promConfig;
prometheusYml = let
- yml = if cfg.configText != null then
+ yml = if cfg.configText != null then
pkgs.writeText "prometheus.yml" cfg.configText
else generatedPrometheusYml;
in promtoolCheck "check-config" "prometheus.yml" yml;
cmdlineArgs = cfg.extraFlags ++ [
- "-storage.local.path=${cfg.dataDir}/metrics"
+ "-storage.local.path=${workingDir}/metrics"
"-config.file=${prometheusYml}"
"-web.listen-address=${cfg.listenAddress}"
"-alertmanager.notification-queue-capacity=${toString cfg.alertmanagerNotificationQueueCapacity}"
"-alertmanager.timeout=${toString cfg.alertmanagerTimeout}s"
- (optionalString (cfg.alertmanagerURL != []) "-alertmanager.url=${concatStringsSep "," cfg.alertmanagerURL}")
- (optionalString (cfg.webExternalUrl != null) "-web.external-url=${cfg.webExternalUrl}")
- ];
+ ] ++
+ optional (cfg.alertmanagerURL != []) "-alertmanager.url=${concatStringsSep "," cfg.alertmanagerURL}" ++
+ optional (cfg.webExternalUrl != null) "-web.external-url=${cfg.webExternalUrl}";
+
+ # This becomes the main config file for Prometheus 2
+ promConfig2 = {
+ global = cfg2.globalConfig;
+ rule_files = map (prom2toolCheck "check rules" "rules") (cfg2.ruleFiles ++ [
+ (pkgs.writeText "prometheus.rules" (concatStringsSep "\n" cfg2.rules))
+ ]);
+ scrape_configs = cfg2.scrapeConfigs;
+ alerting = optionalAttrs (cfg2.alertmanagerURL != []) {
+ alertmanagers = [{
+ static_configs = [{
+ targets = cfg2.alertmanagerURL;
+ }];
+ }];
+ };
+ };
+
+ generatedPrometheus2Yml = writePrettyJSON "prometheus.yml" promConfig2;
+
+ prometheus2Yml = let
+ yml = if cfg2.configText != null then
+ pkgs.writeText "prometheus.yml" cfg2.configText
+ else generatedPrometheus2Yml;
+ in prom2toolCheck "check config" "prometheus.yml" yml;
+
+ cmdlineArgs2 = cfg2.extraFlags ++ [
+ "--storage.tsdb.path=${workingDir2}/data/"
+ "--config.file=${prometheus2Yml}"
+ "--web.listen-address=${cfg2.listenAddress}"
+ "--alertmanager.notification-queue-capacity=${toString cfg2.alertmanagerNotificationQueueCapacity}"
+ "--alertmanager.timeout=${toString cfg2.alertmanagerTimeout}s"
+ ] ++
+ optional (cfg2.webExternalUrl != null) "--web.external-url=${cfg2.webExternalUrl}";
promTypes.globalConfig = types.submodule {
options = {
@@ -403,10 +461,21 @@ in {
};
dataDir = mkOption {
- type = types.path;
- default = "/var/lib/prometheus";
+ type = types.nullOr types.path;
+ default = null;
description = ''
Directory to store Prometheus metrics data.
+ This option is deprecated, please use .
+ '';
+ };
+
+ stateDir = mkOption {
+ type = types.nullOr types.str;
+ default = null;
+ description = ''
+ Directory below ${stateDirBase} to store Prometheus metrics data.
+ This directory will be created automatically using systemd's StateDirectory mechanism.
+ Defaults to prometheus.
'';
};
@@ -497,30 +566,201 @@ in {
'';
};
};
- };
+ services.prometheus2 = {
- config = mkIf cfg.enable {
- users.groups.${promGroup}.gid = config.ids.gids.prometheus;
- users.users.${promUser} = {
- description = "Prometheus daemon user";
- uid = config.ids.uids.prometheus;
- group = promGroup;
- home = cfg.dataDir;
- createHome = true;
- };
- systemd.services.prometheus = {
- wantedBy = [ "multi-user.target" ];
- after = [ "network.target" ];
- script = ''
- #!/bin/sh
- exec ${cfg.package}/bin/prometheus \
- ${concatStringsSep " \\\n " cmdlineArgs}
- '';
- serviceConfig = {
- User = promUser;
- Restart = "always";
- WorkingDirectory = cfg.dataDir;
+ enable = mkOption {
+ type = types.bool;
+ default = false;
+ description = ''
+ Enable the Prometheus 2 monitoring daemon.
+ '';
+ };
+
+ package = mkOption {
+ type = types.package;
+ default = pkgs.prometheus_2;
+ defaultText = "pkgs.prometheus_2";
+ description = ''
+ The prometheus2 package that should be used.
+ '';
+ };
+
+ listenAddress = mkOption {
+ type = types.str;
+ default = "0.0.0.0:9090";
+ description = ''
+ Address to listen on for the web interface, API, and telemetry.
+ '';
+ };
+
+ stateDir = mkOption {
+ type = types.str;
+ default = "prometheus2";
+ description = ''
+ Directory below ${stateDirBase} to store Prometheus metrics data.
+ This directory will be created automatically using systemd's StateDirectory mechanism.
+ Defaults to prometheus2.
+ '';
+ };
+
+ extraFlags = mkOption {
+ type = types.listOf types.str;
+ default = [];
+ description = ''
+ Extra commandline options when launching Prometheus 2.
+ '';
+ };
+
+ configText = mkOption {
+ type = types.nullOr types.lines;
+ default = null;
+ description = ''
+ If non-null, this option defines the text that is written to
+ prometheus.yml. If null, the contents of prometheus.yml is generated
+ from the structured config options.
+ '';
+ };
+
+ globalConfig = mkOption {
+ type = promTypes.globalConfig;
+ default = {};
+ apply = _filter;
+ description = ''
+ Parameters that are valid in all configuration contexts. They
+ also serve as defaults for other configuration sections
+ '';
+ };
+
+ rules = mkOption {
+ type = types.listOf types.str;
+ default = [];
+ description = ''
+ Alerting and/or Recording rules to evaluate at runtime.
+ '';
+ };
+
+ ruleFiles = mkOption {
+ type = types.listOf types.path;
+ default = [];
+ description = ''
+ Any additional rules files to include in this configuration.
+ '';
+ };
+
+ scrapeConfigs = mkOption {
+ type = types.listOf promTypes.scrape_config;
+ default = [];
+ apply = x: map _filter x;
+ description = ''
+ A list of scrape configurations.
+ '';
+ };
+
+ alertmanagerURL = mkOption {
+ type = types.listOf types.str;
+ default = [];
+ description = ''
+ List of Alertmanager URLs to send notifications to.
+ '';
+ };
+
+ alertmanagerNotificationQueueCapacity = mkOption {
+ type = types.int;
+ default = 10000;
+ description = ''
+ The capacity of the queue for pending alert manager notifications.
+ '';
+ };
+
+ alertmanagerTimeout = mkOption {
+ type = types.int;
+ default = 10;
+ description = ''
+ Alert manager HTTP API timeout (in seconds).
+ '';
+ };
+
+ webExternalUrl = mkOption {
+ type = types.nullOr types.str;
+ default = null;
+ example = "https://example.com/";
+ description = ''
+ The URL under which Prometheus is externally reachable (for example,
+ if Prometheus is served via a reverse proxy).
+ '';
};
};
- };
+ };
+
+ config = mkMerge [
+ (mkIf (cfg.enable || cfg2.enable) {
+ users.groups.${promGroup}.gid = config.ids.gids.prometheus;
+ users.users.${promUser} = {
+ description = "Prometheus daemon user";
+ uid = config.ids.uids.prometheus;
+ group = promGroup;
+ };
+ })
+ (mkIf cfg.enable {
+ warnings =
+ optional (cfg.dataDir != null) ''
+ The option services.prometheus.dataDir is deprecated, please use
+ services.prometheus.stateDir.
+ '';
+ assertions = [
+ {
+ assertion = !(cfg.dataDir != null && cfg.stateDir != null);
+ message =
+ "The options services.prometheus.dataDir and services.prometheus.stateDir" +
+ " can't both be set at the same time! It's recommended to only set the latter" +
+ " since the former is deprecated.";
+ }
+ {
+ assertion = cfg.dataDir != null -> hasPrefix stateDirBase cfg.dataDir;
+ message =
+ "The option services.prometheus.dataDir should have ${stateDirBase} as a prefix!";
+ }
+ {
+ assertion = cfg.stateDir != null -> !hasPrefix "/" cfg.stateDir;
+ message =
+ "The option services.prometheus.stateDir shouldn't be an absolute directory." +
+ " It should be a directory relative to ${stateDirBase}.";
+ }
+ {
+ assertion = cfg2.stateDir != null -> !hasPrefix "/" cfg2.stateDir;
+ message =
+ "The option services.prometheus2.stateDir shouldn't be an absolute directory." +
+ " It should be a directory relative to ${stateDirBase}.";
+ }
+ ];
+ systemd.services.prometheus = {
+ wantedBy = [ "multi-user.target" ];
+ after = [ "network.target" ];
+ serviceConfig = {
+ ExecStart = "${cfg.package}/bin/prometheus" +
+ optionalString (length cmdlineArgs != 0) (" \\\n " +
+ concatStringsSep " \\\n " cmdlineArgs);
+ User = promUser;
+ Restart = "always";
+ WorkingDirectory = workingDir;
+ StateDirectory = stateDir;
+ };
+ };
+ })
+ (mkIf cfg2.enable {
+ systemd.services.prometheus2 = {
+ wantedBy = [ "multi-user.target" ];
+ after = [ "network.target" ];
+ serviceConfig = {
+ ExecStart = "${cfg2.package}/bin/prometheus" +
+ optionalString (length cmdlineArgs2 != 0) (" \\\n " +
+ concatStringsSep " \\\n " cmdlineArgs2);
+ User = promUser;
+ Restart = "always";
+ WorkingDirectory = workingDir2;
+ StateDirectory = cfg2.stateDir;
+ };
+ };
+ })
+ ];
}
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 66a877e8bae..14d855f44dc 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -194,6 +194,7 @@ in
predictable-interface-names = handleTest ./predictable-interface-names.nix {};
printing = handleTest ./printing.nix {};
prometheus = handleTest ./prometheus.nix {};
+ prometheus2 = handleTest ./prometheus-2.nix {};
prometheus-exporters = handleTest ./prometheus-exporters.nix {};
prosody = handleTest ./prosody.nix {};
proxy = handleTest ./proxy.nix {};
diff --git a/nixos/tests/prometheus-2.nix b/nixos/tests/prometheus-2.nix
new file mode 100644
index 00000000000..5a4d8668cb8
--- /dev/null
+++ b/nixos/tests/prometheus-2.nix
@@ -0,0 +1,34 @@
+import ./make-test.nix {
+ name = "prometheus-2";
+
+ nodes = {
+ one = { pkgs, ... }: {
+ services.prometheus2 = {
+ enable = true;
+ scrapeConfigs = [{
+ job_name = "prometheus";
+ static_configs = [{
+ targets = [ "127.0.0.1:9090" ];
+ labels = { instance = "localhost"; };
+ }];
+ }];
+ rules = [
+ ''
+ groups:
+ - name: test
+ rules:
+ - record: testrule
+ expr: count(up{job="prometheus"})
+ ''
+ ];
+ };
+ };
+ };
+
+ testScript = ''
+ startAll;
+ $one->waitForUnit("prometheus2.service");
+ $one->waitForOpenPort(9090);
+ $one->succeed("curl -s http://127.0.0.1:9090/metrics");
+ '';
+}