Merge pull request #68221 from WilliButz/remove-prometheus1

remove prometheus 1

nixos/doc/manual/release-notes/rl-1909.xml

@@ -464,7 +464,14 @@
      <literal>packetbeat5</literal>) of the ELK-stack and Elastic beats have been removed.
     </para>
    </listitem>
-
+   <listitem>
+    <para>
+     For NixOS 19.03, both Prometheus 1 and 2 were available to allow for
+     a seamless transition from version 1 to 2 with existing setups.
+     Because Prometheus 1 is no longer developed, it was removed.
+     Prometheus 2 is now configured with <literal>services.prometheus</literal>.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 

nixos/modules/rename.nix

@@ -52,10 +52,11 @@ with lib;
     (mkRemovedOptionModule [ "services" "misc" "nzbget" "openFirewall" ] "The port used by nzbget is managed through the web interface so you should adjust your firewall rules accordingly.")
     (mkRemovedOptionModule [ "services" "prometheus" "alertmanager" "user" ] "The alertmanager service is now using systemd's DynamicUser mechanism which obviates a user setting.")
     (mkRemovedOptionModule [ "services" "prometheus" "alertmanager" "group" ] "The alertmanager service is now using systemd's DynamicUser mechanism which obviates a group setting.")
-    (mkRemovedOptionModule [ "services" "prometheus2" "alertmanagerURL" ] ''
+    (mkRemovedOptionModule [ "services" "prometheus" "alertmanagerURL" ] ''
       Due to incompatibility, the alertmanagerURL option has been removed,
       please use 'services.prometheus2.alertmanagers' instead.
     '')
+    (mkRenamedOptionModule [ "services" "prometheus2" ] [ "services" "prometheus" ])
     (mkRenamedOptionModule [ "services" "tor" "relay" "portSpec" ] [ "services" "tor" "relay" "port" ])
     (mkRenamedOptionModule [ "services" "vmwareGuest" ] [ "virtualisation" "vmware" "guest" ])
     (mkRenamedOptionModule [ "jobs" ] [ "systemd" "services" ])

nixos/modules/services/monitoring/prometheus/default.nix

@@ -4,37 +4,14 @@ with lib;
 
 let
   cfg = config.services.prometheus;
-  cfg2 = config.services.prometheus2;
-  promUser = "prometheus";
-  promGroup = "prometheus";
 
-  stateDir =
+  workingDir = "/var/lib/" + cfg.stateDir;
-    if cfg.stateDir != null
-    then cfg.stateDir
-    else
-      if cfg.dataDir != null
-      then
-        # This assumes /var/lib/ is a prefix of cfg.dataDir.
-        # This is checked as an assertion below.
-        removePrefix stateDirBase cfg.dataDir
-      else "prometheus";
-  stateDirBase = "/var/lib/";
-  workingDir  = stateDirBase + stateDir;
-  workingDir2 = stateDirBase + cfg2.stateDir;
 
   # a wrapper that verifies that the configuration is valid
-  promtoolCheck = what: name: file: pkgs.runCommand "${name}-${what}-checked"
+  promtoolCheck = what: name: file:
-    { buildInputs = [ cfg.package ]; } ''
-    ln -s ${file} $out
-    promtool ${what} $out
-  '';
-
-  # a wrapper that verifies that the configuration is valid for
-  # prometheus 2
-  prom2toolCheck = what: name: file:
     pkgs.runCommand
       "${name}-${replaceStrings [" "] [""] what}-checked"
-      { buildInputs = [ cfg2.package ]; } ''
+      { buildInputs = [ cfg.package ]; } ''
     ln -s ${file} $out
     promtool ${what} $out
   '';
@@ -45,61 +22,34 @@ let
       echo '${builtins.toJSON x}' | ${pkgs.jq}/bin/jq . > $out
     '';
 
-  # This becomes the main config file for Prometheus 1
+  generatedPrometheusYml = writePrettyJSON "prometheus.yml" promConfig;
+
+  # This becomes the main config file for Prometheus
   promConfig = {
     global = filterValidPrometheus cfg.globalConfig;
-    rule_files = map (promtoolCheck "check-rules" "rules") (cfg.ruleFiles ++ [
+    rule_files = map (promtoolCheck "check rules" "rules") (cfg.ruleFiles ++ [
       (pkgs.writeText "prometheus.rules" (concatStringsSep "\n" cfg.rules))
     ]);
     scrape_configs = filterValidPrometheus cfg.scrapeConfigs;
+    alerting = {
+      inherit (cfg) alertmanagers;
+    };
   };
-
-  generatedPrometheusYml = writePrettyJSON "prometheus.yml" promConfig;
 
   prometheusYml = let
     yml = if cfg.configText != null then
       pkgs.writeText "prometheus.yml" cfg.configText
       else generatedPrometheusYml;
-    in promtoolCheck "check-config" "prometheus.yml" yml;
+    in promtoolCheck "check config" "prometheus.yml" yml;
 
   cmdlineArgs = cfg.extraFlags ++ [
-    "-storage.local.path=${workingDir}/metrics"
+    "--storage.tsdb.path=${workingDir}/data/"
-    "-config.file=${prometheusYml}"
+    "--config.file=${prometheusYml}"
-    "-web.listen-address=${cfg.listenAddress}"
+    "--web.listen-address=${cfg.listenAddress}"
-    "-alertmanager.notification-queue-capacity=${toString cfg.alertmanagerNotificationQueueCapacity}"
+    "--alertmanager.notification-queue-capacity=${toString cfg.alertmanagerNotificationQueueCapacity}"
-    "-alertmanager.timeout=${toString cfg.alertmanagerTimeout}s"
+    "--alertmanager.timeout=${toString cfg.alertmanagerTimeout}s"
   ] ++
-  optional (cfg.alertmanagerURL != []) "-alertmanager.url=${concatStringsSep "," cfg.alertmanagerURL}" ++
+  optional (cfg.webExternalUrl != null) "--web.external-url=${cfg.webExternalUrl}";
-  optional (cfg.webExternalUrl != null) "-web.external-url=${cfg.webExternalUrl}";
-
-  # This becomes the main config file for Prometheus 2
-  promConfig2 = {
-    global = filterValidPrometheus cfg2.globalConfig;
-    rule_files = map (prom2toolCheck "check rules" "rules") (cfg2.ruleFiles ++ [
-      (pkgs.writeText "prometheus.rules" (concatStringsSep "\n" cfg2.rules))
-    ]);
-    scrape_configs = filterValidPrometheus cfg2.scrapeConfigs;
-    alerting = {
-      inherit (cfg2) alertmanagers;
-    };
-  };
-
-  generatedPrometheus2Yml = writePrettyJSON "prometheus.yml" promConfig2;
-
-  prometheus2Yml = let
-    yml = if cfg2.configText != null then
-      pkgs.writeText "prometheus.yml" cfg2.configText
-      else generatedPrometheus2Yml;
-    in prom2toolCheck "check config" "prometheus.yml" yml;
-
-  cmdlineArgs2 = cfg2.extraFlags ++ [
-    "--storage.tsdb.path=${workingDir2}/data/"
-    "--config.file=${prometheus2Yml}"
-    "--web.listen-address=${cfg2.listenAddress}"
-    "--alertmanager.notification-queue-capacity=${toString cfg2.alertmanagerNotificationQueueCapacity}"
-    "--alertmanager.timeout=${toString cfg2.alertmanagerTimeout}s"
-  ] ++
-  optional (cfg2.webExternalUrl != null) "--web.external-url=${cfg2.webExternalUrl}";
 
   filterValidPrometheus = filterAttrsListRecursive (n: v: !(n == "_module" || v == null));
   filterAttrsListRecursive = pred: x:
@@ -514,8 +464,7 @@ let
   };
 
 in {
-  options = {
+  options.services.prometheus = {
-    services.prometheus = {
 
     enable = mkOption {
       type = types.bool;
@@ -542,22 +491,12 @@ in {
       '';
     };
 
-      dataDir = mkOption {
-        type = types.nullOr types.path;
-        default = null;
-        description = ''
-          Directory to store Prometheus metrics data.
-          This option is deprecated, please use <option>services.prometheus.stateDir</option>.
-        '';
-      };
-
     stateDir = mkOption {
-        type = types.nullOr types.str;
+      type = types.str;
-        default = null;
+      default = "prometheus2";
       description = ''
-          Directory below <literal>${stateDirBase}</literal> to store Prometheus metrics data.
+        Directory below <literal>/var/lib</literal> to store Prometheus metrics data.
         This directory will be created automatically using systemd's StateDirectory mechanism.
-          Defaults to <literal>prometheus</literal>.
       '';
     };
 
@@ -612,128 +551,6 @@ in {
       '';
     };
 
-      alertmanagerURL = mkOption {
-        type = types.listOf types.str;
-        default = [];
-        description = ''
-          List of Alertmanager URLs to send notifications to.
-        '';
-      };
-
-      alertmanagerNotificationQueueCapacity = mkOption {
-        type = types.int;
-        default = 10000;
-        description = ''
-          The capacity of the queue for pending alert manager notifications.
-        '';
-      };
-
-      alertmanagerTimeout = mkOption {
-        type = types.int;
-        default = 10;
-        description = ''
-          Alert manager HTTP API timeout (in seconds).
-        '';
-      };
-
-      webExternalUrl = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        example = "https://example.com/";
-        description = ''
-          The URL under which Prometheus is externally reachable (for example,
-          if Prometheus is served via a reverse proxy).
-        '';
-      };
-    };
-    services.prometheus2 = {
-
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Enable the Prometheus 2 monitoring daemon.
-        '';
-      };
-
-      package = mkOption {
-        type = types.package;
-        default = pkgs.prometheus_2;
-        defaultText = "pkgs.prometheus_2";
-        description = ''
-          The prometheus2 package that should be used.
-        '';
-      };
-
-      listenAddress = mkOption {
-        type = types.str;
-        default = "0.0.0.0:9090";
-        description = ''
-          Address to listen on for the web interface, API, and telemetry.
-        '';
-      };
-
-      stateDir = mkOption {
-        type = types.str;
-        default = "prometheus2";
-        description = ''
-          Directory below <literal>${stateDirBase}</literal> to store Prometheus metrics data.
-          This directory will be created automatically using systemd's StateDirectory mechanism.
-          Defaults to <literal>prometheus2</literal>.
-        '';
-      };
-
-      extraFlags = mkOption {
-        type = types.listOf types.str;
-        default = [];
-        description = ''
-          Extra commandline options when launching Prometheus 2.
-        '';
-      };
-
-      configText = mkOption {
-        type = types.nullOr types.lines;
-        default = null;
-        description = ''
-          If non-null, this option defines the text that is written to
-          prometheus.yml. If null, the contents of prometheus.yml is generated
-          from the structured config options.
-        '';
-      };
-
-      globalConfig = mkOption {
-        type = promTypes.globalConfig;
-        default = {};
-        description = ''
-          Parameters that are valid in all  configuration contexts. They
-          also serve as defaults for other configuration sections
-        '';
-      };
-
-      rules = mkOption {
-        type = types.listOf types.str;
-        default = [];
-        description = ''
-          Alerting and/or Recording rules to evaluate at runtime.
-        '';
-      };
-
-      ruleFiles = mkOption {
-        type = types.listOf types.path;
-        default = [];
-        description = ''
-          Any additional rules files to include in this configuration.
-        '';
-      };
-
-      scrapeConfigs = mkOption {
-        type = types.listOf promTypes.scrape_config;
-        default = [];
-        description = ''
-          A list of scrape configurations.
-        '';
-      };
-
     alertmanagers = mkOption {
       type = types.listOf types.attrs;
       example = literalExample ''
@@ -780,49 +597,14 @@ in {
       '';
     };
   };
-   };
 
-  config = mkMerge [
+  config = mkIf cfg.enable {
-    (mkIf (cfg.enable || cfg2.enable) {
+    users.groups.prometheus.gid = config.ids.gids.prometheus;
-      users.groups.${promGroup}.gid = config.ids.gids.prometheus;
+    users.users.prometheus = {
-      users.users.${promUser} = {
       description = "Prometheus daemon user";
       uid = config.ids.uids.prometheus;
-        group = promGroup;
+      group = "prometheus";
     };
-    })
-    (mkIf cfg.enable {
-      warnings =
-        optional (cfg.dataDir != null) ''
-          The option services.prometheus.dataDir is deprecated, please use
-          services.prometheus.stateDir.
-        '';
-      assertions = [
-        {
-          assertion = !(cfg.dataDir != null && cfg.stateDir != null);
-          message =
-            "The options services.prometheus.dataDir and services.prometheus.stateDir" +
-            " can't both be set at the same time! It's recommended to only set the latter" +
-            " since the former is deprecated.";
-        }
-        {
-          assertion = cfg.dataDir != null -> hasPrefix stateDirBase cfg.dataDir;
-          message =
-            "The option services.prometheus.dataDir should have ${stateDirBase} as a prefix!";
-        }
-        {
-          assertion = cfg.stateDir != null -> !hasPrefix "/" cfg.stateDir;
-          message =
-            "The option services.prometheus.stateDir shouldn't be an absolute directory." +
-            " It should be a directory relative to ${stateDirBase}.";
-        }
-        {
-          assertion = cfg2.stateDir != null -> !hasPrefix "/" cfg2.stateDir;
-          message =
-            "The option services.prometheus2.stateDir shouldn't be an absolute directory." +
-            " It should be a directory relative to ${stateDirBase}.";
-        }
-      ];
     systemd.services.prometheus = {
       wantedBy = [ "multi-user.target" ];
       after    = [ "network.target" ];
@@ -830,27 +612,11 @@ in {
         ExecStart = "${cfg.package}/bin/prometheus" +
           optionalString (length cmdlineArgs != 0) (" \\\n  " +
             concatStringsSep " \\\n  " cmdlineArgs);
-          User = promUser;
+        User = "prometheus";
         Restart  = "always";
         WorkingDirectory = workingDir;
-          StateDirectory = stateDir;
+        StateDirectory = cfg.stateDir;
       };
     };
-    })
-    (mkIf cfg2.enable {
-      systemd.services.prometheus2 = {
-        wantedBy = [ "multi-user.target" ];
-        after    = [ "network.target" ];
-        serviceConfig = {
-          ExecStart = "${cfg2.package}/bin/prometheus" +
-            optionalString (length cmdlineArgs2 != 0) (" \\\n  " +
-              concatStringsSep " \\\n  " cmdlineArgs2);
-          User = promUser;
-          Restart  = "always";
-          WorkingDirectory = workingDir2;
-          StateDirectory = cfg2.stateDir;
   };
-      };
-    })
-  ];
 }

nixos/modules/services/monitoring/thanos.nix

@@ -218,8 +218,8 @@ let
         toArgs = optionToArgs;
         option = mkOption {
           type = types.str;
-          default = "/var/lib/${config.services.prometheus2.stateDir}/data";
+          default = "/var/lib/${config.services.prometheus.stateDir}/data";
-          defaultText = "/var/lib/\${config.services.prometheus2.stateDir}/data";
+          defaultText = "/var/lib/\${config.services.prometheus.stateDir}/data";
           description = ''
             Data directory of TSDB.
           '';
@@ -679,22 +679,22 @@ in {
     (mkIf cfg.sidecar.enable {
       assertions = [
         {
-          assertion = config.services.prometheus2.enable;
+          assertion = config.services.prometheus.enable;
           message =
-            "Please enable services.prometheus2 when enabling services.thanos.sidecar.";
+            "Please enable services.prometheus when enabling services.thanos.sidecar.";
         }
         {
-          assertion = !(config.services.prometheus2.globalConfig.external_labels == null ||
+          assertion = !(config.services.prometheus.globalConfig.external_labels == null ||
-                        config.services.prometheus2.globalConfig.external_labels == {});
+                        config.services.prometheus.globalConfig.external_labels == {});
           message =
             "services.thanos.sidecar requires uniquely identifying external labels " +
             "to be configured in the Prometheus server. " +
-            "Please set services.prometheus2.globalConfig.external_labels.";
+            "Please set services.prometheus.globalConfig.external_labels.";
         }
       ];
       systemd.services.thanos-sidecar = {
         wantedBy = [ "multi-user.target" ];
-        after    = [ "network.target" "prometheus2.service" ];
+        after    = [ "network.target" "prometheus.service" ];
         serviceConfig = {
           User = "prometheus";
           Restart = "always";

nixos/tests/all-tests.nix

@@ -224,7 +224,6 @@ in
   predictable-interface-names = handleTest ./predictable-interface-names.nix {};
   printing = handleTest ./printing.nix {};
   prometheus = handleTest ./prometheus.nix {};
-  prometheus2 = handleTest ./prometheus-2.nix {};
   prometheus-exporters = handleTest ./prometheus-exporters.nix {};
   prosody = handleTest ./xmpp/prosody.nix {};
   prosodyMysql = handleTest ./xmpp/prosody-mysql.nix {};

nixos/tests/prometheus-2.nix

@@ -1,239 +0,0 @@
-let
-  grpcPort   = 19090;
-  queryPort  =  9090;
-  minioPort  =  9000;
-  pushgwPort =  9091;
-
-  s3 = {
-    accessKey = "BKIKJAA5BMMU2RHO6IBB";
-    secretKey = "V7f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12";
-  };
-
-  objstore.config = {
-    type = "S3";
-    config = {
-      bucket = "thanos-bucket";
-      endpoint = "s3:${toString minioPort}";
-      region =  "us-east-1";
-      access_key = s3.accessKey;
-      secret_key = s3.secretKey;
-      insecure = true;
-      signature_version2 = false;
-      encrypt_sse =  false;
-      put_user_metadata = {};
-      http_config = {
-        idle_conn_timeout = "0s";
-        insecure_skip_verify = false;
-      };
-      trace = {
-        enable = false;
-      };
-    };
-  };
-
-in import ./make-test.nix {
-  name = "prometheus-2";
-
-  nodes = {
-    prometheus = { pkgs, ... }: {
-      virtualisation.diskSize = 2 * 1024;
-      environment.systemPackages = [ pkgs.jq ];
-      networking.firewall.allowedTCPPorts = [ grpcPort ];
-      services.prometheus2 = {
-        enable = true;
-        scrapeConfigs = [
-          {
-            job_name = "prometheus";
-            static_configs = [
-              {
-                targets = [ "127.0.0.1:${toString queryPort}" ];
-                labels = { instance = "localhost"; };
-              }
-            ];
-          }
-          {
-            job_name = "pushgateway";
-            scrape_interval = "1s";
-            static_configs = [
-              {
-                targets = [ "127.0.0.1:${toString pushgwPort}" ];
-              }
-            ];
-          }
-        ];
-        rules = [
-          ''
-            groups:
-              - name: test
-                rules:
-                  - record: testrule
-                    expr: count(up{job="prometheus"})
-          ''
-        ];
-        globalConfig = {
-          external_labels = {
-            some_label = "required by thanos";
-          };
-        };
-        extraFlags = [
-          # Required by thanos
-          "--storage.tsdb.min-block-duration=5s"
-          "--storage.tsdb.max-block-duration=5s"
-        ];
-      };
-      services.prometheus.pushgateway = {
-        enable = true;
-        web.listen-address = ":${toString pushgwPort}";
-        persistMetrics = true;
-        persistence.interval = "1s";
-        stateDir = "prometheus-pushgateway";
-      };
-      services.thanos = {
-        sidecar = {
-          enable = true;
-          grpc-address = "0.0.0.0:${toString grpcPort}";
-          inherit objstore;
-        };
-
-        # TODO: Add some tests for these services:
-        #rule = {
-        #  enable = true;
-        #  http-address = "0.0.0.0:19194";
-        #  grpc-address = "0.0.0.0:19193";
-        #  query.addresses = [
-        #    "localhost:19191"
-        #  ];
-        #  labels = {
-        #    just = "some";
-        #    nice = "labels";
-        #  };
-        #};
-        #
-        #receive = {
-        #  http-address = "0.0.0.0:19195";
-        #  enable = true;
-        #  labels = {
-        #    just = "some";
-        #    nice = "labels";
-        #  };
-        #};
-      };
-    };
-
-    query = { pkgs, ... }: {
-      environment.systemPackages = [ pkgs.jq ];
-      services.thanos.query = {
-        enable = true;
-        http-address = "0.0.0.0:${toString queryPort}";
-        store.addresses = [
-          "prometheus:${toString grpcPort}"
-        ];
-      };
-    };
-
-    store = { pkgs, ... }: {
-      virtualisation.diskSize = 2 * 1024;
-      environment.systemPackages = with pkgs; [ jq thanos ];
-      services.thanos.store = {
-        enable = true;
-        http-address = "0.0.0.0:10902";
-        grpc-address = "0.0.0.0:${toString grpcPort}";
-        inherit objstore;
-        sync-block-duration = "1s";
-      };
-      services.thanos.compact = {
-        enable = true;
-        http-address = "0.0.0.0:10903";
-        inherit objstore;
-        consistency-delay = "5s";
-      };
-      services.thanos.query = {
-        enable = true;
-        http-address = "0.0.0.0:${toString queryPort}";
-        store.addresses = [
-          "localhost:${toString grpcPort}"
-        ];
-      };
-    };
-
-    s3 = { pkgs, ... } : {
-      # Minio requires at least 1GiB of free disk space to run.
-      virtualisation.diskSize = 2 * 1024;
-      networking.firewall.allowedTCPPorts = [ minioPort ];
-
-      services.minio = {
-        enable = true;
-        inherit (s3) accessKey secretKey;
-      };
-
-      environment.systemPackages = [ pkgs.minio-client ];
-    };
-  };
-
-  testScript = { nodes, ... } : ''
-    # Before starting the other machines we first make sure that our S3 service is online
-    # and has a bucket added for thanos:
-    $s3->start;
-    $s3->waitForUnit("minio.service");
-    $s3->waitForOpenPort(${toString minioPort});
-    $s3->succeed(
-      "mc config host add minio " .
-      "http://localhost:${toString minioPort} ${s3.accessKey} ${s3.secretKey} S3v4");
-    $s3->succeed("mc mb minio/thanos-bucket");
-
-    # Now that s3 has started we can start the other machines:
-    $prometheus->start;
-    $query->start;
-    $store->start;
-
-    # Check if prometheus responds to requests:
-    $prometheus->waitForUnit("prometheus2.service");
-    $prometheus->waitForOpenPort(${toString queryPort});
-    $prometheus->succeed("curl -s http://127.0.0.1:${toString queryPort}/metrics");
-
-    # Let's test if pushing a metric to the pushgateway succeeds:
-    $prometheus->waitForUnit("pushgateway.service");
-    $prometheus->succeed(
-      "echo 'some_metric 3.14' | " .
-      "curl --data-binary \@- http://127.0.0.1:${toString pushgwPort}/metrics/job/some_job");
-
-    # Now check whether that metric gets ingested by prometheus.
-    # Since we'll check for the metric several times on different machines
-    # we abstract the test using the following function:
-
-    # Function to check if the metric "some_metric" has been received and returns the correct value.
-    local *Machine::waitForMetric = sub {
-      my ($self) = @_;
-      $self->waitUntilSucceeds(
-        "curl -sf 'http://127.0.0.1:${toString queryPort}/api/v1/query?query=some_metric' " .
-        "| jq '.data.result[0].value[1]' | grep '\"3.14\"'");
-    };
-
-    $prometheus->waitForMetric;
-
-    # Let's test if the pushgateway persists metrics to the configured location.
-    $prometheus->waitUntilSucceeds("test -e /var/lib/prometheus-pushgateway/metrics");
-
-    # Test thanos
-    $prometheus->waitForUnit("thanos-sidecar.service");
-
-    # Test if the Thanos query service can correctly retrieve the metric that was send above.
-    $query->waitForUnit("thanos-query.service");
-    $query->waitForMetric;
-
-    # Test if the Thanos sidecar has correctly uploaded its TSDB to S3, if the
-    # Thanos storage service has correctly downloaded it from S3 and if the Thanos
-    # query service running on $store can correctly retrieve the metric:
-    $store->waitForUnit("thanos-store.service");
-    $store->waitForMetric;
-
-    $store->waitForUnit("thanos-compact.service");
-
-    # Test if the Thanos bucket command is able to retrieve blocks from the S3 bucket
-    # and check if the blocks have the correct labels:
-    $store->succeed(
-      "thanos bucket ls" .
-      " --objstore.config-file=${nodes.store.config.services.thanos.store.objstore.config-file}" .
-      " --output=json | jq .thanos.labels.some_label | grep 'required by thanos'");
-  '';
-}

nixos/tests/prometheus.nix

@@ -1,48 +1,239 @@
-import ./make-test.nix {
+let
+  grpcPort   = 19090;
+  queryPort  =  9090;
+  minioPort  =  9000;
+  pushgwPort =  9091;
+
+  s3 = {
+    accessKey = "BKIKJAA5BMMU2RHO6IBB";
+    secretKey = "V7f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12";
+  };
+
+  objstore.config = {
+    type = "S3";
+    config = {
+      bucket = "thanos-bucket";
+      endpoint = "s3:${toString minioPort}";
+      region =  "us-east-1";
+      access_key = s3.accessKey;
+      secret_key = s3.secretKey;
+      insecure = true;
+      signature_version2 = false;
+      encrypt_sse =  false;
+      put_user_metadata = {};
+      http_config = {
+        idle_conn_timeout = "0s";
+        insecure_skip_verify = false;
+      };
+      trace = {
+        enable = false;
+      };
+    };
+  };
+
+in import ./make-test.nix {
   name = "prometheus";
 
   nodes = {
-    one = { ... }: {
+    prometheus = { pkgs, ... }: {
+      virtualisation.diskSize = 2 * 1024;
+      environment.systemPackages = [ pkgs.jq ];
+      networking.firewall.allowedTCPPorts = [ grpcPort ];
       services.prometheus = {
         enable = true;
-        scrapeConfigs = [{
+        scrapeConfigs = [
-          job_name = "prometheus";
-          static_configs = [{
-            targets = [ "127.0.0.1:9090" ];
-            labels = { instance = "localhost"; };
-          }];
-        }];
-        rules = [ ''testrule = count(up{job="prometheus"})'' ];
-
-        # a very simple version of the alertmanager configuration just to see if
-        # configuration checks & service startup are working
-        alertmanager = {
-          enable = true;
-          listenAddress = "[::1]";
-          port = 9093;
-          configuration = {
-            route.receiver = "webhook";
-            receivers = [
           {
-                name = "webhook";
+            job_name = "prometheus";
-                webhook_configs = [
+            static_configs = [
-                  { url = "http://localhost"; }
+              {
+                targets = [ "127.0.0.1:${toString queryPort}" ];
+                labels = { instance = "localhost"; };
+              }
+            ];
+          }
+          {
+            job_name = "pushgateway";
+            scrape_interval = "1s";
+            static_configs = [
+              {
+                targets = [ "127.0.0.1:${toString pushgwPort}" ];
+              }
             ];
           }
         ];
+        rules = [
+          ''
+            groups:
+              - name: test
+                rules:
+                  - record: testrule
+                    expr: count(up{job="prometheus"})
+          ''
+        ];
+        globalConfig = {
+          external_labels = {
+            some_label = "required by thanos";
           };
         };
+        extraFlags = [
+          # Required by thanos
+          "--storage.tsdb.min-block-duration=5s"
+          "--storage.tsdb.max-block-duration=5s"
+        ];
       };
+      services.prometheus.pushgateway = {
+        enable = true;
+        web.listen-address = ":${toString pushgwPort}";
+        persistMetrics = true;
+        persistence.interval = "1s";
+        stateDir = "prometheus-pushgateway";
+      };
+      services.thanos = {
+        sidecar = {
+          enable = true;
+          grpc-address = "0.0.0.0:${toString grpcPort}";
+          inherit objstore;
+        };
+
+        # TODO: Add some tests for these services:
+        #rule = {
+        #  enable = true;
+        #  http-address = "0.0.0.0:19194";
+        #  grpc-address = "0.0.0.0:19193";
+        #  query.addresses = [
+        #    "localhost:19191"
+        #  ];
+        #  labels = {
+        #    just = "some";
+        #    nice = "labels";
+        #  };
+        #};
+        #
+        #receive = {
+        #  http-address = "0.0.0.0:19195";
+        #  enable = true;
+        #  labels = {
+        #    just = "some";
+        #    nice = "labels";
+        #  };
+        #};
       };
     };
 
-  testScript = ''
+    query = { pkgs, ... }: {
-    startAll;
+      environment.systemPackages = [ pkgs.jq ];
-    $one->waitForUnit("prometheus.service");
+      services.thanos.query = {
-    $one->waitForOpenPort(9090);
+        enable = true;
-    $one->succeed("curl -s http://127.0.0.1:9090/metrics");
+        http-address = "0.0.0.0:${toString queryPort}";
-    $one->waitForUnit("alertmanager.service");
+        store.addresses = [
-    $one->waitForOpenPort("9093");
+          "prometheus:${toString grpcPort}"
-    $one->succeed("curl -f -s http://localhost:9093/");
+        ];
+      };
+    };
+
+    store = { pkgs, ... }: {
+      virtualisation.diskSize = 2 * 1024;
+      environment.systemPackages = with pkgs; [ jq thanos ];
+      services.thanos.store = {
+        enable = true;
+        http-address = "0.0.0.0:10902";
+        grpc-address = "0.0.0.0:${toString grpcPort}";
+        inherit objstore;
+        sync-block-duration = "1s";
+      };
+      services.thanos.compact = {
+        enable = true;
+        http-address = "0.0.0.0:10903";
+        inherit objstore;
+        consistency-delay = "5s";
+      };
+      services.thanos.query = {
+        enable = true;
+        http-address = "0.0.0.0:${toString queryPort}";
+        store.addresses = [
+          "localhost:${toString grpcPort}"
+        ];
+      };
+    };
+
+    s3 = { pkgs, ... } : {
+      # Minio requires at least 1GiB of free disk space to run.
+      virtualisation.diskSize = 2 * 1024;
+      networking.firewall.allowedTCPPorts = [ minioPort ];
+
+      services.minio = {
+        enable = true;
+        inherit (s3) accessKey secretKey;
+      };
+
+      environment.systemPackages = [ pkgs.minio-client ];
+    };
+  };
+
+  testScript = { nodes, ... } : ''
+    # Before starting the other machines we first make sure that our S3 service is online
+    # and has a bucket added for thanos:
+    $s3->start;
+    $s3->waitForUnit("minio.service");
+    $s3->waitForOpenPort(${toString minioPort});
+    $s3->succeed(
+      "mc config host add minio " .
+      "http://localhost:${toString minioPort} ${s3.accessKey} ${s3.secretKey} S3v4");
+    $s3->succeed("mc mb minio/thanos-bucket");
+
+    # Now that s3 has started we can start the other machines:
+    $prometheus->start;
+    $query->start;
+    $store->start;
+
+    # Check if prometheus responds to requests:
+    $prometheus->waitForUnit("prometheus.service");
+    $prometheus->waitForOpenPort(${toString queryPort});
+    $prometheus->succeed("curl -s http://127.0.0.1:${toString queryPort}/metrics");
+
+    # Let's test if pushing a metric to the pushgateway succeeds:
+    $prometheus->waitForUnit("pushgateway.service");
+    $prometheus->succeed(
+      "echo 'some_metric 3.14' | " .
+      "curl --data-binary \@- http://127.0.0.1:${toString pushgwPort}/metrics/job/some_job");
+
+    # Now check whether that metric gets ingested by prometheus.
+    # Since we'll check for the metric several times on different machines
+    # we abstract the test using the following function:
+
+    # Function to check if the metric "some_metric" has been received and returns the correct value.
+    local *Machine::waitForMetric = sub {
+      my ($self) = @_;
+      $self->waitUntilSucceeds(
+        "curl -sf 'http://127.0.0.1:${toString queryPort}/api/v1/query?query=some_metric' " .
+        "| jq '.data.result[0].value[1]' | grep '\"3.14\"'");
+    };
+
+    $prometheus->waitForMetric;
+
+    # Let's test if the pushgateway persists metrics to the configured location.
+    $prometheus->waitUntilSucceeds("test -e /var/lib/prometheus-pushgateway/metrics");
+
+    # Test thanos
+    $prometheus->waitForUnit("thanos-sidecar.service");
+
+    # Test if the Thanos query service can correctly retrieve the metric that was send above.
+    $query->waitForUnit("thanos-query.service");
+    $query->waitForMetric;
+
+    # Test if the Thanos sidecar has correctly uploaded its TSDB to S3, if the
+    # Thanos storage service has correctly downloaded it from S3 and if the Thanos
+    # query service running on $store can correctly retrieve the metric:
+    $store->waitForUnit("thanos-store.service");
+    $store->waitForMetric;
+
+    $store->waitForUnit("thanos-compact.service");
+
+    # Test if the Thanos bucket command is able to retrieve blocks from the S3 bucket
+    # and check if the blocks have the correct labels:
+    $store->succeed(
+      "thanos bucket ls" .
+      " --objstore.config-file=${nodes.store.config.services.thanos.store.objstore.config-file}" .
+      " --output=json | jq .thanos.labels.some_label | grep 'required by thanos'");
   '';
 }

pkgs/servers/monitoring/prometheus/default.nix

@@ -1,12 +1,11 @@
-{ stdenv, go, buildGoPackage, fetchFromGitHub }:
+{ lib, go, buildGoPackage, fetchFromGitHub }:
 
 let
   goPackagePath = "github.com/prometheus/prometheus";
-in rec {
+in
-  buildPrometheus = { version, sha256, doCheck ? true, ... }@attrs:
+buildGoPackage rec {
-    let attrs' = builtins.removeAttrs attrs ["version" "sha256"]; in
+  pname = "prometheus";
-      buildGoPackage ({
+  version = "2.12.0";
-        name = "prometheus-${version}";
 
   inherit goPackagePath;
 
@@ -14,17 +13,19 @@ in rec {
     rev = "v${version}";
     owner = "prometheus";
     repo = "prometheus";
-          inherit sha256;
+    sha256 = "1ci9dc512c1hry1b8jqif0mrnks6w3yagwm3jf69ihcwilr2n7vs";
   };
 
-        buildFlagsArray = let t = "${goPackagePath}/vendor/github.com/prometheus/common/version"; in ''
+  buildFlagsArray = let
+    t = "${goPackagePath}/vendor/github.com/prometheus/common/version";
+  in ''
     -ldflags=
        -X ${t}.Version=${version}
        -X ${t}.Revision=unknown
        -X ${t}.Branch=unknown
        -X ${t}.BuildUser=nix@nixpkgs
        -X ${t}.BuildDate=unknown
-             -X ${t}.GoVersion=${stdenv.lib.getVersion go}
+       -X ${t}.GoVersion=${lib.getVersion go}
   '';
 
   preInstall = ''
@@ -33,22 +34,13 @@ in rec {
     cp -a $src/console_libraries $src/consoles $bin/etc/prometheus
   '';
 
-        meta = with stdenv.lib; {
+  doCheck = true;
+
+  meta = with lib; {
     description = "Service monitoring system and time series database";
-          homepage = https://prometheus.io;
+    homepage = "https://prometheus.io";
     license = licenses.asl20;
-          maintainers = with maintainers; [ benley fpletz globin ];
+    maintainers = with maintainers; [ benley fpletz globin willibutz ];
     platforms = platforms.unix;
   };
-    } // attrs');
-
-  prometheus_1 = buildPrometheus {
-    version = "1.8.2";
-    sha256 = "088flpg3qgnj9afl9vbaa19v2s1d21yxy38nrlv5m7cxwy2pi5pv";
-  };
-
-  prometheus_2 = buildPrometheus {
-    version = "2.12.0";
-    sha256 = "1ci9dc512c1hry1b8jqif0mrnks6w3yagwm3jf69ihcwilr2n7vs";
-  };
 }

pkgs/top-level/all-packages.nix

@@ -15014,13 +15014,8 @@ in
 
   postgresql_jdbc = callPackage ../development/java-modules/postgresql_jdbc { };
 
-  inherit (callPackage ../servers/monitoring/prometheus { }) prometheus_1;
-
-  inherit (callPackage ../servers/monitoring/prometheus { })
-    prometheus_2;
-
   prom2json = callPackage ../servers/monitoring/prometheus/prom2json.nix { };
-  prometheus = prometheus_1;
+  prometheus = callPackage ../servers/monitoring/prometheus { };
   prometheus-alertmanager = callPackage ../servers/monitoring/prometheus/alertmanager.nix { };
   prometheus-aws-s3-exporter = callPackage ../servers/monitoring/prometheus/aws-s3-exporter.nix { };
   prometheus-bind-exporter = callPackage ../servers/monitoring/prometheus/bind-exporter.nix { };