Merge pull request #65088 from WilliButz/prometheus-exporters-cleanup

prometheus-nginx-exporter: switch to official version & general prometheus-exporters cleanup

nixos/doc/manual/release-notes/rl-1909.xml

@@ -195,6 +195,13 @@
      <xref linkend="opt-programs.zsh.loginShellInit" /> and <xref linkend="opt-programs.zsh.promptInit" /> may break if it relies on those options being set.
     </para>
    </listitem>
+   <listitem>
+    <para>
+      The <literal>prometheus-nginx-exporter</literal> package now uses the offical exporter provided by NGINX Inc.
+      Its metrics are differently structured and are incompatible to the old ones. For information about the metrics,
+      have a look at the <link xlink:href="https://github.com/nginxinc/nginx-prometheus-exporter">official repo</link>.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 

nixos/modules/services/monitoring/prometheus/exporters.nix

@@ -1,8 +1,10 @@
-{ config, pkgs, lib, ... }:
+{ config, pkgs, lib, options, ... }:
-
-with lib;
 
 let
+  inherit (lib) concatStrings foldl foldl' genAttrs literalExample maintainers
+                mapAttrsToList mkDefault mkEnableOption mkIf mkMerge mkOption
+                optional types;
+
   cfg = config.services.prometheus.exporters;
 
   # each attribute in `exporterOpts` is expected to have specified:
@@ -17,25 +19,28 @@ let
   #  Note that `extraOpts` is optional, but a script for the exporter's
   #  systemd service must be provided by specifying either
   #  `serviceOpts.script` or `serviceOpts.serviceConfig.ExecStart`
-  exporterOpts = {
+
-    blackbox  = import ./exporters/blackbox.nix  { inherit config lib pkgs; };
+  exporterOpts = genAttrs [
-    collectd  = import ./exporters/collectd.nix  { inherit config lib pkgs; };
+    "bind"
-    dnsmasq   = import ./exporters/dnsmasq.nix   { inherit config lib pkgs; };
+    "blackbox"
-    dovecot   = import ./exporters/dovecot.nix   { inherit config lib pkgs; };
+    "collectd"
-    fritzbox  = import ./exporters/fritzbox.nix  { inherit config lib pkgs; };
+    "dnsmasq"
-    json      = import ./exporters/json.nix      { inherit config lib pkgs; };
+    "dovecot"
-    minio     = import ./exporters/minio.nix     { inherit config lib pkgs; };
+    "fritzbox"
-    nginx     = import ./exporters/nginx.nix     { inherit config lib pkgs; };
+    "json"
-    node      = import ./exporters/node.nix      { inherit config lib pkgs; };
+    "minio"
-    postfix   = import ./exporters/postfix.nix   { inherit config lib pkgs; };
+    "nginx"
-    snmp      = import ./exporters/snmp.nix      { inherit config lib pkgs; };
+    "node"
-    surfboard = import ./exporters/surfboard.nix { inherit config lib pkgs; };
+    "postfix"
-    tor       = import ./exporters/tor.nix       { inherit config lib pkgs; };
+    "snmp"
-    unifi     = import ./exporters/unifi.nix     { inherit config lib pkgs; };
+    "surfboard"
-    varnish   = import ./exporters/varnish.nix   { inherit config lib pkgs; };
+    "tor"
-    bind      = import ./exporters/bind.nix      { inherit config lib pkgs; };
+    "unifi"
-    wireguard = import ./exporters/wireguard.nix { inherit config lib pkgs; };
+    "varnish"
-  };
+    "wireguard"
+  ] (name:
+    import (./. + "/exporters/${name}.nix") { inherit config lib pkgs options; }
+  );
 
   mkExporterOpts = ({ name, port }: {
     enable = mkEnableOption "the prometheus ${name} exporter";
@@ -97,9 +102,10 @@ let
     };
   });
 
-  mkSubModule = { name, port, extraOpts, ... }: {
+  mkSubModule = { name, port, extraOpts, imports }: {
     ${name} = mkOption {
       type = types.submodule {
+        inherit imports;
         options = (mkExporterOpts {
           inherit name port;
         } // extraOpts);
@@ -112,13 +118,15 @@ let
   mkSubModules = (foldl' (a: b: a//b) {}
     (mapAttrsToList (name: opts: mkSubModule {
       inherit name;
-      inherit (opts) port serviceOpts;
+      inherit (opts) port;
       extraOpts = opts.extraOpts or {};
+      imports = opts.imports or [];
     }) exporterOpts)
   );
 
   mkExporterConf = { name, conf, serviceOpts }:
     mkIf conf.enable {
+      warnings = conf.warnings or [];
       networking.firewall.extraCommands = mkIf conf.openFirewall (concatStrings [
         "ip46tables -A nixos-fw ${conf.firewallFilter} "
         "-m comment --comment ${name}-exporter -j nixos-fw-accept"

nixos/modules/services/monitoring/prometheus/exporters.xml

@@ -113,7 +113,7 @@
      specific options and configuration:
 <programlisting>
 # nixpgs/nixos/modules/services/prometheus/exporters/postfix.nix
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 
@@ -184,4 +184,42 @@ in
    </listitem>
   </itemizedlist>
  </section>
+ <section xml:id="module-services-prometheus-exporters-update-exporter-module">
+  <title>Updating an exporter module</title>
+   <para>
+     Should an exporter option change at some point, it is possible to add
+     information about the change to the exporter definition similar to
+     <literal>nixpkgs/nixos/modules/rename.nix</literal>:
+<programlisting>
+{ config, lib, pkgs, options }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.exporters.nginx;
+in
+{
+  port = 9113;
+  extraOpts = {
+    # additional module options
+    # ...
+  };
+  serviceOpts = {
+    # service configuration
+    # ...
+  };
+  imports = [
+    # 'services.prometheus.exporters.nginx.telemetryEndpoint' -> 'services.prometheus.exporters.nginx.telemetryPath'
+    (mkRenamedOptionModule [ "telemetryEndpoint" ] [ "telemetryPath" ])
+
+    # removed option 'services.prometheus.exporters.nginx.insecure'
+    (mkRemovedOptionModule [ "insecure" ] ''
+      This option was replaced by 'prometheus.exporters.nginx.sslVerify' which defaults to true.
+    '')
+    ({ options.warnings = options.warnings; })
+  ];
+}
+</programlisting>
+    </para>
+  </section>
 </chapter>

nixos/modules/services/monitoring/prometheus/exporters/bind.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/modules/services/monitoring/prometheus/exporters/blackbox.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/modules/services/monitoring/prometheus/exporters/collectd.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/modules/services/monitoring/prometheus/exporters/dnsmasq.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/modules/services/monitoring/prometheus/exporters/dovecot.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/modules/services/monitoring/prometheus/exporters/fritzbox.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/modules/services/monitoring/prometheus/exporters/json.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/modules/services/monitoring/prometheus/exporters/minio.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/modules/services/monitoring/prometheus/exporters/nginx.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 
@@ -16,32 +16,40 @@ in
         Can be enabled with services.nginx.statusPage = true.
       '';
     };
-    telemetryEndpoint = mkOption {
+    telemetryPath = mkOption {
       type = types.str;
       default = "/metrics";
       description = ''
         Path under which to expose metrics.
       '';
     };
-    insecure = mkOption {
+    sslVerify = mkOption {
       type = types.bool;
       default = true;
       description = ''
-        Ignore server certificate if using https.
+        Whether to perform certificate verification for https.
       '';
     };
+
   };
   serviceOpts = {
     serviceConfig = {
       DynamicUser = true;
       ExecStart = ''
-        ${pkgs.prometheus-nginx-exporter}/bin/nginx_exporter \
+        ${pkgs.prometheus-nginx-exporter}/bin/nginx-prometheus-exporter \
-          --nginx.scrape_uri '${cfg.scrapeUri}' \
+          --nginx.scrape-uri '${cfg.scrapeUri}' \
-          --telemetry.address ${cfg.listenAddress}:${toString cfg.port} \
+          --nginx.ssl-verify ${toString cfg.sslVerify} \
-          --telemetry.endpoint ${cfg.telemetryEndpoint} \
+          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
-          --insecure ${toString cfg.insecure} \
+          --web.telemetry-path ${cfg.telemetryPath} \
           ${concatStringsSep " \\\n  " cfg.extraFlags}
       '';
     };
   };
+  imports = [
+    (mkRenamedOptionModule [ "telemetryEndpoint" ] [ "telemetryPath" ])
+    (mkRemovedOptionModule [ "insecure" ] ''
+      This option was replaced by 'prometheus.exporters.nginx.sslVerify'.
+    '')
+    ({ options.warnings = options.warnings; })
+  ];
 }

nixos/modules/services/monitoring/prometheus/exporters/node.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/modules/services/monitoring/prometheus/exporters/postfix.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/modules/services/monitoring/prometheus/exporters/snmp.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/modules/services/monitoring/prometheus/exporters/surfboard.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/modules/services/monitoring/prometheus/exporters/tor.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/modules/services/monitoring/prometheus/exporters/unifi.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/modules/services/monitoring/prometheus/exporters/varnish.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 
@@ -68,7 +68,6 @@ in
   serviceOpts = {
     path = [ pkgs.varnish ];
     serviceConfig = {
-      DynamicUser = true;
       RestartSec = mkDefault 1;
       ExecStart = ''
         ${pkgs.prometheus-varnish-exporter}/bin/prometheus_varnish_exporter \

nixos/modules/services/monitoring/prometheus/exporters/wireguard.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs }:
+{ config, lib, pkgs, options }:
 
 with lib;
 

nixos/tests/prometheus-exporters.nix

@@ -3,10 +3,11 @@
 , pkgs ? import ../.. { inherit system config; }
 }:
 
-with pkgs.lib;
-with import ../lib/testing.nix { inherit system pkgs; };
-
 let
+  inherit (import ../lib/testing.nix { inherit system pkgs; }) makeTest;
+  inherit (pkgs.lib) concatStringsSep maintainers mapAttrs mkMerge
+                     removeSuffix replaceChars singleton splitString;
+
   escape' = str: replaceChars [''"'' "$" "\n"] [''\\\"'' "\\$" ""] str;
 
 /*
@@ -73,7 +74,7 @@ let
       exporterTest = ''
         waitForUnit("prometheus-bind-exporter.service");
         waitForOpenPort(9119);
-        succeed("curl -sSf http://localhost:9119/metrics" | grep -q 'bind_query_recursions_total 0');
+        succeed("curl -sSf http://localhost:9119/metrics | grep -q 'bind_query_recursions_total 0'");
       '';
     };
 
@@ -311,6 +312,7 @@ let
       };
       exporterTest = ''
         waitForUnit("prometheus-varnish-exporter.service");
+        waitForOpenPort(6081);
         waitForOpenPort(9131);
         succeed("curl -sSf http://localhost:9131/metrics | grep -q 'varnish_up 1'");
       '';

pkgs/servers/monitoring/prometheus/nginx-exporter.nix

@@ -2,23 +2,25 @@
 
 buildGoPackage rec {
   name = "nginx_exporter-${version}";
-  version = "0.1.0";
+  version = "0.4.1";
 
-  goPackagePath = "github.com/discordianfish/nginx_exporter";
+  goPackagePath = "github.com/nginxinc/nginx-prometheus-exporter";
+
+  buildFlagsArray = [
+    "-ldflags=" "-X main.version=${version}"
+  ];
 
   src = fetchFromGitHub {
     rev = "v${version}";
-    owner = "discordianfish";
+    owner = "nginxinc";
-    repo = "nginx_exporter";
+    repo = "nginx-prometheus-exporter";
-    sha256 = "1xwxnvkzslaj44r44ag24a9qfzjdxwz67hhpkdq42193zqpnlim7";
+    sha256 = "0c5bxl9xrd4gh2w5wyrzghmbcy9k1khydzml5cm0rsyqhwsvs8m5";
   };
 
-  goDeps = ./nginx-exporter_deps.nix;
-
   meta = with stdenv.lib; {
-    description = "Metrics relay from nginx stats to Prometheus";
+    description = "NGINX Prometheus Exporter for NGINX and NGINX Plus";
-    homepage = https://github.com/discordianfish/nginx_exporter;
+    homepage = "https://github.com/nginxinc/nginx-prometheus-exporter";
-    license = licenses.mit;
+    license = licenses.asl20;
     maintainers = with maintainers; [ benley fpletz willibutz ];
     platforms = platforms.unix;
   };

pkgs/servers/monitoring/prometheus/nginx-exporter_deps.nix

@@ -1,119 +0,0 @@
-[
-  {
-    goPackagePath = "github.com/alecthomas/template";
-    fetch = {
-      type = "git";
-      url = "https://github.com/alecthomas/template";
-      rev = "a0175ee3bccc567396460bf5acd36800cb10c49c";
-      sha256 = "0qjgvvh26vk1cyfq9fadyhfgdj36f1iapbmr5xp6zqipldz8ffxj";
-    };
-  }
-  {
-    goPackagePath = "github.com/alecthomas/units";
-    fetch = {
-      type = "git";
-      url = "https://github.com/alecthomas/units";
-      rev = "2efee857e7cfd4f3d0138cc3cbb1b4966962b93a";
-      sha256 = "1j65b91qb9sbrml9cpabfrcf07wmgzzghrl7809hjjhrmbzri5bl";
-    };
-  }
-  {
-    goPackagePath = "github.com/beorn7/perks";
-    fetch = {
-      type = "git";
-      url = "https://github.com/beorn7/perks";
-      rev = "3a771d992973f24aa725d07868b467d1ddfceafb";
-      sha256 = "1l2lns4f5jabp61201sh88zf3b0q793w4zdgp9nll7mmfcxxjif3";
-    };
-  }
-  {
-    goPackagePath = "github.com/golang/protobuf";
-    fetch = {
-      type = "git";
-      url = "https://github.com/golang/protobuf";
-      rev = "b4deda0973fb4c70b50d226b1af49f3da59f5265";
-      sha256 = "0ya4ha7m20bw048m1159ppqzlvda4x0vdprlbk5sdgmy74h3xcdq";
-    };
-  }
-  {
-    goPackagePath = "github.com/matttproud/golang_protobuf_extensions";
-    fetch = {
-      type = "git";
-      url = "https://github.com/matttproud/golang_protobuf_extensions";
-      rev = "c12348ce28de40eed0136aa2b644d0ee0650e56c";
-      sha256 = "1d0c1isd2lk9pnfq2nk0aih356j30k3h1gi2w0ixsivi5csl7jya";
-    };
-  }
-  {
-    goPackagePath = "github.com/prometheus/client_golang";
-    fetch = {
-      type = "git";
-      url = "https://github.com/prometheus/client_golang";
-      rev = "82f5ff156b29e276022b1a958f7d385870fb9814";
-      sha256 = "111j329yrlgvh73dm80gawwxsh9dgjkw74254kyj5c2rfmra7znz";
-    };
-  }
-  {
-    goPackagePath = "github.com/prometheus/client_model";
-    fetch = {
-      type = "git";
-      url = "https://github.com/prometheus/client_model";
-      rev = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c";
-      sha256 = "19y4ywsivhpxj7ikf2j0gm9k3cmyw37qcbfi78n526jxcc7kw998";
-    };
-  }
-  {
-    goPackagePath = "github.com/prometheus/common";
-    fetch = {
-      type = "git";
-      url = "https://github.com/prometheus/common";
-      rev = "d811d2e9bf898806ecfb6ef6296774b13ffc314c";
-      sha256 = "0r4067r4ysmljksqw3awcxx5qplqhykahc5igdzgkky7i4bvaik1";
-    };
-  }
-  {
-    goPackagePath = "github.com/prometheus/procfs";
-    fetch = {
-      type = "git";
-      url = "https://github.com/prometheus/procfs";
-      rev = "8b1c2da0d56deffdbb9e48d4414b4e674bd8083e";
-      sha256 = "0x128p15h35mgwqxkigfkk1lfrcz9g697ahl8v6xp9kwvcqvjrrf";
-    };
-  }
-  {
-    goPackagePath = "github.com/sirupsen/logrus";
-    fetch = {
-      type = "git";
-      url = "https://github.com/sirupsen/logrus";
-      rev = "778f2e774c725116edbc3d039dc0dfc1cc62aae8";
-      sha256 = "0drlrl192k4qkpcf1b6nw2qlixply31x2jhcckjzl3hn4mzwi6nf";
-    };
-  }
-  {
-    goPackagePath = "golang.org/x/crypto";
-    fetch = {
-      type = "git";
-      url = "https://go.googlesource.com/crypto";
-      rev = "ae8bce0030810cf999bb2b9868ae5c7c58e6343b";
-      sha256 = "0aihwcf0g8mq1sb96pwhpgvwxlf862pdhsfr7mdppz7bii4i9shw";
-    };
-  }
-  {
-    goPackagePath = "golang.org/x/sys";
-    fetch = {
-      type = "git";
-      url = "https://go.googlesource.com/sys";
-      rev = "78d5f264b493f125018180c204871ecf58a2dce1";
-      sha256 = "0x23n60wskys39dwybz5za77ldky9i518kp58ragpd5528kcc68s";
-    };
-  }
-  {
-    goPackagePath = "gopkg.in/alecthomas/kingpin.v2";
-    fetch = {
-      type = "git";
-      url = "https://gopkg.in/alecthomas/kingpin.v2";
-      rev = "947dcec5ba9c011838740e680966fd7087a71d0d";
-      sha256 = "0mndnv3hdngr3bxp7yxfd47cas4prv98sqw534mx7vp38gd88n5r";
-    };
-  }
-]