nixos/journalbeat: support journalbeat >= 6 & add test

nixos/modules/services/logging/journalbeat.nix

@@ -5,11 +5,13 @@ with lib;
 let
   cfg = config.services.journalbeat;
 
+  lt6 = builtins.compareVersions cfg.package.version "6" < 0;
+
   journalbeatYml = pkgs.writeText "journalbeat.yml" ''
     name: ${cfg.name}
     tags: ${builtins.toJSON cfg.tags}
 
-    journalbeat.cursor_state_file: ${cfg.stateDir}/cursor-state
+    ${optionalString lt6 "journalbeat.cursor_state_file: /var/lib/${cfg.stateDir}/cursor-state"}
 
     ${cfg.extraConfig}
   '';
@@ -22,6 +24,16 @@ in
 
       enable = mkEnableOption "journalbeat";
 
+      package = mkOption {
+        type = types.package;
+        default = pkgs.journalbeat;
+        defaultText = "pkgs.journalbeat";
+        example = literalExample "pkgs.journalbeat7";
+        description = ''
+          The journalbeat package to use
+        '';
+      };
+
       name = mkOption {
         type = types.str;
         default = "journalbeat";
@@ -36,13 +48,17 @@ in
 
       stateDir = mkOption {
         type = types.str;
-        default = "/var/lib/journalbeat";
+        default = "journalbeat";
-        description = "The state directory. Journalbeat's own logs and other data are stored here.";
+        description = ''
+          Directory below <literal>/var/lib/</literal> to store journalbeat's
+          own logs and other data. This directory will be created automatically
+          using systemd's StateDirectory mechanism.
+        '';
       };
 
       extraConfig = mkOption {
         type = types.lines;
-        default = ''
+        default = optionalString lt6 ''
           journalbeat:
             seek_position: cursor
             cursor_seek_fallback: tail
@@ -61,7 +77,16 @@ in
 
   config = mkIf cfg.enable {
 
-    systemd.services.journalbeat = with pkgs; {
+    assertions = [
+      {
+        assertion = !hasPrefix "/" cfg.stateDir;
+        message =
+          "The option services.journalbeat.stateDir shouldn't be an absolute directory." +
+          " It should be a directory relative to /var/lib/.";
+      }
+    ];
+
+    systemd.services.journalbeat = {
       description = "Journalbeat log shipper";
       wantedBy = [ "multi-user.target" ];
       preStart = ''
@@ -69,7 +94,13 @@ in
         mkdir -p ${cfg.stateDir}/logs
       '';
       serviceConfig = {
-        ExecStart = "${pkgs.journalbeat}/bin/journalbeat -c ${journalbeatYml} -path.data ${cfg.stateDir}/data -path.logs ${cfg.stateDir}/logs";
+        StateDirectory = cfg.stateDir;
+        ExecStart = ''
+          ${cfg.package}/bin/journalbeat \
+            -c ${journalbeatYml} \
+            -path.data /var/lib/${cfg.stateDir}/data \
+            -path.logs /var/lib/${cfg.stateDir}/logs'';
+        Restart = "always";
       };
     };
   };

nixos/tests/elk.nix

@@ -12,6 +12,11 @@ with pkgs.lib;
 let
   esUrl = "http://localhost:9200";
 
+  totalHits = message :
+    "curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' " +
+    ''-d '{\"query\" : { \"match\" : { \"message\" : \"${message}\"}}}' '' +
+    "| jq .hits.total";
+
   mkElkTest = name : elk :
    let elasticsearchGe7 = builtins.compareVersions elk.elasticsearch.version "7" >= 0;
    in makeTest {
@@ -21,7 +26,7 @@ let
     };
     nodes = {
       one =
-        { pkgs, ... }: {
+        { pkgs, lib, ... }: {
             # Not giving the machine at least 2060MB results in elasticsearch failing with the following error:
             #
             #   OpenJDK 64-Bit Server VM warning:
@@ -40,6 +45,26 @@ let
             environment.systemPackages = [ pkgs.jq ];
 
             services = {
+
+              journalbeat = let lt6 = builtins.compareVersions
+                                        elk.journalbeat.version "6" < 0; in {
+                enable = true;
+                package = elk.journalbeat;
+                extraConfig = mkOptionDefault (''
+                  logging:
+                    to_syslog: true
+                    level: warning
+                    metrics.enabled: false
+                  output.elasticsearch:
+                    hosts: [ "127.0.0.1:9200" ]
+                    ${optionalString lt6 "template.enabled: false"}
+                '' + optionalString (!lt6) ''
+                  journalbeat.inputs:
+                  - paths: []
+                    seek: cursor
+                '');
+              };
+
               logstash = {
                 enable = true;
                 package = elk.logstash;
@@ -107,14 +132,19 @@ let
     testScript = ''
       startAll;
 
+      # Wait until elasticsearch is listening for connections.
       $one->waitForUnit("elasticsearch.service");
+      $one->waitForOpenPort(9200);
 
       # Continue as long as the status is not "red". The status is probably
       # "yellow" instead of "green" because we are using a single elasticsearch
       # node which elasticsearch considers risky.
       #
-      # TODO: extend this test with multiple elasticsearch nodes and see if the status turns "green".
+      # TODO: extend this test with multiple elasticsearch nodes
-      $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red");
+      #       and see if the status turns "green".
+      $one->waitUntilSucceeds(
+        "curl --silent --show-error '${esUrl}/_cluster/health' " .
+        "| jq .status | grep -v red");
 
       # Perform some simple logstash tests.
       $one->waitForUnit("logstash.service");
@@ -123,16 +153,28 @@ let
 
       # See if kibana is healthy.
       $one->waitForUnit("kibana.service");
-      $one->waitUntilSucceeds("curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green");
+      $one->waitUntilSucceeds(
+        "curl --silent --show-error 'http://localhost:5601/api/status' " .
+        "| jq .status.overall.state | grep green");
 
       # See if logstash messages arive in elasticsearch.
-      $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
+      $one->waitUntilSucceeds("${totalHits "flowers"} | grep -v 0");
-      $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
+      $one->waitUntilSucceeds("${totalHits "dragons"} | grep 0");
+
+      # Test if a message logged to the journal
+      # is ingested by elasticsearch via journalbeat.
+      $one->waitForUnit("journalbeat.service");
+      $one->execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat");
+      $one->waitUntilSucceeds(
+        "${totalHits "Supercalifragilisticexpialidocious"} | grep -v 0");
+
     '' + optionalString (!elasticsearchGe7) ''
       # Test elasticsearch-curator.
       $one->systemctl("stop logstash");
       $one->systemctl("start elasticsearch-curator");
-      $one->waitUntilSucceeds("! curl --silent --show-error '${esUrl}/_cat/indices' | grep logstash | grep -q ^$1");
+      $one->waitUntilSucceeds(
+        "! curl --silent --show-error '${esUrl}/_cat/indices' " .
+        "| grep logstash | grep -q ^$1");
     '';
   };
 in mapAttrs mkElkTest {
@@ -140,6 +182,7 @@ in mapAttrs mkElkTest {
     elasticsearch = pkgs.elasticsearch5;
     logstash      = pkgs.logstash5;
     kibana        = pkgs.kibana5;
+    journalbeat   = pkgs.journalbeat5;
   };
   "ELK-6" =
     if enableUnfree
@@ -147,11 +190,13 @@ in mapAttrs mkElkTest {
       elasticsearch = pkgs.elasticsearch6;
       logstash      = pkgs.logstash6;
       kibana        = pkgs.kibana6;
+      journalbeat   = pkgs.journalbeat6;
     }
     else {
       elasticsearch = pkgs.elasticsearch6-oss;
       logstash      = pkgs.logstash6-oss;
       kibana        = pkgs.kibana6-oss;
+      journalbeat   = pkgs.journalbeat6;
     };
   "ELK-7" =
     if enableUnfree
@@ -159,10 +204,12 @@ in mapAttrs mkElkTest {
       elasticsearch = pkgs.elasticsearch7;
       logstash      = pkgs.logstash7;
       kibana        = pkgs.kibana7;
+      journalbeat   = pkgs.journalbeat7;
     }
     else {
       elasticsearch = pkgs.elasticsearch7-oss;
       logstash      = pkgs.logstash7-oss;
       kibana        = pkgs.kibana7-oss;
+      journalbeat   = pkgs.journalbeat7;
     };
 }