Merge pull request #77303 from tfc/port-elk-initdb-kafka

Port elk initdb kafka

nixos/tests/elk.nix

@@ -6,20 +6,12 @@
   # NIXPKGS_ALLOW_UNFREE=1 nix-build nixos/tests/elk.nix -A ELK-6 --arg enableUnfree true
 }:
 
-with import ../lib/testing.nix { inherit system pkgs; };
-with pkgs.lib;
-
 let
   esUrl = "http://localhost:9200";
 
-  totalHits = message :
-    "curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' " +
-    ''-d '{\"query\" : { \"match\" : { \"message\" : \"${message}\"}}}' '' +
-    "| jq .hits.total";
-
   mkElkTest = name : elk :
    let elasticsearchGe7 = builtins.compareVersions elk.elasticsearch.version "7" >= 0;
-   in makeTest {
+   in import ./make-test-python.nix ({
     inherit name;
     meta = with pkgs.stdenv.lib.maintainers; {
       maintainers = [ eelco offline basvandijk ];
@@ -50,15 +42,15 @@ let
                                         elk.journalbeat.version "6" < 0; in {
                 enable = true;
                 package = elk.journalbeat;
-                extraConfig = mkOptionDefault (''
+                extraConfig = pkgs.lib.mkOptionDefault (''
                   logging:
                     to_syslog: true
                     level: warning
                     metrics.enabled: false
                   output.elasticsearch:
                     hosts: [ "127.0.0.1:9200" ]
-                    ${optionalString lt6 "template.enabled: false"}
+                    ${pkgs.lib.optionalString lt6 "template.enabled: false"}
-                '' + optionalString (!lt6) ''
+                '' + pkgs.lib.optionalString (!lt6) ''
                   journalbeat.inputs:
                   - paths: []
                     seek: cursor
@@ -130,11 +122,23 @@ let
       };
 
     testScript = ''
-      startAll;
+      import json
 
-      # Wait until elasticsearch is listening for connections.
+
-      $one->waitForUnit("elasticsearch.service");
+      def total_hits(message):
-      $one->waitForOpenPort(9200);
+          dictionary = {"query": {"match": {"message": message}}}
+          return (
+              "curl --silent --show-error '${esUrl}/_search' "
+              + "-H 'Content-Type: application/json' "
+              + "-d '{}' ".format(json.dumps(dictionary))
+              + "| jq .hits.total"
+          )
+
+
+      start_all()
+
+      one.wait_for_unit("elasticsearch.service")
+      one.wait_for_open_port(9200)
 
       # Continue as long as the status is not "red". The status is probably
       # "yellow" instead of "green" because we are using a single elasticsearch
@@ -142,42 +146,43 @@ let
       #
       # TODO: extend this test with multiple elasticsearch nodes
       #       and see if the status turns "green".
-      $one->waitUntilSucceeds(
+      one.wait_until_succeeds(
-        "curl --silent --show-error '${esUrl}/_cluster/health' " .
+          "curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"
-        "| jq .status | grep -v red");
+      )
 
-      # Perform some simple logstash tests.
+      with subtest("Perform some simple logstash tests"):
-      $one->waitForUnit("logstash.service");
+          one.wait_for_unit("logstash.service")
-      $one->waitUntilSucceeds("cat /tmp/logstash.out | grep flowers");
+          one.wait_until_succeeds("cat /tmp/logstash.out | grep flowers")
-      $one->waitUntilSucceeds("cat /tmp/logstash.out | grep -v dragons");
+          one.wait_until_succeeds("cat /tmp/logstash.out | grep -v dragons")
 
-      # See if kibana is healthy.
+      with subtest("Kibana is healthy"):
-      $one->waitForUnit("kibana.service");
+          one.wait_for_unit("kibana.service")
-      $one->waitUntilSucceeds(
+          one.wait_until_succeeds(
-        "curl --silent --show-error 'http://localhost:5601/api/status' " .
+              "curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"
-        "| jq .status.overall.state | grep green");
+          )
 
-      # See if logstash messages arive in elasticsearch.
+      with subtest("Logstash messages arive in elasticsearch"):
-      $one->waitUntilSucceeds("${totalHits "flowers"} | grep -v 0");
+          one.wait_until_succeeds(total_hits("flowers") + " | grep -v 0")
-      $one->waitUntilSucceeds("${totalHits "dragons"} | grep 0");
+          one.wait_until_succeeds(total_hits("dragons") + " | grep 0")
 
-      # Test if a message logged to the journal
+      with subtest(
-      # is ingested by elasticsearch via journalbeat.
+          "A message logged to the journal is ingested by elasticsearch via journalbeat"
-      $one->waitForUnit("journalbeat.service");
+      ):
-      $one->execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat");
+          one.wait_for_unit("journalbeat.service")
-      $one->waitUntilSucceeds(
+          one.execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat")
-        "${totalHits "Supercalifragilisticexpialidocious"} | grep -v 0");
+          one.wait_until_succeeds(
-
+              total_hits("Supercalifragilisticexpialidocious") + " | grep -v 0"
-    '' + optionalString (!elasticsearchGe7) ''
+          )
-      # Test elasticsearch-curator.
+    '' + pkgs.lib.optionalString (!elasticsearchGe7) ''
-      $one->systemctl("stop logstash");
+      with subtest("Elasticsearch-curator works"):
-      $one->systemctl("start elasticsearch-curator");
+          one.systemctl("stop logstash")
-      $one->waitUntilSucceeds(
+          one.systemctl("start elasticsearch-curator")
-        "! curl --silent --show-error '${esUrl}/_cat/indices' " .
+          one.wait_until_succeeds(
-        "| grep logstash | grep -q ^$1");
+              '! curl --silent --show-error "${esUrl}/_cat/indices" | grep logstash | grep -q ^'
+          )
     '';
-  };
+  }) {};
-in mapAttrs mkElkTest {
+in pkgs.lib.mapAttrs mkElkTest {
   ELK-6 =
     if enableUnfree
     then {

nixos/tests/initdb.nix

@@ -1,26 +0,0 @@
-let
-  pkgs = import <nixpkgs> { };
-in
-with import <nixpkgs/nixos/lib/testing.nix> { inherit pkgs; system = builtins.currentSystem; };
-with pkgs.lib;
-
-makeTest {
-    name = "pg-initdb";
-
-    machine = {...}:
-      {
-        documentation.enable = false;
-        services.postgresql.enable = true;
-        services.postgresql.package = pkgs.postgresql_9_6;
-        environment.pathsToLink = [
-          "/share/postgresql"
-        ];
-      };
-
-    testScript = ''
-      $machine->start;
-      $machine->succeed("sudo -u postgres initdb -D /tmp/testpostgres2");
-      $machine->shutdown;
-    '';
-
-  }

nixos/tests/kafka.nix

@@ -3,11 +3,10 @@
   pkgs ? import ../.. { inherit system config; }
 }:
 
-with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
 
 let
-  makeKafkaTest = name: kafkaPackage: (makeTest {
+  makeKafkaTest = name: kafkaPackage: (import ./make-test-python.nix ({
     inherit name;
     meta = with pkgs.stdenv.lib.maintainers; {
       maintainers = [ nequissimus ];
@@ -45,24 +44,40 @@ let
     };
 
     testScript = ''
-      startAll;
+      start_all()
 
-      $zookeeper1->waitForUnit("default.target");
+      zookeeper1.wait_for_unit("default.target")
-      $zookeeper1->waitForUnit("zookeeper.service");
+      zookeeper1.wait_for_unit("zookeeper.service")
-      $zookeeper1->waitForOpenPort(2181);
+      zookeeper1.wait_for_open_port(2181)
 
-      $kafka->waitForUnit("default.target");
+      kafka.wait_for_unit("default.target")
-      $kafka->waitForUnit("apache-kafka.service");
+      kafka.wait_for_unit("apache-kafka.service")
-      $kafka->waitForOpenPort(9092);
+      kafka.wait_for_open_port(9092)
 
-      $kafka->waitUntilSucceeds("${kafkaPackage}/bin/kafka-topics.sh --create --zookeeper zookeeper1:2181 --partitions 1 --replication-factor 1 --topic testtopic");
+      kafka.wait_until_succeeds(
-      $kafka->mustSucceed("echo 'test 1' | ${kafkaPackage}/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic testtopic");
+          "${kafkaPackage}/bin/kafka-topics.sh --create "
+          + "--zookeeper zookeeper1:2181 --partitions 1 "
+          + "--replication-factor 1 --topic testtopic"
+      )
+      kafka.succeed(
+          "echo 'test 1' | "
+          + "${kafkaPackage}/bin/kafka-console-producer.sh "
+          + "--broker-list localhost:9092 --topic testtopic"
+      )
     '' + (if name == "kafka_0_9" then ''
-      $kafka->mustSucceed("${kafkaPackage}/bin/kafka-console-consumer.sh --zookeeper zookeeper1:2181 --topic testtopic --from-beginning --max-messages 1 | grep 'test 1'");
+      assert "test 1" in kafka.succeed(
+          "${kafkaPackage}/bin/kafka-console-consumer.sh "
+          + "--zookeeper zookeeper1:2181 --topic testtopic "
+          + "--from-beginning --max-messages 1"
+      )
     '' else ''
-      $kafka->mustSucceed("${kafkaPackage}/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic testtopic --from-beginning --max-messages 1 | grep 'test 1'");
+      assert "test 1" in kafka.succeed(
+          "${kafkaPackage}/bin/kafka-console-consumer.sh "
+          + "--bootstrap-server localhost:9092 --topic testtopic "
+          + "--from-beginning --max-messages 1"
+      )
     '');
-  });
+  }) {});
 
 in with pkgs; {
   kafka_0_9  = makeKafkaTest "kafka_0_9"  apacheKafka_0_9;

nixos/tests/postgresql.nix

@@ -29,11 +29,15 @@ let
 
     machine = {...}:
       {
-        services.postgresql.enable = true;
+        services.postgresql = {
-        services.postgresql.package = postgresql-package;
+          enable = true;
+          package = postgresql-package;
+        };
 
-        services.postgresqlBackup.enable = true;
+        services.postgresqlBackup = {
-        services.postgresqlBackup.databases = optional (!backup-all) "postgres";
+          enable = true;
+          databases = optional (!backup-all) "postgres";
+        };
       };
 
     testScript = let
@@ -49,23 +53,32 @@ let
       machine.start()
       machine.wait_for_unit("postgresql")
 
-      # postgresql should be available just after unit start
+      with subtest("Postgresql is available just after unit start"):
-      machine.succeed(
+          machine.succeed(
-          "cat ${test-sql} | sudo -u postgres psql"
+              "cat ${test-sql} | sudo -u postgres psql"
-      )
+          )
-      machine.shutdown()  # make sure that postgresql survive restart (bug #1735)
+
-      time.sleep(2)
+      with subtest("Postgresql survives restart (bug #1735)"):
-      machine.start()
+          machine.shutdown()
-      machine.wait_for_unit("postgresql")
+          time.sleep(2)
+          machine.start()
+          machine.wait_for_unit("postgresql")
+
       machine.fail(check_count("SELECT * FROM sth;", 3))
       machine.succeed(check_count("SELECT * FROM sth;", 5))
       machine.fail(check_count("SELECT * FROM sth;", 4))
       machine.succeed(check_count("SELECT xpath('/test/text()', doc) FROM xmltest;", 1))
 
-      # Check backup service
+      with subtest("Backup service works"):
-      machine.succeed("systemctl start ${backupService}.service")
+          machine.succeed(
-      machine.succeed("zcat /var/backup/postgresql/${backupName}.sql.gz | grep '<test>ok</test>'")
+              "systemctl start ${backupService}.service",
-      machine.succeed("stat -c '%a' /var/backup/postgresql/${backupName}.sql.gz | grep 600")
+              "zcat /var/backup/postgresql/${backupName}.sql.gz | grep '<test>ok</test>'",
+              "stat -c '%a' /var/backup/postgresql/${backupName}.sql.gz | grep 600",
+          )
+
+      with subtest("Initdb works"):
+          machine.succeed("sudo -u postgres initdb -D /tmp/testpostgres2")
+
       machine.shutdown()
     '';