Merge pull request #98446 from srhb/fix-kafka

apache-kafka: Fix, update and adjust JRE/JVM

nixos/doc/manual/release-notes/rl-2105.xml

@@ -562,6 +562,21 @@ self: super:
      <xref linkend="opt-services.xserver.videoDrivers" /> no longer uses the deprecated <literal>cirrus</literal> and <literal>vesa</literal> device dependent X drivers by default. It also enables both <literal>amdgpu</literal> and <literal>nouveau</literal> drivers by default now.
     </para>
    </listitem>
+   <listitem>
+    <para>
+      The <package>apacheKafka</package> packages are now built with
+      version-matched JREs. Versions 2.6 and above, the ones that recommend it,
+      use jdk11, while versions below remain on jdk8. The NixOS service has
+      been adjusted to start the service using the same version as the package,
+      adjustable with the new
+      <link linkend="opt-services.apache-kafka.jre">services.apache-kafka.jre</link>
+      option. Furthermore, the default list of
+      <link linkend="opt-services.apache-kafka.jvmOptions">services.apache-kafka.jvmOptions</link>
+      have been removed. You should set your own according to the
+      <link xlink:href="https://kafka.apache.org/documentation/#java">upstream documentation</link>
+      for your Kafka version.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 

nixos/modules/services/misc/apache-kafka.nix

@@ -90,19 +90,7 @@ in {
 
     jvmOptions = mkOption {
       description = "Extra command line options for the JVM running Kafka.";
-      default = [
+      default = [];
-        "-server"
-        "-Xmx1G"
-        "-Xms1G"
-        "-XX:+UseCompressedOops"
-        "-XX:+UseParNewGC"
-        "-XX:+UseConcMarkSweepGC"
-        "-XX:+CMSClassUnloadingEnabled"
-        "-XX:+CMSScavengeBeforeRemark"
-        "-XX:+DisableExplicitGC"
-        "-Djava.awt.headless=true"
-        "-Djava.net.preferIPv4Stack=true"
-      ];
       type = types.listOf types.str;
       example = [
         "-Djava.net.preferIPv4Stack=true"
@@ -118,6 +106,13 @@ in {
       type = types.package;
     };
 
+    jre = mkOption {
+      description = "The JRE with which to run Kafka";
+      default = cfg.package.passthru.jre;
+      defaultText = "pkgs.apacheKafka.passthru.jre";
+      type = types.package;
+    };
+
   };
 
   config = mkIf cfg.enable {
@@ -138,7 +133,7 @@ in {
       after = [ "network.target" ];
       serviceConfig = {
         ExecStart = ''
-          ${pkgs.jre}/bin/java \
+          ${cfg.jre}/bin/java \
             -cp "${cfg.package}/libs/*" \
             -Dlog4j.configuration=file:${logConfig} \
             ${toString cfg.jvmOptions} \

nixos/tests/kafka.nix

@@ -30,11 +30,6 @@ let
           '';
           package = kafkaPackage;
           zookeeper = "zookeeper1:2181";
-          # These are the default options, but UseCompressedOops doesn't work with 32bit JVM
-          jvmOptions = [
-            "-server" "-Xmx1G" "-Xms1G" "-XX:+UseParNewGC" "-XX:+UseConcMarkSweepGC" "-XX:+CMSClassUnloadingEnabled"
-            "-XX:+CMSScavengeBeforeRemark" "-XX:+DisableExplicitGC" "-Djava.awt.headless=true" "-Djava.net.preferIPv4Stack=true"
-          ] ++ optionals (! pkgs.stdenv.isi686 ) [ "-XX:+UseCompressedOops" ];
         };
 
         networking.firewall.allowedTCPPorts = [ 9092 ];
@@ -82,4 +77,5 @@ let
 in with pkgs; {
   kafka_2_4  = makeKafkaTest "kafka_2_4"  apacheKafka_2_4;
   kafka_2_5  = makeKafkaTest "kafka_2_5"  apacheKafka_2_5;
+  kafka_2_6  = makeKafkaTest "kafka_2_6"  apacheKafka_2_6;
 }

pkgs/servers/apache-kafka/default.nix

@@ -1,21 +1,29 @@
-{ lib, stdenv, fetchurl, jre8, makeWrapper, bash, coreutils, gnugrep, gnused, ps,
+{ lib, stdenv, fetchurl, jdk8_headless, jdk11_headless, makeWrapper, bash, coreutils, gnugrep, gnused, ps,
   majorVersion ? "1.0" }:
 
 let
+  jre8 = jdk8_headless;
+  jre11 = jdk11_headless;
   versionMap = {
     "2.4" = {
       kafkaVersion = "2.4.1";
       scalaVersion = "2.12";
       sha256 = "0ahsprmpjz026mhbr79187wfdrxcg352iipyfqfrx68q878wnxr1";
+      jre = jre8;
     };
     "2.5" = {
-      kafkaVersion = "2.5.0";
+      kafkaVersion = "2.5.1";
+      scalaVersion = "2.12";
+      sha256 = "1wn4iszrm2rvsfyyr515zx79k5m86davjkcwcwpxcgc4k3q0z7lv";
+      jre = jre8;
+    };
+    "2.6" = {
+      kafkaVersion = "2.6.1";
       scalaVersion = "2.13";
-      sha256 = "0w3g7ii8x63m2blv2a8c491d0diczpliaqm9f7w5yn98hikh0aqi";
+      sha256 = "1a2kd4r6f8z7qf886nnq9f350sblzzdi230j2hll7x156888573y";
+      jre = jre11;
     };
   };
-
-  jre = jre8; # TODO: remove override https://github.com/NixOS/nixpkgs/pull/89731
 in
 
 with versionMap.${majorVersion};
@@ -63,5 +71,5 @@ stdenv.mkDerivation rec {
     maintainers = [ maintainers.ragge ];
     platforms = platforms.unix;
   };
-
+  passthru = { inherit jre; };
 }

pkgs/top-level/all-packages.nix

@@ -11861,9 +11861,10 @@ in
   apacheAnt_1_9 = callPackage ../development/tools/build-managers/apache-ant/1.9.nix { };
   ant = apacheAnt;
 
-  apacheKafka = apacheKafka_2_5;
+  apacheKafka = apacheKafka_2_6;
   apacheKafka_2_4 = callPackage ../servers/apache-kafka { majorVersion = "2.4"; };
   apacheKafka_2_5 = callPackage ../servers/apache-kafka { majorVersion = "2.5"; };
+  apacheKafka_2_6 = callPackage ../servers/apache-kafka { majorVersion = "2.6"; };
 
   kt = callPackage ../tools/misc/kt {};