Merge pull request #104889 from ztzg/x-16104-zookeeper-update

zookeeper: 3.4.12 -> 3.6.2 & assorted changes

maintainers/maintainer-list.nix

@@ -9906,6 +9906,12 @@
     githubId = 8100652;
     name = "David Mell";
   };
+  ztzg = {
+    email = "dd@crosstwine.com";
+    github = "ztzg";
+    githubId = 393108;
+    name = "Damien Diederen";
+  };
   zx2c4 = {
     email = "Jason@zx2c4.com";
     github = "zx2c4";

nixos/doc/manual/release-notes/rl-2103.xml

@@ -370,6 +370,13 @@
      and rebooting.
     </para>
    </listitem>
+   <listitem>
+    <para>
+     The <package>zookeeper</package> package does not provide
+     <literal>zooInspector.sh</literal> anymore, as that "contrib" has
+     been dropped from upstream releases.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 </section>

nixos/modules/services/misc/zookeeper.nix

@@ -76,6 +76,7 @@ in {
       default = ''
         zookeeper.root.logger=INFO, CONSOLE
         log4j.rootLogger=INFO, CONSOLE
+        log4j.logger.org.apache.zookeeper.audit.Log4jAuditLogger=INFO, CONSOLE
         log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
         log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
         log4j.appender.CONSOLE.layout.ConversionPattern=[myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n
@@ -128,11 +129,10 @@ in {
       description = "Zookeeper Daemon";
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" ];
-      environment = { ZOOCFGDIR = configDir; };
       serviceConfig = {
         ExecStart = ''
           ${pkgs.jre}/bin/java \
-            -cp "${cfg.package}/lib/*:${cfg.package}/${cfg.package.name}.jar:${configDir}" \
+            -cp "${cfg.package}/lib/*:${configDir}" \
             ${escapeShellArgs cfg.extraCmdLineOptions} \
             -Dzookeeper.datadir.autocreate=false \
             ${optionalString cfg.preferIPv4 "-Djava.net.preferIPv4Stack=true"} \
@@ -143,6 +143,7 @@ in {
       };
       preStart = ''
         echo "${toString cfg.id}" > ${cfg.dataDir}/myid
+        mkdir -p ${cfg.dataDir}/version-2
       '';
     };
 

nixos/tests/zookeeper.nix

@@ -1,7 +1,12 @@
-import ./make-test-python.nix ({ pkgs, ...} : {
+import ./make-test-python.nix ({ pkgs, ...} :
+let
+
+  perlEnv = pkgs.perl.withPackages (p: [p.NetZooKeeper]);
+
+in {
   name = "zookeeper";
   meta = with pkgs.stdenv.lib.maintainers; {
-    maintainers = [ nequissimus ];
+    maintainers = [ nequissimus ztzg ];
   };
 
   nodes = {
@@ -30,5 +35,12 @@ import ./make-test-python.nix ({ pkgs, ...} : {
     server.wait_until_succeeds(
         "${pkgs.zookeeper}/bin/zkCli.sh -server localhost:2181 get /foo | grep hello"
     )
+
+    server.wait_until_succeeds(
+        "${perlEnv}/bin/perl -E 'use Net::ZooKeeper qw(:acls); $z=Net::ZooKeeper->new(q(localhost:2181)); $z->create(qw(/perl foo acl), ZOO_OPEN_ACL_UNSAFE) || die $z->get_error()'"
+    )
+    server.wait_until_succeeds(
+        "${perlEnv}/bin/perl -E 'use Net::ZooKeeper qw(:acls); $z=Net::ZooKeeper->new(q(localhost:2181)); $z->get(qw(/perl)) eq qw(foo) || die $z->get_error()'"
+    )
   '';
 })

pkgs/development/libraries/zookeeper_mt/default.nix

@@ -1,21 +1,56 @@
-{ stdenv, zookeeper, bash }:
+{ stdenv
+, fetchurl
+, autoreconfHook
+, jre
+, openssl
+, pkg-config
+# We depend on ZooKeeper for the Jute compiler.
+, zookeeper
+}:
 
 stdenv.mkDerivation rec {
-  name = "zookeeper_mt-${stdenv.lib.getVersion zookeeper}";
+  pname = "zookeeper_mt";
+  version = stdenv.lib.getVersion zookeeper;
 
-  src = zookeeper.src;
+  src = fetchurl {
+    url = "mirror://apache/zookeeper/${zookeeper.pname}-${version}/apache-${zookeeper.pname}-${version}.tar.gz";
+    sha512 = "16994067d460a1b6af6a71f3458c64ee32629e876a1ff6646d57be62f1a5adab57462af84074ecaded4186dd3fde035ee24cd9d578b8e5044073eb05f4ab9c3e";
+  };
 
-  setSourceRoot = "export sourceRoot=${zookeeper.name}/src/c";
+  sourceRoot = "apache-${zookeeper.pname}-${version}/zookeeper-client/zookeeper-client-c";
 
-  NIX_CFLAGS_COMPILE = stdenv.lib.optionalString (!stdenv.isDarwin) "-Wno-error=format-overflow -Wno-error=stringop-truncation";
+  nativeBuildInputs = [
+    autoreconfHook
+    jre
+  ];
 
-  buildInputs = [ zookeeper bash ];
+  buildInputs = [
+    openssl
+    pkg-config
+    zookeeper
+  ];
+
+  # Generate the C marshallers/unmarshallers for the Jute-encoded
+  # definitions.
+  preConfigure = ''
+    mkdir generated
+    cd generated
+    java -cp ${zookeeper}/lib/${zookeeper.pname}-jute-${version}.jar \
+        org.apache.jute.compiler.generated.Rcc -l c \
+        ../../../zookeeper-jute/src/main/resources/zookeeper.jute
+    cd ..
+  '';
+
+  configureFlags = [
+    # We're not going to start test servers in the sandbox anyway.
+    "--without-cppunit"
+  ];
 
   meta = with stdenv.lib; {
-    homepage = "http://zookeeper.apache.org";
+    homepage = "https://zookeeper.apache.org";
     description = "Apache Zookeeper";
     license = licenses.asl20;
-    maintainers = [ maintainers.commandodev ];
+    maintainers = with maintainers; [ commandodev ztzg ];
     platforms = platforms.unix;
   };
 }

pkgs/servers/zookeeper/default.nix

@@ -2,11 +2,11 @@
 
 stdenv.mkDerivation rec {
   pname = "zookeeper";
-  version = "3.4.13";
+  version = "3.6.2";
 
   src = fetchurl {
-    url = "mirror://apache/zookeeper/${pname}-${version}/${pname}-${version}.tar.gz";
+    url = "mirror://apache/zookeeper/${pname}-${version}/apache-${pname}-${version}-bin.tar.gz";
-    sha256 = "0karf13zks3ba2rdmma2lyabvmasc04cjmgxp227f0nj8677kvbw";
+    sha512 = "caff5111bb6876b7124760bc006e6fa2523efa54b99321a3c9cd8192ea0d5596abc7d70a054b1aac9b20a411407dae7611c7aba870c23bff28eb1643ba499199";
   };
 
   buildInputs = [ makeWrapper jre ];
@@ -15,40 +15,28 @@ stdenv.mkDerivation rec {
 
   installPhase = ''
     mkdir -p $out
-    cp -R conf docs lib ${pname}-${version}.jar $out
+    cp -R conf docs lib $out
+    # Without this, zkCli.sh tries creating a log file in the Nix store.
+    substituteInPlace $out/conf/log4j.properties \
+        --replace 'INFO, RFAAUDIT' 'INFO, CONSOLE'
     mkdir -p $out/bin
-    cp -R bin/{zkCli,zkCleanup,zkEnv,zkServer}.sh $out/bin
+    cp -R bin/{zkCli,zkCleanup,zkEnv,zkServer,zkSnapShotToolkit,zkTxnLogToolkit}.sh $out/bin
     patchShebangs $out/bin
     substituteInPlace $out/bin/zkServer.sh \
         --replace /bin/echo ${coreutils}/bin/echo
-    for i in $out/bin/{zkCli,zkCleanup,zkServer}.sh; do
+    for i in $out/bin/{zkCli,zkCleanup,zkServer,zkSnapShotToolkit,zkTxnLogToolkit}.sh; do
       wrapProgram $i \
         --set JAVA_HOME "${jre}" \
         --prefix PATH : "${bash}/bin"
     done
     chmod -x $out/bin/zkEnv.sh
-
-    mkdir -p $out/share/zooinspector
-    cp -r contrib/ZooInspector/{${pname}-${version}-ZooInspector.jar,icons,lib,config} $out/share/zooinspector
-
-    classpath="$out/${pname}-${version}.jar:$out/share/zooinspector/${pname}-${version}-ZooInspector.jar"
-    for jar in $out/lib/*.jar $out/share/zooinspector/lib/*.jar; do
-      classpath="$classpath:$jar"
-    done
-
-    cat << EOF > $out/bin/zooInspector.sh
-    #!${runtimeShell}
-    cd $out/share/zooinspector
-    exec ${jre}/bin/java -cp $classpath org.apache.zookeeper.inspector.ZooInspector
-    EOF
-    chmod +x $out/bin/zooInspector.sh
   '';
 
   meta = with stdenv.lib; {
-    homepage = "http://zookeeper.apache.org";
+    homepage = "https://zookeeper.apache.org";
     description = "Apache Zookeeper";
     license = licenses.asl20;
-    maintainers = with maintainers; [ nathan-gs cstrahan pradeepchhetri ];
+    maintainers = with maintainers; [ nathan-gs cstrahan pradeepchhetri ztzg ];
     platforms = platforms.unix;
   };
 }

pkgs/tools/filesystems/zkfuse/default.nix

@@ -1,14 +1,30 @@
-{ stdenv, lib, zookeeper, zookeeper_mt, fuse, autoreconfHook, log4cxx, boost }:
+{ stdenv
+, lib
+, autoreconfHook
+, gnused
+, boost
+, fuse
+, log4cxx
+, zookeeper
+, zookeeper_mt
+}:
 
-stdenv.mkDerivation {
+stdenv.mkDerivation rec {
   pname = "zkfuse";
-  inherit (zookeeper) version src;
+  inherit (zookeeper_mt) version src;
 
-  sourceRoot = "${zookeeper.name}/src/contrib/zkfuse";
+  sourceRoot = "apache-${zookeeper.pname}-${version}/zookeeper-contrib/zookeeper-contrib-zkfuse";
 
-  nativeBuildInputs = [ autoreconfHook ];
+  nativeBuildInputs = [ autoreconfHook gnused ];
   buildInputs = [ zookeeper_mt log4cxx boost fuse ];
 
+  postPatch = ''
+    # Make the async API accessible, and use the proper include path.
+    sed -i src/zkadapter.h \
+        -e '/"zookeeper\.h"/i#define THREADED' \
+        -e 's,"zookeeper\.h",<zookeeper/zookeeper.h>,'
+  '';
+
   installPhase = ''
     mkdir -p $out/bin
     cp -v src/zkfuse $out/bin
@@ -16,7 +32,7 @@ stdenv.mkDerivation {
 
   meta = with lib; {
     platforms = platforms.linux;
-    maintainers = with maintainers; [ cstrahan ];
+    maintainers = with maintainers; [ cstrahan ztzg ];
     license = licenses.asl20;
   };
 }

pkgs/top-level/perl-packages.nix

@@ -15752,18 +15752,31 @@ let
 
   NetZooKeeper = buildPerlPackage {
     pname = "Net-ZooKeeper";
-    version = "0.41";
+    version = "0.42pre";
-    src = fetchurl {
+    src = fetchFromGitHub {
-      url = "mirror://cpan/authors/id/M/MA/MAF/Net-ZooKeeper-0.41.tar.gz";
+      owner = "mark-5";
-      sha256 = "91c177f30f82302eaf3173356eef05c21bc82163df752acb469177bd14a72db9";
+      repo = "p5-net-zookeeper";
+      rev = "66e1a360aff9c39af728c36092b540a4b6045f70";
+      sha256 = "0xl8lcv9gfv0nn8vrrxa4az359whqdhmzw4r51nn3add8pn3s9ip";
     };
     buildInputs = [ pkgs.zookeeper_mt ];
+    nativeBuildInputs = [ pkgs.gnused ];
     # fix "error: format not a string literal and no format arguments [-Werror=format-security]"
     hardeningDisable = [ "format" ];
-    NIX_CFLAGS_COMPILE = "-I${pkgs.zookeeper_mt}/include";
+    # Make the async API accessible
+    NIX_CFLAGS_COMPILE = "-DTHREADED";
     NIX_CFLAGS_LINK = "-L${pkgs.zookeeper_mt.out}/lib -lzookeeper_mt";
+    # Most tests are skipped as no server is available in the sandbox.
+    # `t/35_log.t` seems to suffer from a race condition; remove it.  See
+    # https://github.com/NixOS/nixpkgs/pull/104889#issuecomment-737144513
+    preCheck = ''
+      rm t/35_log.t
+    '' + stdenv.lib.optionalString stdenv.isDarwin ''
+      rm t/30_connect.t
+      rm t/45_class.t
+    '';
     meta = {
-      maintainers = [ maintainers.limeytexan ];
+      maintainers = with maintainers; [ limeytexan ztzg ];
       homepage = "https://github.com/mark-5/p5-net-zookeeper";
       license = stdenv.lib.licenses.asl20;
     };