Merge pull request #75798 from tobim/pkgsStatic/arrow

arrow-cpp: add pkgsStatic support

pkgs/development/libraries/arrow-cpp/default.nix

@@ -1,6 +1,7 @@
-{ stdenv, fetchurl, fetchFromGitHub, fixDarwinDylibNames, autoconf, boost
+{ stdenv, lib, fetchurl, fetchFromGitHub, fixDarwinDylibNames, autoconf, boost
 , brotli, cmake, double-conversion, flatbuffers, gflags, glog, gtest, lz4, perl
-, python, rapidjson, snappy, thrift, uriparser, which, zlib, zstd }:
+, python, rapidjson, snappy, thrift, uriparser, which, zlib, zstd
+, enableShared ? true }:
 
 let
   parquet-testing = fetchFromGitHub {
@@ -34,13 +35,16 @@ in stdenv.mkDerivation rec {
   patches = [
     # patch to fix python-test
     ./darwin.patch
+  ] ++ lib.optionals (!enableShared) [
+    # The shared jemalloc lib is unused and breaks in static mode due to missing -fpic.
+    ./jemalloc-disable-shared.patch
   ];
 
   nativeBuildInputs = [
     cmake
     autoconf # for vendored jemalloc
     flatbuffers
-  ] ++ stdenv.lib.optional stdenv.isDarwin fixDarwinDylibNames;
+  ] ++ lib.optional stdenv.isDarwin fixDarwinDylibNames;
   buildInputs = [
     boost
     brotli
@@ -71,21 +75,26 @@ in stdenv.mkDerivation rec {
     "-DARROW_DEPENDENCY_SOURCE=SYSTEM"
     "-DARROW_PARQUET=ON"
     "-DARROW_PLASMA=ON"
-    "-DARROW_PYTHON=ON"
+    # Disable Python for static mode because openblas is currently broken there.
+    "-DARROW_PYTHON=${if enableShared then "ON" else "OFF"}"
     "-Duriparser_SOURCE=SYSTEM"
-  ] ++ stdenv.lib.optional (!stdenv.isx86_64) "-DARROW_USE_SIMD=OFF";
+  ] ++ lib.optionals (!enableShared) [
+    "-DARROW_BUILD_SHARED=OFF"
+    "-DARROW_TEST_LINKAGE=static"
+    "-DOPENSSL_USE_STATIC_LIBS=ON"
+  ] ++ lib.optional (!stdenv.isx86_64) "-DARROW_USE_SIMD=OFF";
 
   doInstallCheck = true;
   PARQUET_TEST_DATA =
     if doInstallCheck then "${parquet-testing}/data" else null;
   installCheckInputs = [ perl which ];
-  installCheckPhase = (stdenv.lib.optionalString stdenv.isDarwin ''
+  installCheckPhase = (lib.optionalString stdenv.isDarwin ''
     for f in release/*test{,s}; do
       install_name_tool -add_rpath "$out"/lib  "$f"
     done
   '')
   + (let
-    excludedTests = stdenv.lib.optionals stdenv.isDarwin [
+    excludedTests = lib.optionals stdenv.isDarwin [
       # Some plasma tests need to be patched to use a shorter AF_UNIX socket
       # path on Darwin. See https://github.com/NixOS/nix/pull/1085
       "plasma-external-store-tests"
@@ -99,8 +108,8 @@ in stdenv.mkDerivation rec {
   meta = {
     description = "A  cross-language development platform for in-memory data";
     homepage = "https://arrow.apache.org/";
-    license = stdenv.lib.licenses.asl20;
+    license = lib.licenses.asl20;
-    platforms = stdenv.lib.platforms.unix;
+    platforms = lib.platforms.unix;
-    maintainers = with stdenv.lib.maintainers; [ tobim veprbl ];
+    maintainers = with lib.maintainers; [ tobim veprbl ];
   };
 }

pkgs/development/libraries/arrow-cpp/jemalloc-disable-shared.patch

@@ -0,0 +1,11 @@
+diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
+--- a/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cmake_modules/ThirdpartyToolchain.cmake
+@@ -1428,6 +1428,7 @@ if(ARROW_JEMALLOC)
+                       "--with-jemalloc-prefix=je_arrow_"
+                       "--with-private-namespace=je_arrow_private_"
+                       "--without-export"
++                      "--disable-shared"
+                       # Don't override operator new()
+                       "--disable-cxx" "--disable-libdl"
+                       # See https://github.com/jemalloc/jemalloc/issues/1237

pkgs/development/libraries/double-conversion/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, lib, fetchFromGitHub, cmake }:
+{ stdenv, lib, fetchFromGitHub, cmake, static ? false }:
 
 stdenv.mkDerivation rec {
   pname = "double-conversion";
@@ -13,7 +13,7 @@ stdenv.mkDerivation rec {
 
   nativeBuildInputs = [ cmake ];
 
-  cmakeFlags = [ "-DBUILD_SHARED_LIBS=ON" ];
+  cmakeFlags = [ "-DBUILD_SHARED_LIBS=${if static then "OFF" else "ON"}" ];
 
   # Case sensitivity issue
   preConfigure = lib.optionalString stdenv.isDarwin ''
@@ -24,7 +24,7 @@ stdenv.mkDerivation rec {
 
   meta = with stdenv.lib; {
     description = "Binary-decimal and decimal-binary routines for IEEE doubles";
-    homepage = https://github.com/google/double-conversion;
+    homepage = "https://github.com/google/double-conversion";
     license = licenses.bsd3;
     platforms = platforms.unix;
     maintainers = with maintainers; [ abbradar ];

pkgs/development/libraries/gflags/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, fetchFromGitHub, cmake }:
+{ stdenv, fetchFromGitHub, cmake, enableShared ? true}:
 
 stdenv.mkDerivation rec {
   pname = "gflags";
@@ -17,8 +17,8 @@ stdenv.mkDerivation rec {
   preConfigure = "rm BUILD";
 
   cmakeFlags = [
-    "-DBUILD_SHARED_LIBS=ON"
+    "-DGFLAGS_BUILD_SHARED_LIBS=${if enableShared then "ON" else "OFF"}"
-    "-DBUILD_STATIC_LIBS=ON"
+    "-DGFLAGS_BUILD_STATIC_LIBS=ON"
   ];
 
   doCheck = false;

pkgs/development/libraries/glog/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, lib, fetchFromGitHub, fetchpatch, autoreconfHook, perl }:
+{ stdenv, lib, fetchFromGitHub, fetchpatch, cmake, perl, static ? false }:
 
 stdenv.mkDerivation rec {
   pname = "glog";
@@ -20,13 +20,15 @@ stdenv.mkDerivation rec {
     })
   ];
 
-  nativeBuildInputs = [ autoreconfHook ];
+  nativeBuildInputs = [ cmake ];
+
+  cmakeFlags = [ "-DBUILD_SHARED_LIBS=${if static then "OFF" else "ON"}" ];
 
   checkInputs = [ perl ];
   doCheck = false; # fails with "Mangled symbols (28 out of 380) found in demangle.dm"
 
   meta = with stdenv.lib; {
-    homepage = https://github.com/google/glog;
+    homepage = "https://github.com/google/glog";
     license = licenses.bsd3;
     description = "Library for application-level logging";
     platforms = platforms.unix;

pkgs/development/libraries/science/math/openblas/default.nix

@@ -115,6 +115,9 @@ stdenv.mkDerivation rec {
   nativeBuildInputs = [
     perl
     which
+  ];
+
+  depsBuildBuild = [
     buildPackages.gfortran
     buildPackages.stdenv.cc
   ];

pkgs/development/libraries/snappy/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, fetchFromGitHub, cmake }:
+{ stdenv, fetchFromGitHub, cmake, static ? false }:
 
 stdenv.mkDerivation rec {
   pname = "snappy";
@@ -17,7 +17,10 @@ stdenv.mkDerivation rec {
 
   nativeBuildInputs = [ cmake ];
 
-  cmakeFlags = [ "-DBUILD_SHARED_LIBS=ON" "-DCMAKE_SKIP_BUILD_RPATH=OFF" ];
+  cmakeFlags = [
+    "-DBUILD_SHARED_LIBS=${if static then "OFF" else "ON"}"
+    "-DCMAKE_SKIP_BUILD_RPATH=OFF"
+  ];
 
   postInstall = ''
     substituteInPlace "$out"/lib/cmake/Snappy/SnappyTargets.cmake \
@@ -29,7 +32,7 @@ stdenv.mkDerivation rec {
   doCheck = true;
 
   meta = with stdenv.lib; {
-    homepage = https://google.github.io/snappy/;
+    homepage = "https://google.github.io/snappy/";
     license = licenses.bsd3;
     description = "Compression/decompression library for very high speeds";
     platforms = platforms.all;

pkgs/development/libraries/thrift/default.nix

@@ -1,6 +1,5 @@
-{ stdenv, fetchurl, boost, zlib, libevent, openssl, python, pkgconfig, bison
+{ stdenv, fetchurl, boost, zlib, libevent, openssl, python, cmake, pkgconfig
-, flex, twisted
+, bison, flex, twisted, static ? false }:
-}:
 
 stdenv.mkDerivation rec {
   pname = "thrift";
@@ -11,27 +10,36 @@ stdenv.mkDerivation rec {
     sha256 = "0yai9c3bdsrkkjshgim7zk0i7malwfprg00l9774dbrkh2w4ilvs";
   };
 
-  #enableParallelBuilding = true; problems on hydra
-
   # Workaround to make the python wrapper not drop this package:
   # pythonFull.buildEnv.override { extraLibs = [ thrift ]; }
   pythonPath = [];
 
-  nativeBuildInputs = [ pkgconfig ];
+  nativeBuildInputs = [ cmake pkgconfig ];
-  buildInputs = [
+  buildInputs = [ boost zlib libevent openssl python bison flex ]
-    boost zlib libevent openssl python bison flex twisted
+    ++ stdenv.lib.optional (!static) twisted;
-  ];
 
   preConfigure = "export PY_PREFIX=$out";
 
-  # TODO: package boost-test, so we can run the test suite. (Currently it fails
+  cmakeFlags = [
-  # to find libboost_unit_test_framework.a.)
+    # FIXME: Fails to link in static mode with undefined reference to
-  configureFlags = [ "--enable-tests=no" ];
+    # `boost::unit_test::unit_test_main(bool (*)(), int, char**)'
-  doCheck = false;
+    "-DBUILD_TESTING:BOOL=${if static then "OFF" else "ON"}"
+  ] ++ stdenv.lib.optionals static [
+    "-DWITH_STATIC_LIB:BOOL=ON"
+    "-DOPENSSL_USE_STATIC_LIBS=ON"
+  ];
+
+  doCheck = !static;
+  checkPhase = ''
+    runHook preCheck
+    LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/lib ctest -E PythonTestSSLSocket
+    runHook postCheck
+  '';
+  enableParallelChecking = false;
 
   meta = with stdenv.lib; {
     description = "Library for scalable cross-language services";
-    homepage = http://thrift.apache.org/;
+    homepage = "http://thrift.apache.org/";
     license = licenses.asl20;
     platforms = platforms.linux ++ platforms.darwin;
     maintainers = [ maintainers.bjornfor ];

pkgs/development/web/woff2/brotli-static.patch

@@ -0,0 +1,36 @@
+diff a/cmake/FindBrotliDec.cmake b/cmake/FindBrotliDec.cmake
+--- a/cmake/FindBrotliDec.cmake
++++ b/cmake/FindBrotliDec.cmake
+@@ -18,10 +18,10 @@ find_path(BROTLIDEC_INCLUDE_DIRS
+     HINTS ${PC_BROTLIDEC_INCLUDEDIR}
+ )
+ 
+-find_library(BROTLIDEC_LIBRARIES
+-    NAMES brotlidec
+-    HINTS ${PC_BROTLIDEC_LIBDIR}
+-)
++if(NOT BUILD_SHARED_LIBS)
++  set(_S "STATIC_")
++endif()
++set(BROTLIDEC_LIBRARIES ${PC_BROTLIDEC_${_S}LIBRARIES})
+ 
+ include(FindPackageHandleStandardArgs)
+ find_package_handle_standard_args(BrotliDec
+diff a/cmake/FindBrotliEnc.cmake b/cmake/FindBrotliEnc.cmake
+--- a/cmake/FindBrotliEnc.cmake
++++ b/cmake/FindBrotliEnc.cmake
+@@ -18,10 +18,10 @@ find_path(BROTLIENC_INCLUDE_DIRS
+     HINTS ${PC_BROTLIENC_INCLUDEDIR}
+ )
+ 
+-find_library(BROTLIENC_LIBRARIES
+-    NAMES brotlienc
+-    HINTS ${PC_BROTLIENC_LIBDIR}
+-)
++if(NOT BUILD_SHARED_LIBS)
++  set(_S "STATIC_")
++endif()
++set(BROTLIENC_LIBRARIES ${PC_BROTLIENC_${_S}LIBRARIES})
+ 
+ include(FindPackageHandleStandardArgs)
+ find_package_handle_standard_args(BrotliEnc

pkgs/development/web/woff2/default.nix

@@ -1,4 +1,4 @@
-{ brotli, cmake, fetchFromGitHub, stdenv }:
+{ brotli, cmake, pkgconfig, fetchFromGitHub, stdenv, static ? false }:
 
 stdenv.mkDerivation rec {
   pname = "woff2";
@@ -13,18 +13,24 @@ stdenv.mkDerivation rec {
 
   outputs = [ "out" "dev" "lib" ];
 
-  nativeBuildInputs = [ cmake ];
+  # Need to explicitly link to brotlicommon
+  patches = stdenv.lib.optional static ./brotli-static.patch;
+
+  nativeBuildInputs = [ cmake pkgconfig ];
+
+  cmakeFlags = [ "-DBUILD_SHARED_LIBS=${if static then "OFF" else "ON"}" ]
+    ++ stdenv.lib.optional static "-DCMAKE_SKIP_RPATH:BOOL=TRUE";
 
   propagatedBuildInputs = [ brotli ];
 
+  postPatch = ''
     # without this binaries only get built if shared libs are disable
-  patchPhase = ''
     sed 's@^if (NOT BUILD_SHARED_LIBS)$@if (TRUE)@g' -i CMakeLists.txt
   '';
 
   meta = with stdenv.lib; {
     description = "Webfont compression reference code";
-    homepage = https://github.com/google/woff2;
+    homepage = "https://github.com/google/woff2";
     license = licenses.mit;
     maintainers = [ maintainers.hrdinka ];
     platforms = platforms.unix;

pkgs/tools/compression/zstd/default.nix

@@ -1,7 +1,8 @@
-{ stdenv, fetchFromGitHub, fetchpatch, gnugrep
+{ stdenv, fetchFromGitHub, fetchpatch, cmake, gnugrep
 , fixDarwinDylibNames
 , file
-, legacySupport ? false }:
+, legacySupport ? false
+, enableShared ? true }:
 
 stdenv.mkDerivation rec {
   pname = "zstd";
@@ -14,56 +15,46 @@ stdenv.mkDerivation rec {
     owner = "facebook";
   };
 
+  nativeBuildInputs = [ cmake ]
+   ++ stdenv.lib.optional stdenv.isDarwin fixDarwinDylibNames;
+
   patches = [
-    # All 3 from https://github.com/facebook/zstd/pull/1883
+    # From https://github.com/facebook/zstd/pull/1883
     (fetchpatch {
       url = "https://github.com/facebook/zstd/commit/106278e7e5fafaea3b7deb4147bdc8071562d2f0.diff";
       sha256 = "13z7id1qbc05cv1rmak7c8xrchp7jh1i623bq5pwcihg57wzcyr8";
     })
-    (fetchpatch {
-      url = "https://github.com/facebook/zstd/commit/0ede342acc2c26f87ae962fa88e158904d4198c4.diff";
-      sha256 = "12l5xbvnzkvr76mvl1ls767paqfwbd9q1pzq44ckacfpz4f6iaap";
-      excludes = [
-        # I think line endings are causing problems, or something like that
-        "programs/windres/generate_res.bat"
-      ];
-    })
-    (fetchpatch {
-      url = "https://github.com/facebook/zstd/commit/10552eaffef84c011f67af0e04f0780b50a5ab26.diff";
-      sha256 = "1s27ravar3rn7q8abybp9733jhpsfcaci51k04da94ahahvxwiqw";
-    })
   ] # This I didn't upstream because if you use posix threads with MinGW it will
     # work find, and I'm not sure how to write the condition.
     ++ stdenv.lib.optional stdenv.hostPlatform.isWindows ./mcfgthreads-no-pthread.patch;
 
-  nativeBuildInputs = stdenv.lib.optional stdenv.isDarwin fixDarwinDylibNames;
+  cmakeFlags = [
-
+    "-DZSTD_BUILD_SHARED:BOOL=${if enableShared then "ON" else "OFF"}"
-  makeFlags = [
+    "-DZSTD_LEGACY_SUPPORT:BOOL=${if legacySupport then "ON" else "OFF"}"
-    "ZSTD_LEGACY_SUPPORT=${if legacySupport then "1" else "0"}"
+    "-DZSTD_BUILD_TESTS:BOOL=ON"
-  ] ++ stdenv.lib.optional stdenv.hostPlatform.isWindows "OS=Windows";
+  ];
+  cmakeDir = "../build/cmake";
+  dontUseCmakeBuildDir = true;
+  preConfigure = ''
+    mkdir -p build_ && cd $_
+  '';
 
   checkInputs = [ file ];
   doCheck = true;
   preCheck = ''
-    substituteInPlace tests/playTests.sh \
+    substituteInPlace ../tests/playTests.sh \
       --replace 'MD5SUM="md5 -r"' 'MD5SUM="md5sum"'
   '';
 
-  installFlags = [
+  preInstall = stdenv.lib.optionalString enableShared ''
-    "PREFIX=$(out)"
+    substituteInPlace ../programs/zstdgrep \
-  ];
-
-  preInstall = ''
-    substituteInPlace programs/zstdgrep \
       --replace ":-grep" ":-${gnugrep}/bin/grep" \
       --replace ":-zstdcat" ":-$out/bin/zstdcat"
 
-    substituteInPlace programs/zstdless \
+    substituteInPlace ../programs/zstdless \
       --replace "zstdcat" "$out/bin/zstdcat"
   '';
 
-  enableParallelBuilding = true;
-
   meta = with stdenv.lib; {
     description = "Zstandard real-time compression algorithm";
     longDescription = ''

pkgs/top-level/static.nix

@@ -162,6 +162,10 @@ in {
     # --disable-shared flag
     stdenv = super.stdenv;
   };
+  arrow-cpp = super.arrow-cpp.override {
+    enableShared = false;
+    python = { pkgs = { python = null; numpy = null; }; };
+  };
   boost = super.boost.override {
     enableStatic = true;
     enableShared = false;
@@ -170,9 +174,25 @@ in {
     # --disable-shared flag
     stdenv = super.stdenv;
   };
+  thrift = super.thrift.override {
+    static = true;
+    twisted = null;
+  };
+  double-conversion = super.double-conversion.override {
+    static = true;
+  };
   gmp = super.gmp.override {
     withStatic = true;
   };
+  gflags = super.gflags.override {
+    enableShared = false;
+  };
+  glog = super.glog.override {
+    static = true;
+  };
+  gtest = super.gtest.override {
+    static = true;
+  };
   cdo = super.cdo.override {
     enable_all_static = true;
   };
@@ -191,6 +211,12 @@ in {
     # it doesn’t like the --disable-shared flag
     stdenv = super.stdenv;
   };
+  woff2 = super.woff2.override {
+    static = true;
+  };
+  snappy = super.snappy.override {
+    static = true;
+  };
   lz4 = super.lz4.override {
     enableShared = false;
     enableStatic = true;
@@ -223,6 +249,10 @@ in {
     staticOnly = true;
   };
 
+  zstd = super.zstd.override {
+    enableShared = false;
+  };
+
   llvmPackages_8 = super.llvmPackages_8 // {
     libraries = super.llvmPackages_8.libraries // rec {
       libcxxabi = super.llvmPackages_8.libraries.libcxxabi.override {