Merge pull request #95824 from obsidiansystems/python3.tensorflow_2-update-to-2.3.0

tensorflow: update to 2.3.0

pkgs/development/python-modules/h5py/default.nix

@@ -1,6 +1,6 @@
 { stdenv, fetchPypi, isPy27, python, buildPythonPackage
 , numpy, hdf5, cython, six, pkgconfig, unittest2, fetchpatch
-, mpi4py ? null, openssh }:
+, mpi4py ? null, openssh, pytest }:
 
 assert hdf5.mpiSupport -> mpi4py != null && hdf5.mpi == mpi4py.mpi;
 
@@ -10,20 +10,15 @@ let
   mpi = hdf5.mpi;
   mpiSupport = hdf5.mpiSupport;
 in buildPythonPackage rec {
-  version = "2.9.0";
+  version = "2.10.0";
   pname = "h5py";
+  disabled = isPy27;
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "9d41ca62daf36d6b6515ab8765e4c8c4388ee18e2a665701fef2b41563821002";
+    sha256 = "84412798925dc870ffd7107f045d7659e60f5d46d1c70c700375248bf6bf512d";
   };
 
-  patches = [ ( fetchpatch {
-    # Skip a test that probes an already fixed bug in HDF5 (upstream patch)
-    url = "https://github.com/h5py/h5py/commit/141eafa531c6c09a06efe6a694251a1eea84908d.patch";
-    sha256 = "0lmdn0gznr7gadx7qkxybl945fvwk6r0cc4lg3ylpf8ril1975h8";
-  })];
-
   configure_flags = "--hdf5=${hdf5}" + optionalString mpiSupport " --mpi";
 
   postConfigure = ''
@@ -36,7 +31,7 @@ in buildPythonPackage rec {
 
   preBuild = if mpiSupport then "export CC=${mpi}/bin/mpicc" else "";
 
-  checkInputs = optional isPy27 unittest2 ++ [ openssh ];
+  checkInputs = optional isPy27 unittest2 ++ [ pytest openssh ];
   nativeBuildInputs = [ pkgconfig ];
   buildInputs = [ hdf5 cython ]
     ++ optional mpiSupport mpi;

pkgs/development/python-modules/tensorflow-estimator/2/default.nix

@@ -6,13 +6,13 @@
 
 buildPythonPackage rec {
   pname = "tensorflow-estimator";
-  version = "2.2.0";
+  version = "2.3.0";
   format = "wheel";
 
   src = fetchPypi {
     pname = "tensorflow_estimator";
     inherit version format;
-    sha256 = "1hkx4k6927xn4qpwiba6wa56n0qqm7s23bymm377j9bz2bfsr7fh";
+    sha256 = "11n4sl9wfr00fv1i837b7a36ink86ggmlsgj7i06kcfc011h6pmp";
   };
 
   propagatedBuildInputs = [ mock numpy absl-py ];

pkgs/development/python-modules/tensorflow/2/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, pkgs, bazel_0_29, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin
+{ stdenv, pkgs, bazel_3, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin
 , addOpenGLRunpath
 # Python deps
 , buildPythonPackage, isPy3k, isPy27, pythonOlder, pythonAtLeast, python
@@ -6,7 +6,7 @@
 , numpy, tensorflow-tensorboard_2, backports_weakref, mock, enum34, absl-py
 , future, setuptools, wheel, keras-preprocessing, keras-applications, google-pasta
 , functools32
-, opt-einsum
+, opt-einsum, astunparse, h5py
 , termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator_2
 # Common deps
 , git, swig, which, binutils, glibcLocales, cython
@@ -17,7 +17,7 @@
 # that in nix as well. It would make some things easier and less confusing, but
 # it would also make the default tensorflow package unfree. See
 # https://groups.google.com/a/tensorflow.org/forum/#!topic/developers/iRCt5m4qUz0
-, cudaSupport ? false, nvidia_x11 ? null, cudatoolkit ? null, cudnn ? null, nccl ? null
+, cudaSupport ? false, cudatoolkit ? null, cudnn ? null, nccl ? null
 , mklSupport ? false, mkl ? null
 # XLA without CUDA is broken
 , xlaSupport ? cudaSupport
@@ -30,8 +30,7 @@
 , Foundation, Security
 }:
 
-assert cudaSupport -> nvidia_x11 != null
+assert cudaSupport -> cudatoolkit != null
-                   && cudatoolkit != null
                    && cudnn != null;
 
 # unsupported combination
@@ -47,6 +46,7 @@ let
     paths = [
       cudatoolkit.lib
       cudatoolkit.out
+    ] ++ lib.optionals (lib.versionOlder cudatoolkit.version "11") [
       # for some reason some of the required libs are in the targets/x86_64-linux
       # directory; not sure why but this works around it
       "${cudatoolkit}/targets/${stdenv.system}"
@@ -72,7 +72,7 @@ let
 
   tfFeature = x: if x then "1" else "0";
 
-  version = "2.1.0";
+  version = "2.3.0";
   variant = if cudaSupport then "-gpu" else "";
   pname = "tensorflow${variant}";
 
@@ -97,40 +97,32 @@ let
 
   bazel-build = buildBazelPackage {
     name = "${pname}-${version}";
-    bazel = bazel_0_29;
+    bazel = bazel_3;
 
     src = fetchFromGitHub {
       owner = "tensorflow";
       repo = "tensorflow";
       rev = "v${version}";
-      sha256 = "1g79xi8yl4sjia8ysk9b7xfzrz83zy28v5dlb2wzmcf0k5pmz60p";
+      sha256 = "1dd5fgyiazyfy7y2iv4v42qnap51fr6dzwb26inrsj7aaas06j71";
     };
 
     patches = [
-      # Work around https://github.com/tensorflow/tensorflow/issues/24752
-      ../no-saved-proto.patch
       # Fixes for NixOS jsoncpp
       ../system-jsoncpp.patch
 
-      (fetchpatch {
-        name = "backport-pr-18950.patch";
-        url = "https://github.com/tensorflow/tensorflow/commit/73640aaec2ab0234d9fff138e3c9833695570c0a.patch";
-        sha256 = "1n9ypbrx36fc1kc9cz5b3p9qhg15xxhq4nz6ap3hwqba535nakfz";
-      })
-
-      (fetchpatch {
-        # Don't try to fetch things that don't exist
-        name = "prune-missing-deps.patch";
-        url = "https://github.com/tensorflow/tensorflow/commit/b39b1ed24b4814db27d2f748dc85c10730ae851d.patch";
-        sha256 = "1skysz53nancvw1slij6s7flar2kv3gngnsq60ff4lap88kx5s6c";
-        excludes = [ "tensorflow/cc/saved_model/BUILD" ];
-      })
-
       ./lift-gast-restriction.patch
 
-      # cuda 10.2 does not have "-bin2c-path" option anymore
+      # see https://github.com/tensorflow/tensorflow/issues/40688
-      # https://github.com/tensorflow/tensorflow/issues/34429
+      (fetchpatch {
-      ../cuda-10.2-no-bin2c-path.patch
+        url = "https://github.com/tensorflow/tensorflow/commit/75ea0b31477d6ba9e990e296bbbd8ca4e7eebadf.patch";
+        sha256 = "1xp1icacig0xm0nmb05sbrf4nw4xbln9fhc308birrv8286zx7wv";
+      })
+
+      # see https://github.com/tensorflow/tensorflow/issues/40884
+      (fetchpatch {
+        url = "https://github.com/tensorflow/tensorflow/pull/41867/commits/65341f73d110bf173325768947343e1bb8f699fc.patch";
+        sha256 = "18ykkycaag1pcarz53bz6ydxjlah92j4178qn58gcayx1fy7hvh3";
+      })
     ];
 
     # On update, it can be useful to steal the changes from gentoo
@@ -147,7 +139,7 @@ let
       git
 
       # libs taken from system through the TF_SYS_LIBS mechanism
-      # grpc
+      grpc
       sqlite
       openssl
       jsoncpp
@@ -165,7 +157,6 @@ let
     ] ++ lib.optionals cudaSupport [
       cudatoolkit
       cudnn
-      nvidia_x11
     ] ++ lib.optionals mklSupport [
       mkl
     ] ++ lib.optionals stdenv.isDarwin [
@@ -182,24 +173,26 @@ let
     TF_SYSTEM_LIBS = lib.concatStringsSep "," [
       "absl_py"
       "astor_archive"
+      "astunparse_archive"
       "boringssl"
       # Not packaged in nixpkgs
       # "com_github_googleapis_googleapis"
       # "com_github_googlecloudplatform_google_cloud_cpp"
+      "com_github_grpc_grpc"
       "com_google_protobuf"
       "com_googlesource_code_re2"
       "curl"
       "cython"
       "double_conversion"
+      "enum34_archive"
       "flatbuffers"
+      "functools32_archive"
       "gast_archive"
-      # Lots of errors, requires an older version
+      "gif"
-      # "grpc"
       "hwloc"
       "icu"
-      "jpeg"
       "jsoncpp_git"
-      "keras_applications_archive"
+      "libjpeg_turbo"
       "lmdb"
       "nasm"
       # "nsync" # not packaged in nixpkgs
@@ -207,12 +200,14 @@ let
       "org_sqlite"
       "pasta"
       "pcre"
+      "png"
+      "pybind11"
       "six_archive"
       "snappy"
       "swig"
       "termcolor_archive"
       "wrapt"
-      "zlib_archive"
+      "zlib"
     ];
 
     INCLUDEDIR = "${includes_joined}/include";
@@ -235,13 +230,16 @@ let
     TF_CUDA_COMPUTE_CAPABILITIES = lib.concatStringsSep "," cudaCapabilities;
 
     postPatch = ''
-      # https://github.com/tensorflow/tensorflow/issues/20919
-      sed -i '/androidndk/d' tensorflow/lite/kernels/internal/BUILD
-
       # Tensorboard pulls in a bunch of dependencies, some of which may
       # include security vulnerabilities. So we make it optional.
       # https://github.com/tensorflow/tensorflow/issues/20280#issuecomment-400230560
       sed -i '/tensorboard >=/d' tensorflow/tools/pip_package/setup.py
+
+      # numpy 1.19 added in https://github.com/tensorflow/tensorflow/commit/75ea0b31477d6ba9e990e296bbbd8ca4e7eebadf.patch
+      sed -i 's/numpy >= 1.16.0, < 1.19.0/numpy >= 1.16.0/' tensorflow/tools/pip_package/setup.py
+
+      # bazel 3.3 should work just as well as bazel 3.1
+      rm -f .bazelversion
     '';
 
     preConfigure = let
@@ -272,15 +270,8 @@ let
       runHook postConfigure
     '';
 
-    # FIXME: Tensorflow uses dlopen() for CUDA libraries.
-    NIX_LDFLAGS = lib.optionalString cudaSupport "-lcudart -lcublas -lcufft -lcurand -lcusolver -lcusparse -lcudnn";
-
     hardeningDisable = [ "format" ];
 
-    bazelFlags = [
-      # temporary fixes to make the build work with bazel 0.27
-      "--incompatible_no_support_tools_in_action_inputs=false"
-    ];
     bazelBuildFlags = [
       "--config=opt" # optimize using the flags set in the configure phase
     ]
@@ -288,15 +279,17 @@ let
 
     bazelTarget = "//tensorflow/tools/pip_package:build_pip_package //tensorflow/tools/lib_package:libtensorflow";
 
+    removeRulesCC = false;
+
     fetchAttrs = {
       # So that checksums don't depend on these.
       TF_SYSTEM_LIBS = null;
 
       # cudaSupport causes fetch of ncclArchive, resulting in different hashes
       sha256 = if cudaSupport then
-        "1kqk1gx5g63kb2zdj392x6mnpbrmgqghrdv597aipn7s23xzj8pd"
+        "0pf8128chkm6fxnhd4956n6gvijlj00mjmvry33gq3xx3bayhs9g"
       else
-        "1plpcm2ydpajsrxdvmmpfy7l0gfdir78hap72w4k7ddm6d3rm2fv";
+        "0mkgss2nyk21zlj8hp24cs3dmpdnxk8qi6qq4hyc18lp82p09xwa";
     };
 
     buildAttrs = {
@@ -347,7 +340,7 @@ let
 
 in buildPythonPackage {
   inherit version pname;
-  disabled = isPy27 || (pythonAtLeast "3.8");
+  disabled = isPy27;
 
   src = bazel-build.python;
 
@@ -377,6 +370,8 @@ in buildPythonPackage {
     wrapt
     grpcio
     opt-einsum
+    astunparse
+    h5py
   ] ++ lib.optionals (!isPy3k) [
     mock
     future
@@ -392,6 +387,8 @@ in buildPythonPackage {
   postFixup = lib.optionalString cudaSupport ''
     find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do
       addOpenGLRunpath "$lib"
+
+      patchelf --set-rpath "${cudatoolkit}/lib:${cudatoolkit.lib}/lib:${cudnn}/lib:${nccl}/lib:$(patchelf --print-rpath "$lib")" "$lib"
     done
   '';
 

pkgs/development/python-modules/tensorflow/2/lift-gast-restriction.patch

@@ -2,10 +2,10 @@ diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_packag
 index 992f2eae22..d9386f9b13 100644
 --- a/tensorflow/tools/pip_package/setup.py
 +++ b/tensorflow/tools/pip_package/setup.py
-@@ -54,5 +54,5 @@ REQUIRED_PACKAGES = [
+@@ -56,5 +56,5 @@ REQUIRED_PACKAGES = [
-     'enum34 >= 1.1.6;python_version<"3.4"',
+     'astunparse == 1.6.3',
--    'gast == 0.2.2',
+-    'gast == 0.3.3',
-+    'gast >= 0.2.2',
++    'gast >= 0.3.3',
-     'google_pasta >= 0.1.6',
+     'google_pasta >= 0.1.8',
-     'keras_applications >= 1.0.8',
+     'h5py >= 2.10.0, < 2.11.0',
-     'keras_preprocessing >= 1.0.5',
+     'keras_preprocessing >= 1.1.1, < 1.2',

pkgs/top-level/python-packages.nix

@@ -6932,10 +6932,9 @@ in {
 
   tensorflow-build_2 = callPackage ../development/python-modules/tensorflow/2 {
     cudaSupport = pkgs.config.cudaSupport or false;
-    inherit (pkgs.linuxPackages) nvidia_x11;
+    cudatoolkit = pkgs.cudatoolkit_11;
-    cudatoolkit = pkgs.cudatoolkit_10;
+    cudnn = pkgs.cudnn_cudatoolkit_11;
-    cudnn = pkgs.cudnn_cudatoolkit_10;
+    nccl = pkgs.nccl_cudatoolkit_11;
-    nccl = pkgs.nccl_cudatoolkit_10;
     openssl = pkgs.openssl_1_1;
     inherit (pkgs.darwin.apple_sdk.frameworks) Foundation Security;
   };