Merge pull request #71282 from timokau/tensorflow-1.15.0

python.pkgs.tensorflow: 1.14.0 -> 1.15.0

pkgs/development/python-modules/tensorflow-estimator/default.nix

@@ -6,13 +6,14 @@
 
 buildPythonPackage rec {
   pname = "tensorflow-estimator";
-  version = "1.14.0";
+  # This is effectively 1.15.0. Upstream tagged 1.15.0 by mistake before actually updating the version in setup.py, which is why this tag is called 1.15.1.
+  version = "1.15.1";
   format = "wheel";
 
   src = fetchPypi {
     pname = "tensorflow_estimator";
     inherit version format;
-    sha256 = "14irpsyj14vn2dpwr601f54058wywci1pv0hss8s01rl0rk3y1ya";
+    sha256 = "1fc61wmc0w22frs79j2x4g6wnv5g21xc6rix1g4bsvy9qfvvylw8";
   };
 
   propagatedBuildInputs = [ mock numpy absl-py ];

pkgs/development/python-modules/tensorflow-tensorboard/default.nix

@@ -14,7 +14,7 @@
 
 buildPythonPackage rec {
   pname = "tensorflow-tensorboard";
-  version = "1.14.0";
+  version = "1.15.0";
   format = "wheel";
 
   src = fetchPypi ({
@@ -23,10 +23,10 @@ buildPythonPackage rec {
     format = "wheel";
   } // (if isPy3k then {
     python = "py3";
-    sha256 = "1z631614jk5zgasgmwfr33gz8bwv11p9f5llzlwvx3a8rnyv3q2h";
+    sha256 = "1g62i3nrgp8q9wfsyqqjkkfnsz7x2k018c26kdh527h1yrjjrbac";
   } else {
     python = "py2";
-    sha256 = "1clv29yy942l3mfar2z6wkkk6l18fz7j6mi2dfz24j9dln0scny3";
+    sha256 = "0l3zc8j2sh7h1z4qpy8kfvclv3kzndri55p10i42q6xahs9phav1";
   }));
 
   propagatedBuildInputs = [

pkgs/development/python-modules/tensorflow/default.nix

@@ -5,12 +5,18 @@
 # Python libraries
 , numpy, tensorflow-tensorboard, backports_weakref, mock, enum34, absl-py
 , future, setuptools, wheel, keras-preprocessing, keras-applications, google-pasta
+, functools32
+, opt-einsum
 , termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator
 # Common deps
 , git, swig, which, binutils, glibcLocales, cython
 # Common libraries
 , jemalloc, openmpi, astor, gast, grpc, sqlite, openssl, jsoncpp, re2
 , curl, snappy, flatbuffers, icu, double-conversion, libpng, libjpeg, giflib
+# Upsteam by default includes cuda support since tensorflow 1.15. We could do
+# that in nix as well. It would make some things easier and less confusing, but
+# it would also make the default tensorflow package unfree. See
+# https://groups.google.com/a/tensorflow.org/forum/#!topic/developers/iRCt5m4qUz0
 , cudaSupport ? false, nvidia_x11 ? null, cudatoolkit ? null, cudnn ? null, nccl ? null
 # XLA without CUDA is broken
 , xlaSupport ? cudaSupport
@@ -33,7 +39,13 @@ let
 
   cudatoolkit_joined = symlinkJoin {
     name = "${cudatoolkit.name}-merged";
-    paths = [ cudatoolkit.out cudatoolkit.lib ];
+    paths = [
+      cudatoolkit.lib
+      cudatoolkit.out
+      # for some reason some of the required libs are in the targets/x86_64-linux
+      # directory; not sure why but this works around it
+      "${cudatoolkit}/targets/${stdenv.system}"
+    ];
   };
 
   cudatoolkit_cc_joined = symlinkJoin {
@@ -55,12 +67,12 @@ let
 
   tfFeature = x: if x then "1" else "0";
 
-  version = "1.14.0";
+  version = "1.15.0";
   variant = if cudaSupport then "-gpu" else "";
   pname = "tensorflow${variant}";
 
   pythonEnv = python.withPackages (_:
-    [ # python deps needed during wheel build time
+    [ # python deps needed during wheel build time (not runtime, see the buildPythonPackage part for that)
       numpy
       keras-preprocessing
       protobuf
@@ -74,6 +86,7 @@ let
       wheel
   ] ++ lib.optionals (!isPy3k)
   [ future
+    functools32
     mock
   ]);
 
@@ -84,7 +97,7 @@ let
       owner = "tensorflow";
       repo = "tensorflow";
       rev = "v${version}";
-      sha256 = "06jvwlsm14b8rqwd8q8796r0vmn0wk64s4ps2zg0sapkmp9vvcmi";
+      sha256 = "1j8vysfblkyydrr67qr3i7kvaq5ygnjlx8hw9a9pc95ac462jq7i";
     };
 
     patches = [
@@ -100,12 +113,15 @@ let
         sha256 = "1m2qmwv1ysqa61z6255xggwbq6mnxbig749bdvrhnch4zydxb4di";
       })
 
-      # https://github.com/tensorflow/tensorflow/issues/29220
+      ./tf-1.15-bazel-1.0.patch
+
       (fetchpatch {
-        name = "bazel-0.27.patch";
+        # be compatible with gast >0.2 instead of only gast 0.2.2
-        url = "https://github.com/tensorflow/tensorflow/commit/cfccbdb8c4a92dd26382419dceb4d934c2380391.patch";
+        name = "gast-update.patch";
-        sha256 = "1l56wjia2c4685flsfkkgy471wx3c66wyv8khspv06zchj0k0liw";
+        url = "https://github.com/tensorflow/tensorflow/commit/85751ad6c7f5fd12c6c79545d96896cba92fa8b4.patch";
+        sha256 = "077cpj0kzyqxzdya1dwh8df17zfzhqn7c685hx6iskvw2979zg2n";
       })
+      ./lift-gast-restriction.patch
     ];
 
     # On update, it can be useful to steal the changes from gentoo
@@ -157,7 +173,6 @@ let
       # "com_github_googleapis_googleapis"
       # "com_github_googlecloudplatform_google_cloud_cpp"
       "com_google_protobuf"
-      "com_google_protobuf_cc"
       "com_googlesource_code_re2"
       "curl"
       "cython"
@@ -175,11 +190,11 @@ let
       "lmdb"
       "nasm"
       # "nsync" # not packaged in nixpkgs
+      "opt_einsum_archive"
       "org_sqlite"
       "pasta"
       "pcre"
       "png_archive"
-      "protobuf_archive"
       "six_archive"
       "snappy"
       "swig"
@@ -253,6 +268,7 @@ let
     bazelFlags = [
       # temporary fixes to make the build work with bazel 0.27
       "--incompatible_no_support_tools_in_action_inputs=false"
+      "--incompatible_use_native_patch=false"
     ];
     bazelBuildFlags = [
       "--config=opt" # optimize using the flags set in the configure phase
@@ -266,9 +282,9 @@ let
 
       # cudaSupport causes fetch of ncclArchive, resulting in different hashes
       sha256 = if cudaSupport then
-        "196pm3ynfafqlcxah07hkvphf536hpix1ydgsynr1yg08aynlvvx"
+        "1rbg8w8pjf15hpvzrclsi19lhsrwdns6f8psb1wz35ay0ggdw8c0"
       else
-        "138r85n27ijzwxfwb5pcfyb79v14368jpckw0vmciz6pwf11bd9g";
+        "0d8wq89iz9vrzvr971mgdclxxjcjr32r7aj817h019x3pc53qnwx";
     };
 
     buildAttrs = {
@@ -311,7 +327,9 @@ let
       license = licenses.asl20;
       maintainers = with maintainers; [ jyp abbradar ];
       platforms = platforms.linux;
-      broken = !(xlaSupport -> cudaSupport);
+      # The py2 build fails due to some issue importing protobuf. Possibly related to the fix in
+      # https://github.com/akesandgren/easybuild-easyblocks/commit/1f2e517ddfd1b00a342c6abb55aef3fd93671a2b
+      broken = !(xlaSupport -> cudaSupport) || !isPy3k;
     };
   };
 
@@ -345,9 +363,11 @@ in buildPythonPackage {
     termcolor
     wrapt
     grpcio
+    opt-einsum
   ] ++ lib.optionals (!isPy3k) [
     mock
-    future # FIXME
+    future
+    functools32
   ] ++ lib.optionals (pythonOlder "3.4") [
     backports_weakref enum34
   ] ++ lib.optionals withTensorboard [
@@ -366,7 +386,26 @@ in buildPythonPackage {
   # TODO try to run them anyway
   # TODO better test (files in tensorflow/tools/ci_build/builds/*test)
   checkPhase = ''
-    ${python.interpreter} -c "import tensorflow"
+    ${python.interpreter} <<EOF
+    # A simple "Hello world"
+    import tensorflow as tf
+    hello = tf.constant("Hello, world!")
+    sess = tf.Session()
+    sess.run(hello)
+
+    # Fit a simple model to random data
+    import numpy as np
+    np.random.seed(0)
+    tf.random.set_random_seed(0)
+    model = tf.keras.models.Sequential([
+        tf.keras.layers.Dense(1, activation="linear")
+    ])
+    model.compile(optimizer="sgd", loss="mse")
+
+    x = np.random.uniform(size=(1,1))
+    y = np.random.uniform(size=(1,))
+    model.fit(x, y, epochs=1)
+    EOF
   '';
 
   passthru.libtensorflow = bazel-build.out;

pkgs/development/python-modules/tensorflow/lift-gast-restriction.patch

@@ -0,0 +1,13 @@
+diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
+index 992f2eae22..d9386f9b13 100644
+--- a/tensorflow/tools/pip_package/setup.py
++++ b/tensorflow/tools/pip_package/setup.py
+@@ -54,7 +54,7 @@ REQUIRED_PACKAGES = [
+     'astor >= 0.6.0',
+     'backports.weakref >= 1.0rc1;python_version<"3.4"',
+     'enum34 >= 1.1.6;python_version<"3.4"',
+-    'gast == 0.2.2',
++    'gast >= 0.2.2',
+     'google_pasta >= 0.1.6',
+     'keras_applications >= 1.0.8',
+     'keras_preprocessing >= 1.0.5',

pkgs/development/python-modules/tensorflow/tf-1.15-bazel-1.0.patch

@@ -0,0 +1,213 @@
+diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
+index f740ba66b5..6cc9003787 100644
+--- a/tensorflow/c/BUILD
++++ b/tensorflow/c/BUILD
+@@ -270,6 +270,7 @@ tf_cuda_library(
+         "//tensorflow/core/platform",
+         "@com_google_absl//absl/strings",
+     ],
++    alwayslink = 1,
+ )
+ 
+ exports_files(
+diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
+index 5c42e508f7..16b421862c 100644
+--- a/tensorflow/c/eager/BUILD
++++ b/tensorflow/c/eager/BUILD
+@@ -79,6 +79,7 @@ tf_cuda_library(
+         "//tensorflow/core/profiler/lib:profiler_session",
+         "//tensorflow/core:gpu_runtime",
+     ],
++    alwayslink = 1,
+ )
+ 
+ tf_cuda_library(
+@@ -226,6 +227,7 @@ tf_cuda_library(
+         "//tensorflow/core/profiler/rpc/client:capture_profile",
+         "//tensorflow/core:gpu_runtime",
+     ],
++    alwayslink = 1,
+ )
+ 
+ tf_cuda_cc_test(
+diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD
+index 39b84922d1..b2affdd999 100644
+--- a/tensorflow/cc/saved_model/BUILD
++++ b/tensorflow/cc/saved_model/BUILD
+@@ -123,6 +123,7 @@ cc_library(
+         "//tensorflow/core/util/tensor_bundle:naming",
+         # mobile not supported yet
+     ]),
++    alwayslink = 1,
+ )
+ 
+ tf_cc_test(
+diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
+index c23c1f9b39..805643b217 100644
+--- a/tensorflow/core/BUILD
++++ b/tensorflow/core/BUILD
+@@ -777,6 +777,7 @@ cc_library(
+         ":lib_proto_parsing",
+         ":protos_all_cc",
+     ],
++    alwayslink = 1,
+ )
+ 
+ # DEPRECATED: use platform:stringpiece instead.
+@@ -2496,6 +2497,7 @@ cc_library(
+                "@com_google_protobuf//:protobuf",
+            ] + tf_protos_all_impl() + tf_protos_grappler_impl() +
+            tf_additional_numa_deps(),
++    alwayslink = 1,
+ )
+ 
+ # File compiled with extra flags to get cpu-specific acceleration.
+diff --git a/tensorflow/core/lib/random/BUILD b/tensorflow/core/lib/random/BUILD
+index 3bd933261b..e1e589e76d 100644
+--- a/tensorflow/core/lib/random/BUILD
++++ b/tensorflow/core/lib/random/BUILD
+@@ -50,6 +50,7 @@ cc_library(
+         "//tensorflow/core/platform:types",
+         "//third_party/eigen3",
+     ],
++    alwayslink = 1,
+ )
+ 
+ filegroup(
+diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl
+index 5459d8d428..feba3a5686 100644
+--- a/tensorflow/core/platform/default/build_config.bzl
++++ b/tensorflow/core/platform/default/build_config.bzl
+@@ -228,6 +228,7 @@ def cc_proto_library(
+         hdrs = gen_hdrs,
+         deps = cc_libs + deps,
+         includes = includes,
++        alwayslink = 1,
+         **kargs
+     )
+     native.cc_library(
+diff --git a/tensorflow/lite/java/src/test/native/BUILD b/tensorflow/lite/java/src/test/native/BUILD
+index 6dcdab2aee..32bb0a8d85 100644
+--- a/tensorflow/lite/java/src/test/native/BUILD
++++ b/tensorflow/lite/java/src/test/native/BUILD
+@@ -19,6 +19,7 @@ cc_library(
+         "//tensorflow/lite/java/jni",
+         "//tensorflow/lite/kernels:kernel_util",
+     ],
++    alwayslink = 1,
+ )
+ 
+ tflite_jni_binary(
+diff --git a/tensorflow/lite/python/testdata/BUILD b/tensorflow/lite/python/testdata/BUILD
+index 7bda81358f..ac1188d844 100644
+--- a/tensorflow/lite/python/testdata/BUILD
++++ b/tensorflow/lite/python/testdata/BUILD
+@@ -60,6 +60,7 @@ cc_library(
+     deps = [
+         "//tensorflow/lite/c:c_api_internal",
+     ],
++    alwayslink = 1,
+ )
+ 
+ cc_binary(
+diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
+index 6fd9b4f273..29df3a3dff 100644
+--- a/tensorflow/python/BUILD
++++ b/tensorflow/python/BUILD
+@@ -375,6 +375,7 @@ cc_library(
+         "//tensorflow/core:lib",
+         "//tensorflow/core:protos_all_cc",
+     ],
++    alwayslink = 1,
+ )
+ 
+ cc_library(
+@@ -411,6 +412,7 @@ cc_library(
+         "//third_party/py/numpy:headers",
+         "//third_party/python_runtime:headers",
+     ],
++    alwayslink = 1,
+ )
+ 
+ cc_library(
+@@ -617,6 +619,7 @@ cc_library(
+         "//tensorflow/core:op_gen_lib",
+         "//tensorflow/core:protos_all_cc",
+     ],
++    alwayslink = 1,
+ )
+ 
+ py_library(
+diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
+index a3956322fe..32752f59ad 100644
+--- a/tensorflow/tensorflow.bzl
++++ b/tensorflow/tensorflow.bzl
+@@ -2331,6 +2331,7 @@ def tf_generate_proto_text_sources(name, srcs_relative_dir, srcs, protodeps = []
+         hdrs = out_hdrs,
+         visibility = visibility,
+         deps = deps,
++        alwayslink = 1,
+     )
+ 
+ def tf_genrule_cmd_append_to_srcs(to_append):
+diff --git a/tensorflow/tools/graph_transforms/BUILD b/tensorflow/tools/graph_transforms/BUILD
+index adafe2aca1..8965316b12 100644
+--- a/tensorflow/tools/graph_transforms/BUILD
++++ b/tensorflow/tools/graph_transforms/BUILD
+@@ -223,6 +223,7 @@ cc_library(
+         "//tensorflow/core:lib_internal",
+         "//tensorflow/core:protos_all_cc",
+     ],
++    alwayslink = 1,
+ )
+ 
+ # This library includes a main function, to make it easy to create other
+diff --git a/third_party/icu/data/BUILD.bazel b/third_party/icu/data/BUILD.bazel
+index 7db21566e4..8e18c7cc3a 100644
+--- a/third_party/icu/data/BUILD.bazel
++++ b/third_party/icu/data/BUILD.bazel
+@@ -43,4 +43,5 @@ cc_library(
+     name = "conversion_data",
+     srcs = [":conversion_data.c"],
+     deps = ["@icu//:headers"],
++    alwayslink = 1,
+ )
+diff --git a/third_party/protobuf/protobuf.patch b/third_party/protobuf/protobuf.patch
+index df0648563d..18fc6cdf35 100644
+--- a/third_party/protobuf/protobuf.patch
++++ b/third_party/protobuf/protobuf.patch
+@@ -11,7 +11,15 @@ index 2fb26050..c2744d5b 100644
+  
+  ################################################################################
+  # Protobuf Runtime Library
+-@@ -218,7 +218,7 @@ cc_library(
++@@ -209,6 +209,7 @@ cc_library(
++     copts = COPTS,
++     includes = ["src/"],
++     linkopts = LINK_OPTS,
+++    alwayslink = 1,
++     visibility = ["//visibility:public"],
++     deps = [":protobuf_lite"] + PROTOBUF_DEPS,
++ )
++@@ -219,7 +220,7 @@ cc_library(
+  # TODO(keveman): Remove this target once the support gets added to Bazel.
+  cc_library(
+      name = "protobuf_headers",
+@@ -20,3 +28,4 @@ index 2fb26050..c2744d5b 100644
+      includes = ["src/"],
+      visibility = ["//visibility:public"],
+  )
++ 
+\ No newline at end of file
+diff --git a/third_party/systemlibs/protobuf.bzl b/third_party/systemlibs/protobuf.bzl
+index 774514f3fd..1c415b018b 100644
+--- a/third_party/systemlibs/protobuf.bzl
++++ b/third_party/systemlibs/protobuf.bzl
+@@ -262,6 +262,7 @@ def cc_proto_library(
+         hdrs = gen_hdrs,
+         deps = cc_libs + deps,
+         includes = includes,
++        alwayslink = 1,
+         **kargs
+     )
+