From a0b992fdf8c5bae41682f9208a1e611dbfd2b60c Mon Sep 17 00:00:00 2001 From: Michael Lingelbach Date: Tue, 25 Feb 2020 08:00:44 -0800 Subject: [PATCH 1/5] tensorflow: 1.15.0 -> 1.15.1 * Apply glibc 2.3 patch * build tensorflow with bazel_1 * Bump openssl version to 1.1 (cherry picked from commit 34296980d1b35ad40752fe9330f9814ba2186428) --- pkgs/build-support/build-bazel-package/default.nix | 5 +++++ .../python-modules/tensorflow/default.nix | 14 ++++++++++---- pkgs/top-level/python-packages.nix | 2 +- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pkgs/build-support/build-bazel-package/default.nix b/pkgs/build-support/build-bazel-package/default.nix index a17712472b1..058f42aa8d5 100644 --- a/pkgs/build-support/build-bazel-package/default.nix +++ b/pkgs/build-support/build-bazel-package/default.nix @@ -4,8 +4,13 @@ , lib }: +let + bazelPkg = bazel; +in + args@{ name +, bazel ? bazelPkg , bazelFlags ? [] , bazelBuildFlags ? [] , bazelFetchFlags ? [] diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index a9f3ebf9e48..9e5acca4749 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -1,4 +1,4 @@ -{ stdenv, pkgs, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin +{ stdenv, pkgs, bazel_1, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin , addOpenGLRunpath # Python deps , buildPythonPackage, isPy3k, pythonOlder, pythonAtLeast, python @@ -69,7 +69,7 @@ let tfFeature = x: if x then "1" else "0"; - version = "1.15.0"; + version = "1.15.1"; variant = if cudaSupport then "-gpu" else ""; pname = "tensorflow${variant}"; @@ -94,6 +94,7 @@ let bazel-build = buildBazelPackage { name = "${pname}-${version}"; + bazel = bazel_1; src = fetchFromGitHub { owner = "tensorflow"; @@ -114,6 +115,11 @@ let url = "https://github.com/tensorflow/tensorflow/pull/29673/commits/498e35a3bfe38dd75cf1416a1a23c07c3b59e6af.patch"; sha256 = "1m2qmwv1ysqa61z6255xggwbq6mnxbig749bdvrhnch4zydxb4di"; }) + (fetchpatch { + name = "backport-pr-18950.patch"; + url = "https://github.com/tensorflow/tensorflow/commit/73640aaec2ab0234d9fff138e3c9833695570c0a.patch"; + sha256 = "1n9ypbrx36fc1kc9cz5b3p9qhg15xxhq4nz6ap3hwqba535nakfz"; + }) ./tf-1.15-bazel-1.0.patch @@ -291,9 +297,9 @@ let # cudaSupport causes fetch of ncclArchive, resulting in different hashes sha256 = if cudaSupport then - "1rbg8w8pjf15hpvzrclsi19lhsrwdns6f8psb1wz35ay0ggdw8c0" + "1p544yk7jcspgc4qr4amw11ds16c2an5yxvagx5pmwawz0s083pf" else - "0d8wq89iz9vrzvr971mgdclxxjcjr32r7aj817h019x3pc53qnwx"; + "1dqbw3k3avqiy9xpgs44l6z65ab5rjjlxwig8z7gcl7fw9h6sbq9"; }; buildAttrs = { diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 610a8d0852f..59ac9274b3e 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -6567,7 +6567,7 @@ in { cudatoolkit = pkgs.cudatoolkit_10; cudnn = pkgs.cudnn_cudatoolkit_10; nccl = pkgs.nccl_cudatoolkit_10; - openssl = pkgs.openssl_1_0_2; + openssl = pkgs.openssl_1_1; inherit (pkgs.darwin.apple_sdk.frameworks) Foundation Security; }; From 4e7dac0a569404d7a7bfa44652ad69d2c4c1e54c Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Mon, 24 Feb 2020 13:32:36 +0100 Subject: [PATCH 2/5] tensorflow: update fixed output hashes (cherry picked from commit 642bea6cecf5ee1258b2a317f6e56278c7a355b9) --- pkgs/development/python-modules/tensorflow/default.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index 9e5acca4749..86fc0df5eb6 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -296,10 +296,12 @@ let TF_SYSTEM_LIBS = null; # cudaSupport causes fetch of ncclArchive, resulting in different hashes + # FIXME: can't (re)produce this output with current bazel. + # FIXME: build log: https://gist.github.com/andir/eff3e9c8eda5b56c8ea84903aed9cc35 sha256 = if cudaSupport then - "1p544yk7jcspgc4qr4amw11ds16c2an5yxvagx5pmwawz0s083pf" + "0gyhjvzshgj59mbns8njlfl9qpz4sdg4j0xs2dva0w2nql7cr7im" else - "1dqbw3k3avqiy9xpgs44l6z65ab5rjjlxwig8z7gcl7fw9h6sbq9"; + "04jvg3mc2si4xdbszc1vnw1rmf22p7snbjphmnklp7bc39jxkcrz"; }; buildAttrs = { From 08f2ed7b027a33bb04205babc22b1b5b159fa846 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Tue, 3 Mar 2020 17:27:44 +0100 Subject: [PATCH 3/5] pythonPackages.tensorflow: use bazel_0 This allows us to get rid of the compatibility hacks that we had to add (tf-1.15-bazel-1.0.patch) and also fixes #77626. (cherry picked from commit c7adb4ee7282672c330b2f8b37ac5f6d74e1a523) --- .../python-modules/tensorflow/default.nix | 13 +- .../tensorflow/tf-1.15-bazel-1.0.patch | 213 ------------------ 2 files changed, 7 insertions(+), 219 deletions(-) delete mode 100644 pkgs/development/python-modules/tensorflow/tf-1.15-bazel-1.0.patch diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index 86fc0df5eb6..05254664628 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -1,4 +1,4 @@ -{ stdenv, pkgs, bazel_1, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin +{ stdenv, pkgs, bazel_0, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin , addOpenGLRunpath # Python deps , buildPythonPackage, isPy3k, pythonOlder, pythonAtLeast, python @@ -94,7 +94,7 @@ let bazel-build = buildBazelPackage { name = "${pname}-${version}"; - bazel = bazel_1; + bazel = bazel_0; src = fetchFromGitHub { owner = "tensorflow"; @@ -121,7 +121,6 @@ let sha256 = "1n9ypbrx36fc1kc9cz5b3p9qhg15xxhq4nz6ap3hwqba535nakfz"; }) - ./tf-1.15-bazel-1.0.patch (fetchpatch { # be compatible with gast >0.2 instead of only gast 0.2.2 @@ -283,7 +282,6 @@ let bazelFlags = [ # temporary fixes to make the build work with bazel 0.27 "--incompatible_no_support_tools_in_action_inputs=false" - "--incompatible_use_native_patch=false" ]; bazelBuildFlags = [ "--config=opt" # optimize using the flags set in the configure phase @@ -299,9 +297,9 @@ let # FIXME: can't (re)produce this output with current bazel. # FIXME: build log: https://gist.github.com/andir/eff3e9c8eda5b56c8ea84903aed9cc35 sha256 = if cudaSupport then - "0gyhjvzshgj59mbns8njlfl9qpz4sdg4j0xs2dva0w2nql7cr7im" + "0bzkqjnw1crf0v91yb1frvy0l7kmjawbfwdhm89h73i8fqjab8jw" else - "04jvg3mc2si4xdbszc1vnw1rmf22p7snbjphmnklp7bc39jxkcrz"; + "1d7czp43a3a4aksvdcskbdy7dgifily1amqbz9fa6d8mkhdj5if5"; }; buildAttrs = { @@ -422,6 +420,9 @@ in buildPythonPackage { x = np.random.uniform(size=(1,1)) y = np.random.uniform(size=(1,)) model.fit(x, y, epochs=1) + + # regression test for #77626 + from tensorflow.contrib import tensor_forest EOF ''; diff --git a/pkgs/development/python-modules/tensorflow/tf-1.15-bazel-1.0.patch b/pkgs/development/python-modules/tensorflow/tf-1.15-bazel-1.0.patch deleted file mode 100644 index 4d70e99108b..00000000000 --- a/pkgs/development/python-modules/tensorflow/tf-1.15-bazel-1.0.patch +++ /dev/null @@ -1,213 +0,0 @@ -diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD -index f740ba66b5..6cc9003787 100644 ---- a/tensorflow/c/BUILD -+++ b/tensorflow/c/BUILD -@@ -270,6 +270,7 @@ tf_cuda_library( - "//tensorflow/core/platform", - "@com_google_absl//absl/strings", - ], -+ alwayslink = 1, - ) - - exports_files( -diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD -index 5c42e508f7..16b421862c 100644 ---- a/tensorflow/c/eager/BUILD -+++ b/tensorflow/c/eager/BUILD -@@ -79,6 +79,7 @@ tf_cuda_library( - "//tensorflow/core/profiler/lib:profiler_session", - "//tensorflow/core:gpu_runtime", - ], -+ alwayslink = 1, - ) - - tf_cuda_library( -@@ -226,6 +227,7 @@ tf_cuda_library( - "//tensorflow/core/profiler/rpc/client:capture_profile", - "//tensorflow/core:gpu_runtime", - ], -+ alwayslink = 1, - ) - - tf_cuda_cc_test( -diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD -index 39b84922d1..b2affdd999 100644 ---- a/tensorflow/cc/saved_model/BUILD -+++ b/tensorflow/cc/saved_model/BUILD -@@ -123,6 +123,7 @@ cc_library( - "//tensorflow/core/util/tensor_bundle:naming", - # mobile not supported yet - ]), -+ alwayslink = 1, - ) - - tf_cc_test( -diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD -index c23c1f9b39..805643b217 100644 ---- a/tensorflow/core/BUILD -+++ b/tensorflow/core/BUILD -@@ -777,6 +777,7 @@ cc_library( - ":lib_proto_parsing", - ":protos_all_cc", - ], -+ alwayslink = 1, - ) - - # DEPRECATED: use platform:stringpiece instead. -@@ -2496,6 +2497,7 @@ cc_library( - "@com_google_protobuf//:protobuf", - ] + tf_protos_all_impl() + tf_protos_grappler_impl() + - tf_additional_numa_deps(), -+ alwayslink = 1, - ) - - # File compiled with extra flags to get cpu-specific acceleration. -diff --git a/tensorflow/core/lib/random/BUILD b/tensorflow/core/lib/random/BUILD -index 3bd933261b..e1e589e76d 100644 ---- a/tensorflow/core/lib/random/BUILD -+++ b/tensorflow/core/lib/random/BUILD -@@ -50,6 +50,7 @@ cc_library( - "//tensorflow/core/platform:types", - "//third_party/eigen3", - ], -+ alwayslink = 1, - ) - - filegroup( -diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl -index 5459d8d428..feba3a5686 100644 ---- a/tensorflow/core/platform/default/build_config.bzl -+++ b/tensorflow/core/platform/default/build_config.bzl -@@ -228,6 +228,7 @@ def cc_proto_library( - hdrs = gen_hdrs, - deps = cc_libs + deps, - includes = includes, -+ alwayslink = 1, - **kargs - ) - native.cc_library( -diff --git a/tensorflow/lite/java/src/test/native/BUILD b/tensorflow/lite/java/src/test/native/BUILD -index 6dcdab2aee..32bb0a8d85 100644 ---- a/tensorflow/lite/java/src/test/native/BUILD -+++ b/tensorflow/lite/java/src/test/native/BUILD -@@ -19,6 +19,7 @@ cc_library( - "//tensorflow/lite/java/jni", - "//tensorflow/lite/kernels:kernel_util", - ], -+ alwayslink = 1, - ) - - tflite_jni_binary( -diff --git a/tensorflow/lite/python/testdata/BUILD b/tensorflow/lite/python/testdata/BUILD -index 7bda81358f..ac1188d844 100644 ---- a/tensorflow/lite/python/testdata/BUILD -+++ b/tensorflow/lite/python/testdata/BUILD -@@ -60,6 +60,7 @@ cc_library( - deps = [ - "//tensorflow/lite/c:c_api_internal", - ], -+ alwayslink = 1, - ) - - cc_binary( -diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD -index 6fd9b4f273..29df3a3dff 100644 ---- a/tensorflow/python/BUILD -+++ b/tensorflow/python/BUILD -@@ -375,6 +375,7 @@ cc_library( - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - ], -+ alwayslink = 1, - ) - - cc_library( -@@ -411,6 +412,7 @@ cc_library( - "//third_party/py/numpy:headers", - "//third_party/python_runtime:headers", - ], -+ alwayslink = 1, - ) - - cc_library( -@@ -617,6 +619,7 @@ cc_library( - "//tensorflow/core:op_gen_lib", - "//tensorflow/core:protos_all_cc", - ], -+ alwayslink = 1, - ) - - py_library( -diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl -index a3956322fe..32752f59ad 100644 ---- a/tensorflow/tensorflow.bzl -+++ b/tensorflow/tensorflow.bzl -@@ -2331,6 +2331,7 @@ def tf_generate_proto_text_sources(name, srcs_relative_dir, srcs, protodeps = [] - hdrs = out_hdrs, - visibility = visibility, - deps = deps, -+ alwayslink = 1, - ) - - def tf_genrule_cmd_append_to_srcs(to_append): -diff --git a/tensorflow/tools/graph_transforms/BUILD b/tensorflow/tools/graph_transforms/BUILD -index adafe2aca1..8965316b12 100644 ---- a/tensorflow/tools/graph_transforms/BUILD -+++ b/tensorflow/tools/graph_transforms/BUILD -@@ -223,6 +223,7 @@ cc_library( - "//tensorflow/core:lib_internal", - "//tensorflow/core:protos_all_cc", - ], -+ alwayslink = 1, - ) - - # This library includes a main function, to make it easy to create other -diff --git a/third_party/icu/data/BUILD.bazel b/third_party/icu/data/BUILD.bazel -index 7db21566e4..8e18c7cc3a 100644 ---- a/third_party/icu/data/BUILD.bazel -+++ b/third_party/icu/data/BUILD.bazel -@@ -43,4 +43,5 @@ cc_library( - name = "conversion_data", - srcs = [":conversion_data.c"], - deps = ["@icu//:headers"], -+ alwayslink = 1, - ) -diff --git a/third_party/protobuf/protobuf.patch b/third_party/protobuf/protobuf.patch -index df0648563d..18fc6cdf35 100644 ---- a/third_party/protobuf/protobuf.patch -+++ b/third_party/protobuf/protobuf.patch -@@ -11,7 +11,15 @@ index 2fb26050..c2744d5b 100644 - - ################################################################################ - # Protobuf Runtime Library --@@ -218,7 +218,7 @@ cc_library( -+@@ -209,6 +209,7 @@ cc_library( -+ copts = COPTS, -+ includes = ["src/"], -+ linkopts = LINK_OPTS, -++ alwayslink = 1, -+ visibility = ["//visibility:public"], -+ deps = [":protobuf_lite"] + PROTOBUF_DEPS, -+ ) -+@@ -219,7 +220,7 @@ cc_library( - # TODO(keveman): Remove this target once the support gets added to Bazel. - cc_library( - name = "protobuf_headers", -@@ -20,3 +28,4 @@ index 2fb26050..c2744d5b 100644 - includes = ["src/"], - visibility = ["//visibility:public"], - ) -+ -\ No newline at end of file -diff --git a/third_party/systemlibs/protobuf.bzl b/third_party/systemlibs/protobuf.bzl -index 774514f3fd..1c415b018b 100644 ---- a/third_party/systemlibs/protobuf.bzl -+++ b/third_party/systemlibs/protobuf.bzl -@@ -262,6 +262,7 @@ def cc_proto_library( - hdrs = gen_hdrs, - deps = cc_libs + deps, - includes = includes, -+ alwayslink = 1, - **kargs - ) - From 436a4bace62d0393194fe9f4cdd07476e9f39e5e Mon Sep 17 00:00:00 2001 From: Dylan Simon Date: Wed, 4 Mar 2020 17:58:04 -0500 Subject: [PATCH 4/5] tensorflow: 1.15.1 -> 1.15.2 (cherry picked from commit 0a5ec494b574ccd3db3b234c7d6637d127bd16da) (cherry picked from commit 1e769bdd2ad136c7be98652960b34e0d81967084) --- pkgs/development/python-modules/tensorflow/default.nix | 10 ++++------ .../tensorflow/lift-gast-restriction.patch | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index 05254664628..a740a5282f4 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -69,7 +69,7 @@ let tfFeature = x: if x then "1" else "0"; - version = "1.15.1"; + version = "1.15.2"; variant = if cudaSupport then "-gpu" else ""; pname = "tensorflow${variant}"; @@ -100,7 +100,7 @@ let owner = "tensorflow"; repo = "tensorflow"; rev = "v${version}"; - sha256 = "1j8vysfblkyydrr67qr3i7kvaq5ygnjlx8hw9a9pc95ac462jq7i"; + sha256 = "1q0848drjvnaaa38dgns8knmpmkj5plzsc98j20m5ybv68s55w78"; }; patches = [ @@ -294,12 +294,10 @@ let TF_SYSTEM_LIBS = null; # cudaSupport causes fetch of ncclArchive, resulting in different hashes - # FIXME: can't (re)produce this output with current bazel. - # FIXME: build log: https://gist.github.com/andir/eff3e9c8eda5b56c8ea84903aed9cc35 sha256 = if cudaSupport then - "0bzkqjnw1crf0v91yb1frvy0l7kmjawbfwdhm89h73i8fqjab8jw" + "1qygfcvvn9vysap9nk6xccxi9mgmzyxiywz6k456f811l1v70p2c" else - "1d7czp43a3a4aksvdcskbdy7dgifily1amqbz9fa6d8mkhdj5if5"; + "0kfjanw0mfbh30vi1ms2xlg8yp429cbyfriik6yxd5cla2pncg2j"; }; buildAttrs = { diff --git a/pkgs/development/python-modules/tensorflow/lift-gast-restriction.patch b/pkgs/development/python-modules/tensorflow/lift-gast-restriction.patch index 24cc118d8f3..30861d92341 100644 --- a/pkgs/development/python-modules/tensorflow/lift-gast-restriction.patch +++ b/pkgs/development/python-modules/tensorflow/lift-gast-restriction.patch @@ -3,9 +3,9 @@ index 992f2eae22..d9386f9b13 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -54,7 +54,7 @@ REQUIRED_PACKAGES = [ - 'astor >= 0.6.0', - 'backports.weakref >= 1.0rc1;python_version<"3.4"', 'enum34 >= 1.1.6;python_version<"3.4"', + # functools comes with python3, need to install the backport for python2 + 'functools32 >= 3.2.3;python_version<"3"', - 'gast == 0.2.2', + 'gast >= 0.2.2', 'google_pasta >= 0.1.6', From ca6ccd4f4c0467b20b28b6aaa444f75631787ae6 Mon Sep 17 00:00:00 2001 From: Jonathan Ringer Date: Wed, 18 Mar 2020 12:04:55 -0700 Subject: [PATCH 5/5] pythonPackages.tensorflow: disable for python2.7 and 3.8 (cherry picked from commit b7bdf48e3fa6b90ea475a03ed159f397bd3c3c65) --- pkgs/development/python-modules/tensorflow/default.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index a740a5282f4..54629bebc11 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -1,7 +1,7 @@ { stdenv, pkgs, bazel_0, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin , addOpenGLRunpath # Python deps -, buildPythonPackage, isPy3k, pythonOlder, pythonAtLeast, python +, buildPythonPackage, isPy3k, isPy27, pythonOlder, pythonAtLeast, python # Python libraries , numpy, tensorflow-tensorboard, backports_weakref, mock, enum34, absl-py , future, setuptools, wheel, keras-preprocessing, keras-applications, google-pasta @@ -348,6 +348,7 @@ let in buildPythonPackage { inherit version pname; + disabled = isPy27 || (pythonAtLeast "3.8"); src = bazel-build.python;