From 152d2fd22c5346fa2614825c9e03051e21bd1d72 Mon Sep 17 00:00:00 2001 From: Timo Kaufmann Date: Mon, 7 Oct 2019 10:14:16 +0200 Subject: [PATCH 1/6] python.pkgs.{tensorflow,tensorflow-estimator,tensorflow-tensorboard}: 1.14.0 -> 1.15.0 --- .../tensorflow-estimator/default.nix | 5 ++- .../tensorflow-tensorboard/default.nix | 6 +-- .../python-modules/tensorflow/default.nix | 38 +++++++++++-------- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/pkgs/development/python-modules/tensorflow-estimator/default.nix b/pkgs/development/python-modules/tensorflow-estimator/default.nix index 127e7560679..9f6be4d1dda 100644 --- a/pkgs/development/python-modules/tensorflow-estimator/default.nix +++ b/pkgs/development/python-modules/tensorflow-estimator/default.nix @@ -6,13 +6,14 @@ buildPythonPackage rec { pname = "tensorflow-estimator"; - version = "1.14.0"; + # This is effectively 1.15.0. Upstream tagged 1.15.0 by mistake before actually updating the version in setup.py, which is why this tag is called 1.15.1. + version = "1.15.1"; format = "wheel"; src = fetchPypi { pname = "tensorflow_estimator"; inherit version format; - sha256 = "14irpsyj14vn2dpwr601f54058wywci1pv0hss8s01rl0rk3y1ya"; + sha256 = "1fc61wmc0w22frs79j2x4g6wnv5g21xc6rix1g4bsvy9qfvvylw8"; }; propagatedBuildInputs = [ mock numpy absl-py ]; diff --git a/pkgs/development/python-modules/tensorflow-tensorboard/default.nix b/pkgs/development/python-modules/tensorflow-tensorboard/default.nix index fa1759119c9..baefd635566 100644 --- a/pkgs/development/python-modules/tensorflow-tensorboard/default.nix +++ b/pkgs/development/python-modules/tensorflow-tensorboard/default.nix @@ -14,7 +14,7 @@ buildPythonPackage rec { pname = "tensorflow-tensorboard"; - version = "1.14.0"; + version = "1.15.0"; format = "wheel"; src = fetchPypi ({ @@ -23,10 +23,10 @@ buildPythonPackage rec { format = "wheel"; } // (if isPy3k then { python = "py3"; - sha256 = "1z631614jk5zgasgmwfr33gz8bwv11p9f5llzlwvx3a8rnyv3q2h"; + sha256 = "1g62i3nrgp8q9wfsyqqjkkfnsz7x2k018c26kdh527h1yrjjrbac"; } else { python = "py2"; - sha256 = "1clv29yy942l3mfar2z6wkkk6l18fz7j6mi2dfz24j9dln0scny3"; + sha256 = "0l3zc8j2sh7h1z4qpy8kfvclv3kzndri55p10i42q6xahs9phav1"; })); propagatedBuildInputs = [ diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index e180c9d70e9..045d610effd 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -5,12 +5,18 @@ # Python libraries , numpy, tensorflow-tensorboard, backports_weakref, mock, enum34, absl-py , future, setuptools, wheel, keras-preprocessing, keras-applications, google-pasta +, functools32 +, opt-einsum , termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator # Common deps , git, swig, which, binutils, glibcLocales, cython # Common libraries , jemalloc, openmpi, astor, gast, grpc, sqlite, openssl, jsoncpp, re2 , curl, snappy, flatbuffers, icu, double-conversion, libpng, libjpeg, giflib +# Upsteam by default includes cuda support since tensorflow 1.15. We could do +# that in nix as well. It would make some things easier and less confusing, but +# it would also make the default tensorflow package unfree. See +# https://groups.google.com/a/tensorflow.org/forum/#!topic/developers/iRCt5m4qUz0 , cudaSupport ? false, nvidia_x11 ? null, cudatoolkit ? null, cudnn ? null, nccl ? null # XLA without CUDA is broken , xlaSupport ? cudaSupport @@ -33,7 +39,13 @@ let cudatoolkit_joined = symlinkJoin { name = "${cudatoolkit.name}-merged"; - paths = [ cudatoolkit.out cudatoolkit.lib ]; + paths = [ + cudatoolkit.lib + cudatoolkit.out + # for some reason some of the required libs are in the targets/x86_64-linux + # directory; not sure why but this works around it + "${cudatoolkit}/targets/${stdenv.system}" + ]; }; cudatoolkit_cc_joined = symlinkJoin { @@ -55,12 +67,12 @@ let tfFeature = x: if x then "1" else "0"; - version = "1.14.0"; + version = "1.15.0"; variant = if cudaSupport then "-gpu" else ""; pname = "tensorflow${variant}"; pythonEnv = python.withPackages (_: - [ # python deps needed during wheel build time + [ # python deps needed during wheel build time (not runtime, see the buildPythonPackage part for that) numpy keras-preprocessing protobuf @@ -74,6 +86,7 @@ let wheel ] ++ lib.optionals (!isPy3k) [ future + functools32 mock ]); @@ -84,7 +97,7 @@ let owner = "tensorflow"; repo = "tensorflow"; rev = "v${version}"; - sha256 = "06jvwlsm14b8rqwd8q8796r0vmn0wk64s4ps2zg0sapkmp9vvcmi"; + sha256 = "1j8vysfblkyydrr67qr3i7kvaq5ygnjlx8hw9a9pc95ac462jq7i"; }; patches = [ @@ -99,13 +112,6 @@ let url = "https://github.com/tensorflow/tensorflow/pull/29673/commits/498e35a3bfe38dd75cf1416a1a23c07c3b59e6af.patch"; sha256 = "1m2qmwv1ysqa61z6255xggwbq6mnxbig749bdvrhnch4zydxb4di"; }) - - # https://github.com/tensorflow/tensorflow/issues/29220 - (fetchpatch { - name = "bazel-0.27.patch"; - url = "https://github.com/tensorflow/tensorflow/commit/cfccbdb8c4a92dd26382419dceb4d934c2380391.patch"; - sha256 = "1l56wjia2c4685flsfkkgy471wx3c66wyv8khspv06zchj0k0liw"; - }) ]; # On update, it can be useful to steal the changes from gentoo @@ -157,7 +163,6 @@ let # "com_github_googleapis_googleapis" # "com_github_googlecloudplatform_google_cloud_cpp" "com_google_protobuf" - "com_google_protobuf_cc" "com_googlesource_code_re2" "curl" "cython" @@ -175,11 +180,11 @@ let "lmdb" "nasm" # "nsync" # not packaged in nixpkgs + "opt_einsum_archive" "org_sqlite" "pasta" "pcre" "png_archive" - "protobuf_archive" "six_archive" "snappy" "swig" @@ -266,9 +271,9 @@ let # cudaSupport causes fetch of ncclArchive, resulting in different hashes sha256 = if cudaSupport then - "196pm3ynfafqlcxah07hkvphf536hpix1ydgsynr1yg08aynlvvx" + "1rbg8w8pjf15hpvzrclsi19lhsrwdns6f8psb1wz35ay0ggdw8c0" else - "138r85n27ijzwxfwb5pcfyb79v14368jpckw0vmciz6pwf11bd9g"; + "0j9r7xgmkc5cj53m32jk8x0qkz05bcvnwjkd9g6l6wy10anx7kvq"; }; buildAttrs = { @@ -345,9 +350,10 @@ in buildPythonPackage { termcolor wrapt grpcio + opt-einsum ] ++ lib.optionals (!isPy3k) [ mock - future # FIXME + future ] ++ lib.optionals (pythonOlder "3.4") [ backports_weakref enum34 ] ++ lib.optionals withTensorboard [ From 1fbd307880425b568fa949490d60f9b987ab2458 Mon Sep 17 00:00:00 2001 From: Timo Kaufmann Date: Thu, 17 Oct 2019 14:48:28 +0200 Subject: [PATCH 2/6] python.pkgs.tensorflow: fix build with bazel 1.0 The tensorflow build was broken by the bazel 1.0 update in 73eb01bde000c34b54764f6f2b9fd1d89a54ccfa. --- .../python-modules/tensorflow/default.nix | 5 +- .../tensorflow/tf-1.15-bazel-1.0.patch | 213 ++++++++++++++++++ 2 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 pkgs/development/python-modules/tensorflow/tf-1.15-bazel-1.0.patch diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index 045d610effd..b3d9d865908 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -112,6 +112,8 @@ let url = "https://github.com/tensorflow/tensorflow/pull/29673/commits/498e35a3bfe38dd75cf1416a1a23c07c3b59e6af.patch"; sha256 = "1m2qmwv1ysqa61z6255xggwbq6mnxbig749bdvrhnch4zydxb4di"; }) + + ./tf-1.15-bazel-1.0.patch ]; # On update, it can be useful to steal the changes from gentoo @@ -258,6 +260,7 @@ let bazelFlags = [ # temporary fixes to make the build work with bazel 0.27 "--incompatible_no_support_tools_in_action_inputs=false" + "--incompatible_use_native_patch=false" ]; bazelBuildFlags = [ "--config=opt" # optimize using the flags set in the configure phase @@ -273,7 +276,7 @@ let sha256 = if cudaSupport then "1rbg8w8pjf15hpvzrclsi19lhsrwdns6f8psb1wz35ay0ggdw8c0" else - "0j9r7xgmkc5cj53m32jk8x0qkz05bcvnwjkd9g6l6wy10anx7kvq"; + "0d8wq89iz9vrzvr971mgdclxxjcjr32r7aj817h019x3pc53qnwx"; }; buildAttrs = { diff --git a/pkgs/development/python-modules/tensorflow/tf-1.15-bazel-1.0.patch b/pkgs/development/python-modules/tensorflow/tf-1.15-bazel-1.0.patch new file mode 100644 index 00000000000..4d70e99108b --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/tf-1.15-bazel-1.0.patch @@ -0,0 +1,213 @@ +diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD +index f740ba66b5..6cc9003787 100644 +--- a/tensorflow/c/BUILD ++++ b/tensorflow/c/BUILD +@@ -270,6 +270,7 @@ tf_cuda_library( + "//tensorflow/core/platform", + "@com_google_absl//absl/strings", + ], ++ alwayslink = 1, + ) + + exports_files( +diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD +index 5c42e508f7..16b421862c 100644 +--- a/tensorflow/c/eager/BUILD ++++ b/tensorflow/c/eager/BUILD +@@ -79,6 +79,7 @@ tf_cuda_library( + "//tensorflow/core/profiler/lib:profiler_session", + "//tensorflow/core:gpu_runtime", + ], ++ alwayslink = 1, + ) + + tf_cuda_library( +@@ -226,6 +227,7 @@ tf_cuda_library( + "//tensorflow/core/profiler/rpc/client:capture_profile", + "//tensorflow/core:gpu_runtime", + ], ++ alwayslink = 1, + ) + + tf_cuda_cc_test( +diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD +index 39b84922d1..b2affdd999 100644 +--- a/tensorflow/cc/saved_model/BUILD ++++ b/tensorflow/cc/saved_model/BUILD +@@ -123,6 +123,7 @@ cc_library( + "//tensorflow/core/util/tensor_bundle:naming", + # mobile not supported yet + ]), ++ alwayslink = 1, + ) + + tf_cc_test( +diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD +index c23c1f9b39..805643b217 100644 +--- a/tensorflow/core/BUILD ++++ b/tensorflow/core/BUILD +@@ -777,6 +777,7 @@ cc_library( + ":lib_proto_parsing", + ":protos_all_cc", + ], ++ alwayslink = 1, + ) + + # DEPRECATED: use platform:stringpiece instead. +@@ -2496,6 +2497,7 @@ cc_library( + "@com_google_protobuf//:protobuf", + ] + tf_protos_all_impl() + tf_protos_grappler_impl() + + tf_additional_numa_deps(), ++ alwayslink = 1, + ) + + # File compiled with extra flags to get cpu-specific acceleration. +diff --git a/tensorflow/core/lib/random/BUILD b/tensorflow/core/lib/random/BUILD +index 3bd933261b..e1e589e76d 100644 +--- a/tensorflow/core/lib/random/BUILD ++++ b/tensorflow/core/lib/random/BUILD +@@ -50,6 +50,7 @@ cc_library( + "//tensorflow/core/platform:types", + "//third_party/eigen3", + ], ++ alwayslink = 1, + ) + + filegroup( +diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl +index 5459d8d428..feba3a5686 100644 +--- a/tensorflow/core/platform/default/build_config.bzl ++++ b/tensorflow/core/platform/default/build_config.bzl +@@ -228,6 +228,7 @@ def cc_proto_library( + hdrs = gen_hdrs, + deps = cc_libs + deps, + includes = includes, ++ alwayslink = 1, + **kargs + ) + native.cc_library( +diff --git a/tensorflow/lite/java/src/test/native/BUILD b/tensorflow/lite/java/src/test/native/BUILD +index 6dcdab2aee..32bb0a8d85 100644 +--- a/tensorflow/lite/java/src/test/native/BUILD ++++ b/tensorflow/lite/java/src/test/native/BUILD +@@ -19,6 +19,7 @@ cc_library( + "//tensorflow/lite/java/jni", + "//tensorflow/lite/kernels:kernel_util", + ], ++ alwayslink = 1, + ) + + tflite_jni_binary( +diff --git a/tensorflow/lite/python/testdata/BUILD b/tensorflow/lite/python/testdata/BUILD +index 7bda81358f..ac1188d844 100644 +--- a/tensorflow/lite/python/testdata/BUILD ++++ b/tensorflow/lite/python/testdata/BUILD +@@ -60,6 +60,7 @@ cc_library( + deps = [ + "//tensorflow/lite/c:c_api_internal", + ], ++ alwayslink = 1, + ) + + cc_binary( +diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD +index 6fd9b4f273..29df3a3dff 100644 +--- a/tensorflow/python/BUILD ++++ b/tensorflow/python/BUILD +@@ -375,6 +375,7 @@ cc_library( + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + ], ++ alwayslink = 1, + ) + + cc_library( +@@ -411,6 +412,7 @@ cc_library( + "//third_party/py/numpy:headers", + "//third_party/python_runtime:headers", + ], ++ alwayslink = 1, + ) + + cc_library( +@@ -617,6 +619,7 @@ cc_library( + "//tensorflow/core:op_gen_lib", + "//tensorflow/core:protos_all_cc", + ], ++ alwayslink = 1, + ) + + py_library( +diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl +index a3956322fe..32752f59ad 100644 +--- a/tensorflow/tensorflow.bzl ++++ b/tensorflow/tensorflow.bzl +@@ -2331,6 +2331,7 @@ def tf_generate_proto_text_sources(name, srcs_relative_dir, srcs, protodeps = [] + hdrs = out_hdrs, + visibility = visibility, + deps = deps, ++ alwayslink = 1, + ) + + def tf_genrule_cmd_append_to_srcs(to_append): +diff --git a/tensorflow/tools/graph_transforms/BUILD b/tensorflow/tools/graph_transforms/BUILD +index adafe2aca1..8965316b12 100644 +--- a/tensorflow/tools/graph_transforms/BUILD ++++ b/tensorflow/tools/graph_transforms/BUILD +@@ -223,6 +223,7 @@ cc_library( + "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", + ], ++ alwayslink = 1, + ) + + # This library includes a main function, to make it easy to create other +diff --git a/third_party/icu/data/BUILD.bazel b/third_party/icu/data/BUILD.bazel +index 7db21566e4..8e18c7cc3a 100644 +--- a/third_party/icu/data/BUILD.bazel ++++ b/third_party/icu/data/BUILD.bazel +@@ -43,4 +43,5 @@ cc_library( + name = "conversion_data", + srcs = [":conversion_data.c"], + deps = ["@icu//:headers"], ++ alwayslink = 1, + ) +diff --git a/third_party/protobuf/protobuf.patch b/third_party/protobuf/protobuf.patch +index df0648563d..18fc6cdf35 100644 +--- a/third_party/protobuf/protobuf.patch ++++ b/third_party/protobuf/protobuf.patch +@@ -11,7 +11,15 @@ index 2fb26050..c2744d5b 100644 + + ################################################################################ + # Protobuf Runtime Library +-@@ -218,7 +218,7 @@ cc_library( ++@@ -209,6 +209,7 @@ cc_library( ++ copts = COPTS, ++ includes = ["src/"], ++ linkopts = LINK_OPTS, +++ alwayslink = 1, ++ visibility = ["//visibility:public"], ++ deps = [":protobuf_lite"] + PROTOBUF_DEPS, ++ ) ++@@ -219,7 +220,7 @@ cc_library( + # TODO(keveman): Remove this target once the support gets added to Bazel. + cc_library( + name = "protobuf_headers", +@@ -20,3 +28,4 @@ index 2fb26050..c2744d5b 100644 + includes = ["src/"], + visibility = ["//visibility:public"], + ) ++ +\ No newline at end of file +diff --git a/third_party/systemlibs/protobuf.bzl b/third_party/systemlibs/protobuf.bzl +index 774514f3fd..1c415b018b 100644 +--- a/third_party/systemlibs/protobuf.bzl ++++ b/third_party/systemlibs/protobuf.bzl +@@ -262,6 +262,7 @@ def cc_proto_library( + hdrs = gen_hdrs, + deps = cc_libs + deps, + includes = includes, ++ alwayslink = 1, + **kargs + ) + From 4fb7831cb4c97f0dc949004a34de72e6ec7d3ec1 Mon Sep 17 00:00:00 2001 From: Timo Kaufmann Date: Tue, 12 Nov 2019 16:28:54 +0100 Subject: [PATCH 3/6] python.pkgs.tensorflow: fix build with newer gast --- .../python-modules/tensorflow/default.nix | 8 ++++++++ .../tensorflow/lift-gast-restriction.patch | 13 +++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 pkgs/development/python-modules/tensorflow/lift-gast-restriction.patch diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index b3d9d865908..469ae6fe3d6 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -114,6 +114,14 @@ let }) ./tf-1.15-bazel-1.0.patch + + (fetchpatch { + # be compatible with gast >0.2 instead of only gast 0.2.2 + name = "gast-update.patch"; + url = "https://github.com/tensorflow/tensorflow/commit/85751ad6c7f5fd12c6c79545d96896cba92fa8b4.patch"; + sha256 = "077cpj0kzyqxzdya1dwh8df17zfzhqn7c685hx6iskvw2979zg2n"; + }) + ./lift-gast-restriction.patch ]; # On update, it can be useful to steal the changes from gentoo diff --git a/pkgs/development/python-modules/tensorflow/lift-gast-restriction.patch b/pkgs/development/python-modules/tensorflow/lift-gast-restriction.patch new file mode 100644 index 00000000000..24cc118d8f3 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/lift-gast-restriction.patch @@ -0,0 +1,13 @@ +diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py +index 992f2eae22..d9386f9b13 100644 +--- a/tensorflow/tools/pip_package/setup.py ++++ b/tensorflow/tools/pip_package/setup.py +@@ -54,7 +54,7 @@ REQUIRED_PACKAGES = [ + 'astor >= 0.6.0', + 'backports.weakref >= 1.0rc1;python_version<"3.4"', + 'enum34 >= 1.1.6;python_version<"3.4"', +- 'gast == 0.2.2', ++ 'gast >= 0.2.2', + 'google_pasta >= 0.1.6', + 'keras_applications >= 1.0.8', + 'keras_preprocessing >= 1.0.5', From 6832f1da6a48bc6fdb69f25723fc0f0365b122eb Mon Sep 17 00:00:00 2001 From: Timo Kaufmann Date: Wed, 13 Nov 2019 15:25:19 +0100 Subject: [PATCH 4/6] python.pkgs.tensorflow: add functools32 dependency Now needed at runtime with the python2 build. --- pkgs/development/python-modules/tensorflow/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index 469ae6fe3d6..de2a84a60ae 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -365,6 +365,7 @@ in buildPythonPackage { ] ++ lib.optionals (!isPy3k) [ mock future + functools32 ] ++ lib.optionals (pythonOlder "3.4") [ backports_weakref enum34 ] ++ lib.optionals withTensorboard [ From e4378d4641ebdb7bb058db22a9dfb6304527d62a Mon Sep 17 00:00:00 2001 From: Timo Kaufmann Date: Wed, 13 Nov 2019 15:25:48 +0100 Subject: [PATCH 5/6] python.pkgs.tensorflow: add more sanity checks --- .../python-modules/tensorflow/default.nix | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index de2a84a60ae..c65c11fce3e 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -384,7 +384,26 @@ in buildPythonPackage { # TODO try to run them anyway # TODO better test (files in tensorflow/tools/ci_build/builds/*test) checkPhase = '' - ${python.interpreter} -c "import tensorflow" + ${python.interpreter} < Date: Wed, 13 Nov 2019 15:27:06 +0100 Subject: [PATCH 6/6] python.pkgs.tensorflow: mark as broken for python2 --- pkgs/development/python-modules/tensorflow/default.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index c65c11fce3e..79be6a60c42 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -327,7 +327,9 @@ let license = licenses.asl20; maintainers = with maintainers; [ jyp abbradar ]; platforms = platforms.linux; - broken = !(xlaSupport -> cudaSupport); + # The py2 build fails due to some issue importing protobuf. Possibly related to the fix in + # https://github.com/akesandgren/easybuild-easyblocks/commit/1f2e517ddfd1b00a342c6abb55aef3fd93671a2b + broken = !(xlaSupport -> cudaSupport) || !isPy3k; }; };