Merge pull request #95824 from obsidiansystems/python3.tensorflow_2-update-to-2.3.0

tensorflow: update to 2.3.0
This commit is contained in:
Daniël de Kok 2020-09-30 11:37:42 +02:00 committed by GitHub
commit b751c120ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 61 additions and 70 deletions

View File

@ -1,6 +1,6 @@
{ stdenv, fetchPypi, isPy27, python, buildPythonPackage
, numpy, hdf5, cython, six, pkgconfig, unittest2, fetchpatch
, mpi4py ? null, openssh }:
, mpi4py ? null, openssh, pytest }:
assert hdf5.mpiSupport -> mpi4py != null && hdf5.mpi == mpi4py.mpi;
@ -10,20 +10,15 @@ let
mpi = hdf5.mpi;
mpiSupport = hdf5.mpiSupport;
in buildPythonPackage rec {
version = "2.9.0";
version = "2.10.0";
pname = "h5py";
disabled = isPy27;
src = fetchPypi {
inherit pname version;
sha256 = "9d41ca62daf36d6b6515ab8765e4c8c4388ee18e2a665701fef2b41563821002";
sha256 = "84412798925dc870ffd7107f045d7659e60f5d46d1c70c700375248bf6bf512d";
};
patches = [ ( fetchpatch {
# Skip a test that probes an already fixed bug in HDF5 (upstream patch)
url = "https://github.com/h5py/h5py/commit/141eafa531c6c09a06efe6a694251a1eea84908d.patch";
sha256 = "0lmdn0gznr7gadx7qkxybl945fvwk6r0cc4lg3ylpf8ril1975h8";
})];
configure_flags = "--hdf5=${hdf5}" + optionalString mpiSupport " --mpi";
postConfigure = ''
@ -36,7 +31,7 @@ in buildPythonPackage rec {
preBuild = if mpiSupport then "export CC=${mpi}/bin/mpicc" else "";
checkInputs = optional isPy27 unittest2 ++ [ openssh ];
checkInputs = optional isPy27 unittest2 ++ [ pytest openssh ];
nativeBuildInputs = [ pkgconfig ];
buildInputs = [ hdf5 cython ]
++ optional mpiSupport mpi;

View File

@ -6,13 +6,13 @@
buildPythonPackage rec {
pname = "tensorflow-estimator";
version = "2.2.0";
version = "2.3.0";
format = "wheel";
src = fetchPypi {
pname = "tensorflow_estimator";
inherit version format;
sha256 = "1hkx4k6927xn4qpwiba6wa56n0qqm7s23bymm377j9bz2bfsr7fh";
sha256 = "11n4sl9wfr00fv1i837b7a36ink86ggmlsgj7i06kcfc011h6pmp";
};
propagatedBuildInputs = [ mock numpy absl-py ];

View File

@ -1,4 +1,4 @@
{ stdenv, pkgs, bazel_0_29, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin
{ stdenv, pkgs, bazel_3, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin
, addOpenGLRunpath
# Python deps
, buildPythonPackage, isPy3k, isPy27, pythonOlder, pythonAtLeast, python
@ -6,7 +6,7 @@
, numpy, tensorflow-tensorboard_2, backports_weakref, mock, enum34, absl-py
, future, setuptools, wheel, keras-preprocessing, keras-applications, google-pasta
, functools32
, opt-einsum
, opt-einsum, astunparse, h5py
, termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator_2
# Common deps
, git, swig, which, binutils, glibcLocales, cython
@ -17,7 +17,7 @@
# that in nix as well. It would make some things easier and less confusing, but
# it would also make the default tensorflow package unfree. See
# https://groups.google.com/a/tensorflow.org/forum/#!topic/developers/iRCt5m4qUz0
, cudaSupport ? false, nvidia_x11 ? null, cudatoolkit ? null, cudnn ? null, nccl ? null
, cudaSupport ? false, cudatoolkit ? null, cudnn ? null, nccl ? null
, mklSupport ? false, mkl ? null
# XLA without CUDA is broken
, xlaSupport ? cudaSupport
@ -30,8 +30,7 @@
, Foundation, Security
}:
assert cudaSupport -> nvidia_x11 != null
&& cudatoolkit != null
assert cudaSupport -> cudatoolkit != null
&& cudnn != null;
# unsupported combination
@ -47,6 +46,7 @@ let
paths = [
cudatoolkit.lib
cudatoolkit.out
] ++ lib.optionals (lib.versionOlder cudatoolkit.version "11") [
# for some reason some of the required libs are in the targets/x86_64-linux
# directory; not sure why but this works around it
"${cudatoolkit}/targets/${stdenv.system}"
@ -72,7 +72,7 @@ let
tfFeature = x: if x then "1" else "0";
version = "2.1.0";
version = "2.3.0";
variant = if cudaSupport then "-gpu" else "";
pname = "tensorflow${variant}";
@ -97,40 +97,32 @@ let
bazel-build = buildBazelPackage {
name = "${pname}-${version}";
bazel = bazel_0_29;
bazel = bazel_3;
src = fetchFromGitHub {
owner = "tensorflow";
repo = "tensorflow";
rev = "v${version}";
sha256 = "1g79xi8yl4sjia8ysk9b7xfzrz83zy28v5dlb2wzmcf0k5pmz60p";
sha256 = "1dd5fgyiazyfy7y2iv4v42qnap51fr6dzwb26inrsj7aaas06j71";
};
patches = [
# Work around https://github.com/tensorflow/tensorflow/issues/24752
../no-saved-proto.patch
# Fixes for NixOS jsoncpp
../system-jsoncpp.patch
(fetchpatch {
name = "backport-pr-18950.patch";
url = "https://github.com/tensorflow/tensorflow/commit/73640aaec2ab0234d9fff138e3c9833695570c0a.patch";
sha256 = "1n9ypbrx36fc1kc9cz5b3p9qhg15xxhq4nz6ap3hwqba535nakfz";
})
(fetchpatch {
# Don't try to fetch things that don't exist
name = "prune-missing-deps.patch";
url = "https://github.com/tensorflow/tensorflow/commit/b39b1ed24b4814db27d2f748dc85c10730ae851d.patch";
sha256 = "1skysz53nancvw1slij6s7flar2kv3gngnsq60ff4lap88kx5s6c";
excludes = [ "tensorflow/cc/saved_model/BUILD" ];
})
./lift-gast-restriction.patch
# cuda 10.2 does not have "-bin2c-path" option anymore
# https://github.com/tensorflow/tensorflow/issues/34429
../cuda-10.2-no-bin2c-path.patch
# see https://github.com/tensorflow/tensorflow/issues/40688
(fetchpatch {
url = "https://github.com/tensorflow/tensorflow/commit/75ea0b31477d6ba9e990e296bbbd8ca4e7eebadf.patch";
sha256 = "1xp1icacig0xm0nmb05sbrf4nw4xbln9fhc308birrv8286zx7wv";
})
# see https://github.com/tensorflow/tensorflow/issues/40884
(fetchpatch {
url = "https://github.com/tensorflow/tensorflow/pull/41867/commits/65341f73d110bf173325768947343e1bb8f699fc.patch";
sha256 = "18ykkycaag1pcarz53bz6ydxjlah92j4178qn58gcayx1fy7hvh3";
})
];
# On update, it can be useful to steal the changes from gentoo
@ -147,7 +139,7 @@ let
git
# libs taken from system through the TF_SYS_LIBS mechanism
# grpc
grpc
sqlite
openssl
jsoncpp
@ -165,7 +157,6 @@ let
] ++ lib.optionals cudaSupport [
cudatoolkit
cudnn
nvidia_x11
] ++ lib.optionals mklSupport [
mkl
] ++ lib.optionals stdenv.isDarwin [
@ -182,24 +173,26 @@ let
TF_SYSTEM_LIBS = lib.concatStringsSep "," [
"absl_py"
"astor_archive"
"astunparse_archive"
"boringssl"
# Not packaged in nixpkgs
# "com_github_googleapis_googleapis"
# "com_github_googlecloudplatform_google_cloud_cpp"
"com_github_grpc_grpc"
"com_google_protobuf"
"com_googlesource_code_re2"
"curl"
"cython"
"double_conversion"
"enum34_archive"
"flatbuffers"
"functools32_archive"
"gast_archive"
# Lots of errors, requires an older version
# "grpc"
"gif"
"hwloc"
"icu"
"jpeg"
"jsoncpp_git"
"keras_applications_archive"
"libjpeg_turbo"
"lmdb"
"nasm"
# "nsync" # not packaged in nixpkgs
@ -207,12 +200,14 @@ let
"org_sqlite"
"pasta"
"pcre"
"png"
"pybind11"
"six_archive"
"snappy"
"swig"
"termcolor_archive"
"wrapt"
"zlib_archive"
"zlib"
];
INCLUDEDIR = "${includes_joined}/include";
@ -235,13 +230,16 @@ let
TF_CUDA_COMPUTE_CAPABILITIES = lib.concatStringsSep "," cudaCapabilities;
postPatch = ''
# https://github.com/tensorflow/tensorflow/issues/20919
sed -i '/androidndk/d' tensorflow/lite/kernels/internal/BUILD
# Tensorboard pulls in a bunch of dependencies, some of which may
# include security vulnerabilities. So we make it optional.
# https://github.com/tensorflow/tensorflow/issues/20280#issuecomment-400230560
sed -i '/tensorboard >=/d' tensorflow/tools/pip_package/setup.py
# numpy 1.19 added in https://github.com/tensorflow/tensorflow/commit/75ea0b31477d6ba9e990e296bbbd8ca4e7eebadf.patch
sed -i 's/numpy >= 1.16.0, < 1.19.0/numpy >= 1.16.0/' tensorflow/tools/pip_package/setup.py
# bazel 3.3 should work just as well as bazel 3.1
rm -f .bazelversion
'';
preConfigure = let
@ -272,15 +270,8 @@ let
runHook postConfigure
'';
# FIXME: Tensorflow uses dlopen() for CUDA libraries.
NIX_LDFLAGS = lib.optionalString cudaSupport "-lcudart -lcublas -lcufft -lcurand -lcusolver -lcusparse -lcudnn";
hardeningDisable = [ "format" ];
bazelFlags = [
# temporary fixes to make the build work with bazel 0.27
"--incompatible_no_support_tools_in_action_inputs=false"
];
bazelBuildFlags = [
"--config=opt" # optimize using the flags set in the configure phase
]
@ -288,15 +279,17 @@ let
bazelTarget = "//tensorflow/tools/pip_package:build_pip_package //tensorflow/tools/lib_package:libtensorflow";
removeRulesCC = false;
fetchAttrs = {
# So that checksums don't depend on these.
TF_SYSTEM_LIBS = null;
# cudaSupport causes fetch of ncclArchive, resulting in different hashes
sha256 = if cudaSupport then
"1kqk1gx5g63kb2zdj392x6mnpbrmgqghrdv597aipn7s23xzj8pd"
"0pf8128chkm6fxnhd4956n6gvijlj00mjmvry33gq3xx3bayhs9g"
else
"1plpcm2ydpajsrxdvmmpfy7l0gfdir78hap72w4k7ddm6d3rm2fv";
"0mkgss2nyk21zlj8hp24cs3dmpdnxk8qi6qq4hyc18lp82p09xwa";
};
buildAttrs = {
@ -347,7 +340,7 @@ let
in buildPythonPackage {
inherit version pname;
disabled = isPy27 || (pythonAtLeast "3.8");
disabled = isPy27;
src = bazel-build.python;
@ -377,6 +370,8 @@ in buildPythonPackage {
wrapt
grpcio
opt-einsum
astunparse
h5py
] ++ lib.optionals (!isPy3k) [
mock
future
@ -392,6 +387,8 @@ in buildPythonPackage {
postFixup = lib.optionalString cudaSupport ''
find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do
addOpenGLRunpath "$lib"
patchelf --set-rpath "${cudatoolkit}/lib:${cudatoolkit.lib}/lib:${cudnn}/lib:${nccl}/lib:$(patchelf --print-rpath "$lib")" "$lib"
done
'';

View File

@ -2,10 +2,10 @@ diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_packag
index 992f2eae22..d9386f9b13 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -54,5 +54,5 @@ REQUIRED_PACKAGES = [
'enum34 >= 1.1.6;python_version<"3.4"',
- 'gast == 0.2.2',
+ 'gast >= 0.2.2',
'google_pasta >= 0.1.6',
'keras_applications >= 1.0.8',
'keras_preprocessing >= 1.0.5',
@@ -56,5 +56,5 @@ REQUIRED_PACKAGES = [
'astunparse == 1.6.3',
- 'gast == 0.3.3',
+ 'gast >= 0.3.3',
'google_pasta >= 0.1.8',
'h5py >= 2.10.0, < 2.11.0',
'keras_preprocessing >= 1.1.1, < 1.2',

View File

@ -6932,10 +6932,9 @@ in {
tensorflow-build_2 = callPackage ../development/python-modules/tensorflow/2 {
cudaSupport = pkgs.config.cudaSupport or false;
inherit (pkgs.linuxPackages) nvidia_x11;
cudatoolkit = pkgs.cudatoolkit_10;
cudnn = pkgs.cudnn_cudatoolkit_10;
nccl = pkgs.nccl_cudatoolkit_10;
cudatoolkit = pkgs.cudatoolkit_11;
cudnn = pkgs.cudnn_cudatoolkit_11;
nccl = pkgs.nccl_cudatoolkit_11;
openssl = pkgs.openssl_1_1;
inherit (pkgs.darwin.apple_sdk.frameworks) Foundation Security;
};