From 82540b5e1a1cf56638f21326999d271eb85c9499 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Bernardy Date: Wed, 30 May 2018 16:38:51 +0200 Subject: [PATCH] pythonPackages.tensorflow: 1.5.0 -> 1.7.1 Re-instates binary build for all versions. --- .../python-modules/tensorflow/bin.nix | 56 +++++++++++++++---- .../python-modules/tensorflow/prefetcher.sh | 29 ++++++++++ .../tensorflow/tf1.7.1-hashes.nix | 34 +++++++++++ pkgs/top-level/python-packages.nix | 6 +- 4 files changed, 111 insertions(+), 14 deletions(-) create mode 100644 pkgs/development/python-modules/tensorflow/prefetcher.sh create mode 100644 pkgs/development/python-modules/tensorflow/tf1.7.1-hashes.nix diff --git a/pkgs/development/python-modules/tensorflow/bin.nix b/pkgs/development/python-modules/tensorflow/bin.nix index c9b89ff7616..9c6b84e8c7a 100644 --- a/pkgs/development/python-modules/tensorflow/bin.nix +++ b/pkgs/development/python-modules/tensorflow/bin.nix @@ -2,7 +2,7 @@ , lib , fetchurl , buildPythonPackage -, isPy3k, isPy36, pythonOlder +, isPy3k, isPy35, isPy36, pythonOlder , numpy , six , protobuf @@ -12,20 +12,40 @@ , enum34 , tensorflow-tensorboard , cudaSupport ? false +, cudatoolkit ? null +, cudnn ? null +, nvidia_x11 ? null +, zlib +, python +, symlinkJoin }: -# tensorflow is built from a downloaded wheel because the source -# build doesn't yet work on Darwin. +# We keep this binary build for two reasons: +# - the source build doesn't work on Darwin. +# - the source build is currently brittle and not easy to maintain -buildPythonPackage rec { +assert cudaSupport -> cudatoolkit != null + && cudnn != null + && nvidia_x11 != null; +let + cudatoolkit_joined = symlinkJoin { + name = "unsplit_cudatoolkit"; + paths = [ cudatoolkit.out + cudatoolkit.lib ];}; + +in buildPythonPackage rec { pname = "tensorflow"; - version = "1.5.0"; + version = "1.7.1"; format = "wheel"; - src = fetchurl { - url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-${version}-py3-none-any.whl"; - sha256 = "1mapv45n9wmgcq3i3im0pv0gmhwkxw5z69nsnxb1gfxbj1mz5h9m"; - }; + src = let + pyVerNoDot = lib.strings.stringAsChars (x: if x == "." then "" else x) "${python.majorVersion}"; + version = if stdenv.isDarwin then builtins.substring 0 1 pyVerNoDot else pyVerNoDot; + platform = if stdenv.isDarwin then "mac" else "linux"; + unit = if cudaSupport then "gpu" else "cpu"; + key = "${platform}_py_${version}_${unit}"; + dls = import ./tf1.7.1-hashes.nix; + in fetchurl dls.${key}; propagatedBuildInputs = [ numpy six protobuf absl-py ] ++ lib.optional (!isPy3k) mock @@ -38,14 +58,28 @@ buildPythonPackage rec { # bleach) Hence we disable dependency checking for now. installFlags = lib.optional isPy36 "--no-dependencies"; + # Note that we need to run *after* the fixup phase because the + # libraries are loaded at runtime. If we run in preFixup then + # patchelf --shrink-rpath will remove the cuda libraries. + postFixup = let + rpath = stdenv.lib.makeLibraryPath + ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit_joined cudnn nvidia_x11 ]); + in + lib.optionalString (stdenv.isLinux) '' + rrPath="$out/${python.sitePackages}/tensorflow/:${rpath}" + internalLibPath="$out/${python.sitePackages}/tensorflow/python/_pywrap_tensorflow_internal.so" + find $out -name '*.${stdenv.hostPlatform.extensions.sharedLibrary}' -exec patchelf --set-rpath "$rrPath" {} \; + ''; + + meta = with stdenv.lib; { description = "Computation using data flow graphs for scalable machine learning"; homepage = http://tensorflow.org; license = licenses.asl20; maintainers = with maintainers; [ jyp abbradar ]; - platforms = platforms.darwin; + platforms = with platforms; linux ++ lib.optionals (!cudaSupport) darwin; # Python 2.7 build uses different string encoding. # See https://github.com/NixOS/nixpkgs/pull/37044#issuecomment-373452253 - broken = cudaSupport || !isPy3k; + broken = stdenv.isDarwin && !isPy3k; }; } diff --git a/pkgs/development/python-modules/tensorflow/prefetcher.sh b/pkgs/development/python-modules/tensorflow/prefetcher.sh new file mode 100644 index 00000000000..ba9aa0bdf35 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/prefetcher.sh @@ -0,0 +1,29 @@ +version=1.7.1 +hashfile=tf${version}-hashes.nix +rm -f $hashfile +echo "{" >> $hashfile +for sys in "linux" "mac"; do + for tfpref in "cpu/tensorflow" "gpu/tensorflow_gpu"; do + for pykind in "py2-none-any" "py3-none-any" "cp27-none-linux_x86_64" "cp35-cp35m-linux_x86_64" "cp36-cp36m-linux_x86_64"; do + if [ $sys == "mac" ]; then + [[ $pykind =~ py.* ]] && [[ $tfpref =~ cpu.* ]] + result=$? + pyver=${pykind:2:1} + flavour=cpu + else + [[ $pykind =~ .*linux.* ]] + result=$? + pyver=${pykind:2:2} + flavour=${tfpref:0:3} + fi + if [ $result == 0 ]; then + url=https://storage.googleapis.com/tensorflow/$sys/$tfpref-$version-$pykind.whl + hash=$(nix-prefetch-url $url) + echo "${sys}_py_${pyver}_${flavour} = {" >> $hashfile + echo " url = \"$url\";" >> $hashfile + echo " sha256 = \"$hash\";" >> $hashfile + echo "};" >> $hashfile + fi + done + done +done diff --git a/pkgs/development/python-modules/tensorflow/tf1.7.1-hashes.nix b/pkgs/development/python-modules/tensorflow/tf1.7.1-hashes.nix new file mode 100644 index 00000000000..c9ed7c18264 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow/tf1.7.1-hashes.nix @@ -0,0 +1,34 @@ +{ +linux_py_27_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.1-cp27-none-linux_x86_64.whl"; + sha256 = "0p8n5x74qmdv9g63y176xqpfdc1gawzjysn79bvk46knrks3pa2b"; +}; +linux_py_35_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.1-cp35-cp35m-linux_x86_64.whl"; + sha256 = "050qv8fjpnw2y8da7s910jv4nsxg56d3xdpl09jim47kbwqabr5m"; +}; +linux_py_36_cpu = { + url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.1-cp36-cp36m-linux_x86_64.whl"; + sha256 = "00d5cij1mh64hh0zc2qfl8z2hpr3nna6lhpsc6qh4am1g7wz4ndn"; +}; +linux_py_27_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.1-cp27-none-linux_x86_64.whl"; + sha256 = "0ami6nlp9cwg631a8f5rfpzpwb9ls9zxhsx61cimw46xljx3l2b5"; +}; +linux_py_35_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.1-cp35-cp35m-linux_x86_64.whl"; + sha256 = "1xfc8dww52fy8g4b0j8r20q7yj2bfg20hlk9p7sk3k9z8swfw0kc"; +}; +linux_py_36_gpu = { + url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.1-cp36-cp36m-linux_x86_64.whl"; + sha256 = "1kkqx8m7h03b8l9l6dki4g4r7sgi3wbb4dp9gvk6l08n4vnlvc50"; +}; +mac_py_2_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.1-py2-none-any.whl"; + sha256 = "1icbsvvwkkc09s6bdd43drvnhc6v6xmnqwjzipgc8rmpj1z71yz5"; +}; +mac_py_3_cpu = { + url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.1-py3-none-any.whl"; + sha256 = "0s5dy956jvwazqflc90v15i912zvhwsbzlf0cl8k7isq52j6g3kp"; +}; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 6bf9f27291a..b6c383bfc99 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -17580,11 +17580,11 @@ EOF tensorflow = if stdenv.isDarwin then callPackage ../development/python-modules/tensorflow/bin.nix { } - else callPackage ../development/python-modules/tensorflow rec { + else callPackage ../development/python-modules/tensorflow/bin.nix rec { cudaSupport = pkgs.config.cudaSupport or false; inherit (pkgs.linuxPackages) nvidia_x11; - cudatoolkit = pkgs.cudatoolkit_9; - cudnn = pkgs.cudnn_cudatoolkit_9; + cudatoolkit = pkgs.cudatoolkit_9_0; + cudnn = pkgs.cudnn_cudatoolkit_9_0; }; tensorflowWithoutCuda = self.tensorflow.override {