Merge pull request #41259 from jyp/tf-1.7.1-bin

python.pkgs.tensorflow: wheel version

pkgs/development/compilers/cudatoolkit/default.nix

@@ -151,35 +151,35 @@ let
 
 in {
 
-  cudatoolkit6 = common {
+  cudatoolkit_6 = common {
     version = "6.0.37";
     url = "http://developer.download.nvidia.com/compute/cuda/6_0/rel/installers/cuda_6.0.37_linux_64.run";
     sha256 = "991e436c7a6c94ec67cf44204d136adfef87baa3ded270544fa211179779bc40";
     gcc = gcc48;
   };
 
-  cudatoolkit65 = common {
+  cudatoolkit_6_5 = common {
     version = "6.5.19";
     url = "http://developer.download.nvidia.com/compute/cuda/6_5/rel/installers/cuda_6.5.19_linux_64.run";
     sha256 = "1x9zdmk8z784d3d35vr2ak1l4h5v4jfjhpxfi9fl9dvjkcavqyaj";
     gcc = gcc48;
   };
 
-  cudatoolkit7 = common {
+  cudatoolkit_7 = common {
     version = "7.0.28";
     url = "http://developer.download.nvidia.com/compute/cuda/7_0/Prod/local_installers/cuda_7.0.28_linux.run";
     sha256 = "1km5hpiimx11jcazg0h3mjzk220klwahs2vfqhjavpds5ff2wafi";
     gcc = gcc49;
   };
 
-  cudatoolkit75 = common {
+  cudatoolkit_7_5 = common {
     version = "7.5.18";
     url = "http://developer.download.nvidia.com/compute/cuda/7.5/Prod/local_installers/cuda_7.5.18_linux.run";
     sha256 = "1v2ylzp34ijyhcxyh5p6i0cwawwbbdhni2l5l4qm21s1cx9ish88";
     gcc = gcc49;
   };
 
-  cudatoolkit8 = common {
+  cudatoolkit_8 = common {
     version = "8.0.61.2";
     url = "https://developer.nvidia.com/compute/cuda/8.0/Prod2/local_installers/cuda_8.0.61_375.26_linux-run";
     sha256 = "1i4xrsqbad283qffvysn88w2pmxzxbbby41lw0j1113z771akv4w";
@@ -192,7 +192,14 @@ in {
     gcc = gcc5;
   };
 
-  cudatoolkit9 = common {
+  cudatoolkit_9_0 = common {
+    version = "9.0.176.1";
+    url = "https://developer.nvidia.com/compute/cuda/9.0/Prod/local_installers/cuda_9.0.176_384.81_linux-run";
+    sha256 = "0308rmmychxfa4inb1ird9bpgfppgr9yrfg1qp0val5azqik91ln";
+    gcc = gcc6;
+  };
+
+  cudatoolkit_9 = common {
     version = "9.1.85.1";
     url = "https://developer.nvidia.com/compute/cuda/9.1/Prod/local_installers/cuda_9.1.85_387.26_linux";
     sha256 = "0lz9bwhck1ax4xf1fyb5nicb7l1kssslj518z64iirpy2qmwg5l4";

pkgs/development/libraries/science/math/cudnn/default.nix

@@ -1,4 +1,4 @@
-{ callPackage, cudatoolkit7, cudatoolkit75, cudatoolkit8, cudatoolkit9 }:
+{ callPackage, cudatoolkit_7, cudatoolkit_7_5, cudatoolkit_8, cudatoolkit_9_0, cudatoolkit_9 }:
 
 let
   generic = args: callPackage (import ./generic.nix (removeAttrs args ["cudatoolkit"])) {
@@ -8,37 +8,44 @@ let
 in
 
 {
-  cudnn_cudatoolkit7 = generic rec {
+  cudnn_cudatoolkit_7 = generic rec {
     version = "4.0";
-    cudatoolkit = cudatoolkit7;
+    cudatoolkit = cudatoolkit_7;
     srcName = "cudnn-${cudatoolkit.majorVersion}-linux-x64-v${version}-prod.tgz";
     sha256 = "0zgr6qdbc29qw6sikhrh6diwwz7150rqc8a49f2qf37j2rvyyr2f";
   };
 
-  cudnn_cudatoolkit75 = generic rec {
+  cudnn_cudatoolkit_7_5 = generic rec {
     version = "6.0";
-    cudatoolkit = cudatoolkit75;
+    cudatoolkit = cudatoolkit_7_5;
     srcName = "cudnn-${cudatoolkit.majorVersion}-linux-x64-v${version}.tgz";
     sha256 = "0b68hv8pqcvh7z8xlgm4cxr9rfbjs0yvg1xj2n5ap4az1h3lp3an";
   };
 
-  cudnn6_cudatoolkit8 = generic rec {
+  cudnn6_cudatoolkit_8 = generic rec {
     version = "6.0";
-    cudatoolkit = cudatoolkit8;
+    cudatoolkit = cudatoolkit_8;
     srcName = "cudnn-${cudatoolkit.majorVersion}-linux-x64-v${version}.tgz";
     sha256 = "173zpgrk55ri8if7s5yngsc89ajd6hz4pss4cdxlv6lcyh5122cv";
   };
 
-  cudnn_cudatoolkit8 = generic rec {
+  cudnn_cudatoolkit_8 = generic rec {
     version = "7.0.5";
-    cudatoolkit = cudatoolkit8;
+    cudatoolkit = cudatoolkit_8;
     srcName = "cudnn-${cudatoolkit.majorVersion}-linux-x64-v7.tgz";
     sha256 = "9e0b31735918fe33a79c4b3e612143d33f48f61c095a3b993023cdab46f6d66e";
   };
 
-  cudnn_cudatoolkit9 = generic rec {
+  cudnn_cudatoolkit_9_0 = generic rec {
     version = "7.0.5";
-    cudatoolkit = cudatoolkit9;
+    cudatoolkit = cudatoolkit_9_0;
+    srcName = "cudnn-${cudatoolkit.majorVersion}-linux-x64-v7.tgz";
+    sha256 = "03mbv4m5lhwnc181xz8li067pjzzhxqbxgnrfc68dffm8xj0fghs";
+  };
+
+  cudnn_cudatoolkit_9 = generic rec {
+    version = "7.0.5";
+    cudatoolkit = cudatoolkit_9;
     srcName = "cudnn-${cudatoolkit.majorVersion}-linux-x64-v7.tgz";
     sha256 = "1rfmdd2v47p83fm3sfyvik31gci0q17qs6kjng6mvcsd6akmvb8y";
   };

pkgs/development/libraries/science/math/nccl/default.nix

@@ -1,4 +1,4 @@
-{ callPackage, cudatoolkit8, cudatoolkit9 }:
+{ callPackage, cudatoolkit_8, cudatoolkit_9 }:
 
 let
   generic = args: callPackage (import ./generic.nix (removeAttrs args ["cudatoolkit"])) {
@@ -8,16 +8,16 @@ let
 in
 
 {
-  nccl_cudatoolkit8 = generic rec {
+  nccl_cudatoolkit_8 = generic rec {
     version = "2.1.4";
-    cudatoolkit = cudatoolkit8;
+    cudatoolkit = cudatoolkit_8;
     srcName = "nccl_${version}-1+cuda${cudatoolkit.majorVersion}_x86_64.txz";
     sha256 = "1lwwm8kdhna5m318yg304kl2gsz1jwhv4zv4gn8av2m57zh848zi";
   };
 
-  nccl_cudatoolkit9 = generic rec {
+  nccl_cudatoolkit_9 = generic rec {
     version = "2.1.4";
-    cudatoolkit = cudatoolkit9;
+    cudatoolkit = cudatoolkit_9;
     srcName = "nccl_${version}-1+cuda${cudatoolkit.majorVersion}_x86_64.txz";
     sha256 = "0pajmqzkacpszs63jh2hw2qqc49kj75kcf7r0ky8hdh560q8xn0p";
   };

pkgs/development/python-modules/tensorflow/bin.nix

@@ -2,7 +2,7 @@
 , lib
 , fetchurl
 , buildPythonPackage
-, isPy3k, isPy36, pythonOlder
+, isPy3k, isPy35, isPy36, pythonOlder
 , numpy
 , six
 , protobuf
@@ -12,20 +12,40 @@
 , enum34
 , tensorflow-tensorboard
 , cudaSupport ? false
+, cudatoolkit ? null
+, cudnn ? null
+, nvidia_x11 ? null
+, zlib
+, python
+, symlinkJoin
 }:
 
-# tensorflow is built from a downloaded wheel because the source
+# We keep this binary build for two reasons:
-# build doesn't yet work on Darwin.
+# - the source build doesn't work on Darwin.
+# - the source build is currently brittle and not easy to maintain
 
-buildPythonPackage rec {
+assert cudaSupport -> cudatoolkit != null
+                   && cudnn != null
+                   && nvidia_x11 != null;
+let
+  cudatoolkit_joined = symlinkJoin {
+    name = "unsplit_cudatoolkit";
+    paths = [ cudatoolkit.out
+              cudatoolkit.lib ];};
+
+in buildPythonPackage rec {
   pname = "tensorflow";
-  version = "1.5.0";
+  version = "1.7.1";
   format = "wheel";
 
-  src = fetchurl {
+  src = let
-    url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-${version}-py3-none-any.whl";
+    pyVerNoDot = lib.strings.stringAsChars (x: if x == "." then "" else x) "${python.majorVersion}";
-    sha256 = "1mapv45n9wmgcq3i3im0pv0gmhwkxw5z69nsnxb1gfxbj1mz5h9m";
+    version = if stdenv.isDarwin then builtins.substring 0 1 pyVerNoDot else pyVerNoDot;
-  };
+    platform = if stdenv.isDarwin then "mac" else "linux";
+    unit = if cudaSupport then "gpu" else "cpu";
+    key = "${platform}_py_${version}_${unit}";
+    dls = import ./tf1.7.1-hashes.nix;
+  in fetchurl dls.${key};
 
   propagatedBuildInputs = [ numpy six protobuf absl-py ]
                  ++ lib.optional (!isPy3k) mock
@@ -38,14 +58,28 @@ buildPythonPackage rec {
   # bleach) Hence we disable dependency checking for now.
   installFlags = lib.optional isPy36 "--no-dependencies";
 
+  # Note that we need to run *after* the fixup phase because the
+  # libraries are loaded at runtime. If we run in preFixup then
+  # patchelf --shrink-rpath will remove the cuda libraries.
+  postFixup = let
+    rpath = stdenv.lib.makeLibraryPath
+      ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit_joined cudnn nvidia_x11 ]);
+  in
+  lib.optionalString (stdenv.isLinux) ''
+    rrPath="$out/${python.sitePackages}/tensorflow/:${rpath}"
+    internalLibPath="$out/${python.sitePackages}/tensorflow/python/_pywrap_tensorflow_internal.so"
+    find $out -name '*.${stdenv.hostPlatform.extensions.sharedLibrary}' -exec patchelf --set-rpath "$rrPath" {} \;
+  '';
+
+
   meta = with stdenv.lib; {
     description = "Computation using data flow graphs for scalable machine learning";
     homepage = http://tensorflow.org;
     license = licenses.asl20;
     maintainers = with maintainers; [ jyp abbradar ];
-    platforms = platforms.darwin;
+    platforms = with platforms; linux ++ lib.optionals (!cudaSupport) darwin;
     # Python 2.7 build uses different string encoding.
     # See https://github.com/NixOS/nixpkgs/pull/37044#issuecomment-373452253
-    broken = cudaSupport || !isPy3k;
+    broken = stdenv.isDarwin && !isPy3k;
   };
 }

pkgs/development/python-modules/tensorflow/prefetcher.sh

@@ -0,0 +1,29 @@
+version=1.7.1
+hashfile=tf${version}-hashes.nix
+rm -f $hashfile
+echo "{" >> $hashfile
+for sys in "linux" "mac"; do
+    for tfpref in "cpu/tensorflow" "gpu/tensorflow_gpu"; do
+        for pykind in "py2-none-any" "py3-none-any" "cp27-none-linux_x86_64" "cp35-cp35m-linux_x86_64" "cp36-cp36m-linux_x86_64"; do
+            if [ $sys == "mac" ]; then
+               [[ $pykind =~ py.* ]] && [[ $tfpref =~ cpu.* ]]
+               result=$?
+               pyver=${pykind:2:1}
+               flavour=cpu
+            else
+               [[ $pykind =~ .*linux.* ]]
+               result=$?
+               pyver=${pykind:2:2}
+               flavour=${tfpref:0:3}
+            fi
+            if [ $result == 0 ]; then
+                url=https://storage.googleapis.com/tensorflow/$sys/$tfpref-$version-$pykind.whl
+                hash=$(nix-prefetch-url $url)
+                echo "${sys}_py_${pyver}_${flavour} = {" >> $hashfile
+                echo "  url = \"$url\";" >> $hashfile
+                echo "  sha256 = \"$hash\";" >> $hashfile
+                echo "};" >> $hashfile
+            fi
+        done
+    done
+done

pkgs/development/python-modules/tensorflow/tf1.7.1-hashes.nix

@@ -0,0 +1,34 @@
+{
+linux_py_27_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.1-cp27-none-linux_x86_64.whl";
+  sha256 = "0p8n5x74qmdv9g63y176xqpfdc1gawzjysn79bvk46knrks3pa2b";
+};
+linux_py_35_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.1-cp35-cp35m-linux_x86_64.whl";
+  sha256 = "050qv8fjpnw2y8da7s910jv4nsxg56d3xdpl09jim47kbwqabr5m";
+};
+linux_py_36_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.1-cp36-cp36m-linux_x86_64.whl";
+  sha256 = "00d5cij1mh64hh0zc2qfl8z2hpr3nna6lhpsc6qh4am1g7wz4ndn";
+};
+linux_py_27_gpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.1-cp27-none-linux_x86_64.whl";
+  sha256 = "0ami6nlp9cwg631a8f5rfpzpwb9ls9zxhsx61cimw46xljx3l2b5";
+};
+linux_py_35_gpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.1-cp35-cp35m-linux_x86_64.whl";
+  sha256 = "1xfc8dww52fy8g4b0j8r20q7yj2bfg20hlk9p7sk3k9z8swfw0kc";
+};
+linux_py_36_gpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.1-cp36-cp36m-linux_x86_64.whl";
+  sha256 = "1kkqx8m7h03b8l9l6dki4g4r7sgi3wbb4dp9gvk6l08n4vnlvc50";
+};
+mac_py_2_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.1-py2-none-any.whl";
+  sha256 = "1icbsvvwkkc09s6bdd43drvnhc6v6xmnqwjzipgc8rmpj1z71yz5";
+};
+mac_py_3_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.1-py3-none-any.whl";
+  sha256 = "0s5dy956jvwazqflc90v15i912zvhwsbzlf0cl8k7isq52j6g3kp";
+};
+}

pkgs/top-level/all-packages.nix

@@ -1868,23 +1868,25 @@ with pkgs;
   cron = callPackage ../tools/system/cron { };
 
   inherit (callPackages ../development/compilers/cudatoolkit { })
-    cudatoolkit6
+    cudatoolkit_6
-    cudatoolkit65
+    cudatoolkit_6_5
-    cudatoolkit7
+    cudatoolkit_7
-    cudatoolkit75
+    cudatoolkit_7_5
-    cudatoolkit8
+    cudatoolkit_8
-    cudatoolkit9;
+    cudatoolkit_9_0
+    cudatoolkit_9;
 
-  cudatoolkit = cudatoolkit9;
+  cudatoolkit = cudatoolkit_9;
 
   inherit (callPackages ../development/libraries/science/math/cudnn { })
-    cudnn_cudatoolkit7
+    cudnn_cudatoolkit_7
-    cudnn_cudatoolkit75
+    cudnn_cudatoolkit_7_5
-    cudnn6_cudatoolkit8
+    cudnn6_cudatoolkit_8
-    cudnn_cudatoolkit8
+    cudnn_cudatoolkit_8
-    cudnn_cudatoolkit9;
+    cudnn_cudatoolkit_9
+    cudnn_cudatoolkit_9_0;
 
-  cudnn = cudnn_cudatoolkit9;
+  cudnn = cudnn_cudatoolkit_9;
 
   curlFull = curl.override {
     idnSupport = true;
@@ -3933,10 +3935,10 @@ with pkgs;
   xnbd = callPackage ../tools/networking/xnbd { };
 
   inherit (callPackages ../development/libraries/science/math/nccl { })
-    nccl_cudatoolkit8
+    nccl_cudatoolkit_8
-    nccl_cudatoolkit9;
+    nccl_cudatoolkit_9;
 
-  nccl = nccl_cudatoolkit9;
+  nccl = nccl_cudatoolkit_9;
 
   ndjbdns = callPackage ../tools/networking/ndjbdns { };
 

pkgs/top-level/python-packages.nix

@@ -1207,8 +1207,8 @@ in {
   cufflinks = callPackage ../development/python-modules/cufflinks { };
 
   cupy = callPackage ../development/python-modules/cupy {
-    cudatoolkit = pkgs.cudatoolkit8;
+    cudatoolkit = pkgs.cudatoolkit_8;
-    cudnn = pkgs.cudnn6_cudatoolkit8;
+    cudnn = pkgs.cudnn6_cudatoolkit_8;
     nccl = pkgs.nccl;
   };
 
@@ -3976,7 +3976,7 @@ in {
   };
 
   pycuda = callPackage ../development/python-modules/pycuda rec {
-    cudatoolkit = pkgs.cudatoolkit75;
+    cudatoolkit = pkgs.cudatoolkit_7_5;
     inherit (pkgs.stdenv) mkDerivation;
   };
 
@@ -5608,14 +5608,14 @@ in {
     # https://github.com/pytorch/pytorch/issues/5831
     # https://devtalk.nvidia.com/default/topic/1028112
     # We should be able to remove this when CUDA 9.2 is released.
-    cudatoolkit9 = pkgs.cudatoolkit9.override {
+    cudatoolkit_9 = pkgs.cudatoolkit_9.override {
       gcc6 = pkgs.gcc5;
     };
   in callPackage ../development/python-modules/pytorch {
     cudaSupport = pkgs.config.cudaSupport or false;
-    cudatoolkit = cudatoolkit9;
+    cudatoolkit = cudatoolkit_9;
-    cudnn = pkgs.cudnn_cudatoolkit9.override {
+    cudnn = pkgs.cudnn_cudatoolkit_9.override {
-      inherit cudatoolkit9;
+      inherit cudatoolkit_9;
     };
   };
 
@@ -17595,11 +17595,11 @@ EOF
   tensorflow =
     if stdenv.isDarwin
     then callPackage ../development/python-modules/tensorflow/bin.nix { }
-    else callPackage ../development/python-modules/tensorflow rec {
+    else callPackage ../development/python-modules/tensorflow/bin.nix rec {
       cudaSupport = pkgs.config.cudaSupport or false;
       inherit (pkgs.linuxPackages) nvidia_x11;
-      cudatoolkit = pkgs.cudatoolkit9;
+      cudatoolkit = pkgs.cudatoolkit_9_0;
-      cudnn = pkgs.cudnn_cudatoolkit9;
+      cudnn = pkgs.cudnn_cudatoolkit_9_0;
     };
 
   tensorflowWithoutCuda = self.tensorflow.override {