python3Packages.tensorflow-bin: 1.14.0 -> 2.1.0

pkgs/development/python-modules/tensorflow/bin.nix

@@ -14,6 +14,9 @@
 , absl-py
 , grpcio
 , mock
+, scipy
+, wheel
+, opt-einsum
 , backports_weakref
 , tensorflow-estimator
 , tensorflow-tensorboard
@@ -55,21 +58,22 @@ in buildPythonPackage {
 
   src = let
     pyVerNoDot = lib.strings.stringAsChars (x: if x == "." then "" else x) python.pythonVersion;
-    pyver = if stdenv.isDarwin then builtins.substring 0 1 pyVerNoDot else pyVerNoDot;
     platform = if stdenv.isDarwin then "mac" else "linux";
     unit = if cudaSupport then "gpu" else "cpu";
-    key = "${platform}_py_${pyver}_${unit}";
+    key = "${platform}_py_${pyVerNoDot}_${unit}";
   in fetchurl packages.${key};
 
   propagatedBuildInputs = [
     protobuf
     numpy
+    scipy
     termcolor
     grpcio
     six
     astor
     absl-py
     gast
+    opt-einsum
     google-pasta
     wrapt
     tensorflow-estimator
@@ -79,40 +83,94 @@ in buildPythonPackage {
   ] ++ lib.optional (!isPy3k) mock
     ++ lib.optionals (pythonOlder "3.4") [ backports_weakref ];
 
-  nativeBuildInputs = lib.optional cudaSupport addOpenGLRunpath;
+  nativeBuildInputs = [ wheel ] ++ lib.optional cudaSupport addOpenGLRunpath;
 
-  # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow
+  preConfigure = ''
-  # and the propageted input tensorflow-tensorboard which causes environment collisions.
+    unset SOURCE_DATE_EPOCH
-  # another possibility would be to have tensorboard only in the buildInputs
+
-  # https://github.com/tensorflow/tensorflow/blob/v1.7.1/tensorflow/tools/pip_package/setup.py#L79
+    # Make sure that dist and the wheel file are writable.
-  postInstall = ''
+    chmod u+rwx -R ./dist
-    rm $out/bin/tensorboard
+
+    pushd dist
+
+    # Unpack the wheel file.
+    wheel unpack --dest unpacked ./*.whl
+
+    # Tensorflow has a hard dependency on gast==0.2.2, but we relax it to
+    # gast==0.3.2.
+    substituteInPlace ./unpacked/tensorflow*/tensorflow_core/tools/pip_package/setup.py --replace "gast == 0.2.2" "gast == 0.3.2"
+    substituteInPlace ./unpacked/tensorflow*/tensorflow_*.dist-info/METADATA --replace "gast (==0.2.2)" "gast (==0.3.2)"
+
+    # Pack the wheel file back up.
+    wheel pack ./unpacked/tensorflow*
+
+    popd
   '';
 
   # Note that we need to run *after* the fixup phase because the
   # libraries are loaded at runtime. If we run in preFixup then
   # patchelf --shrink-rpath will remove the cuda libraries.
-  postFixup = let
+  postFixup =
-    rpath = stdenv.lib.makeLibraryPath
+    let
-      ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit.out cudatoolkit.lib cudnn nvidia_x11 ]);
+      # rpaths we only need to add if CUDA is enabled.
-  in
+      cudapaths = lib.optionals cudaSupport [
-  lib.optionalString stdenv.isLinux ''
+        cudatoolkit.out
-    rrPath="$out/${python.sitePackages}/tensorflow/:$out/${python.sitePackages}/tensorflow/contrib/tensor_forest/:${rpath}"
+        cudatoolkit.lib
-    internalLibPath="$out/${python.sitePackages}/tensorflow/python/_pywrap_tensorflow_internal.so"
+        cudnn
-    find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do
+        nvidia_x11
-      patchelf --set-rpath "$rrPath" "$lib"
+      ];
-      ${lib.optionalString cudaSupport ''
-        addOpenGLRunpath "$lib"
-      ''}
-    done
-  '';
 
+      libpaths = [
+        stdenv.cc.cc.lib
+        zlib
+      ];
+
+      rpath = stdenv.lib.makeLibraryPath (libpaths ++ cudapaths);
+    in
+    lib.optionalString stdenv.isLinux ''
+      # This is an array containing all the directories in the tensorflow2
+      # package that contain .so files.
+      #
+      # TODO: Create this list programmatically, and remove paths that aren't
+      # actually needed.
+      rrPathArr=(
+        "$out/${python.sitePackages}/tensorflow_core/"
+        "$out/${python.sitePackages}/tensorflow_core/compiler/tf2tensorrt/"
+        "$out/${python.sitePackages}/tensorflow_core/compiler/tf2xla/ops/"
+        "$out/${python.sitePackages}/tensorflow_core/lite/experimental/microfrontend/python/ops/"
+        "$out/${python.sitePackages}/tensorflow_core/lite/python/interpreter_wrapper/"
+        "$out/${python.sitePackages}/tensorflow_core/lite/python/optimize/"
+        "$out/${python.sitePackages}/tensorflow_core/python/"
+        "$out/${python.sitePackages}/tensorflow_core/python/framework/"
+        "${rpath}"
+      )
+
+      # The the bash array into a colon-separated list of RPATHs.
+      rrPath=$(IFS=$':'; echo "''${rrPathArr[*]}")
+      echo "about to run patchelf with the following rpath: $rrPath"
+
+      find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do
+        echo "about to patchelf $lib..."
+        chmod a+rx "$lib"
+        patchelf --set-rpath "$rrPath" "$lib"
+        ${lib.optionalString cudaSupport ''
+          addOpenGLRunpath "$lib"
+        ''}
+      done
+    '';
+
+  pythonImportsCheck = [
+    "tensorflow"
+    "tensorflow.keras"
+    "tensorflow.python"
+    "tensorflow.python.framework"
+  ];
 
   meta = with stdenv.lib; {
     description = "Computation using data flow graphs for scalable machine learning";
     homepage = http://tensorflow.org;
     license = licenses.asl20;
-    maintainers = with maintainers; [ jyp abbradar ];
+    maintainers = with maintainers; [ jyp abbradar cdepillabout ];
     platforms = [ "x86_64-linux" "x86_64-darwin" ];
     # Python 2.7 build uses different string encoding.
     # See https://github.com/NixOS/nixpkgs/pull/37044#issuecomment-373452253

pkgs/development/python-modules/tensorflow/binary-hashes.nix

@@ -1,43 +1,51 @@
 {
-version = "1.14.0";
+version = "2.1.0";
-linux_py_27_cpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp27-none-linux_x86_64.whl";
-  sha256 = "0yywdrfk97dh1bxhibspg0raz70fx9lcczj6xlimqy4xb60clx7k";
-};
-linux_py_35_cpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp35-cp35m-linux_x86_64.whl";
-  sha256 = "1xvyb6xcrjhlwvrmrhn5vs9xy7g98smqmpv4i3hhpry4qyasphhj";
-};
-linux_py_36_cpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp36-cp36m-linux_x86_64.whl";
-  sha256 = "1psd9vyxz9f39dwj77nvrg373sxv3p5vdp9fnz81dpsm0b0mwl44";
-};
-linux_py_37_cpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp37-cp37m-linux_x86_64.whl";
-  sha256 = "0bg2sb1n2ag27r7ww695kg5hb0mjrw4kc5893krmixx2j71860c5";
-};
 linux_py_27_gpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp27-none-linux_x86_64.whl";
+  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.1.0-cp27-cp27mu-manylinux2010_x86_64.whl";
-  sha256 = "0y1x91gayg6pjddgl8ndcm63wfzhyv4s5khgl7ffzsgni1ivaqw5";
+  sha256 = "17lnhr7vdrls68c79n3sah5rpd0q1x2v5m84azvlyxxh2wpypfmb";
+};
+linux_py_27_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp27-cp27mu-manylinux2010_x86_64.whl";
+  sha256 = "10lz3i4pcpgqrcbjmxm0n7k1gsqlpna3kdid902j2fy060cpi93z";
 };
 linux_py_35_gpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp35-cp35m-linux_x86_64.whl";
+  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.1.0-cp35-cp35m-manylinux2010_x86_64.whl";
-  sha256 = "03piggpbz1jx8m2b95spq3jrdff4w6xx63ji07am7hyw2nsgx3mx";
+  sha256 = "09s081n08dpmflwgir3zwzfijfpmahbh2gy5fn5bv5ll86g1szsy";
+};
+linux_py_35_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp35-cp35m-manylinux2010_x86_64.whl";
+  sha256 = "1aa7v9fnvx03hqvhl3x3xcn41qy6qxw5xybg54ifjvvicp455c8l";
 };
 linux_py_36_gpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp36-cp36m-linux_x86_64.whl";
+  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl";
-  sha256 = "0ypkp8cfhharsyyikb1qgf44cfm6284km9xswzvzymjzz75vg3gd";
+  sha256 = "1dqp080ljbl9v3115vjp63ls0fimiwym6zxyanyhrlk8kwsq20zc";
+};
+linux_py_36_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl";
+  sha256 = "133z8anx7xm9rr5i9s9dwnp1wf06nr6s7q1lbs4lxpk6kn9nl480";
 };
 linux_py_37_gpu = {
-  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.14.0-cp37-cp37m-linux_x86_64.whl";
+  url = "https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.1.0-cp37-cp37m-manylinux2010_x86_64.whl";
-  sha256 = "0virp8nn2ysx4855hq29kas6fm6b3dsiybwzdxy9nnb9n2d8qlm2";
+  sha256 = "0yabl3xmcpr67w0zksqs3qc68nl9ax0vcd7w7b35nq8f65xl0ghy";
 };
-mac_py_2_cpu = {
+linux_py_37_cpu = {
-  url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py2-none-any.whl";
+  url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp37-cp37m-manylinux2010_x86_64.whl";
-  sha256 = "14f86k3pgq7z6i4s4im55zpp38f0drnm7xlclavsgcc0nxnj3z26";
+  sha256 = "04gngbngyg7p1gwx1q89my0cl8j7lq4kknqh51s2ynrix71zvsy6";
 };
-mac_py_3_cpu = {
+mac_py_27_cpu = {
-  url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.14.0-py3-none-any.whl";
+  url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.1.0-cp27-cp27m-macosx_10_9_x86_64.whl";
-  sha256 = "0f3swpcjfgqhj6h5wnx8snc0xjkx4hnkqx83fmlrwpncs8c131d3";
+  sha256 = "1mprp72w5kk0lyjm2mh4lf57827xk3wsg28c4gizwm00ydfgacg6";
+};
+mac_py_35_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.1.0-cp35-cp35m-macosx_10_6_intel.whl";
+  sha256 = "1as7brf5ai6r7v1di9646jfrbnirpk2b0d1g29mn3shavb62kw8w";
+};
+mac_py_36_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.1.0-cp36-cp36m-macosx_10_9_x86_64.whl";
+  sha256 = "1v1rw9kjrskhcq1yas4ly2yfnzf2i1pjh6qg6zixfbkpkw7sw3wc";
+};
+mac_py_37_cpu = {
+  url = "https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.1.0-cp37-cp37m-macosx_10_9_x86_64.whl";
+  sha256 = "1hh4n0d97mrq35cmmsrnlmcv9vlswsyjy368lj3pda3y9dvck3rf";
 };
 }

pkgs/development/python-modules/tensorflow/prefetcher.sh

@@ -1,33 +1,44 @@
 #!/usr/bin/env bash
 
-version=1.14.0
+version=2.1.0
+
+# List of binary wheels for Tensorflow.  The most recent versions can be found
+# on the following page:
+# https://www.tensorflow.org/install/pip?lang=python3#package-location
+url_and_key_list=(
+  "linux_py_27_gpu https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${version}-cp27-cp27mu-manylinux2010_x86_64.whl"
+  "linux_py_27_cpu https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-${version}-cp27-cp27mu-manylinux2010_x86_64.whl"
+  "linux_py_35_gpu https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${version}-cp35-cp35m-manylinux2010_x86_64.whl"
+  "linux_py_35_cpu https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-${version}-cp35-cp35m-manylinux2010_x86_64.whl"
+  "linux_py_36_gpu https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${version}-cp36-cp36m-manylinux2010_x86_64.whl"
+  "linux_py_36_cpu https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-${version}-cp36-cp36m-manylinux2010_x86_64.whl"
+  "linux_py_37_gpu https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${version}-cp37-cp37m-manylinux2010_x86_64.whl"
+  "linux_py_37_cpu https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-${version}-cp37-cp37m-manylinux2010_x86_64.whl"
+  "mac_py_27_cpu https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-${version}-cp27-cp27m-macosx_10_9_x86_64.whl"
+  "mac_py_35_cpu https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-${version}-cp35-cp35m-macosx_10_6_intel.whl"
+  "mac_py_36_cpu https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-${version}-cp36-cp36m-macosx_10_9_x86_64.whl"
+  "mac_py_37_cpu https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-${version}-cp37-cp37m-macosx_10_9_x86_64.whl"
+)
+
 hashfile=binary-hashes.nix
 rm -f $hashfile
 echo "{" >> $hashfile
 echo "version = \"$version\";" >> $hashfile
-for sys in "linux" "mac"; do
+
-    for tfpref in "cpu/tensorflow" "gpu/tensorflow_gpu"; do
+for url_and_key in "${url_and_key_list[@]}"; do
-        for pykind in "py2-none-any" "py3-none-any" "cp27-none-linux_x86_64" "cp35-cp35m-linux_x86_64" "cp36-cp36m-linux_x86_64" "cp37-cp37m-linux_x86_64"; do
+  key=$(echo "$url_and_key" | cut -d' ' -f1)
-            if [ $sys == "mac" ]; then
+  url=$(echo "$url_and_key" | cut -d' ' -f2)
-               [[ $pykind =~ py.* ]] && [[ $tfpref =~ cpu.* ]]
+
-               result=$?
+  echo "prefetching ${url}..."
-               pyver=${pykind:2:1}
+  hash=$(nix-prefetch-url $url)
-               flavour=cpu
+
-            else
+  echo "$key = {" >> $hashfile
-               [[ $pykind =~ .*linux.* ]]
+  echo "  url = \"$url\";" >> $hashfile
-               result=$?
+  echo "  sha256 = \"$hash\";" >> $hashfile
-               pyver=${pykind:2:2}
+  echo "};" >> $hashfile
-               flavour=${tfpref:0:3}
+
-            fi
+  echo
-            if [ $result == 0 ]; then
-                url=https://storage.googleapis.com/tensorflow/$sys/$tfpref-$version-$pykind.whl
-                hash=$(nix-prefetch-url $url)
-                echo "${sys}_py_${pyver}_${flavour} = {" >> $hashfile
-                echo "  url = \"$url\";" >> $hashfile
-                echo "  sha256 = \"$hash\";" >> $hashfile
-                echo "};" >> $hashfile
-            fi
-        done
-    done
 done
+
 echo "}" >> $hashfile
+echo "done."