Merge remote-tracking branch 'upstream/master' into openssl-1.1

pkgs/development/python-modules/configshell/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, fetchFromGitHub, buildPythonPackage, pyparsing, six }:
+{ stdenv, fetchFromGitHub, buildPythonPackage, pyparsing, six, urwid }:
 
 buildPythonPackage rec {
   pname = "configshell";
@@ -11,7 +11,7 @@ buildPythonPackage rec {
     sha256 = "0zpr2n4105qqsklyfyr9lzl1rhxjcv0mnsl57hgk0m763w6na90h";
   };
 
-  propagatedBuildInputs = [ pyparsing six ];
+  propagatedBuildInputs = [ pyparsing six urwid ];
 
   meta = with stdenv.lib; {
     description = "A Python library for building configuration shells";

pkgs/development/python-modules/datatable/default.nix

@@ -1,11 +1,16 @@
-{ lib
+{ blessed
 , buildPythonPackage
 , fetchPypi
-, pythonOlder
+, lib
+, libcxx
+, libcxxabi
 , llvm
-, typesentry
-, blessed
+, openmp
 , pytest
+, pythonOlder
+, stdenv
+, substituteAll
+, typesentry
 }:
 
 buildPythonPackage rec {
@@ -17,10 +22,25 @@ buildPythonPackage rec {
     sha256 = "1s8z81zffrckvdwrrl0pkjc7gsdvjxw59xgg6ck81dl7gkh5grjk";
   };
 
+  patches = [
+    # Disable the compiler monkey patching, and remove the task that's copying
+    # the native dependencies to the build directory.
+    ./remove-compiler-monkeypatch_disable-native-relocation.patch
+  ] ++ lib.optionals stdenv.isDarwin [
+    # Replace the library auto-detection with hardcoded paths.
+    (substituteAll {
+      src = ./hardcode-library-paths.patch;
+
+      libomp_dylib = "${lib.getLib openmp}/lib/libomp.dylib";
+      libcxx_dylib = "${lib.getLib libcxx}/lib/libc++.1.dylib";
+      libcxxabi_dylib = "${lib.getLib libcxxabi}/lib/libc++abi.dylib";
+    })
+  ];
+
   disabled = pythonOlder "3.5";
 
   propagatedBuildInputs = [ typesentry blessed ];
-  buildInputs = [ llvm ];
+  buildInputs = [ llvm ] ++ lib.optionals stdenv.isDarwin [ openmp ];
   checkInputs = [ pytest ];
 
   LLVM = llvm;

pkgs/development/python-modules/datatable/hardcode-library-paths.patch

@@ -0,0 +1,43 @@
+diff --git a/ci/setup_utils.py b/ci/setup_utils.py
+index 66b385a..6255af0 100644
+--- a/ci/setup_utils.py
++++ b/ci/setup_utils.py
+@@ -600,37 +600,7 @@ def find_linked_dynamic_libraries():
+     them as a list of absolute paths.
+     """
+     with TaskContext("Find the required dynamic libraries") as log:
+-        llvm = get_llvm()
+-        libs = required_link_libraries()
+-        resolved = []
+-        for libname in libs:
+-            if llvm:
+-                fullpath = os.path.join(llvm, "lib", libname)
+-                if os.path.isfile(fullpath):
+-                    resolved.append(fullpath)
+-                    log.info("Library `%s` found at %s" % (libname, fullpath))
+-                    continue
+-                else:
+-                    log.info("%s does not exist" % fullpath)
+-            # Rely on the shell `locate` command to find the dynamic libraries.
+-            proc = subprocess.Popen(["locate", libname], stdout=subprocess.PIPE,
+-                                    stderr=subprocess.PIPE)
+-            stdout, stderr = proc.communicate()
+-            if proc.returncode == 0:
+-                results = stdout.decode().strip().split("\n")
+-                results = [r for r in results if r]
+-                if results:
+-                    results.sort(key=len)
+-                    fullpath = results[0]
+-                    assert os.path.isfile(fullpath), "Invalid path: %r" % (fullpath,)
+-                    resolved.append(fullpath)
+-                    log.info("Library `%s` found at %s" % (libname, fullpath))
+-                    continue
+-                else:
+-                    log.fatal("Cannot locate dynamic library `%s`" % libname)
+-            else:
+-                log.fatal("`locate` command returned the following error:\n%s"
+-                          % stderr.decode())
++        resolved = ["@libomp_dylib@", "@libcxx_dylib@", "@libcxxabi_dylib@"]
+         return resolved
+ 
+ 

pkgs/development/python-modules/datatable/remove-compiler-monkeypatch_disable-native-relocation.patch

@@ -0,0 +1,28 @@
+diff --git a/setup.py b/setup.py
+index 58fc875..8032561 100644
+--- a/setup.py
++++ b/setup.py
+@@ -141,23 +141,6 @@ if cmd in ("build", "bdist_wheel", "build_ext", "install"):
+     extra_link_args = get_extra_link_args()
+     cpp_files = get_c_sources("c")
+ 
+-    with TaskContext("Copy dynamic libraries") as log:
+-        # Copy system libraries into the datatable/lib folder, so that they can
+-        # be packaged with the wheel
+-        libs = find_linked_dynamic_libraries()
+-        for libpath in libs:
+-            trgfile = os.path.join("datatable", "lib",
+-                                   os.path.basename(libpath))
+-            if os.path.exists(trgfile):
+-                log.info("File %s already exists, skipped" % trgfile)
+-            else:
+-                log.info("Copying %s to %s" % (libpath, trgfile))
+-                shutil.copy(libpath, trgfile)
+-
+-    if ismacos():
+-        monkey_patch_compiler()
+-
+-
+ # Create the git version file
+ if cmd in ("build", "sdist", "bdist_wheel", "install"):
+     make_git_version_file(True)

pkgs/development/python-modules/diofant/default.nix

@@ -0,0 +1,44 @@
+{ lib
+, isPy3k
+, buildPythonPackage
+, fetchPypi
+, pytestrunner
+, setuptools_scm
+, isort
+, mpmath
+, strategies
+}:
+
+buildPythonPackage rec {
+  pname = "diofant";
+  version = "0.10.0";
+
+  src = fetchPypi {
+    inherit version;
+    pname = "Diofant";
+    sha256 = "0qjg0mmz2cqxryr610mppx3virf1gslzrsk24304502588z53v8w";
+  };
+
+  nativeBuildInputs = [
+    isort
+    pytestrunner
+    setuptools_scm
+  ];
+
+  propagatedBuildInputs = [
+    mpmath
+    strategies
+  ];
+
+  # tests take ~1h
+  doCheck = false;
+
+  disabled = !isPy3k;
+
+  meta = with lib; {
+    description = "A Python CAS library";
+    homepage = "https://diofant.readthedocs.io/";
+    license = licenses.bsd3;
+    maintainers = with maintainers; [ suhr ];
+  };
+}

pkgs/development/python-modules/dm-sonnet/default.nix

@@ -36,7 +36,7 @@ let
     bazelTarget = ":install";
 
     fetchAttrs = {
-      sha256 = "141k2caj5hlk31gnqryp6w2mzyz4i5s1rf8f3qr18isxy235k3w6";
+      sha256 = "0mxma7jajm42v1hv6agl909xra0azihj588032ivhlmmh403x6wg";
     };
 
     bazelFlags = [

pkgs/development/python-modules/fipy/default.nix

@@ -0,0 +1,55 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, numpy
+, scipy
+, pyamg
+, pysparse
+, future
+, matplotlib
+, tkinter
+, mpi4py
+, scikit-fmm
+, isPy27
+, gmsh
+, python
+, stdenv
+}:
+
+let
+  not_darwin_inputs = lib.optionals (! stdenv.isDarwin) [ gmsh ];
+in
+  buildPythonPackage rec {
+    pname = "fipy";
+    version = "3.3";
+
+    src = fetchPypi {
+      pname = "FiPy";
+      inherit version;
+      sha256 = "11agpg3d6yrns8igkpml1mxy3mkqkjq2yrw1mw12y07dkk12ii19";
+    };
+
+    propagatedBuildInputs = [
+      numpy
+      scipy
+      pyamg
+      matplotlib
+      tkinter
+      mpi4py
+      future
+      scikit-fmm
+    ] ++ lib.optionals isPy27 [ pysparse ] ++ not_darwin_inputs;
+
+    checkInputs = not_darwin_inputs;
+
+    checkPhase = ''
+      ${python.interpreter} setup.py test --modules
+    '';
+
+    meta = with lib; {
+      homepage = https://www.ctcms.nist.gov/fipy/;
+      description = "A Finite Volume PDE Solver Using Python";
+      license = licenses.free;
+      maintainers = with maintainers; [ costrouc wd15 ];
+    };
+  }

pkgs/development/python-modules/imbalanced-learn/0.4.nix

@@ -0,0 +1,29 @@
+{ stdenv, buildPythonPackage, fetchPypi, scikitlearn, pandas, nose, pytest }:
+
+buildPythonPackage rec {
+  pname = "imbalanced-learn";
+  version = "0.4.3";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "5bd9e86e40ce4001a57426541d7c79b18143cbd181e3330c1a3e5c5c43287083";
+  };
+
+  propagatedBuildInputs = [ scikitlearn ];
+  checkInputs = [ nose pytest pandas ];
+  checkPhase = ''
+    export HOME=$PWD
+    # skip some tests that fail because of minimal rounding errors
+    # or large dependencies
+    py.test imblearn -k 'not classification \
+                         and not _generator \
+                         and not _forest \
+                         and not wrong_memory'
+  '';
+
+  meta = with stdenv.lib; {
+    description = "Library offering a number of re-sampling techniques commonly used in datasets showing strong between-class imbalance";
+    homepage = https://github.com/scikit-learn-contrib/imbalanced-learn;
+    license = licenses.mit;
+  };
+}

pkgs/development/python-modules/imbalanced-learn/default.nix

@@ -1,21 +1,30 @@
-{ stdenv, buildPythonPackage, fetchPypi, scikitlearn, pandas, nose, pytest }:
+{ stdenv, buildPythonPackage, fetchPypi, isPy27
+, nose
+, pandas
+, pytest
+, scikitlearn
+, tensorflow
+}:
 
 buildPythonPackage rec {
   pname = "imbalanced-learn";
-  version = "0.4.3";
+  version = "0.5.0";
+  disabled = isPy27; # scikit-learn>=0.21 doesn't work on python2
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "5bd9e86e40ce4001a57426541d7c79b18143cbd181e3330c1a3e5c5c43287083";
+    sha256 = "1m8r055mvkws0s449s1dyrkgricls6basnszwbwqwrw6g19n1xsx";
   };
 
   propagatedBuildInputs = [ scikitlearn ];
   checkInputs = [ nose pytest pandas ];
   checkPhase = ''
-    export HOME=$PWD
+    export HOME=$TMPDIR
     # skip some tests that fail because of minimal rounding errors
-    py.test imblearn --ignore=imblearn/metrics/classification.py
-    py.test doc/*.rst
+    # or very large dependencies (keras + tensorflow)
+    py.test imblearn -k 'not estimator \
+                         and not classification \
+                         and not _generator'
   '';
 
   meta = with stdenv.lib; {

pkgs/development/python-modules/mysql-connector/default.nix

@@ -4,13 +4,13 @@
 
 buildPythonPackage rec {
   pname = "mysql-connector";
-  version = "8.0.16";
+  version = "8.0.17";
 
   src = fetchFromGitHub {
     owner = "mysql";
     repo = "mysql-connector-python";
     rev = version;
-    sha256 = "0yl3fkyws24lc2qrbvn42bqy72aqy8q5v8f0j5zy3mkh9a7wlxdp";
+    sha256 = "1by0g7hrbmb1wj2wh3q9y92mjimck2izh1i4fm1xfbp278p2acbd";
   };
 
   propagatedBuildInputs = [ protobuf ];

pkgs/development/python-modules/openapi-spec-validator/default.nix

@@ -0,0 +1,25 @@
+{ lib, buildPythonPackage, isPy27, fetchPypi
+, jsonschema, pyyaml, six, pathlib
+, mock, pytest, pytestcov, pytest-flake8, tox }:
+
+buildPythonPackage rec {
+  pname = "openapi-spec-validator";
+  version = "0.2.7";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1sz9ls6a7h056nc5q76w4xl43sr1h9in2f23qwkxazcazr3zpi3p";
+  };
+
+  propagatedBuildInputs = [ jsonschema pyyaml six ]
+    ++ (lib.optionals (isPy27) [ pathlib ]);
+
+  checkInputs = [ mock pytest pytestcov pytest-flake8 tox ];
+
+  meta = with lib; {
+    homepage = https://github.com/p1c2u/openapi-spec-validator;
+    description = "Validates OpenAPI Specs against the OpenAPI 2.0 (aka Swagger) and OpenAPI 3.0.0 specification";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ rvl ];
+  };
+}

pkgs/development/python-modules/pyamg/default.nix

@@ -0,0 +1,36 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, numpy
+, scipy
+, pytest
+, pybind11
+}:
+
+buildPythonPackage rec {
+  pname = "pyamg";
+  version = "4.0.0";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "3ceb38ffd86e29774e759486f2961599c8ed847459c68727493cadeaf115a38a";
+  };
+
+  propagatedBuildInputs = [
+    numpy
+    scipy
+    pytest
+    pybind11
+  ];
+
+  preBuild = ''
+    export HOME=$(mktemp -d)
+  '';
+
+  meta = with lib; {
+    description = "Algebraic Multigrid Solvers in Python";
+    homepage = https://github.com/pyamg/pyamg;
+    license = licenses.mit;
+    maintainers = [ maintainers.costrouc ];
+  };
+}

pkgs/development/python-modules/pyproj/default.nix

@@ -1,4 +1,4 @@
-{ lib, buildPythonPackage, fetchPypi, python, pkgs, pythonOlder, substituteAll
+{ lib, buildPythonPackage, fetchFromGitHub, python, pkgs, pythonOlder, substituteAll
 , aenum
 , cython
 , pytest
@@ -8,11 +8,13 @@
 
 buildPythonPackage rec {
   pname = "pyproj";
-  version = "2.2.1";
+  version = "2.2.2";
 
-  src = fetchPypi {
-    inherit pname version;
-    sha256 = "0yigcxwmx5cczipf2mpmy2gq1dnl0635yjvjq86ay47j1j5fd2gc";
+  src = fetchFromGitHub {
+    owner = "pyproj4";
+    repo = "pyproj";
+    rev = "v${version}rel";
+    sha256 = "0mb0jczgqh3sma69k7237i38h09gxgmvmddls9hpw4f3131f5ax7";
   };
 
   # force pyproj to use ${pkgs.proj}

pkgs/development/python-modules/pysparse/default.nix

@@ -0,0 +1,49 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, numpy
+, setuptools
+, liblapack
+, isPy27
+, python
+}:
+
+buildPythonPackage rec {
+  pname = "pysparse";
+  version = "1.3-dev";
+  disabled = !isPy27;
+
+  src = fetchFromGitHub {
+    owner = "PythonOptimizers";
+    repo = "pysparse";
+    rev = "f8430bd99ac2a6209c462657c5792d10033888cc";
+    sha256 = "19xcq8214yndra1xjhna3qjm32wprsqck97dlnw3xcww7rfy6hqh";
+  };
+
+  hardeningDisable = [ "all" ];
+
+  propagatedBuildInputs = [
+    numpy
+    numpy.blas
+    liblapack
+  ];
+
+  # Include patches from working version of PySparse 1.3-dev in
+  # Conda-Forge,
+  # https://github.com/conda-forge/pysparse-feedstock/tree/b69266911a2/recipe
+  # Thanks to https://github.com/guyer
+  patches = [ ./dropPackageLoader.patch ];
+
+  checkPhase = ''
+    cd test
+    ${python.interpreter} -c "import pysparse"
+    ${python.interpreter} test_sparray.py
+  '';
+
+  meta = with lib; {
+    homepage = https://github.com/PythonOptimizers/pysparse;
+    description = "A Sparse Matrix Library for Python";
+    license = licenses.bsd3;
+    maintainers = with maintainers; [ costrouc ];
+  };
+}

pkgs/development/python-modules/pysparse/dropPackageLoader.patch

@@ -0,0 +1,88 @@
+diff --git a/pysparse/__init__.py b/pysparse/__init__.py
+index 6d09b00..ff39084 100644
+--- a/pysparse/__init__.py
++++ b/pysparse/__init__.py
+@@ -1,9 +1,42 @@
+-"PySparse: A Fast Sparse Matrix Library for Python"
++"""
++PySparse: A Fast Sparse Matrix Library for Python
++=================================================
++
++Documentation is available in the docstrings and
++online at http://pysparse.sourceforge.net/.
++
++Contents
++--------
++Pysparse imports
++::
++ spmatrix                     --- sparse matrix types
++
++and, in addition, provides:
++    
++Subpackages
++-----------
++Using any of these subpackages requires an explicit import.  For example,
++``import pysparse.itsolvers``.
++
++::
++
++ itsolvers                    --- Iterative linear algebra solvers
++ precon                       --- Preconditioners
++ direct                       --- Direct solvers
++ direct.superlu               --- Wrappers to SuperLU library
++ direct.umfpack               --- Wrappers to UMFPACK library
++ eigen.jdsym                  --- Jacobi davidson eigenvalue solver for symmetric matrices
++ 
++Utility tools
++-------------
++::
++
++ __version__       --- pysparse version string
++"""
++
+ 
+ __docformat__ = 'restructuredtext'
+ 
+-# Imports
+-from numpy._import_tools import PackageLoader
+ try:
+     from version import version as __version__
+ except ImportError:
+@@ -11,31 +44,6 @@ except ImportError:
+     __version__ = 'undefined'
+     
+ from sparse import spmatrix
+-#from sparse import *
+-from misc import get_include
+-
+-pkgload = PackageLoader()
+-pkgload(verbose=False,postpone=True)
+-
+-if __doc__:
+-    __doc__ += """
+-
+-Available subpackages
+----------------------
+-"""
+-if __doc__:
+-    __doc__ += pkgload.get_pkgdocs()
+-
+-__all__ = filter(lambda s: not s.startswith('_'), dir())
+-__all__ += '__version__'
+-
+-__doc__ += """
+-
+-Miscellaneous
+--------------
+-
+-    __version__  :  pysparse version string
+-"""
+ 
+ from pysparse.misc import Deprecated
+ 
+@@ -47,3 +55,5 @@ class _superlu:
+         return self.factorizeFnc(*args, **kwargs)
+     
+ superlu = _superlu()
++
++__all__ = ['spmatrix', 'superlu', '__version__']

pkgs/development/python-modules/scikit-fmm/default.nix

@@ -0,0 +1,32 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, numpy
+, python
+}:
+
+buildPythonPackage rec {
+  pname = "scikit-fmm";
+  version = "2019.1.30";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "eb64b6d8e30b8df8f8636d5fc4fd7ca6a9b05938ccd62518c80c1d9e823069dd";
+  };
+
+  propagatedBuildInputs = [
+    numpy
+  ];
+
+  checkPhase = ''
+    mkdir testdir; cd testdir
+    ${python.interpreter} -c "import skfmm, sys; sys.exit(skfmm.test())"
+  '';
+
+  meta = with lib; {
+    description = "A Python extension module which implements the fast marching method";
+    homepage = https://github.com/scikit-fmm/scikit-fmm;
+    license = licenses.bsd3;
+    maintainers = with maintainers; [ costrouc ];
+  };
+}

pkgs/development/python-modules/shamir-mnemonic/default.nix

@@ -0,0 +1,22 @@
+{ lib, fetchPypi, buildPythonPackage, isPy3k, click, colorama }:
+
+buildPythonPackage rec {
+  pname = "shamir-mnemonic";
+  version = "0.1.0";
+
+  disabled = !isPy3k;
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1cc08d276e05b13cd32bd3b0c5d1cb6c30254e0086e0f6857ec106d4cceff121";
+  };
+
+  propagatedBuildInputs = [ click colorama ];
+
+  meta = {
+    description = "Reference implementation of SLIP-0039";
+    homepage = "https://github.com/trezor/python-shamir-mnemonic";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ "1000101" ];
+  };
+}

pkgs/development/python-modules/spacy/default.nix

@@ -25,18 +25,16 @@
 
 buildPythonPackage rec {
   pname = "spacy";
-  version = "2.1.6";
+  version = "2.1.7";
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "1s0a0vir9lg5q8n832kkadbajb4i4zl20zmdg3g20qlp4mcbn25p";
+    sha256 = "0k4kh9jnpdawaqjxwcdi2h01s85s5r338ajgv9kkq59iha4hichh";
   };
 
   prePatch = ''
     substituteInPlace setup.py \
-      --replace "plac<1.0.0,>=0.9.6" "plac>=0.9.6" \
-      --replace "regex==" "regex>=" \
-      --replace "wheel>=0.32.0,<0.33.0" "wheel>=0.32.0"
+      --replace "plac<1.0.0,>=0.9.6" "plac>=0.9.6"
   '';
 
   propagatedBuildInputs = [

pkgs/development/python-modules/tensorflow/bin.nix

@@ -35,15 +35,19 @@
 assert cudaSupport -> cudatoolkit != null
                    && cudnn != null
                    && nvidia_x11 != null;
+
+# unsupported combination
+assert ! (stdenv.isDarwin && cudaSupport);
+
 let
-  cudatoolkit_joined = symlinkJoin {
-    name = "unsplit_cudatoolkit";
-    paths = [ cudatoolkit.out
-              cudatoolkit.lib ];};
+  packages = import ./binary-hashes.nix;
+
+  variant = if cudaSupport then "-gpu" else "";
+  pname = "tensorflow${variant}";
 
 in buildPythonPackage rec {
-  pname = "tensorflow";
-  version = "1.14.0";
+  inherit pname;
+  inherit (packages) version;
   format = "wheel";
 
   src = let
@@ -52,8 +56,7 @@ in buildPythonPackage rec {
     platform = if stdenv.isDarwin then "mac" else "linux";
     unit = if cudaSupport then "gpu" else "cpu";
     key = "${platform}_py_${pyver}_${unit}";
-    dls = import (./. + "/tf${version}-hashes.nix");
-  in fetchurl dls.${key};
+  in fetchurl packages.${key};
 
   propagatedBuildInputs = [
     protobuf
@@ -86,12 +89,12 @@ in buildPythonPackage rec {
   # patchelf --shrink-rpath will remove the cuda libraries.
   postFixup = let
     rpath = stdenv.lib.makeLibraryPath
-      ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit_joined cudnn nvidia_x11 ]);
+      ([ stdenv.cc.cc.lib zlib ] ++ lib.optionals cudaSupport [ cudatoolkit.out cudatoolkit.lib cudnn nvidia_x11 ]);
   in
-  lib.optionalString (stdenv.isLinux) ''
+  lib.optionalString stdenv.isLinux ''
     rrPath="$out/${python.sitePackages}/tensorflow/:$out/${python.sitePackages}/tensorflow/contrib/tensor_forest/:${rpath}"
     internalLibPath="$out/${python.sitePackages}/tensorflow/python/_pywrap_tensorflow_internal.so"
-    find $out -name '*${stdenv.hostPlatform.extensions.sharedLibrary}' -exec patchelf --set-rpath "$rrPath" {} \;
+    find $out \( -name '*.so' -or -name '*.so.*' \) -exec patchelf --set-rpath "$rrPath" {} \;
   '';
 
 
@@ -100,7 +103,7 @@ in buildPythonPackage rec {
     homepage = http://tensorflow.org;
     license = licenses.asl20;
     maintainers = with maintainers; [ jyp abbradar ];
-    platforms = with platforms; linux ++ lib.optionals (!cudaSupport) darwin;
+    platforms = [ "x86_64-linux" "x86_64-darwin" ];
     # Python 2.7 build uses different string encoding.
     # See https://github.com/NixOS/nixpkgs/pull/37044#issuecomment-373452253
     broken = stdenv.isDarwin && !isPy3k;

pkgs/development/python-modules/tensorflow/binary-hashes.nix

@@ -1,4 +1,5 @@
 {
+version = "1.14.0";
 linux_py_27_cpu = {
   url = "https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.14.0-cp27-none-linux_x86_64.whl";
   sha256 = "0yywdrfk97dh1bxhibspg0raz70fx9lcczj6xlimqy4xb60clx7k";

pkgs/development/python-modules/tensorflow/default.nix

@@ -1,9 +1,16 @@
-{ stdenv, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin
-, buildPythonPackage, isPy3k, pythonOlder, pythonAtLeast
-, which, swig, binutils, glibcLocales
-, python, jemalloc, openmpi
-, numpy, six, protobuf, tensorflow-tensorboard, backports_weakref, mock, enum34, absl-py
-, cudaSupport ? false, nvidia_x11 ? null, cudatoolkit ? null, cudnn ? null
+{ stdenv, pkgs, buildBazelPackage, lib, fetchFromGitHub, fetchpatch, symlinkJoin
+# Python deps
+, buildPythonPackage, isPy3k, pythonOlder, pythonAtLeast, python
+# Python libraries
+, numpy, tensorflow-tensorboard, backports_weakref, mock, enum34, absl-py
+, future, setuptools, wheel, keras-preprocessing, keras-applications, google-pasta
+, termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator
+# Common deps
+, git, swig, which, binutils, glibcLocales, cython
+# Common libraries
+, jemalloc, openmpi, astor, gast, grpc, sqlite, openssl, jsoncpp, re2
+, curl, snappy, flatbuffers, icu, double-conversion, libpng, libjpeg, giflib
+, cudaSupport ? false, nvidia_x11 ? null, cudatoolkit ? null, cudnn ? null, nccl ? null
 # XLA without CUDA is broken
 , xlaSupport ? cudaSupport
 # Default from ./configure script
@@ -21,128 +28,333 @@ assert cudaSupport -> nvidia_x11 != null
 assert ! (stdenv.isDarwin && cudaSupport);
 
 let
-
   withTensorboard = pythonOlder "3.6";
 
   cudatoolkit_joined = symlinkJoin {
-    name = "${cudatoolkit.name}-unsplit";
+    name = "${cudatoolkit.name}-merged";
     paths = [ cudatoolkit.out cudatoolkit.lib ];
   };
 
+  cudatoolkit_cc_joined = symlinkJoin {
+    name = "${cudatoolkit.cc.name}-merged";
+    paths = [
+      cudatoolkit.cc
+      binutils.bintools # for ar, dwp, nm, objcopy, objdump, strip
+    ];
+  };
+
+  # Needed for _some_ system libraries, grep INCLUDEDIR.
+  includes_joined = symlinkJoin {
+    name = "tensorflow-deps-merged";
+    paths = [
+      pkgs.protobuf
+      jsoncpp
+    ];
+  };
+
   tfFeature = x: if x then "1" else "0";
 
-  version = "1.5.0";
+  version = "1.14.0";
+  variant = if cudaSupport then "-gpu" else "";
+  pname = "tensorflow${variant}";
 
-  pkg = buildBazelPackage rec {
-    name = "tensorflow-build-${version}";
+  # TODO: remove after there's support for setupPyDistFlags
+  setuppy = ../../../development/interpreters/python/run_setup.py;
+
+  bazel-build = buildBazelPackage rec {
+    name = "${pname}-${version}";
 
     src = fetchFromGitHub {
       owner = "tensorflow";
       repo = "tensorflow";
       rev = "v${version}";
-      sha256 = "1c4djsaip901nasm7a6dsimr02bsv70a7b1g0kysb4n39qpdh22q";
+      sha256 = "06jvwlsm14b8rqwd8q8796r0vmn0wk64s4ps2zg0sapkmp9vvcmi";
     };
 
     patches = [
-      # Fix build with Bazel >= 0.10
+      # Work around https://github.com/tensorflow/tensorflow/issues/24752
+      ./no-saved-proto.patch
+      # Fixes for NixOS jsoncpp
+      ./system-jsoncpp.patch
+
+      # https://github.com/tensorflow/tensorflow/pull/29673
       (fetchpatch {
-        url = "https://github.com/tensorflow/tensorflow/commit/6fcfab770c2672e2250e0f5686b9545d99eb7b2b.patch";
-        sha256 = "0p61za1mx3a7gj1s5lsps16fcw18iwnvq2b46v1kyqfgq77a12vb";
+        name = "fix-compile-with-cuda-and-mpi.patch";
+        url = "https://github.com/tensorflow/tensorflow/pull/29673/commits/498e35a3bfe38dd75cf1416a1a23c07c3b59e6af.patch";
+        sha256 = "1m2qmwv1ysqa61z6255xggwbq6mnxbig749bdvrhnch4zydxb4di";
       })
+
+      # https://github.com/tensorflow/tensorflow/issues/29220
       (fetchpatch {
-        url = "https://github.com/tensorflow/tensorflow/commit/3f57956725b553d196974c9ad31badeb3eabf8bb.patch";
-        sha256 = "11dja5gqy0qw27sc9b6yw9r0lfk8dznb32vrqqfcnypk2qmv26va";
+        name = "bazel-0.27.patch";
+        url = "https://github.com/tensorflow/tensorflow/commit/cfccbdb8c4a92dd26382419dceb4d934c2380391.patch";
+        sha256 = "1l56wjia2c4685flsfkkgy471wx3c66wyv8khspv06zchj0k0liw";
       })
     ];
 
-    nativeBuildInputs = [ swig which ];
+    # On update, it can be useful to steal the changes from gentoo
+    # https://gitweb.gentoo.org/repo/gentoo.git/tree/sci-libs/tensorflow
 
-    buildInputs = [ python jemalloc openmpi glibcLocales numpy ]
-      ++ lib.optionals cudaSupport [ cudatoolkit cudnn nvidia_x11 ];
+    nativeBuildInputs = [
+      swig which cython
+    ];
 
-    preConfigure = ''
+    buildInputs = [
+      python
+      jemalloc
+      openmpi
+      glibcLocales
+      git
+
+      # python deps needed during wheel build time
+      numpy
+      keras-preprocessing
+      protobuf
+      wrapt
+      gast
+      astor
+      absl-py
+      termcolor
+      keras-applications
+
+      # libs taken from system through the TF_SYS_LIBS mechanism
+      grpc
+      sqlite
+      openssl
+      jsoncpp
+      pkgs.protobuf
+      curl
+      snappy
+      flatbuffers
+      icu
+      double-conversion
+      libpng
+      libjpeg
+      giflib
+      re2
+      pkgs.lmdb
+
+      # for building the wheel
+      setuptools
+      wheel
+    ] ++ lib.optionals (!isPy3k) [
+      future
+      mock
+    ] ++ lib.optionals cudaSupport [
+      cudatoolkit
+      cudnn
+      nvidia_x11
+    ];
+
+
+    # arbitrarily set to the current latest bazel version, overly careful
+    TF_IGNORE_MAX_BAZEL_VERSION = true;
+
+    # Take as many libraries from the system as possible. Keep in sync with
+    # list of valid syslibs in
+    # https://github.com/tensorflow/tensorflow/blob/master/third_party/systemlibs/syslibs_configure.bzl
+    TF_SYSTEM_LIBS = lib.concatStringsSep "," [
+      "absl_py"
+      "astor_archive"
+      "boringssl"
+      # Not packaged in nixpkgs
+      # "com_github_googleapis_googleapis"
+      # "com_github_googlecloudplatform_google_cloud_cpp"
+      "com_google_protobuf"
+      "com_google_protobuf_cc"
+      "com_googlesource_code_re2"
+      "curl"
+      "cython"
+      "double_conversion"
+      "flatbuffers"
+      "gast_archive"
+      "gif_archive"
+      "grpc"
+      "hwloc"
+      "icu"
+      "jpeg"
+      "jsoncpp_git"
+      "keras_applications_archive"
+      "lmdb"
+      "nasm"
+      # "nsync" # not packaged in nixpkgs
+      "org_sqlite"
+      "pasta"
+      "pcre"
+      "png_archive"
+      "protobuf_archive"
+      "six_archive"
+      "snappy"
+      "swig"
+      "termcolor_archive"
+      "wrapt"
+      "zlib_archive"
+    ];
+
+    INCLUDEDIR = "${includes_joined}/include";
+
+    PYTHON_BIN_PATH = python.interpreter;
+ 
+    TF_NEED_GCP = true;
+    TF_NEED_HDFS = true;
+    TF_ENABLE_XLA = tfFeature xlaSupport;
+
+    CC_OPT_FLAGS = " ";
+
+    # https://github.com/tensorflow/tensorflow/issues/14454
+    TF_NEED_MPI = tfFeature cudaSupport;
+
+    TF_NEED_CUDA = tfFeature cudaSupport;
+    TF_CUDA_PATHS = lib.optionalString cudaSupport "${cudatoolkit_joined},${cudnn},${nccl}";
+    GCC_HOST_COMPILER_PREFIX = lib.optionalString cudaSupport "${cudatoolkit_cc_joined}/bin";
+    TF_CUDA_COMPUTE_CAPABILITIES = lib.concatStringsSep "," cudaCapabilities;
+
+    postPatch = ''
+      # https://github.com/tensorflow/tensorflow/issues/20919
+      sed -i '/androidndk/d' tensorflow/lite/kernels/internal/BUILD
+
+      # Tensorboard pulls in a bunch of dependencies, some of which may
+      # include security vulnerabilities. So we make it optional.
+      # https://github.com/tensorflow/tensorflow/issues/20280#issuecomment-400230560
+      sed -i '/tensorboard >=/d' tensorflow/tools/pip_package/setup.py
+    '';
+
+    preConfigure = let
+      opt_flags = []
+        ++ lib.optionals sse42Support ["-msse4.2"]
+        ++ lib.optionals avx2Support ["-mavx2"]
+        ++ lib.optionals fmaSupport ["-mfma"];
+    in ''
       patchShebangs configure
 
-      export PYTHON_BIN_PATH="${python.interpreter}"
-      export PYTHON_LIB_PATH="$NIX_BUILD_TOP/site-packages"
-      export TF_NEED_GCP=1
-      export TF_NEED_HDFS=1
-      export TF_ENABLE_XLA=${tfFeature xlaSupport}
-      export CC_OPT_FLAGS=" "
-      # https://github.com/tensorflow/tensorflow/issues/14454
-      export TF_NEED_MPI=${tfFeature cudaSupport}
-      export TF_NEED_CUDA=${tfFeature cudaSupport}
-      ${lib.optionalString cudaSupport ''
-        export CUDA_TOOLKIT_PATH=${cudatoolkit_joined}
-        export TF_CUDA_VERSION=${cudatoolkit.majorVersion}
-        export CUDNN_INSTALL_PATH=${cudnn}
-        export TF_CUDNN_VERSION=${cudnn.majorVersion}
-        export GCC_HOST_COMPILER_PATH=${cudatoolkit.cc}/bin/gcc
-        export TF_CUDA_COMPUTE_CAPABILITIES=${lib.concatStringsSep "," cudaCapabilities}
-      ''}
+      # dummy ldconfig
+      mkdir dummy-ldconfig
+      echo "#!${stdenv.shell}" > dummy-ldconfig/ldconfig
+      chmod +x dummy-ldconfig/ldconfig
+      export PATH="$PWD/dummy-ldconfig:$PATH"
 
+      export PYTHON_LIB_PATH="$NIX_BUILD_TOP/site-packages"
+      export CC_OPT_FLAGS="${lib.concatStringsSep " " opt_flags}"
       mkdir -p "$PYTHON_LIB_PATH"
     '';
 
-    NIX_LDFLAGS = lib.optionals cudaSupport [ "-lcublas" "-lcudnn" "-lcuda" "-lcudart" ];
+    configurePhase = ''
+      runHook preConfigure
+      ./configure
+      runHook postConfigure
+    '';
 
-    hardeningDisable = [ "all" ];
+    # FIXME: Tensorflow uses dlopen() for CUDA libraries.
+    # No idea why gpr isn't linked properly; perhaps Tensorflow expects a static library?
+    NIX_LDFLAGS = [ "-lgpr" ] ++ lib.optionals cudaSupport [ "-lcudart" "-lcublas" "-lcufft" "-lcurand" "-lcusolver" "-lcusparse" "-lcudnn" ];
 
-    bazelFlags = [ "--config=opt" ]
-                 ++ lib.optional sse42Support "--copt=-msse4.2"
-                 ++ lib.optional avx2Support "--copt=-mavx2"
-                 ++ lib.optional fmaSupport "--copt=-mfma"
-                 ++ lib.optional cudaSupport "--config=cuda";
+    hardeningDisable = [ "format" ];
 
-    bazelTarget = "//tensorflow/tools/pip_package:build_pip_package";
+    bazelFlags = [
+      # temporary fixes to make the build work with bazel 0.27
+      "--incompatible_no_support_tools_in_action_inputs=false"
+    ];
+    bazelBuildFlags = [
+      "--config=opt" # optimize using the flags set in the configure phase
+    ];
+
+    bazelTarget = "//tensorflow/tools/pip_package:build_pip_package //tensorflow/tools/lib_package:libtensorflow";
 
     fetchAttrs = {
       preInstall = ''
         rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker,local_*,\@local_*}
       '';
 
-      sha256 = "1nc98aqrp14q7llypcwaa0kdn9xi7r0p1mnd3vmmn1m299py33ca";
+      # cudaSupport causes fetch of ncclArchive, resulting in different hashes
+      sha256 = if cudaSupport then
+        "1rbzvir569ch33bpvm47byx954vr80rjnzrbgs4rg3bqww73hr3k"
+      else
+        "1wnhma450zh72raiq7ddd9lyq3a056cjqiy6i7y4sidiy09ncvcg";
     };
 
     buildAttrs = {
+      outputs = [ "out" "python" ];
+
       preBuild = ''
         patchShebangs .
-        find -type f -name CROSSTOOL\* -exec sed -i \
-          -e 's,/usr/bin/ar,${binutils.bintools}/bin/ar,g' \
-          {} \;
       '';
 
       installPhase = ''
-        sed -i 's,.*bdist_wheel.*,cp -rL . "$out"; exit 0,' bazel-bin/tensorflow/tools/pip_package/build_pip_package 
-        bazel-bin/tensorflow/tools/pip_package/build_pip_package $PWD/dist
+        mkdir -p "$out"
+        tar -xf bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz -C "$out"
+        # Write pkgconfig file.
+        mkdir "$out/lib/pkgconfig"
+        cat > "$out/lib/pkgconfig/tensorflow.pc" << EOF
+        Name: TensorFlow
+        Version: ${version}
+        Description: Library for computation using data flow graphs for scalable machine learning
+        Requires:
+        Libs: -L$out/lib -ltensorflow
+        Cflags: -I$out/include/tensorflow
+        EOF
+
+        # build the source code, then copy it to $python (build_pip_package
+        # actually builds a symlink farm so we must dereference them).
+        bazel-bin/tensorflow/tools/pip_package/build_pip_package --src "$PWD/dist"
+        cp -Lr "$PWD/dist" "$python"
       '';
     };
-
-    dontFixup = true;
   };
 
 in buildPythonPackage rec {
-  pname = "tensorflow";
-  inherit version;
+  inherit version pname;
 
-  src = pkg;
+  src = bazel-build.python;
 
-  installFlags = lib.optional (!withTensorboard) "--no-dependencies";
-
-  postPatch = lib.optionalString (pythonAtLeast "3.4") ''
-    sed -i '/enum34/d' setup.py
+  # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow
+  # and the propagated input tensorflow-tensorboard, which causes environment collisions.
+  # Another possibility would be to have tensorboard only in the buildInputs
+  # https://github.com/tensorflow/tensorflow/blob/v1.7.1/tensorflow/tools/pip_package/setup.py#L79
+  postInstall = ''
+    rm $out/bin/tensorboard
   '';
 
-  propagatedBuildInputs = [ numpy six protobuf absl-py ]
-                 ++ lib.optional (!isPy3k) mock
-                 ++ lib.optionals (pythonOlder "3.4") [ backports_weakref enum34 ]
-                 ++ lib.optional withTensorboard tensorflow-tensorboard;
+  # TODO: remove after there's support for setupPyDistFlags
+  buildPhase = ''
+    runHook preBuild
+    cp ${setuppy} nix_run_setup
+    ${python.interpreter} nix_run_setup --project_name ${pname} bdist_wheel
+    runHook postBuild
+  '';
+
+  # tensorflow/tools/pip_package/setup.py
+  propagatedBuildInputs = [
+    absl-py
+    astor
+    gast
+    google-pasta
+    keras-applications
+    keras-preprocessing
+    numpy
+    six
+    protobuf
+    tensorflow-estimator
+    termcolor
+    wrapt
+    grpcio
+  ] ++ lib.optionals (!isPy3k) [
+    mock
+    future # FIXME
+  ] ++ lib.optionals (pythonOlder "3.4") [
+    backports_weakref enum34
+  ] ++ lib.optionals withTensorboard [
+    tensorflow-tensorboard
+  ];
 
   # Actual tests are slow and impure.
+  # TODO try to run them anyway
+  # TODO better test (files in tensorflow/tools/ci_build/builds/*test)
   checkPhase = ''
     ${python.interpreter} -c "import tensorflow"
   '';
 
+  passthru.libtensorflow = bazel-build.out;
+
   meta = with stdenv.lib; {
     description = "Computation using data flow graphs for scalable machine learning";
     homepage = http://tensorflow.org;

pkgs/development/python-modules/tensorflow/no-saved-proto.patch

@@ -0,0 +1,14 @@
+diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD
+index 8626ed0087..27deb34387 100644
+--- a/tensorflow/cc/saved_model/BUILD
++++ b/tensorflow/cc/saved_model/BUILD
+@@ -49,9 +49,6 @@ cc_library(
+         # tf_lib depending on the build platform.
+         "//tensorflow/core:lib",
+         "//tensorflow/core:protos_all_cc",
+-    ]) + if_mobile([
+-        # Mobile-friendly SavedModel proto. See go/portable-proto for more info.
+-        "//tensorflow/core:saved_model_portable_proto",
+     ]) + if_android([
+         "//tensorflow/core:android_tensorflow_lib",
+     ]) + if_ios([

pkgs/development/python-modules/tensorflow/prefetcher.sh

@@ -1,7 +1,10 @@
+#!/usr/bin/env bash
+
 version=1.14.0
-hashfile=tf${version}-hashes.nix
+hashfile=binary-hashes.nix
 rm -f $hashfile
 echo "{" >> $hashfile
+echo "version = \"$version\";" >> $hashfile
 for sys in "linux" "mac"; do
     for tfpref in "cpu/tensorflow" "gpu/tensorflow_gpu"; do
         for pykind in "py2-none-any" "py3-none-any" "cp27-none-linux_x86_64" "cp35-cp35m-linux_x86_64" "cp36-cp36m-linux_x86_64" "cp37-cp37m-linux_x86_64"; do

pkgs/development/python-modules/tensorflow/system-jsoncpp.patch

@@ -0,0 +1,21 @@
+diff --git a/third_party/systemlibs/jsoncpp.BUILD b/third_party/systemlibs/jsoncpp.BUILD
+index 526fd0c418..646f3fdcea 100644
+--- a/third_party/systemlibs/jsoncpp.BUILD
++++ b/third_party/systemlibs/jsoncpp.BUILD
+@@ -7,6 +7,7 @@ filegroup(
+ 
+ HEADERS = [
+     "include/json/autolink.h",
++    "include/json/allocator.h",
+     "include/json/config.h",
+     "include/json/features.h",
+     "include/json/forwards.h",
+@@ -23,7 +24,7 @@ genrule(
+     cmd = """
+       for i in $(OUTS); do
+         i=$${i##*/}
+-        ln -sf $(INCLUDEDIR)/jsoncpp/json/$$i $(@D)/include/json/$$i
++        ln -sf $(INCLUDEDIR)/json/$$i $(@D)/include/json/$$i
+       done
+     """,
+ )

pkgs/development/python-modules/textacy/default.nix

@@ -30,8 +30,6 @@ buildPythonPackage rec {
     sha256 = "50402545ac92b1a931c2365e341cb35c4ebe5575525f1dcc5265901ff3895a5f";
   };
 
-  disabled = isPy27; # 2.7 requires backports.csv
-
   propagatedBuildInputs = [
     cachetools
     cld2-cffi
@@ -64,5 +62,8 @@ buildPythonPackage rec {
     homepage = "http://textacy.readthedocs.io/";
     license = licenses.asl20;
     maintainers = with maintainers; [ rvl ];
+    # ftfy and jellyfish no longer support python2
+    # latest scikitlearn not supported for this: https://github.com/chartbeat-labs/textacy/issues/260
+    broken = true;
   };
 }

pkgs/development/python-modules/trezor/default.nix

@@ -10,20 +10,21 @@
 , construct
 , libusb1
 , rlp
+, shamir-mnemonic
 }:
 
 buildPythonPackage rec {
   pname = "trezor";
-  version = "0.11.3";
+  version = "0.11.4";
 
   disabled = !isPy3k;
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "c79a500e90d003073c8060d319dceb042caaba9472f13990c77ed37d04a82108";
+    sha256 = "aeb3f56a4c389495617f27bf218471b7969f636d25ddc491dfefeb8a1b3cd499";
   };
 
-  propagatedBuildInputs = [ typing-extensions protobuf hidapi ecdsa mnemonic requests pyblake2 click construct libusb1 rlp ];
+  propagatedBuildInputs = [ typing-extensions protobuf hidapi ecdsa mnemonic requests pyblake2 click construct libusb1 rlp shamir-mnemonic ];
 
   # build requires UTF-8 locale
   LANG = "en_US.UTF-8";
@@ -41,8 +42,8 @@ buildPythonPackage rec {
 
   meta = {
     description = "Python library for communicating with TREZOR Bitcoin Hardware Wallet";
-    homepage = https://github.com/trezor/python-trezor;
+    homepage = "https://github.com/trezor/trezor-firmware/tree/master/python";
     license = lib.licenses.gpl3;
-    maintainers = with lib.maintainers; [ np prusnak mmahut ];
+    maintainers = with lib.maintainers; [ np prusnak mmahut "1000101" ];
   };
 }

pkgs/development/python-modules/xgboost/default.nix

@@ -3,6 +3,7 @@
 , nose
 , scipy
 , scikitlearn
+, stdenv
 , xgboost
 , substituteAll
 , pandas
@@ -19,6 +20,7 @@ buildPythonPackage rec {
     (substituteAll {
       src = ./lib-path-for-python.patch;
       libpath = "${xgboost}/lib";
+      extention = stdenv.hostPlatform.extensions.sharedLibrary;
     })
   ];
 

pkgs/development/python-modules/xgboost/lib-path-for-python.patch

@@ -35,4 +35,4 @@ index d87922c0..859a30fb 100644
 -            'did you install compilers and run build.sh in root path?\n'
 -            'List of candidates:\n' + ('\n'.join(dll_path)))
 -    return lib_path
-+    return ["@libpath@/libxgboost.so"]
++    return ["@libpath@/libxgboost@extention@"]

pkgs/development/python-modules/zstd/default.nix

@@ -3,11 +3,11 @@
 
 buildPythonPackage rec {
   pname = "zstd";
-  version = "1.4.0.0";
+  version = "1.4.1.0";
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "01prq9rwz1gh42idnj2162w79dzs8gf3ac8pn12lz347w280kjbk";
+    sha256 = "0laxg0pag1bzmqmg4x81jb32412pn98p9zg2b0li035m779nka95";
   };
 
   postPatch = ''