Split buildPythonPackage into setup hooks

This commit splits the `buildPythonPackage` into multiple setup hooks.

Generally, Python packages are built from source to wheels using `setuptools`.
The wheels are then installed with `pip`. Tests were often called with
`python setup.py test` but this is less common nowadays. Most projects
now use a different entry point for running tests, typically `pytest`
or `nosetests`.

Since the wheel format was introduced more tools were built to generate these,
e.g. `flit`. Since PEP 517 is provisionally accepted, defining a build-system
independent format (`pyproject.toml`), `pip` can now use that format to
execute the correct build-system.

In the past I've added support for PEP 517 (`pyproject`) to the Python
builder, resulting in a now rather large builder. Furthermore, it was not possible
to reuse components elsewhere. Therefore, the builder is now split into multiple
setup hooks.

The `setuptoolsCheckHook` is included now by default but in time it should
be removed from `buildPythonPackage` to make it easier to use another hook
(curently one has to pass in `dontUseSetuptoolsCheck`).

pkgs/development/python-modules/atomicwrites/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, buildPythonPackage, fetchPypi }:
+{ stdenv, buildPythonPackage, fetchPypi, pytest }:
 
 buildPythonPackage rec {
   pname = "atomicwrites";
@@ -9,6 +9,10 @@ buildPythonPackage rec {
     sha256 = "75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6";
   };
 
+  # Tests depend on pytest but atomicwrites is a dependency of pytest
+  doCheck = false;
+  checkInputs = [ pytest ];
+
   meta = with stdenv.lib; {
     description = "Atomic file writes on POSIX";
     homepage = https://pypi.python.org/pypi/atomicwrites;

pkgs/development/python-modules/bootstrapped-pip/default.nix

@@ -1,4 +1,8 @@
-{ stdenv, python, fetchPypi, makeWrapper, unzip }:
+{ stdenv, python, fetchPypi, makeWrapper, unzip, makeSetupHook
+, pipInstallHook
+, setuptoolsBuildHook
+
+}:
 
 let
   wheel_source = fetchPypi {
@@ -25,6 +29,15 @@ in stdenv.mkDerivation rec {
     sha256 = "993134f0475471b91452ca029d4390dc8f298ac63a712814f101cd1b6db46676";
   };
 
+  dontUseSetuptoolsBuild = true;
+
+  # Should be propagatedNativeBuildInputs
+  propagatedBuildInputs = [
+    # Override to remove dependencies to prevent infinite recursion.
+    (pipInstallHook.override{pip=null;})
+    (setuptoolsBuildHook.override{setuptools=null; wheel=null;})
+  ];
+
   unpackPhase = ''
     mkdir -p $out/${python.sitePackages}
     unzip -d $out/${python.sitePackages} $src
@@ -32,7 +45,7 @@ in stdenv.mkDerivation rec {
     unzip -d $out/${python.sitePackages} ${wheel_source}
   '';
 
-  patchPhase = ''
+  postPatch = ''
     mkdir -p $out/bin
   '';
 
@@ -52,4 +65,5 @@ in stdenv.mkDerivation rec {
       wrapProgram $f --prefix PYTHONPATH ":" $out/${python.sitePackages}/
     done
   '';
+
 }

pkgs/development/python-modules/pip/default.nix

@@ -1,25 +1,32 @@
 { lib
+, python
 , buildPythonPackage
+, bootstrapped-pip
 , fetchPypi
 , mock
 , scripttest
 , virtualenv
 , pretend
 , pytest
+, setuptools
+, wheel
 }:
 
 buildPythonPackage rec {
   pname = "pip";
   version = "19.1.1";
+  format = "other";
 
   src = fetchPypi {
     inherit pname version;
     sha256 = "44d3d7d3d30a1eb65c7e5ff1173cdf8f7467850605ac7cc3707b6064bddd0958";
   };
 
+  nativeBuildInputs = [ bootstrapped-pip ];
+
   # pip detects that we already have bootstrapped_pip "installed", so we need
   # to force it a little.
-  installFlags = [ "--ignore-installed" ];
+  pipInstallFlags = [ "--ignore-installed" ];
 
   checkInputs = [ mock scripttest virtualenv pretend pytest ];
   # Pip wants pytest, but tests are not distributed

pkgs/development/python-modules/py/default.nix

@@ -12,7 +12,11 @@ buildPythonPackage rec {
   # Circular dependency on pytest
   doCheck = false;
 
-  buildInputs = [ setuptools_scm ];
+  nativeBuildInputs = [ setuptools_scm ];
+
+  pythonImportsCheck = [
+    "py"
+  ];
 
   meta = with stdenv.lib; {
     description = "Library with cross-python path, ini-parsing, io, code, log facilities";

pkgs/development/python-modules/pytest/default.nix

@@ -1,6 +1,6 @@
 { stdenv, buildPythonPackage, pythonOlder, fetchPypi, attrs, hypothesis, py
 , setuptools_scm, setuptools, six, pluggy, funcsigs, isPy3k, more-itertools
-, atomicwrites, mock, writeText, pathlib2, wcwidth, packaging, isPyPy
+, atomicwrites, mock, writeText, pathlib2, wcwidth, packaging, isPyPy, python
 }:
 buildPythonPackage rec {
   version = "5.1.0";
@@ -17,12 +17,13 @@ buildPythonPackage rec {
   };
 
   checkInputs = [ hypothesis mock ];
-  buildInputs = [ setuptools_scm ];
+  nativeBuildInputs = [ setuptools_scm ];
   propagatedBuildInputs = [ attrs py setuptools six pluggy more-itertools atomicwrites wcwidth packaging ]
     ++ stdenv.lib.optionals (!isPy3k) [ funcsigs ]
     ++ stdenv.lib.optionals (pythonOlder "3.6") [ pathlib2 ];
 
   doCheck = !isPyPy; # https://github.com/pytest-dev/pytest/issues/3460
+
   # Ignored file https://github.com/pytest-dev/pytest/pull/5605#issuecomment-522243929
   checkPhase = ''
     runHook preCheck
@@ -35,15 +36,17 @@ buildPythonPackage rec {
     pytestcachePhase() {
         find $out -name .pytest_cache -type d -exec rm -rf {} +
     }
-
     preDistPhases+=" pytestcachePhase"
   '';
 
+  pythonImportsCheck = [
+    "pytest"
+  ];
+
   meta = with stdenv.lib; {
     homepage = https://docs.pytest.org;
     description = "Framework for writing tests";
     maintainers = with maintainers; [ domenkozar lovek323 madjar lsix ];
     license = licenses.mit;
-    platforms = platforms.unix;
   };
 }

pkgs/development/python-modules/setuptools/default.nix

@@ -1,15 +1,17 @@
 { stdenv
+, buildPythonPackage
 , fetchPypi
 , python
 , wrapPython
 , unzip
+, callPackage
+, bootstrapped-pip
 }:
 
-# Should use buildPythonPackage here somehow
-stdenv.mkDerivation rec {
+buildPythonPackage rec {
   pname = "setuptools";
   version = "41.0.1";
-  name = "${python.libPrefix}-${pname}-${version}";
+  format = "other";
 
   src = fetchPypi {
     inherit pname version;
@@ -17,8 +19,11 @@ stdenv.mkDerivation rec {
     sha256 = "a222d126f5471598053c9a77f4b5d4f26eaa1f150ad6e01dcf1a42e185d05613";
   };
 
-  nativeBuildInputs = [ unzip wrapPython python.pythonForBuild ];
-  doCheck = false;  # requires pytest
+  # There is nothing to build
+  dontBuild = true;
+
+  nativeBuildInputs = [ bootstrapped-pip ];
+
   installPhase = ''
       dst=$out/${python.sitePackages}
       mkdir -p $dst
@@ -27,13 +32,11 @@ stdenv.mkDerivation rec {
       wrapPythonPrograms
   '';
 
-  pythonPath = [];
-
-  dontPatchShebangs = true;
-
-  # Python packages built through cross-compilation are always for the host platform.
-  disallowedReferences = stdenv.lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ python.pythonForBuild ];
+  # Adds setuptools to nativeBuildInputs causing infinite recursion.
+  catchConflicts = false;
 
+  # Requires pytest, causing infinite recursion.
+  doCheck = false;
 
   meta = with stdenv.lib; {
     description = "Utilities to facilitate the installation of Python packages";

pkgs/development/python-modules/setuptools_scm/default.nix

@@ -8,8 +8,6 @@ buildPythonPackage rec {
     sha256 = "52ab47715fa0fc7d8e6cd15168d1a69ba995feb1505131c3e814eb7087b57358";
   };
 
-  buildInputs = [ pip ];
-
   # Seems to fail due to chroot and would cause circular dependency
   # with pytest
   doCheck = false;

pkgs/development/python-modules/wheel/default.nix

@@ -1,15 +1,19 @@
 { lib
+, setuptools
+, pip
 , buildPythonPackage
 , fetchPypi
 , pytest
 , pytestcov
 , coverage
 , jsonschema
+, bootstrapped-pip
 }:
 
 buildPythonPackage rec {
   pname = "wheel";
   version = "0.33.4";
+  format = "other";
 
   src = fetchPypi {
     inherit pname version;
@@ -17,14 +21,14 @@ buildPythonPackage rec {
   };
 
   checkInputs = [ pytest pytestcov coverage ];
+  nativeBuildInputs = [ bootstrapped-pip setuptools ];
 
-  propagatedBuildInputs = [ jsonschema ];
-
+  catchConflicts = false;
   # No tests in archive
   doCheck = false;
 
   # We add this flag to ignore the copy installed by bootstrapped-pip
-  installFlags = [ "--ignore-installed" ];
+  pipInstallFlags = [ "--ignore-installed" ];
 
   meta = {
     description = "A built-package format for Python";