Split buildPythonPackage into setup hooks

This commit splits the `buildPythonPackage` into multiple setup hooks.

Generally, Python packages are built from source to wheels using `setuptools`.
The wheels are then installed with `pip`. Tests were often called with
`python setup.py test` but this is less common nowadays. Most projects
now use a different entry point for running tests, typically `pytest`
or `nosetests`.

Since the wheel format was introduced more tools were built to generate these,
e.g. `flit`. Since PEP 517 is provisionally accepted, defining a build-system
independent format (`pyproject.toml`), `pip` can now use that format to
execute the correct build-system.

In the past I've added support for PEP 517 (`pyproject`) to the Python
builder, resulting in a now rather large builder. Furthermore, it was not possible
to reuse components elsewhere. Therefore, the builder is now split into multiple
setup hooks.

The `setuptoolsCheckHook` is included now by default but in time it should
be removed from `buildPythonPackage` to make it easier to use another hook
(curently one has to pass in `dontUseSetuptoolsCheck`).

pkgs/development/interpreters/python/mk-python-derivation.nix

@@ -4,13 +4,22 @@
 , config
 , python
 , wrapPython
-, setuptools
 , unzip
 , ensureNewerSourcesForZipFilesHook
 # Whether the derivation provides a Python module or not.
 , toPythonModule
 , namePrefix
 , update-python-libraries
+, setuptools
+, flitBuildHook
+, pipBuildHook
+, pipInstallHook
+, pythonCatchConflictsHook
+, pythonImportsCheckHook
+, pythonRemoveBinBytecodeHook
+, setuptoolsBuildHook
+, setuptoolsCheckHook
+, wheelUnpackHook
 }:
 
 { name ? "${attrs.pname}-${attrs.version}"
@@ -48,6 +57,11 @@
 # Skip wrapping of python programs altogether
 , dontWrapPythonPrograms ? false
 
+# Don't use Pip to install a wheel
+# Note this is actually a variable for the pipInstallPhase in pip's setupHook.
+# It's included here to prevent an infinite recursion.
+, dontUsePipInstall ? false
+
 # Skip setting the PYTHONNOUSERSITE environment variable in wrapped programs
 , permitUserSite ? false
 
@@ -57,6 +71,13 @@
 # However, some packages do provide executables with extensions, and thus bytecode is generated.
 , removeBinBytecode ? true
 
+# Several package formats are supported.
+# "setuptools" : Install a common setuptools/distutils based package. This builds a wheel.
+# "wheel" : Install from a pre-compiled wheel.
+# "flit" : Install a flit package. This builds a wheel.
+# "other" : Provide your own buildPhase and installPhase.
+, format ? "setuptools"
+
 , meta ? {}
 
 , passthru ? {}
@@ -71,26 +92,43 @@ if disabled
 then throw "${name} not supported for interpreter ${python.executable}"
 else
 
-let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attrs [
-    "disabled" "checkInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts"
-  ] // {
+let
+  inherit (python) stdenv;
+
+  self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [
+    "disabled" "checkPhase" "checkInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "format"
+  ]) // {
 
   name = namePrefix + name;
 
   nativeBuildInputs = [
     python
     wrapPython
-    ensureNewerSourcesForZipFilesHook
-    setuptools
-#     ++ lib.optional catchConflicts setuptools # If we no longer propagate setuptools
+    ensureNewerSourcesForZipFilesHook  # move to wheel installer (pip) or builder (setuptools, flit, ...)?
+  ] ++ lib.optionals catchConflicts [
+    setuptools pythonCatchConflictsHook
+  ] ++ lib.optionals removeBinBytecode [
+    pythonRemoveBinBytecodeHook
   ] ++ lib.optionals (lib.hasSuffix "zip" (attrs.src.name or "")) [
     unzip
+  ] ++ lib.optionals (format == "setuptools") [
+    setuptoolsBuildHook
+  ] ++ lib.optionals (format == "flit") [
+    flitBuildHook
+  ] ++ lib.optionals (format == "pyproject") [
+    pipBuildHook
+  ] ++ lib.optionals (format == "wheel") [
+    wheelUnpackHook
+  ] ++ lib.optionals (!(format == "other") || dontUsePipInstall) [
+    pipInstallHook
+  ] ++ lib.optionals (stdenv.buildPlatform == stdenv.hostPlatform) [
+    # This is a test, however, it should be ran independent of the checkPhase and checkInputs
+    pythonImportsCheckHook
   ] ++ nativeBuildInputs;
 
   buildInputs = buildInputs ++ pythonPath;
 
-  # Propagate python and setuptools. We should stop propagating setuptools.
-  propagatedBuildInputs = propagatedBuildInputs ++ [ python setuptools ];
+  propagatedBuildInputs = propagatedBuildInputs ++ [ python ];
 
   inherit strictDeps;
 
@@ -98,21 +136,17 @@ let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attr
 
   # Python packages don't have a checkPhase, only an installCheckPhase
   doCheck = false;
-  doInstallCheck = doCheck;
-  installCheckInputs = checkInputs;
+  doInstallCheck = attrs.doCheck or true;
+  installCheckInputs = [
+  ] ++ lib.optionals (format == "setuptools") [
+    # Longer-term we should get rid of this and require
+    # users of this function to set the `installCheckPhase` or
+    # pass in a hook that sets it.
+    setuptoolsCheckHook
+  ] ++ checkInputs;
 
   postFixup = lib.optionalString (!dontWrapPythonPrograms) ''
     wrapPythonPrograms
-  '' + lib.optionalString removeBinBytecode ''
-    if [ -d "$out/bin" ]; then
-      rm -rf "$out/bin/__pycache__"                 # Python 3
-      find "$out/bin" -type f -name "*.pyc" -delete # Python 2
-    fi
-  '' + lib.optionalString catchConflicts ''
-    # Check if we have two packages with the same name in the closure and fail.
-    # If this happens, something went wrong with the dependencies specs.
-    # Intentionally kept in a subdirectory, see catch_conflicts/README.md.
-    ${python.pythonForBuild.interpreter} ${./catch_conflicts}/catch_conflicts.py
   '' + attrs.postFixup or '''';
 
   # Python packages built through cross-compilation are always for the host platform.
@@ -123,6 +157,10 @@ let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attr
     platforms = python.meta.platforms;
     isBuildPythonPackage = python.meta.platforms;
   } // meta;
+} // lib.optionalAttrs (attrs?checkPhase) {
+  # If given use the specified checkPhase, otherwise use the setup hook.
+  # Longer-term we should get rid of `checkPhase` and use `installCheckPhase`.
+  installCheckPhase = attrs.checkPhase;
 }));
 
 passthru.updateScript = let