Split buildPythonPackage into setup hooks

This commit splits the `buildPythonPackage` into multiple setup hooks.

Generally, Python packages are built from source to wheels using `setuptools`.
The wheels are then installed with `pip`. Tests were often called with
`python setup.py test` but this is less common nowadays. Most projects
now use a different entry point for running tests, typically `pytest`
or `nosetests`.

Since the wheel format was introduced more tools were built to generate these,
e.g. `flit`. Since PEP 517 is provisionally accepted, defining a build-system
independent format (`pyproject.toml`), `pip` can now use that format to
execute the correct build-system.

In the past I've added support for PEP 517 (`pyproject`) to the Python
builder, resulting in a now rather large builder. Furthermore, it was not possible
to reuse components elsewhere. Therefore, the builder is now split into multiple
setup hooks.

The `setuptoolsCheckHook` is included now by default but in time it should
be removed from `buildPythonPackage` to make it easier to use another hook
(curently one has to pass in `dontUseSetuptoolsCheck`).

doc/languages-frameworks/python.section.md

@@ -540,7 +540,8 @@ and the aliases
 #### `buildPythonPackage` function
 
 The `buildPythonPackage` function is implemented in
-`pkgs/development/interpreters/python/build-python-package.nix`
+`pkgs/development/interpreters/python/mk-python-derivation`
+using setup hooks.
 
 The following is an example:
 ```nix
@@ -797,6 +798,22 @@ such as `ignoreCollisions = true` or `postBuild`. If you need them, you have to
 Python 2 namespace packages may provide `__init__.py` that collide. In that case `python.buildEnv`
 should be used with `ignoreCollisions = true`.
 
+#### Setup hooks
+
+The following are setup hooks specifically for Python packages. Most of these are
+used in `buildPythonPackage`.
+
+- `flitBuildHook` to build a wheel using `flit`.
+- `pipBuildHook` to build a wheel using `pip` and PEP 517. Note a build system (e.g. `setuptools` or `flit`) should still be added as `nativeBuildInput`.
+- `pipInstallHook` to install wheels.
+- `pytestCheckHook` to run tests with `pytest`.
+- `pythonCatchConflictsHook` to check whether a Python package is not already existing.
+- `pythonImportsCheckHook` to check whether importing the listed modules works.
+- `pythonRemoveBinBytecode` to remove bytecode from the `/bin` folder.
+- `setuptoolsBuildHook` to build a wheel using `setuptools`.
+- `setuptoolsCheckHook` to run tests with `python setup.py test`.
+- `wheelUnpackHook` to move a wheel to the correct folder so it can be installed with the `pipInstallHook`.
+
 ### Development mode
 
 Development or editable mode is supported. To develop Python packages

pkgs/development/interpreters/python/build-python-package-common.nix

@@ -1,31 +0,0 @@
-# This function provides generic bits to install a Python wheel.
-
-{ python
-}:
-
-{ buildInputs ? []
-# Additional flags to pass to "pip install".
-, installFlags ? []
-, ... } @ attrs:
-
-attrs // {
-  buildInputs = buildInputs ++ [ python.pythonForBuild.pkgs.bootstrapped-pip ];
-
-  configurePhase = attrs.configurePhase or ''
-    runHook preConfigure
-    runHook postConfigure
-  '';
-
-  installPhase = attrs.installPhase or ''
-    runHook preInstall
-
-    mkdir -p "$out/${python.sitePackages}"
-    export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
-
-    pushd dist
-    ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild
-    popd
-
-    runHook postInstall
-  '';
-}

pkgs/development/interpreters/python/build-python-package-flit.nix

@@ -1,22 +0,0 @@
-# This function provides specific bits for building a flit-based Python package.
-
-{ python
-, flit
-}:
-
-{ ... } @ attrs:
-
-attrs // {
-  nativeBuildInputs = [ flit ];
-  buildPhase = attrs.buildPhase or ''
-    runHook preBuild
-    flit build --format wheel
-    runHook postBuild
-  '';
-
-  # Flit packages, like setuptools packages, might have tests.
-  installCheckPhase = attrs.checkPhase or ''
-    ${python.interpreter} -m unittest discover
-  '';
-  doCheck = attrs.doCheck or true;
-}

pkgs/development/interpreters/python/build-python-package-pyproject.nix

@@ -1,56 +0,0 @@
-# This function provides specific bits for building a setuptools-based Python package.
-
-{ lib
-, python
-}:
-
-{
-# Global options passed to "python setup.py"
-  setupPyGlobalFlags ? []
-# Build options passed to "build_ext"
-# https://github.com/pypa/pip/issues/881
-# Rename to `buildOptions` because it is not setuptools specific?
-, setupPyBuildFlags ? []
-# Execute before shell hook
-, preShellHook ? ""
-# Execute after shell hook
-, postShellHook ? ""
-, ... } @ attrs:
-
-let
-  pipGlobalFlagsString = lib.concatMapStringsSep " " (option: "--global-option ${option}") setupPyGlobalFlags;
-  pipBuildFlagsString = lib.concatMapStringsSep " " (option: "--build-option ${option}") setupPyBuildFlags;
-in attrs // {
-  buildPhase = attrs.buildPhase or ''
-    runHook preBuild
-    mkdir -p dist
-    echo "Creating a wheel..."
-    ${python.pythonForBuild.interpreter} -m pip wheel --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist ${pipGlobalFlagsString} ${pipBuildFlagsString} .
-    echo "Finished creating a wheel..."
-    runHook postBuild
-  '';
-
-  installCheckPhase = ''
-    runHook preCheck
-    echo "No checkPhase defined. Either provide a checkPhase or disable tests in case tests are not available."; exit 1
-    runHook postCheck
-  '';
-
-  # With Python it's a common idiom to run the tests
-  # after the software has been installed.
-  doCheck = attrs.doCheck or true;
-
-  shellHook = attrs.shellHook or ''
-    ${preShellHook}
-    # Long-term setup.py should be dropped.
-    if [ -e pyproject.toml ]; then
-      tmp_path=$(mktemp -d)
-      export PATH="$tmp_path/bin:$PATH"
-      export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH"
-      mkdir -p $tmp_path/${python.pythonForBuild.sitePackages}
-      ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
-    fi
-    ${postShellHook}
-  '';
-
-}

pkgs/development/interpreters/python/build-python-package-setuptools.nix

@@ -1,60 +0,0 @@
-# This function provides specific bits for building a setuptools-based Python package.
-
-{ lib
-, python
-}:
-
-{
-# Global options passed to "python setup.py"
-  setupPyGlobalFlags ? []
-# Build options passed to "python setup.py build_ext"
-# https://github.com/pypa/pip/issues/881
-, setupPyBuildFlags ? []
-# Execute before shell hook
-, preShellHook ? ""
-# Execute after shell hook
-, postShellHook ? ""
-, ... } @ attrs:
-
-let
-  # use setuptools shim (so that setuptools is imported before distutils)
-  # pip does the same thing: https://github.com/pypa/pip/pull/3265
-  setuppy = ./run_setup.py;
-
-  setupPyGlobalFlagsString = lib.concatStringsSep " " setupPyGlobalFlags;
-  setupPyBuildExtString = lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags));
-
-in attrs // {
-  # we copy nix_run_setup over so it's executed relative to the root of the source
-  # many project make that assumption
-  buildPhase = attrs.buildPhase or ''
-    runHook preBuild
-    cp ${setuppy} nix_run_setup
-    ${python.pythonForBuild.interpreter} nix_run_setup ${setupPyGlobalFlagsString} ${setupPyBuildExtString} bdist_wheel
-    runHook postBuild
-  '';
-
-  installCheckPhase = attrs.checkPhase or ''
-    runHook preCheck
-    ${python.pythonForBuild.interpreter} nix_run_setup test
-    runHook postCheck
-  '';
-
-  # Python packages that are installed with setuptools
-  # are typically distributed with tests.
-  # With Python it's a common idiom to run the tests
-  # after the software has been installed.
-  doCheck = attrs.doCheck or true;
-
-  shellHook = attrs.shellHook or ''
-    ${preShellHook}
-    if test -e setup.py; then
-      tmp_path=$(mktemp -d)
-      export PATH="$tmp_path/bin:$PATH"
-      export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH"
-      mkdir -p $tmp_path/${python.pythonForBuild.sitePackages}
-      ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
-    fi
-    ${postShellHook}
-  '';
-}

pkgs/development/interpreters/python/build-python-package-wheel.nix

@@ -1,20 +0,0 @@
-# This function provides specific bits for building a wheel-based Python package.
-
-{
-}:
-
-{ ... } @ attrs:
-
-attrs // {
-  unpackPhase = ''
-    mkdir dist
-    cp "$src" "dist/$(stripHash "$src")"
-  '';
-
-  # Wheels are pre-compiled
-  buildPhase = attrs.buildPhase or ":";
-  installCheckPhase = attrs.checkPhase or ":";
-
-  # Wheels don't have any checks to run
-  doCheck = attrs.doCheck or false;
-}

pkgs/development/interpreters/python/build-python-package.nix

@@ -1,48 +0,0 @@
-# This function provides a generic Python package builder,
-# and can build packages that use distutils, setuptools or flit.
-
-{ lib
-, config
-, python
-, wrapPython
-, setuptools
-, unzip
-, ensureNewerSourcesForZipFilesHook
-, toPythonModule
-, namePrefix
-, flit
-, writeScript
-, update-python-libraries
-}:
-
-let
-  setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python; };
-  pyproject-specific = import ./build-python-package-pyproject.nix { inherit lib python; };
-  flit-specific = import ./build-python-package-flit.nix { inherit python flit; };
-  wheel-specific = import ./build-python-package-wheel.nix { };
-  common = import ./build-python-package-common.nix { inherit python; };
-  mkPythonDerivation = import ./mk-python-derivation.nix {
-    inherit lib config python wrapPython setuptools unzip ensureNewerSourcesForZipFilesHook;
-    inherit toPythonModule namePrefix update-python-libraries;
-  };
-in
-
-{
-# Several package formats are supported.
-# "setuptools" : Install a common setuptools/distutils based package. This builds a wheel.
-# "wheel" : Install from a pre-compiled wheel.
-# "flit" : Install a flit package. This builds a wheel.
-# "other" : Provide your own buildPhase and installPhase.
-format ? "setuptools"
-, ... } @ attrs:
-
-let
-  formatspecific =
-    if format == "pyproject" then common (pyproject-specific attrs)
-    else if format == "setuptools" then common (setuptools-specific attrs)
-    else if format == "flit" then common (flit-specific attrs)
-    else if format == "wheel" then common (wheel-specific attrs)
-    else if format == "other" then {}
-    else throw "Unsupported format ${format}";
-
-in mkPythonDerivation ( attrs // formatspecific )

pkgs/development/interpreters/python/hooks/default.nix

@@ -0,0 +1,95 @@
+# Hooks for building Python packages.
+{ python
+, callPackage
+, makeSetupHook
+}:
+
+let
+  pythonInterpreter = python.pythonForBuild.interpreter;
+  pythonSitePackages = python.sitePackages;
+  pythonCheckInterpreter = python.interpreter;
+  setuppy = ../run_setup.py;
+in rec {
+
+  flitBuildHook = callPackage ({ flit }:
+    makeSetupHook {
+      name = "flit-build-hook";
+      deps = [ flit ];
+      substitutions = {
+        inherit pythonInterpreter;
+      };
+    } ./flit-build-hook.sh) {};
+
+  pipBuildHook = callPackage ({ pip }:
+    makeSetupHook {
+      name = "pip-build-hook.sh";
+      deps = [ pip ];
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages;
+      };
+    } ./pip-build-hook.sh) {};
+
+  pipInstallHook = callPackage ({ pip }:
+    makeSetupHook {
+      name = "pip-install-hook";
+      deps = [ pip ];
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages;
+      };
+    } ./pip-install-hook.sh) {};
+
+  pytestCheckHook = callPackage ({ pytest }:
+    makeSetupHook {
+      name = "pytest-check-hook";
+      deps = [ pytest ];
+      substitutions = {
+        inherit pythonCheckInterpreter;
+      };
+    } ./pytest-check-hook.sh) {};
+
+  pythonCatchConflictsHook = callPackage ({ setuptools }:
+    makeSetupHook {
+      name = "python-catch-conflicts-hook";
+      deps = [ setuptools ];
+      substitutions = {
+        inherit pythonInterpreter;
+        catchConflicts=../catch_conflicts/catch_conflicts.py;
+      };
+    } ./python-catch-conflicts-hook.sh) {};
+
+  pythonImportsCheckHook = callPackage ({}:
+    makeSetupHook {
+      name = "python-imports-check-hook.sh";
+      substitutions = {
+        inherit pythonCheckInterpreter;
+      };
+    } ./python-imports-check-hook.sh) {};
+
+  pythonRemoveBinBytecodeHook = callPackage ({ }:
+    makeSetupHook {
+      name = "python-remove-bin-bytecode-hook";
+    } ./python-remove-bin-bytecode-hook.sh) {};
+
+  setuptoolsBuildHook = callPackage ({ setuptools, wheel }:
+    makeSetupHook {
+      name = "setuptools-setup-hook";
+      deps = [ setuptools wheel ];
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages setuppy;
+      };
+    } ./setuptools-build-hook.sh) {};
+
+  setuptoolsCheckHook = callPackage ({ setuptools }:
+    makeSetupHook {
+      name = "setuptools-check-hook";
+      deps = [ setuptools ];
+      substitutions = {
+        inherit pythonCheckInterpreter setuppy;
+      };
+    } ./setuptools-check-hook.sh) {};
+
+  wheelUnpackHook = callPackage ({ }:
+    makeSetupHook {
+      name = "wheel-unpack-hook.sh";
+    } ./wheel-unpack-hook.sh) {};
+}

pkgs/development/interpreters/python/hooks/flit-build-hook.sh

@@ -0,0 +1,15 @@
+# Setup hook for flit
+echo "Sourcing flit-build-hook"
+
+flitBuildPhase () {
+    echo "Executing flitBuildPhase"
+    preBuild
+    @pythonInterpreter@ -m flit build --format wheel
+    postBuild
+    echo "Finished executing flitBuildPhase"
+}
+
+if [ -z "$dontUseFlitBuild" ] && [ -z "$buildPhase" ]; then
+    echo "Using flitBuildPhase"
+    buildPhase=flitBuildPhase
+fi

pkgs/development/interpreters/python/hooks/pip-build-hook.sh

@@ -0,0 +1,42 @@
+# Setup hook to use for pip projects
+echo "Sourcing pip-build-hook"
+
+pipBuildPhase() {
+    echo "Executing pipBuildPhase"
+    runHook preBuild
+
+    mkdir -p dist
+    echo "Creating a wheel..."
+    @pythonInterpreter@ -m pip wheel --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist "$options" .
+    echo "Finished creating a wheel..."
+
+    runHook postBuild
+    echo "Finished executing pipBuildPhase"
+}
+
+pipShellHook() {
+    echo "Executing pipShellHook"
+    runHook preShellHook
+
+    # Long-term setup.py should be dropped.
+    if [ -e pyproject.toml ]; then
+      tmp_path=$(mktemp -d)
+      export PATH="$tmp_path/bin:$PATH"
+      export PYTHONPATH="$tmp_path/@pythonSitePackages@:$PYTHONPATH"
+      mkdir -p "$tmp_path/@pythonSitePackages@"
+      @pythonInterpreter@ -m pip install -e . --prefix "$tmp_path" >&2
+    fi
+
+    runHook postShellHook
+    echo "Finished executing pipShellHook"
+}
+
+if [ -z "$dontUsePipBuild" ] && [ -z "$buildPhase" ]; then
+    echo "Using pipBuildPhase"
+    buildPhase=pipBuildPhase
+fi
+
+if [ -z "$shellHook" ]; then
+    echo "Using pipShellHook"
+    shellHook=pipShellHook
+fi

pkgs/development/interpreters/python/hooks/pip-install-hook.sh

@@ -0,0 +1,24 @@
+# Setup hook for pip.
+echo "Sourcing pip-install-hook"
+
+declare -a pipInstallFlags
+
+pipInstallPhase() {
+    echo "Executing pipInstallPhase"
+    runHook preInstall
+
+    mkdir -p "$out/@pythonSitePackages@"
+    export PYTHONPATH="$out/@pythonSitePackages@:$PYTHONPATH"
+
+    pushd dist || return 1
+    @pythonInterpreter@ -m pip install ./*.whl --no-index --prefix="$out" --no-cache $pipInstallFlags --build tmpbuild
+    popd || return 1
+
+    runHook postInstall
+    echo "Finished executing pipInstallPhase"
+}
+
+if [ -z "$dontUsePipInstall" ] && [ -z "$installPhase" ]; then
+    echo "Using pipInstallPhase"
+    installPhase=pipInstallPhase
+fi

pkgs/development/interpreters/python/hooks/pytest-check-hook.sh

@@ -0,0 +1,49 @@
+# Setup hook for pytest
+echo "Sourcing pytest-check-hook"
+
+declare -ar disabledTests
+
+function _concatSep {
+    local result
+    local sep="$1"
+    local -n arr=$2
+    for index in ${!arr[*]}; do
+        if [ $index -eq 0 ]; then
+            result="${arr[index]}"
+        else
+            result+=" $sep ${arr[index]}"
+        fi
+    done
+    echo "$result"
+}
+
+function _pytestComputeDisabledTestsString () {
+    declare -a tests
+    local tests=($1)
+    local prefix="not "
+    prefixed=( "${tests[@]/#/$prefix}" )
+    result=$(_concatSep "and" prefixed)
+    echo "$result"
+}
+
+function pytestCheckPhase() {
+    echo "Executing pytestCheckPhase"
+    runHook preCheck
+
+    # Compose arguments
+    args=" -m pytest"
+    if [ -n "$disabledTests" ]; then
+        disabledTestsString=$(_pytestComputeDisabledTestsString "${disabledTests[@]}")
+      args+=" -k \""$disabledTestsString"\""
+    fi
+    args+=" ${pytestFlagsArray[@]}"
+    eval "@pythonCheckInterpreter@ $args"
+
+    runHook postCheck
+    echo "Finished executing pytestCheckPhase"
+}
+
+if [ -z "$dontUsePytestCheck" ] && [ -z "$installCheckPhase" ]; then
+    echo "Using pytestCheckPhase"
+    preDistPhases+=" pytestCheckPhase"
+fi

pkgs/development/interpreters/python/hooks/python-catch-conflicts-hook.sh

@@ -0,0 +1,10 @@
+# Setup hook for detecting conflicts in Python packages
+echo "Sourcing python-catch-conflicts-hook.sh"
+
+pythonCatchConflictsPhase() {
+    @pythonInterpreter@ @catchConflicts@
+}
+
+if [ -z "$dontUsePythonCatchConflicts" ]; then
+    preDistPhases+=" pythonCatchConflictsPhase"
+fi

pkgs/development/interpreters/python/hooks/python-imports-check-hook.sh

@@ -0,0 +1,16 @@
+# Setup hook for checking whether Python imports succeed
+echo "Sourcing python-imports-check-hook.sh"
+
+pythonImportsCheckPhase () {
+    echo "Executing pythonImportsCheckPhase"
+
+    if [ -n "$pythonImportsCheck" ]; then
+        echo "Check whether the following modules can be imported: $pythonImportsCheck"
+        cd $out && eval "@pythonCheckInterpreter@ -c 'import os; import importlib; list(map(lambda mod: importlib.import_module(mod), os.environ[\"pythonImportsCheck\"].split()))'"
+    fi
+}
+
+if [ -z "$dontUsePythonImportsCheck" ]; then
+    echo "Using pythonImportsCheckPhase"
+    preDistPhases+=" pythonImportsCheckPhase"
+fi

pkgs/development/interpreters/python/hooks/python-remove-bin-bytecode-hook.sh

@@ -0,0 +1,17 @@
+# Setup hook for detecting conflicts in Python packages
+echo "Sourcing python-remove-bin-bytecode-hook.sh"
+
+# Check if we have two packages with the same name in the closure and fail.
+# If this happens, something went wrong with the dependencies specs.
+# Intentionally kept in a subdirectory, see catch_conflicts/README.md.
+
+pythonRemoveBinBytecodePhase () {
+    if [ -d "$out/bin" ]; then
+      rm -rf "$out/bin/__pycache__"                 # Python 3
+      find "$out/bin" -type f -name "*.pyc" -delete # Python 2
+    fi
+}
+
+if [ -z "$dontUsePythonRemoveBinBytecode" ]; then
+    preDistPhases+=" pythonRemoveBinBytecodePhase"
+fi

pkgs/development/interpreters/python/hooks/setuptools-build-hook.sh

@@ -0,0 +1,47 @@
+# Setup hook for setuptools.
+echo "Sourcing setuptools-build-hook"
+
+setuptoolsBuildPhase() {
+    echo "Executing setuptoolsBuildPhase"
+    local args
+    runHook preBuild
+
+    cp -f @setuppy@ nix_run_setup
+    args=""
+    if [ -n "$setupPyGlobalFlags" ]; then
+        args+="$setupPyGlobalFlags"
+    fi
+    if [ -n "$setupPyBuildFlags" ]; then
+        args+="build_ext $setupPyBuildFlags"
+    fi
+    eval "@pythonInterpreter@ nix_run_setup $args bdist_wheel"
+
+    runHook postBuild
+    echo "Finished executing setuptoolsInstallPhase"
+}
+
+setuptoolsShellHook() {
+    echo "Executing setuptoolsShellHook"
+    runHook preShellHook
+
+    if test -e setup.py; then
+        tmp_path=$(mktemp -d)
+        export PATH="$tmp_path/bin:$PATH"
+        export PYTHONPATH="@pythonSitePackages@:$PYTHONPATH"
+        mkdir -p "$tmp_path/@pythonSitePackages@"
+        eval "@pythonInterpreter@ -m pip -e . --prefix $tmp_path >&2"
+    fi
+
+    runHook postShellHook
+    echo "Finished executing setuptoolsShellHook"
+}
+
+if [ -z "$dontUseSetuptoolsBuild" ] && [ -z "$buildPhase" ]; then
+    echo "Using setuptoolsBuildPhase"
+    buildPhase=setuptoolsBuildPhase
+fi
+
+if [ -z "$dontUseSetuptoolsShellHook" ] && [ -z "$shellHook" ]; then
+    echo "Using setuptoolsShellHook"
+    shellHook=setuptoolsShellHook
+fi

pkgs/development/interpreters/python/hooks/setuptools-check-hook.sh

@@ -0,0 +1,18 @@
+# Setup hook for setuptools.
+echo "Sourcing setuptools-check-hook"
+
+setuptoolsCheckPhase() {
+    echo "Executing setuptoolsCheckPhase"
+    runHook preCheck
+
+    cp -f @setuppy@ nix_run_setup
+    @pythonCheckInterpreter@ nix_run_setup test
+
+    runHook postCheck
+    echo "Finished executing setuptoolsCheckPhase"
+}
+
+if [ -z "$dontUseSetuptoolsCheck" ] && [ -z "$installCheckPhase" ]; then
+    echo "Using setuptoolsCheckPhase"
+    preDistPhases+=" setuptoolsCheckPhase"
+fi

pkgs/development/interpreters/python/hooks/wheel-unpack-hook.sh

@@ -0,0 +1,18 @@
+# Setup hook to use in case a wheel is fetched
+echo "Sourcing wheel setup hook"
+
+wheelUnpackPhase(){
+    echo "Executing wheelUnpackPhase"
+    runHook preUnpack
+
+    mkdir -p dist
+    cp "$src" "dist/$(stripHash "$src")"
+
+#     runHook postUnpack # Calls find...?
+    echo "Finished executing wheelUnpackPhase"
+}
+
+if [ -z "$dontUseWheelUnpack" ] && [ -z "$unpackPhase" ]; then
+    echo "Using wheelUnpackPhase"
+    unpackPhase=wheelUnpackPhase
+fi

pkgs/development/interpreters/python/mk-python-derivation.nix

@@ -4,13 +4,22 @@
 , config
 , python
 , wrapPython
-, setuptools
 , unzip
 , ensureNewerSourcesForZipFilesHook
 # Whether the derivation provides a Python module or not.
 , toPythonModule
 , namePrefix
 , update-python-libraries
+, setuptools
+, flitBuildHook
+, pipBuildHook
+, pipInstallHook
+, pythonCatchConflictsHook
+, pythonImportsCheckHook
+, pythonRemoveBinBytecodeHook
+, setuptoolsBuildHook
+, setuptoolsCheckHook
+, wheelUnpackHook
 }:
 
 { name ? "${attrs.pname}-${attrs.version}"
@@ -48,6 +57,11 @@
 # Skip wrapping of python programs altogether
 , dontWrapPythonPrograms ? false
 
+# Don't use Pip to install a wheel
+# Note this is actually a variable for the pipInstallPhase in pip's setupHook.
+# It's included here to prevent an infinite recursion.
+, dontUsePipInstall ? false
+
 # Skip setting the PYTHONNOUSERSITE environment variable in wrapped programs
 , permitUserSite ? false
 
@@ -57,6 +71,13 @@
 # However, some packages do provide executables with extensions, and thus bytecode is generated.
 , removeBinBytecode ? true
 
+# Several package formats are supported.
+# "setuptools" : Install a common setuptools/distutils based package. This builds a wheel.
+# "wheel" : Install from a pre-compiled wheel.
+# "flit" : Install a flit package. This builds a wheel.
+# "other" : Provide your own buildPhase and installPhase.
+, format ? "setuptools"
+
 , meta ? {}
 
 , passthru ? {}
@@ -71,26 +92,43 @@ if disabled
 then throw "${name} not supported for interpreter ${python.executable}"
 else
 
-let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attrs [
+let
-    "disabled" "checkInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts"
+  inherit (python) stdenv;
-  ] // {
+
+  self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [
+    "disabled" "checkPhase" "checkInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "format"
+  ]) // {
 
   name = namePrefix + name;
 
   nativeBuildInputs = [
     python
     wrapPython
-    ensureNewerSourcesForZipFilesHook
+    ensureNewerSourcesForZipFilesHook  # move to wheel installer (pip) or builder (setuptools, flit, ...)?
-    setuptools
+  ] ++ lib.optionals catchConflicts [
-#     ++ lib.optional catchConflicts setuptools # If we no longer propagate setuptools
+    setuptools pythonCatchConflictsHook
+  ] ++ lib.optionals removeBinBytecode [
+    pythonRemoveBinBytecodeHook
   ] ++ lib.optionals (lib.hasSuffix "zip" (attrs.src.name or "")) [
     unzip
+  ] ++ lib.optionals (format == "setuptools") [
+    setuptoolsBuildHook
+  ] ++ lib.optionals (format == "flit") [
+    flitBuildHook
+  ] ++ lib.optionals (format == "pyproject") [
+    pipBuildHook
+  ] ++ lib.optionals (format == "wheel") [
+    wheelUnpackHook
+  ] ++ lib.optionals (!(format == "other") || dontUsePipInstall) [
+    pipInstallHook
+  ] ++ lib.optionals (stdenv.buildPlatform == stdenv.hostPlatform) [
+    # This is a test, however, it should be ran independent of the checkPhase and checkInputs
+    pythonImportsCheckHook
   ] ++ nativeBuildInputs;
 
   buildInputs = buildInputs ++ pythonPath;
 
-  # Propagate python and setuptools. We should stop propagating setuptools.
+  propagatedBuildInputs = propagatedBuildInputs ++ [ python ];
-  propagatedBuildInputs = propagatedBuildInputs ++ [ python setuptools ];
 
   inherit strictDeps;
 
@@ -98,21 +136,17 @@ let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attr
 
   # Python packages don't have a checkPhase, only an installCheckPhase
   doCheck = false;
-  doInstallCheck = doCheck;
+  doInstallCheck = attrs.doCheck or true;
-  installCheckInputs = checkInputs;
+  installCheckInputs = [
+  ] ++ lib.optionals (format == "setuptools") [
+    # Longer-term we should get rid of this and require
+    # users of this function to set the `installCheckPhase` or
+    # pass in a hook that sets it.
+    setuptoolsCheckHook
+  ] ++ checkInputs;
 
   postFixup = lib.optionalString (!dontWrapPythonPrograms) ''
     wrapPythonPrograms
-  '' + lib.optionalString removeBinBytecode ''
-    if [ -d "$out/bin" ]; then
-      rm -rf "$out/bin/__pycache__"                 # Python 3
-      find "$out/bin" -type f -name "*.pyc" -delete # Python 2
-    fi
-  '' + lib.optionalString catchConflicts ''
-    # Check if we have two packages with the same name in the closure and fail.
-    # If this happens, something went wrong with the dependencies specs.
-    # Intentionally kept in a subdirectory, see catch_conflicts/README.md.
-    ${python.pythonForBuild.interpreter} ${./catch_conflicts}/catch_conflicts.py
   '' + attrs.postFixup or '''';
 
   # Python packages built through cross-compilation are always for the host platform.
@@ -123,6 +157,10 @@ let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attr
     platforms = python.meta.platforms;
     isBuildPythonPackage = python.meta.platforms;
   } // meta;
+} // lib.optionalAttrs (attrs?checkPhase) {
+  # If given use the specified checkPhase, otherwise use the setup hook.
+  # Longer-term we should get rid of `checkPhase` and use `installCheckPhase`.
+  installCheckPhase = attrs.checkPhase;
 }));
 
 passthru.updateScript = let

pkgs/development/python-modules/atomicwrites/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, buildPythonPackage, fetchPypi }:
+{ stdenv, buildPythonPackage, fetchPypi, pytest }:
 
 buildPythonPackage rec {
   pname = "atomicwrites";
@@ -9,6 +9,10 @@ buildPythonPackage rec {
     sha256 = "75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6";
   };
 
+  # Tests depend on pytest but atomicwrites is a dependency of pytest
+  doCheck = false;
+  checkInputs = [ pytest ];
+
   meta = with stdenv.lib; {
     description = "Atomic file writes on POSIX";
     homepage = https://pypi.python.org/pypi/atomicwrites;

pkgs/development/python-modules/bootstrapped-pip/default.nix

@@ -1,4 +1,8 @@
-{ stdenv, python, fetchPypi, makeWrapper, unzip }:
+{ stdenv, python, fetchPypi, makeWrapper, unzip, makeSetupHook
+, pipInstallHook
+, setuptoolsBuildHook
+
+}:
 
 let
   wheel_source = fetchPypi {
@@ -25,6 +29,15 @@ in stdenv.mkDerivation rec {
     sha256 = "993134f0475471b91452ca029d4390dc8f298ac63a712814f101cd1b6db46676";
   };
 
+  dontUseSetuptoolsBuild = true;
+
+  # Should be propagatedNativeBuildInputs
+  propagatedBuildInputs = [
+    # Override to remove dependencies to prevent infinite recursion.
+    (pipInstallHook.override{pip=null;})
+    (setuptoolsBuildHook.override{setuptools=null; wheel=null;})
+  ];
+
   unpackPhase = ''
     mkdir -p $out/${python.sitePackages}
     unzip -d $out/${python.sitePackages} $src
@@ -32,7 +45,7 @@ in stdenv.mkDerivation rec {
     unzip -d $out/${python.sitePackages} ${wheel_source}
   '';
 
-  patchPhase = ''
+  postPatch = ''
     mkdir -p $out/bin
   '';
 
@@ -52,4 +65,5 @@ in stdenv.mkDerivation rec {
       wrapProgram $f --prefix PYTHONPATH ":" $out/${python.sitePackages}/
     done
   '';
+
 }

pkgs/development/python-modules/pip/default.nix

@@ -1,25 +1,32 @@
 { lib
+, python
 , buildPythonPackage
+, bootstrapped-pip
 , fetchPypi
 , mock
 , scripttest
 , virtualenv
 , pretend
 , pytest
+, setuptools
+, wheel
 }:
 
 buildPythonPackage rec {
   pname = "pip";
   version = "19.1.1";
+  format = "other";
 
   src = fetchPypi {
     inherit pname version;
     sha256 = "44d3d7d3d30a1eb65c7e5ff1173cdf8f7467850605ac7cc3707b6064bddd0958";
   };
 
+  nativeBuildInputs = [ bootstrapped-pip ];
+
   # pip detects that we already have bootstrapped_pip "installed", so we need
   # to force it a little.
-  installFlags = [ "--ignore-installed" ];
+  pipInstallFlags = [ "--ignore-installed" ];
 
   checkInputs = [ mock scripttest virtualenv pretend pytest ];
   # Pip wants pytest, but tests are not distributed

pkgs/development/python-modules/py/default.nix

@@ -12,7 +12,11 @@ buildPythonPackage rec {
   # Circular dependency on pytest
   doCheck = false;
 
-  buildInputs = [ setuptools_scm ];
+  nativeBuildInputs = [ setuptools_scm ];
+
+  pythonImportsCheck = [
+    "py"
+  ];
 
   meta = with stdenv.lib; {
     description = "Library with cross-python path, ini-parsing, io, code, log facilities";

pkgs/development/python-modules/pytest/default.nix

@@ -1,6 +1,6 @@
 { stdenv, buildPythonPackage, pythonOlder, fetchPypi, attrs, hypothesis, py
 , setuptools_scm, setuptools, six, pluggy, funcsigs, isPy3k, more-itertools
-, atomicwrites, mock, writeText, pathlib2, wcwidth, packaging, isPyPy
+, atomicwrites, mock, writeText, pathlib2, wcwidth, packaging, isPyPy, python
 }:
 buildPythonPackage rec {
   version = "5.1.0";
@@ -17,12 +17,13 @@ buildPythonPackage rec {
   };
 
   checkInputs = [ hypothesis mock ];
-  buildInputs = [ setuptools_scm ];
+  nativeBuildInputs = [ setuptools_scm ];
   propagatedBuildInputs = [ attrs py setuptools six pluggy more-itertools atomicwrites wcwidth packaging ]
     ++ stdenv.lib.optionals (!isPy3k) [ funcsigs ]
     ++ stdenv.lib.optionals (pythonOlder "3.6") [ pathlib2 ];
 
   doCheck = !isPyPy; # https://github.com/pytest-dev/pytest/issues/3460
+
   # Ignored file https://github.com/pytest-dev/pytest/pull/5605#issuecomment-522243929
   checkPhase = ''
     runHook preCheck
@@ -35,15 +36,17 @@ buildPythonPackage rec {
     pytestcachePhase() {
         find $out -name .pytest_cache -type d -exec rm -rf {} +
     }
-
     preDistPhases+=" pytestcachePhase"
   '';
 
+  pythonImportsCheck = [
+    "pytest"
+  ];
+
   meta = with stdenv.lib; {
     homepage = https://docs.pytest.org;
     description = "Framework for writing tests";
     maintainers = with maintainers; [ domenkozar lovek323 madjar lsix ];
     license = licenses.mit;
-    platforms = platforms.unix;
   };
 }

pkgs/development/python-modules/setuptools/default.nix

@@ -1,15 +1,17 @@
 { stdenv
+, buildPythonPackage
 , fetchPypi
 , python
 , wrapPython
 , unzip
+, callPackage
+, bootstrapped-pip
 }:
 
-# Should use buildPythonPackage here somehow
+buildPythonPackage rec {
-stdenv.mkDerivation rec {
   pname = "setuptools";
   version = "41.0.1";
-  name = "${python.libPrefix}-${pname}-${version}";
+  format = "other";
 
   src = fetchPypi {
     inherit pname version;
@@ -17,8 +19,11 @@ stdenv.mkDerivation rec {
     sha256 = "a222d126f5471598053c9a77f4b5d4f26eaa1f150ad6e01dcf1a42e185d05613";
   };
 
-  nativeBuildInputs = [ unzip wrapPython python.pythonForBuild ];
+  # There is nothing to build
-  doCheck = false;  # requires pytest
+  dontBuild = true;
+
+  nativeBuildInputs = [ bootstrapped-pip ];
+
   installPhase = ''
       dst=$out/${python.sitePackages}
       mkdir -p $dst
@@ -27,13 +32,11 @@ stdenv.mkDerivation rec {
       wrapPythonPrograms
   '';
 
-  pythonPath = [];
+  # Adds setuptools to nativeBuildInputs causing infinite recursion.
-
+  catchConflicts = false;
-  dontPatchShebangs = true;
-
-  # Python packages built through cross-compilation are always for the host platform.
-  disallowedReferences = stdenv.lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ python.pythonForBuild ];
 
+  # Requires pytest, causing infinite recursion.
+  doCheck = false;
 
   meta = with stdenv.lib; {
     description = "Utilities to facilitate the installation of Python packages";

pkgs/development/python-modules/setuptools_scm/default.nix

@@ -8,8 +8,6 @@ buildPythonPackage rec {
     sha256 = "52ab47715fa0fc7d8e6cd15168d1a69ba995feb1505131c3e814eb7087b57358";
   };
 
-  buildInputs = [ pip ];
-
   # Seems to fail due to chroot and would cause circular dependency
   # with pytest
   doCheck = false;

pkgs/development/python-modules/wheel/default.nix

@@ -1,15 +1,19 @@
 { lib
+, setuptools
+, pip
 , buildPythonPackage
 , fetchPypi
 , pytest
 , pytestcov
 , coverage
 , jsonschema
+, bootstrapped-pip
 }:
 
 buildPythonPackage rec {
   pname = "wheel";
   version = "0.33.4";
+  format = "other";
 
   src = fetchPypi {
     inherit pname version;
@@ -17,14 +21,14 @@ buildPythonPackage rec {
   };
 
   checkInputs = [ pytest pytestcov coverage ];
+  nativeBuildInputs = [ bootstrapped-pip setuptools ];
 
-  propagatedBuildInputs = [ jsonschema ];
+  catchConflicts = false;
-
   # No tests in archive
   doCheck = false;
 
   # We add this flag to ignore the copy installed by bootstrapped-pip
-  installFlags = [ "--ignore-installed" ];
+  pipInstallFlags = [ "--ignore-installed" ];
 
   meta = {
     description = "A built-package format for Python";

pkgs/top-level/python-packages.nix

@@ -42,17 +42,14 @@ let
       }
       else ff;
 
-  buildPythonPackage = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix {
+  buildPythonPackage = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/mk-python-derivation.nix {
-    flit = self.flit;
+    inherit namePrefix;     # We want Python libraries to be named like e.g. "python3.6-${name}"
-    # We want Python libraries to be named like e.g. "python3.6-${name}"
+    inherit toPythonModule; # Libraries provide modules
-    inherit namePrefix;
-    inherit toPythonModule;
   }));
 
-  buildPythonApplication = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix {
+  buildPythonApplication = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/mk-python-derivation.nix {
-    flit = self.flit;
+    namePrefix = "";        # Python applications should not have any prefix
-    namePrefix = "";
+    toPythonModule = x: x;  # Application does not provide modules.
-    toPythonModule = x: x; # Application does not provide modules.
   }));
 
   # See build-setupcfg/default.nix for documentation.
@@ -110,6 +107,9 @@ in {
   inherit toPythonModule toPythonApplication;
   inherit buildSetupcfg;
 
+  inherit (callPackage ../development/interpreters/python/hooks { })
+    flitBuildHook pipBuildHook pipInstallHook pytestCheckHook pythonCatchConflictsHook pythonImportsCheckHook pythonRemoveBinBytecodeHook setuptoolsBuildHook setuptoolsCheckHook wheelUnpackHook;
+
   # helpers
 
   wrapPython = callPackage ../development/interpreters/python/wrap-python.nix {inherit python; inherit (pkgs) makeSetupHook makeWrapper; };
@@ -121,7 +121,7 @@ in {
 
   recursivePthLoader = callPackage ../development/python-modules/recursive-pth-loader { };
 
-  setuptools = toPythonModule (callPackage ../development/python-modules/setuptools { });
+  setuptools = callPackage ../development/python-modules/setuptools { };
 
   vowpalwabbit = callPackage ../development/python-modules/vowpalwabbit { };