Revert "Revert "Revert "cpython: fix finding headers when cross-compiling extension modules"""

This still does not function without issues. E.g., bootstrapped-pip fails with Python 3.6 and 3.7
as well as 3.8 on 32-bit.

Because this is a stdenv-rebuild it needs to be tested significantly better

This reverts commit 6100bc29f7.

pkgs/development/interpreters/python/cpython/3.6/fix-finding-headers-when-cross-compiling.patch

@@ -1,54 +0,0 @@
-From 45dfbbb4f5b67ab83e4365564ea569334e979f8e Mon Sep 17 00:00:00 2001
-From: Ben Wolsieffer <benwolsieffer@gmail.com>
-Date: Fri, 25 Sep 2020 16:49:16 -0400
-Subject: [PATCH] Fix finding headers when cross compiling
-
-When cross-compiling third-party extensions, get_python_inc() may be called to
-return the path to Python's headers. However, it uses the sys.prefix or
-sys.exec_prefix of the build Python, which returns incorrect paths when
-cross-compiling (paths pointing to build system headers).
-
-To fix this, we use the INCLUDEPY and CONFINCLUDEPY conf variables, which can
-be configured to point at host Python by setting _PYTHON_SYSCONFIGDATA_NAME.
-The existing behavior is maintained on non-POSIX platforms or if a prefix is
-manually specified.
----
- Lib/distutils/sysconfig.py | 14 ++++++++++----
- 1 file changed, 10 insertions(+), 4 deletions(-)
-
-diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py
-index 2bcd1dd288..567375e488 100644
---- a/Lib/distutils/sysconfig.py
-+++ b/Lib/distutils/sysconfig.py
-@@ -84,8 +84,6 @@ def get_python_inc(plat_specific=0, prefix=None):
-     If 'prefix' is supplied, use it instead of sys.base_prefix or
-     sys.base_exec_prefix -- i.e., ignore 'plat_specific'.
-     """
--    if prefix is None:
--        prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX
-     if os.name == "posix":
-         if python_build:
-             # Assume the executable is in the build directory.  The
-@@ -98,9 +96,17 @@ def get_python_inc(plat_specific=0, prefix=None):
-             else:
-                 incdir = os.path.join(get_config_var('srcdir'), 'Include')
-                 return os.path.normpath(incdir)
--        python_dir = 'python' + get_python_version() + build_flags
--        return os.path.join(prefix, "include", python_dir)
-+        if prefix is None:
-+          if plat_specific:
-+            return get_config_var('CONFINCLUDEPY')
-+          else:
-+            return get_config_var('INCLUDEPY')
-+        else:
-+          python_dir = 'python' + get_python_version() + build_flags
-+          return os.path.join(prefix, "include", python_dir)
-     elif os.name == "nt":
-+        if prefix is None:
-+          prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX
-         return os.path.join(prefix, "include")
-     else:
-         raise DistutilsPlatformError(
--- 
-2.28.0
-

pkgs/development/interpreters/python/cpython/3.7/fix-finding-headers-when-cross-compiling.patch

@@ -1,54 +0,0 @@
-From debccd4be0a8d619770f63622d9de1b451dd02ac Mon Sep 17 00:00:00 2001
-From: Ben Wolsieffer <benwolsieffer@gmail.com>
-Date: Fri, 25 Sep 2020 16:49:16 -0400
-Subject: [PATCH] Fix finding headers when cross compiling
-
-When cross-compiling third-party extensions, get_python_inc() may be called to
-return the path to Python's headers. However, it uses the sys.prefix or
-sys.exec_prefix of the build Python, which returns incorrect paths when
-cross-compiling (paths pointing to build system headers).
-
-To fix this, we use the INCLUDEPY and CONFINCLUDEPY conf variables, which can
-be configured to point at host Python by setting _PYTHON_SYSCONFIGDATA_NAME.
-The existing behavior is maintained on non-POSIX platforms or if a prefix is
-manually specified.
----
- Lib/distutils/sysconfig.py | 14 ++++++++++----
- 1 file changed, 10 insertions(+), 4 deletions(-)
-
-diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py
-index 37feae5df7..6d4ad06696 100644
---- a/Lib/distutils/sysconfig.py
-+++ b/Lib/distutils/sysconfig.py
-@@ -95,8 +95,6 @@ def get_python_inc(plat_specific=0, prefix=None):
-     If 'prefix' is supplied, use it instead of sys.base_prefix or
-     sys.base_exec_prefix -- i.e., ignore 'plat_specific'.
-     """
--    if prefix is None:
--        prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX
-     if os.name == "posix":
-         if python_build:
-             # Assume the executable is in the build directory.  The
-@@ -109,9 +107,17 @@ def get_python_inc(plat_specific=0, prefix=None):
-             else:
-                 incdir = os.path.join(get_config_var('srcdir'), 'Include')
-                 return os.path.normpath(incdir)
--        python_dir = 'python' + get_python_version() + build_flags
--        return os.path.join(prefix, "include", python_dir)
-+        if prefix is None:
-+          if plat_specific:
-+            return get_config_var('CONFINCLUDEPY')
-+          else:
-+            return get_config_var('INCLUDEPY')
-+        else:
-+          python_dir = 'python' + get_python_version() + build_flags
-+          return os.path.join(prefix, "include", python_dir)
-     elif os.name == "nt":
-+        if prefix is None:
-+          prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX
-         if python_build:
-             # Include both the include and PC dir to ensure we can find
-             # pyconfig.h
--- 
-2.28.0
-

pkgs/development/interpreters/python/cpython/default.nix

@@ -101,44 +101,6 @@ let
     "$out/bin/python"
   else pythonForBuild.interpreter;
 
-  # The CPython interpreter contains a _sysconfigdata_<platform specific suffix>
-  # module that is imported by the sysconfig and distutils.sysconfig modules.
-  # The sysconfigdata module is generated at build time and contains settings
-  # required for building Python extension modules, such as include paths and
-  # other compiler flags. By default, the sysconfigdata module is loaded from
-  # the currently running interpreter (ie. the build platform interpreter), but
-  # when cross-compiling we want to load it from the host platform interpreter.
-  # This can be done using the _PYTHON_SYSCONFIGDATA_NAME environment variable.
-  # The _PYTHON_HOST_PLATFORM variable also needs to be set to get the correct
-  # platform suffix on extension modules. The correct values for these variables
-  # are not documented, and must be derived from the configure script (see links
-  # below).
-  sysconfigdataHook = with stdenv.hostPlatform; let
-    # https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L428
-    pythonHostPlatform = "${parsed.kernel.name}-${parsed.cpu.name}";
-
-    # https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L724
-    multiarchCpu =
-      if isAarch32 then
-        if parsed.cpu.significantByte.name == "littleEndian" then "arm" else "armeb"
-      else parsed.cpu.name;
-    multiarch =
-      if isDarwin then "darwin"
-      else "${multiarchCpu}-${parsed.kernel.name}-${parsed.abi.name}";
-
-    # https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L78
-    pythonSysconfigdataName = "_sysconfigdata__${parsed.kernel.name}_${multiarch}";
-  in ''
-    sysconfigdataHook() {
-      if [ "$1" = '${placeholder "out"}' ]; then
-        export _PYTHON_HOST_PLATFORM='${pythonHostPlatform}'
-        export _PYTHON_SYSCONFIGDATA_NAME='${pythonSysconfigdataName}'
-      fi
-    }
-
-    addEnvHooks "$hostOffset" sysconfigdataHook
-  '';
-
 in with passthru; stdenv.mkDerivation {
   pname = "python3";
   inherit version;
@@ -204,13 +166,6 @@ in with passthru; stdenv.mkDerivation {
   ] ++ [
     # LDSHARED now uses $CC instead of gcc. Fixes cross-compilation of extension modules.
     ./3.8/0001-On-all-posix-systems-not-just-Darwin-set-LDSHARED-if.patch
-    # Use sysconfigdata to find headers. Fixes cross-compilation of extension modules.
-    (
-      if isPy36 then
-        ./3.6/fix-finding-headers-when-cross-compiling.patch
-      else
-        ./3.7/fix-finding-headers-when-cross-compiling.patch
-    )
   ];
 
   postPatch = ''
@@ -324,10 +279,6 @@ in with passthru; stdenv.mkDerivation {
     find $out/lib/python*/config-* -type f -print -exec nuke-refs -e $out '{}' +
     find $out/lib -name '_sysconfigdata*.py*' -print -exec nuke-refs -e $out '{}' +
 
-    # Make the sysconfigdata module accessible on PYTHONPATH
-    # This allows build Python to import host Python's sysconfigdata
-    mkdir -p "$out/${sitePackages}"
-    ln -s "$out/lib/${libPrefix}/"_sysconfigdata*.py "$out/${sitePackages}/"
     '' + optionalString stripConfig ''
     rm -R $out/bin/python*-config $out/lib/python*/config-*
     '' + optionalString stripIdlelib ''
@@ -360,14 +311,6 @@ in with passthru; stdenv.mkDerivation {
     export PATH=${stdenv.lib.makeBinPath [ "$out" bash ]}:$PATH
   '';
 
-  # Add CPython specific setup-hook that configures distutils.sysconfig to
-  # always load sysconfigdata from host Python.
-  postFixup = ''
-    cat << "EOF" >> "$out/nix-support/setup-hook"
-    ${sysconfigdataHook}
-    EOF
-  '';
-
   # Enforce that we don't have references to the OpenSSL -dev package, which we
   # explicitly specify in our configure flags above.
   disallowedReferences =

pkgs/development/python-modules/bootstrapped-pip/default.nix

@@ -45,7 +45,7 @@ stdenv.mkDerivation rec {
     mv wheel* wheel
     # Set up PYTHONPATH. The above folders need to be on PYTHONPATH
     # $out is where we are installing to and takes precedence
-    export PYTHONPATH="$out/${python.sitePackages}:$(pwd)/pip/src:$(pwd)/setuptools:$(pwd)/setuptools/pkg_resources:$(pwd)/wheel:$PYTHONPATH"
+    export PYTHONPATH="$out/${python.sitePackages}:$(pwd)/pip/src:$(pwd)/setuptools:$(pwd)/setuptools/pkg_resources:$(pwd)/wheel"
 
     echo "Building setuptools wheel..."
     pushd setuptools