Merge pull request #82453 from adisbladis/nix-pythonprefix

Python: introduce NIX_PYTHONPREFIX in order to set site.PREFIXES

pkgs/development/interpreters/python/sitecustomize.py

@@ -21,6 +21,19 @@ paths = os.environ.pop('NIX_PYTHONPATH', None)
 if paths:
     functools.reduce(lambda k, p: site.addsitedir(p, k), paths.split(':'), site._init_pathinfo())
 
-executable = os.environ.pop('NIX_PYTHONEXECUTABLE', None)
-if 'PYTHONEXECUTABLE' not in os.environ and executable:
-    sys.executable = executable
+# Check whether we are in a venv. 
+# Note Python 2 does not support base_prefix so we assume we are not in a venv.
+in_venv = sys.version_info.major == 3 and sys.prefix != sys.base_prefix
+
+if not in_venv:
+    executable = os.environ.pop('NIX_PYTHONEXECUTABLE', None)
+    prefix = os.environ.pop('NIX_PYTHONPREFIX', None)
+
+    if 'PYTHONEXECUTABLE' not in os.environ and executable is not None:
+        sys.executable = executable
+    if prefix is not None:
+        # Because we cannot check with Python 2 whether we are in a venv, 
+        # creating a venv from a Nix env won't work as well with Python 2.
+        # Also, note that sysconfig does not like it when sys.prefix is set to None
+        sys.prefix = sys.exec_prefix = prefix
+        site.PREFIXES.insert(0, prefix)

pkgs/development/interpreters/python/tests.nix

@@ -2,6 +2,7 @@
 , runCommand
 , substituteAll
 , lib
+, callPackage
 }:
 
 let
@@ -36,6 +37,7 @@ let
       is_venv = "True";
       is_nixenv = "False";
     };
+
     # Venv built using Python Nix environment (python.buildEnv)
     # TODO: Cannot create venv from a  nix env
     # Error: Command '['/nix/store/ddc8nqx73pda86ibvhzdmvdsqmwnbjf7-python3-3.7.6-venv/bin/python3.7', '-Im', 'ensurepip', '--upgrade', '--default-pip']' returned non-zero exit status 1.
@@ -49,6 +51,14 @@ let
     # };
   };
 
+  # All PyPy package builds are broken at the moment
+  integrationTests = lib.optionalAttrs (python.isPy3k  && (!python.isPyPy)) rec {
+    # Before the addition of NIX_PYTHONPREFIX mypy was broken with typed packages
+    nix-pythonprefix-mypy = callPackage ./tests/test_nix_pythonprefix {
+      interpreter = python;
+    };
+  };
+
   testfun = name: attrs: runCommand "${python.name}-tests-${name}" ({
     inherit (python) pythonVersion;
   } // attrs) ''
@@ -60,4 +70,4 @@ let
     touch $out/success
   '';
 
-in lib.mapAttrs testfun envs 
+in lib.mapAttrs testfun envs // integrationTests

pkgs/development/interpreters/python/tests/test_nix_pythonprefix/default.nix

@@ -0,0 +1,25 @@
+{ interpreter, writeText, runCommandNoCC }:
+
+let
+
+  python = let
+    packageOverrides = self: super: {
+      typeddep = super.callPackage ./typeddep {};
+    };
+  in interpreter.override {inherit packageOverrides; self = python;};
+
+  pythonEnv = python.withPackages(ps: [
+    ps.typeddep
+    ps.mypy
+  ]);
+
+  pythonScript = writeText "myscript.py" ''
+    from typeddep import util
+    s: str = util.echo("hello")
+    print(s)
+  '';
+
+in runCommandNoCC "${interpreter.name}-site-prefix-mypy-test" {} ''
+  ${pythonEnv}/bin/mypy ${pythonScript}
+  touch $out
+''

pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/default.nix

@@ -0,0 +1,11 @@
+{ buildPythonPackage }:
+
+
+buildPythonPackage {
+
+  pname = "typeddep";
+  version = "1.3.3.7";
+
+  src = ./.;
+
+}

pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/setup.py

@@ -0,0 +1,18 @@
+from setuptools import setup
+
+setup(**{
+    'name': 'typeddep',
+    'version': '1.3.3.7',
+    'description': 'Minimal repro to test mypy and site prefixes with Nix',
+    'long_description': None,
+    'author': 'adisbladis',
+    'author_email': 'adisbladis@gmail.com',
+    'maintainer': None,
+    'maintainer_email': None,
+    'url': None,
+    'packages': ['typeddep'],
+    'package_data': {'': ['*']},
+    'install_requires': [],
+    'entry_points': {},
+    'python_requires': '>=3.7,<4.0',
+})

pkgs/development/interpreters/python/tests/test_nix_pythonprefix/typeddep/typeddep/util.py

@@ -0,0 +1,2 @@
+def echo(s: str) -> str:
+    return s

pkgs/development/interpreters/python/tests/test_python.py

@@ -27,7 +27,7 @@ class TestCasePython(unittest.TestCase):
     def test_interpreter(self):
         self.assertEqual(sys.executable, INTERPRETER)
 
-    @unittest.skipIf(IS_NIXENV or IS_PYPY, "Prefix is incorrect and needs to be fixed.")
+    @unittest.skipIf(IS_PYPY, "Prefix is incorrect and needs to be fixed.")
     def test_prefix(self):
         self.assertEqual(sys.prefix, ENV)
         self.assertEqual(sys.prefix, sys.exec_prefix)
@@ -35,9 +35,9 @@ class TestCasePython(unittest.TestCase):
     def test_site_prefix(self):
         self.assertTrue(sys.prefix in site.PREFIXES)
 
-    @unittest.skipIf(sys.version_info.major==2, "Python 2 does not have base_prefix")
+    @unittest.skipIf(IS_PYPY or sys.version_info.major==2, "Python 2 does not have base_prefix")
     def test_base_prefix(self):
-        if IS_VENV:
+        if IS_VENV or IS_NIXENV:
             self.assertNotEqual(sys.prefix, sys.base_prefix)
         else:
             self.assertEqual(sys.prefix, sys.base_prefix)

pkgs/development/interpreters/python/wrapper.nix

@@ -37,7 +37,7 @@ let
             if [ -f "$prg" ]; then
               rm -f "$out/bin/$prg"
               if [ -x "$prg" ]; then
-                makeWrapper "$path/bin/$prg" "$out/bin/$prg" --set NIX_PYTHONEXECUTABLE ${pythonExecutable} --set NIX_PYTHONPATH ${pythonPath} ${if permitUserSite then "" else ''--set PYTHONNOUSERSITE "true"''} ${stdenv.lib.concatStringsSep " " makeWrapperArgs}
+                makeWrapper "$path/bin/$prg" "$out/bin/$prg" --set NIX_PYTHONPREFIX "$out" --set NIX_PYTHONEXECUTABLE ${pythonExecutable} --set NIX_PYTHONPATH ${pythonPath} ${if permitUserSite then "" else ''--set PYTHONNOUSERSITE "true"''} ${stdenv.lib.concatStringsSep " " makeWrapperArgs}
               fi
             fi
           done