From 40851a4d26daabf1613c9a79e638aaf9f94b476c Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Sun, 28 May 2017 09:20:47 +0200 Subject: [PATCH] Python: the pythonModule attribute Python libraries or modules now have an attribute `pythonModule = interpreter;` to indicate they provide Python modules for the specified `interpreter`. The package set provides the following helper functions: - hasPythonModule: Check whether a derivation provides a Python module. - requiredPythonModules: Recurse into a list of Python modules, returning all Python modules that are required. - makePythonPath: Create a PYTHONPATH from a list of Python modules. Also included in this commit is: - disabledIf: Helper function for disabling non-buildPythonPackage functions. --- .../python/build-python-package.nix | 10 +++- .../python/cpython/2.7/default.nix | 2 +- .../python/cpython/3.4/default.nix | 2 +- .../python/cpython/3.5/default.nix | 2 +- .../python/cpython/3.6/default.nix | 2 +- .../python/mk-python-derivation.nix | 9 ++-- .../interpreters/python/pypy/2.7/default.nix | 2 +- .../interpreters/python/wrapper.nix | 7 +-- pkgs/top-level/python-packages.nix | 52 ++++++++++++++++--- 9 files changed, 67 insertions(+), 21 deletions(-) diff --git a/pkgs/development/interpreters/python/build-python-package.nix b/pkgs/development/interpreters/python/build-python-package.nix index b26bf1539cd..29b57834368 100644 --- a/pkgs/development/interpreters/python/build-python-package.nix +++ b/pkgs/development/interpreters/python/build-python-package.nix @@ -5,7 +5,12 @@ { lib , python -, mkPythonDerivation +, wrapPython +, setuptools +, unzip +, ensureNewerSourcesHook +, pythonModule +, namePrefix , bootstrapped-pip , flit }: @@ -15,6 +20,9 @@ let flit-specific = import ./build-python-package-flit.nix { inherit python flit; }; wheel-specific = import ./build-python-package-wheel.nix { }; common = import ./build-python-package-common.nix { inherit python bootstrapped-pip; }; + mkPythonDerivation = import ./mk-python-derivation.nix { + inherit lib python wrapPython setuptools unzip ensureNewerSourcesHook pythonModule namePrefix; + }; in { diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix index eb2a46cb3b7..c7483a81529 100644 --- a/pkgs/development/interpreters/python/cpython/2.7/default.nix +++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix @@ -201,7 +201,7 @@ in stdenv.mkDerivation { in rec { inherit libPrefix sitePackages x11Support hasDistutilsCxxPatch; executable = libPrefix; - buildEnv = callPackage ../../wrapper.nix { python = self; }; + buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; }; withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; pkgs = pythonPackages; isPy2 = true; diff --git a/pkgs/development/interpreters/python/cpython/3.4/default.nix b/pkgs/development/interpreters/python/cpython/3.4/default.nix index a924b543fe3..5c13035be1b 100644 --- a/pkgs/development/interpreters/python/cpython/3.4/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.4/default.nix @@ -160,7 +160,7 @@ in stdenv.mkDerivation { in rec { inherit libPrefix sitePackages x11Support; executable = "${libPrefix}m"; - buildEnv = callPackage ../../wrapper.nix { python = self; }; + buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; }; withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; pkgs = pythonPackages; isPy3 = true; diff --git a/pkgs/development/interpreters/python/cpython/3.5/default.nix b/pkgs/development/interpreters/python/cpython/3.5/default.nix index abe220e0a3e..951cb367528 100644 --- a/pkgs/development/interpreters/python/cpython/3.5/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.5/default.nix @@ -154,7 +154,7 @@ in stdenv.mkDerivation { in rec { inherit libPrefix sitePackages x11Support; executable = "${libPrefix}m"; - buildEnv = callPackage ../../wrapper.nix { python = self; }; + buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; }; withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; pkgs = pythonPackages; isPy3 = true; diff --git a/pkgs/development/interpreters/python/cpython/3.6/default.nix b/pkgs/development/interpreters/python/cpython/3.6/default.nix index 1614159a7ef..b44e167b9f0 100644 --- a/pkgs/development/interpreters/python/cpython/3.6/default.nix +++ b/pkgs/development/interpreters/python/cpython/3.6/default.nix @@ -153,7 +153,7 @@ in stdenv.mkDerivation { in rec { inherit libPrefix sitePackages x11Support; executable = "${libPrefix}m"; - buildEnv = callPackage ../../wrapper.nix { python = self; }; + buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; }; withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; pkgs = pythonPackages; isPy3 = true; diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index 098ab0b1719..386875a0ab4 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -6,13 +6,13 @@ , setuptools , unzip , ensureNewerSourcesHook +# Whether the derivation provides a Python module or not. +, pythonModule +, namePrefix }: { name ? "${attrs.pname}-${attrs.version}" -# by default prefix `name` e.g. "python3.3-${name}" -, namePrefix ? python.libPrefix + "-" - # Dependencies for building the package , buildInputs ? [] @@ -54,7 +54,7 @@ if disabled then throw "${name} not supported for interpreter ${python.executable}" else -python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled" "checkInputs"] // { +python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled" "checkInputs" "pythonModule"] // { name = namePrefix + name; @@ -83,6 +83,7 @@ python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled" "checkInputs" passthru = { inherit python; # The python interpreter + inherit pythonModule; } // passthru; meta = with lib.maintainers; { diff --git a/pkgs/development/interpreters/python/pypy/2.7/default.nix b/pkgs/development/interpreters/python/pypy/2.7/default.nix index f5ee13cfc12..aea389d160f 100644 --- a/pkgs/development/interpreters/python/pypy/2.7/default.nix +++ b/pkgs/development/interpreters/python/pypy/2.7/default.nix @@ -137,7 +137,7 @@ in stdenv.mkDerivation rec { inherit zlibSupport libPrefix sitePackages; executable = "pypy"; isPypy = true; - buildEnv = callPackage ../../wrapper.nix { python = self; }; + buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; }; interpreter = "${self}/bin/${executable}"; withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; pkgs = pythonPackages; diff --git a/pkgs/development/interpreters/python/wrapper.nix b/pkgs/development/interpreters/python/wrapper.nix index f42caf92c17..fc521828ffc 100644 --- a/pkgs/development/interpreters/python/wrapper.nix +++ b/pkgs/development/interpreters/python/wrapper.nix @@ -2,13 +2,14 @@ , extraLibs ? [] , extraOutputsToInstall ? [] , postBuild ? "" -, ignoreCollisions ? false }: +, ignoreCollisions ? false +, requiredPythonModules +, }: # Create a python executable that knows about additional packages. let - recursivePthLoader = import ../../python-modules/recursive-pth-loader/default.nix { stdenv = stdenv; python = python; }; env = let - paths = stdenv.lib.closePropagation (extraLibs ++ [ python recursivePthLoader ] ) ; + paths = requiredPythonModules (extraLibs ++ [ python ] ) ; in buildEnv { name = "${python.name}-env"; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index ad1d50324f6..ea81c4924c1 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -31,10 +31,9 @@ let callPackage = pkgs.newScope self; - bootstrapped-pip = callPackage ../development/python-modules/bootstrapped-pip { }; + namePrefix = python.libPrefix + "-"; - mkPythonDerivation = makeOverridable( callPackage ../development/interpreters/python/mk-python-derivation.nix { - }); + bootstrapped-pip = callPackage ../development/python-modules/bootstrapped-pip { }; # Derivations built with `buildPythonPackage` can already be overriden with `override`, `overrideAttrs`, and `overrideDerivation`. # This function introduces `overridePythonAttrs` and it overrides the call to `buildPythonPackage`. @@ -52,13 +51,20 @@ let } else ff; - buildPythonPackage = makeOverridablePythonPackage (callPackage ../development/interpreters/python/build-python-package.nix { - inherit mkPythonDerivation; + buildPythonPackage = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix { inherit bootstrapped-pip; flit = self.flit; - }); + # We want Python libraries to be named like e.g. "python3.6-${name}" + inherit namePrefix; + pythonModule = python; + })); - buildPythonApplication = args: buildPythonPackage ({namePrefix="";} // args ); + buildPythonApplication = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix { + inherit bootstrapped-pip; + flit = self.flit; + namePrefix = ""; + pythonModule = false; + })); graphiteVersion = "1.0.2"; @@ -80,10 +86,40 @@ let else throw "Unsupported kind ${kind}"); in fetcher (builtins.removeAttrs attrs ["format"]) ); + # Check whether a derivation provides a Python module. + hasPythonModule = drv: (hasAttr "pythonModule" drv) && ( (getAttr "pythonModule" drv) == python); + + # Get list of required Python modules given a list of derivations. + requiredPythonModules = drvs: let + filterNull = list: filter (x: !isNull x) list; + conditionalGetRecurse = attr: condition: drv: let f = conditionalGetRecurse attr condition; in + (if (condition drv) then unique [drv]++(concatMap f (filterNull(getAttr attr drv))) else []); + _required = drv: conditionalGetRecurse "propagatedBuildInputs" hasPythonModule drv; + in [python] ++ (unique (concatMap _required (filterNull drvs))); + + # Create a PYTHONPATH from a list of derivations. This function recurses into the items to find derivations + # providing Python modules. + makePythonPath = drvs: stdenv.lib.makeSearchPath python.sitePackages (requiredPythonModules drvs); + + # Convert derivation to a Python module. + toPythonModule = drv: + drv.overrideAttrs( oldAttrs: { + # Use passthru in order to prevent rebuilds when possible. + passthru = (oldAttrs.passthru or {})// { + name = namePrefix + oldAttrs.name; + pythonModule = python; + pythonPath = [ ]; # Deprecated, for compatibility. + }; + }); + + disabledIf = x: drv: + if x then throw "${removePrefix namePrefix (drv.pname or drv.name)} not supported for interpreter ${python.executable}" else drv; + in { - inherit python bootstrapped-pip pythonAtLeast pythonOlder isPy26 isPy27 isPy33 isPy34 isPy35 isPy36 isPyPy isPy3k mkPythonDerivation buildPythonPackage buildPythonApplication; + inherit python bootstrapped-pip pythonAtLeast pythonOlder isPy26 isPy27 isPy33 isPy34 isPy35 isPy36 isPyPy isPy3k buildPythonPackage buildPythonApplication; inherit fetchPypi callPackage; + inherit hasPythonModule requiredPythonModules makePythonPath disabledIf; # helpers