Merge pull request #89126 from adisbladis/poetry2nix-1_9_0

poetry2nix: 1.8.0 -> 1.9.0

pkgs/development/tools/poetry2nix/poetry2nix/default.nix

@@ -69,28 +69,28 @@ let
       baseOverlay = self: super:
         let
           getDep = depName: self.${depName};
-          lockPkgs = builtins.listToAttrs
+          lockPkgs = builtins.listToAttrs (
-            (
+            builtins.map
-              builtins.map
+              (
-                (
+                pkgMeta: rec {
-                  pkgMeta: rec {
+                  name = moduleName pkgMeta.name;
-                    name = moduleName pkgMeta.name;
+                  value = self.mkPoetryDep (
-                    value = self.mkPoetryDep
+                    pkgMeta // {
-                      (
+                      inherit pwd preferWheels;
-                        pkgMeta // {
+                      source = pkgMeta.source or null;
-                          inherit pwd preferWheels;
+                      files = lockFiles.${name};
-                          source = pkgMeta.source or null;
+                      pythonPackages = self;
-                          files = lockFiles.${name};
+                      sourceSpec = pyProject.tool.poetry.dependencies.${name} or pyProject.tool.poetry.dev-dependencies.${name} or { };
-                          pythonPackages = self;
+                    }
-                          sourceSpec = pyProject.tool.poetry.dependencies.${name} or pyProject.tool.poetry.dev-dependencies.${name};
+                  );
-                        }
+                }
-                      );
+              )
-                  }
+              compatible
-                ) compatible
+          );
-            );
         in
         lockPkgs;
-      overlays = builtins.map getFunctorFn
+      overlays = builtins.map
+        getFunctorFn
         (
           [
             (
@@ -127,6 +127,8 @@ let
     };
 
   /* Returns a package with a python interpreter and all packages specified in the poetry.lock lock file.
+     In editablePackageSources you can pass a mapping from package name to source directory to have
+     those packages available in the resulting environment, whose source changes are immediately available.
 
      Example:
        poetry2nix.mkPoetryEnv { poetrylock = ./poetry.lock; python = python3; }
@@ -139,18 +141,31 @@ let
     , pwd ? projectDir
     , python ? pkgs.python3
     , preferWheels ? false
+      # Example: { my-app = ./src; }
+    , editablePackageSources ? { }
     }:
     let
-      py = mkPoetryPackages
+      py = mkPoetryPackages (
-        (
+        {
-          {
+          inherit pyproject poetrylock overrides python pwd preferWheels;
-            inherit pyproject poetrylock overrides python pwd preferWheels;
+        }
-          }
+      );
-        );
-    in
-    py.python.withPackages (_: py.poetryPackages);
 
-  /* Creates a Python application from pyproject.toml and poetry.lock */
+      editablePackage = import ./editable.nix {
+        inherit pkgs lib poetryLib editablePackageSources;
+        inherit (py) pyProject python;
+      };
+
+    in
+    py.python.withPackages (_: py.poetryPackages ++ lib.optional (editablePackageSources != { }) editablePackage);
+
+  /* Creates a Python application from pyproject.toml and poetry.lock
+
+     The result also contains a .dependencyEnv attribute which is a python
+     environment of all dependencies and this apps modules. This is useful if
+     you rely on dependencies to invoke your modules for deployment: e.g. this
+     allows `gunicorn my-module:app`.
+  */
   mkPoetryApplication =
     { projectDir ? null
     , src ? poetryLib.cleanPythonSources { src = projectDir; }
@@ -194,17 +209,17 @@ let
               pkg = py.pkgs."${dep}";
               constraints = deps.${dep}.python or "";
               isCompat = compat constraints;
-            in if isCompat then pkg else null
+            in
-          ) depAttrs;
+            if isCompat then pkg else null
+          )
+          depAttrs;
       getInputs = attr: attrs.${attr} or [ ];
       mkInput = attr: extraInputs: getInputs attr ++ extraInputs;
       buildSystemPkgs = poetryLib.getBuildSystemPkgs {
         inherit pyProject;
         pythonPackages = py.pkgs;
       };
-    in
+      app = py.pkgs.buildPythonPackage (
-    py.pkgs.buildPythonApplication
-      (
         passedAttrs // {
           pname = moduleName pyProject.tool.poetry.name;
           version = pyProject.tool.poetry.version;
@@ -212,6 +227,10 @@ let
           inherit src;
 
           format = "pyproject";
+          # Like buildPythonApplication, but without the toPythonModule part
+          # Meaning this ends up looking like an application but it also
+          # provides python modules
+          namePrefix = "";
 
           buildInputs = mkInput "buildInputs" buildSystemPkgs;
           propagatedBuildInputs = mkInput "propagatedBuildInputs" (getDeps "dependencies") ++ ([ py.pkgs.setuptools ]);
@@ -220,16 +239,30 @@ let
 
           passthru = {
             python = py;
+            dependencyEnv = (
+              lib.makeOverridable ({ app, ... }@attrs:
+                let
+                  args = builtins.removeAttrs attrs [ "app" ] // {
+                    extraLibs = [ app ];
+                  };
+                in
+                py.buildEnv.override args)
+            ) { inherit app; };
           };
 
-          meta = meta // {
+          meta = lib.optionalAttrs (lib.hasAttr "description" pyProject.tool.poetry) {
-            inherit (pyProject.tool.poetry) description homepage;
+            inherit (pyProject.tool.poetry) description;
+          } // lib.optionalAttrs (lib.hasAttr "homepage" pyProject.tool.poetry) {
+            inherit (pyProject.tool.poetry) homepage;
+          } // {
             inherit (py.meta) platforms;
             license = getLicenseBySpdxId (pyProject.tool.poetry.license or "unknown");
-          };
+          } // meta;
 
         }
       );
+    in
+    app;
 
   /* Poetry2nix CLI used to supplement SHA-256 hashes for git dependencies  */
   cli = import ./cli.nix { inherit pkgs lib version; };

pkgs/development/tools/poetry2nix/poetry2nix/editable.nix

@@ -0,0 +1,54 @@
+{ pkgs
+, lib
+, poetryLib
+, pyProject
+, python
+, editablePackageSources
+}:
+let
+  name = poetryLib.moduleName pyProject.tool.poetry.name;
+
+  # Just enough standard PKG-INFO fields for an editable installation
+  pkgInfoFields = {
+    Metadata-Version = "2.1";
+    Name = name;
+    # While the pyproject.toml could contain arbitrary version strings, for
+    # simplicity we just use the same one for PKG-INFO, even though that
+    # should follow follow PEP 440: https://www.python.org/dev/peps/pep-0345/#version
+    # This is how poetry transforms it: https://github.com/python-poetry/poetry/blob/6cd3645d889f47c10425961661b8193b23f0ed79/poetry/version/version.py
+    Version = pyProject.tool.poetry.version;
+    Summary = pyProject.tool.poetry.description;
+  };
+
+  pkgInfoFile = builtins.toFile "${name}-PKG-INFO"
+    (lib.concatStringsSep "\n" (lib.mapAttrsToList (key: value: "${key}: ${value}") pkgInfoFields));
+
+  entryPointsFile = builtins.toFile "${name}-entry_points.txt"
+    (lib.generators.toINI { } pyProject.tool.poetry.plugins);
+
+  # A python package that contains simple .egg-info and .pth files for an editable installation
+  editablePackage = python.pkgs.toPythonModule (pkgs.runCommandNoCC "${name}-editable"
+    { } ''
+    mkdir -p "$out/${python.sitePackages}"
+    cd "$out/${python.sitePackages}"
+
+    # See https://docs.python.org/3.8/library/site.html for info on such .pth files
+    # These add another site package path for each line
+    touch poetry2nix-editable.pth
+    ${lib.concatMapStringsSep "\n" (src: ''
+      echo "${toString src}" >> poetry2nix-editable.pth
+    '')
+      (lib.attrValues editablePackageSources)}
+
+    # Create a very simple egg so pkg_resources can find this package
+    # See https://setuptools.readthedocs.io/en/latest/formats.html for more info on the egg format
+    mkdir "${name}.egg-info"
+    cd "${name}.egg-info"
+    ln -s ${pkgInfoFile} PKG-INFO
+    ${lib.optionalString (pyProject.tool.poetry ? plugins) ''
+      ln -s ${entryPointsFile} entry_points.txt
+    ''}
+  ''
+  );
+in
+editablePackage

pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix

@@ -14,36 +14,39 @@ in
   removePathDependenciesHook = callPackage
     (
       {}:
-      makeSetupHook {
+      makeSetupHook
-        name = "remove-path-dependencies.sh";
+        {
-        deps = [ ];
+          name = "remove-path-dependencies.sh";
-        substitutions = {
+          deps = [ ];
-          inherit pythonInterpreter;
+          substitutions = {
-          yj = "${yj}/bin/yj";
+            inherit pythonInterpreter;
-          pyprojectPatchScript = "${./pyproject-without-path.py}";
+            yj = "${yj}/bin/yj";
-        };
+            pyprojectPatchScript = "${./pyproject-without-path.py}";
-      } ./remove-path-dependencies.sh
+          };
+        } ./remove-path-dependencies.sh
     ) { };
 
   pipBuildHook = callPackage
     (
       { pip, wheel }:
-      makeSetupHook {
+      makeSetupHook
-        name = "pip-build-hook.sh";
+        {
-        deps = [ pip wheel ];
+          name = "pip-build-hook.sh";
-        substitutions = {
+          deps = [ pip wheel ];
-          inherit pythonInterpreter pythonSitePackages;
+          substitutions = {
-        };
+            inherit pythonInterpreter pythonSitePackages;
-      } ./pip-build-hook.sh
+          };
+        } ./pip-build-hook.sh
     ) { };
 
   poetry2nixFixupHook = callPackage
     (
       {}:
-      makeSetupHook {
+      makeSetupHook
-        name = "fixup-hook.sh";
+        {
-        deps = [ ];
+          name = "fixup-hook.sh";
-      } ./fixup-hook.sh
+          deps = [ ];
+        } ./fixup-hook.sh
     ) { };
 
 }

pkgs/development/tools/poetry2nix/poetry2nix/lib.nix

@@ -20,7 +20,7 @@ let
       minor = l: lib.elemAt l 1;
       joinVersion = v: lib.concatStringsSep "." v;
     in
-    joinVersion ( if major pyVer == major ver && minor pyVer == minor ver then ver else pyVer);
+    joinVersion (if major pyVer == major ver && minor pyVer == minor ver then ver else pyVer);
 
   # Compare a semver expression with a version
   isCompatible = version:
@@ -45,28 +45,27 @@ let
           state = operators."${operator}" acc.state (satisfiesSemver version v);
         };
       initial = { operator = "&&"; state = true; };
-    in if expr == "" then true else (builtins.foldl' combine initial tokens).state;
+    in
+    if expr == "" then true else (builtins.foldl' combine initial tokens).state;
   fromTOML = builtins.fromTOML or
     (
-      toml: builtins.fromJSON
+      toml: builtins.fromJSON (
-        (
+        builtins.readFile (
-          builtins.readFile
+          pkgs.runCommand "from-toml"
-            (
+            {
-              pkgs.runCommand "from-toml"
+              inherit toml;
-                {
+              allowSubstitutes = false;
-                  inherit toml;
+              preferLocalBuild = true;
-                  allowSubstitutes = false;
+            }
-                  preferLocalBuild = true;
+            ''
-                }
+              ${pkgs.remarshal}/bin/remarshal \
-                ''
+                -if toml \
-                  ${pkgs.remarshal}/bin/remarshal \
+                -i <(echo "$toml") \
-                    -if toml \
+                -of json \
-                    -i <(echo "$toml") \
+                -o $out
-                    -of json \
+            ''
-                    -o $out
-                ''
-            )
         )
+      )
     );
   readTOML = path: fromTOML (builtins.readFile path);
 
@@ -88,11 +87,10 @@ let
   #   file: filename including extension
   #   hash: SRI hash
   #   kind: Language implementation and version tag
-  predictURLFromPypi = lib.makeOverridable
+  predictURLFromPypi = lib.makeOverridable (
-    (
+    { pname, file, hash, kind }:
-      { pname, file, hash, kind }:
+    "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}"
-      "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}"
+  );
-    );
 
 
   # Fetch the wheels from the PyPI index.
@@ -102,36 +100,35 @@ let
   #   file: filename including extension
   #   hash: SRI hash
   #   kind: Language implementation and version tag
-  fetchWheelFromPypi = lib.makeOverridable
+  fetchWheelFromPypi = lib.makeOverridable (
-    (
+    { pname, file, hash, kind, curlOpts ? "" }:
-      { pname, file, hash, kind, curlOpts ? "" }:
+    let
-      let
+      version = builtins.elemAt (builtins.split "-" file) 2;
-        version = builtins.elemAt (builtins.split "-" file) 2;
+    in
-      in
+    (pkgs.stdenvNoCC.mkDerivation {
-      (pkgs.stdenvNoCC.mkDerivation {
+      name = file;
-        name = file;
+      nativeBuildInputs = [
-        nativeBuildInputs = [
+        pkgs.curl
-          pkgs.curl
+        pkgs.jq
-          pkgs.jq
+      ];
-        ];
+      isWheel = true;
-        isWheel = true;
+      system = "builtin";
-        system = "builtin";
 
-        preferLocalBuild = true;
+      preferLocalBuild = true;
-        impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [
+      impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [
-          "NIX_CURL_FLAGS"
+        "NIX_CURL_FLAGS"
-        ];
+      ];
 
-        predictedURL = predictURLFromPypi { inherit pname file hash kind; };
+      predictedURL = predictURLFromPypi { inherit pname file hash kind; };
-        inherit pname file version curlOpts;
+      inherit pname file version curlOpts;
 
-        builder = ./fetch-wheel.sh;
+      builder = ./fetch-wheel.sh;
 
-        outputHashMode = "flat";
+      outputHashMode = "flat";
-        outputHashAlgo = "sha256";
+      outputHashAlgo = "sha256";
-        outputHash = hash;
+      outputHash = hash;
-      })
+    })
-    );
+  );
 
   # Fetch the artifacts from the PyPI index. Since we get all
   # info we need from the lock file we don't use nixpkgs' fetchPyPi
@@ -143,16 +140,15 @@ let
   #   file: filename including extension
   #   hash: SRI hash
   #   kind: Language implementation and version tag https://www.python.org/dev/peps/pep-0427/#file-name-convention
-  fetchFromPypi = lib.makeOverridable
+  fetchFromPypi = lib.makeOverridable (
-    (
+    { pname, file, hash, kind }:
-      { pname, file, hash, kind }:
+    if lib.strings.hasSuffix "whl" file then fetchWheelFromPypi { inherit pname file hash kind; }
-      if lib.strings.hasSuffix "whl" file then fetchWheelFromPypi { inherit pname file hash kind; }
+    else
-      else
+      pkgs.fetchurl {
-        pkgs.fetchurl {
+        url = predictURLFromPypi { inherit pname file hash kind; };
-          url = predictURLFromPypi { inherit pname file hash kind; };
+        inherit hash;
-          inherit hash;
+      }
-        }
+  );
-    );
   getBuildSystemPkgs =
     { pythonPackages
     , pyProject

pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix

@@ -58,7 +58,7 @@ pythonPackages.callPackage
           binaryDist = selectWheel fileCandidates;
           sourceDist = builtins.filter isSdist fileCandidates;
           eggs = builtins.filter isEgg fileCandidates;
-          entries = ( if preferWheel then binaryDist ++ sourceDist else sourceDist ++ binaryDist) ++ eggs;
+          entries = (if preferWheel then binaryDist ++ sourceDist else sourceDist ++ binaryDist) ++ eggs;
           lockFileEntry = builtins.head entries;
           _isEgg = isEgg lockFileEntry;
         in
@@ -111,7 +111,8 @@ pythonPackages.callPackage
       propagatedBuildInputs =
         let
           compat = isCompatible (poetryLib.getPythonVersion python);
-          deps = lib.filterAttrs (n: v: v)
+          deps = lib.filterAttrs
+            (n: v: v)
             (
               lib.mapAttrs
                 (
@@ -120,7 +121,8 @@ pythonPackages.callPackage
                       constraints = v.python or "";
                     in
                     compat constraints
-                ) dependencies
+                )
+                dependencies
             );
           depAttrs = lib.attrNames deps;
         in

pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix

@@ -37,14 +37,17 @@ let
   # Make a tree out of expression groups (parens)
   findSubExpressions = expr:
     let
-      acc = builtins.foldl' findSubExpressionsFun {
+      acc = builtins.foldl'
-        exprs = [ ];
+        findSubExpressionsFun
-        expr = expr;
+        {
-        pos = 0;
+          exprs = [ ];
-        openP = 0;
+          expr = expr;
-        exprPos = 0;
+          pos = 0;
-        startPos = 0;
+          openP = 0;
-      } (lib.stringToCharacters expr);
+          exprPos = 0;
+          startPos = 0;
+        }
+        (lib.stringToCharacters expr);
       tailExpr = (substr acc.exprPos acc.pos expr);
       tailExprs = if tailExpr != "" then [ tailExpr ] else [ ];
     in
@@ -53,7 +56,7 @@ let
     let
       splitCond = (
         s: builtins.map
-          (x: stripStr ( if builtins.typeOf x == "list" then (builtins.elemAt x 0) else x))
+          (x: stripStr (if builtins.typeOf x == "list" then (builtins.elemAt x 0) else x))
           (builtins.split " (and|or) " (s + " "))
       );
       mapfn = expr: (
@@ -71,8 +74,9 @@ let
     in
     builtins.foldl'
       (
-        acc: v: acc ++ ( if builtins.typeOf v == "string" then parse v else [ (parseExpressions v) ])
+        acc: v: acc ++ (if builtins.typeOf v == "string" then parse v else [ (parseExpressions v) ])
-      ) [ ] exprs;
+      ) [ ]
+      exprs;
 
   # Transform individual expressions to structured expressions
   # This function also performs variable substitution, replacing environment markers with their explicit values
@@ -159,10 +163,9 @@ let
           let
             parts = builtins.splitVersion c;
             pruned = lib.take ((builtins.length parts) - 1) parts;
-            upper = builtins.toString
+            upper = builtins.toString (
-              (
+              (lib.toInt (builtins.elemAt pruned (builtins.length pruned - 1))) + 1
-                (lib.toInt (builtins.elemAt pruned (builtins.length pruned - 1))) + 1
+            );
-              );
             upperConstraint = builtins.concatStringsSep "." (ireplace (builtins.length pruned - 1) upper pruned);
           in
           op.">=" v c && op."<" v upperConstraint;
@@ -207,10 +210,13 @@ let
           ) else throw "Unsupported type"
         ) else if builtins.typeOf v == "list" then (
           let
-            ret = builtins.foldl' reduceExpressionsFun {
+            ret = builtins.foldl'
-              value = true;
+              reduceExpressionsFun
-              cond = "and";
+              {
-            } v;
+                value = true;
+                cond = "and";
+              }
+              v;
           in
           acc // {
             value = cond."${acc.cond}" acc.value ret.value;
@@ -219,10 +225,13 @@ let
       );
     in
     (
-      builtins.foldl' reduceExpressionsFun {
+      builtins.foldl'
-        value = true;
+        reduceExpressionsFun
-        cond = "and";
+        {
-      } exprs
+          value = true;
+          cond = "and";
+        }
+        exprs
     ).value;
 in
 e: builtins.foldl' (acc: v: v acc) e [

pkgs/development/tools/poetry2nix/poetry2nix/semver.nix

@@ -39,10 +39,9 @@ let
           # Prune constraint
           parts = builtins.splitVersion c;
           pruned = lib.take ((builtins.length parts) - 1) parts;
-          upper = builtins.toString
+          upper = builtins.toString (
-            (
+            (lib.toInt (builtins.elemAt pruned (builtins.length pruned - 1))) + 1
-              (lib.toInt (builtins.elemAt pruned (builtins.length pruned - 1))) + 1
+          );
-            );
           upperConstraint = builtins.concatStringsSep "." (ireplace (builtins.length pruned - 1) upper pruned);
         in
         operators.">=" v c && operators."<" v upperConstraint;
@@ -69,7 +68,7 @@ let
         op = elemAt mPre 0;
         v = elemAt mPre 1;
       }
-        # Infix operators are range matches
+      # Infix operators are range matches
       else if mIn != null then {
         op = elemAt mIn 1;
         v = {
@@ -82,6 +81,7 @@ let
   satisfiesSemver = version: constraint:
     let
       inherit (parseConstraint constraint) op v;
-    in if constraint == "*" then true else operators."${op}" version v;
+    in
+    if constraint == "*" then true else operators."${op}" version v;
 in
 { inherit satisfiesSemver; }