poetry2nix: 1.5.0 -> 1.6.0

pkgs/development/tools/poetry2nix/poetry2nix/default.nix

@@ -3,12 +3,11 @@
 , poetry ? null
 , poetryLib ? import ./lib.nix { inherit lib pkgs; }
 }:
-
 let
   inherit (poetryLib) isCompatible readTOML;
 
   # Poetry2nix version
-  version = "1.1.0";
+  version = "1.6.0";
 
   /* The default list of poetry2nix override overlays */
   defaultPoetryOverrides = (import ./overrides.nix { inherit pkgs lib; });
@@ -29,14 +28,15 @@ let
      Returns an attrset { python, poetryPackages, pyProject, poetryLock } for the given pyproject/lockfile.
   */
   mkPoetryPackages =
-    { pyproject
+    { projectDir ? null
-    , poetrylock
+    , pyproject ? projectDir + "/pyproject.toml"
-    , poetryPkg
+    , poetrylock ? projectDir + "/poetry.lock"
     , overrides ? [ defaultPoetryOverrides ]
-    , meta ? {}
     , python ? pkgs.python3
-    , pwd ? null
+    , pwd ? projectDir
     }@attrs: let
+      poetryPkg = poetry.override { inherit python; };
+
       pyProject = readTOML pyproject;
       poetryLock = readTOML poetrylock;
       lockFiles = lib.getAttrFromPath [ "metadata" "files" ] poetryLock;
@@ -52,14 +52,7 @@ let
 
       # Filter packages by their PEP508 markers & pyproject interpreter version
       partitions = let
-        supportsPythonVersion = pkgMeta: let
+        supportsPythonVersion = pkgMeta: if pkgMeta ? marker then (evalPep508 pkgMeta.marker) else true;
-          pep508Result = if pkgMeta ? marker then (evalPep508 pkgMeta.marker) else true;
-
-          flatDeps = (pyProject.tool.poetry.dependencies or {}) // (pyProject.tool.poetry.dev-dependencies or {});
-          constraints = flatDeps.${pkgMeta.name}.python or "";
-          pyprojectResult = isCompatible python.pythonVersion constraints;
-        in
-          pyprojectResult && pep508Result;
       in
         lib.partition supportsPythonVersion poetryLock.package;
 
@@ -105,7 +98,7 @@ let
                 # The canonical name is setuptools-scm
                 setuptools-scm = super.setuptools_scm;
 
-                inherit (hooks) removePathDependenciesHook;
+                inherit (hooks) removePathDependenciesHook poetry2nixFixupHook;
               }
           )
           # Null out any filtered packages, we don't want python.pkgs from nixpkgs
@@ -133,18 +126,17 @@ let
        poetry2nix.mkPoetryEnv { poetrylock = ./poetry.lock; python = python3; }
   */
   mkPoetryEnv =
-    { pyproject
+    { projectDir ? null
-    , poetrylock
+    , pyproject ? projectDir + "/pyproject.toml"
+    , poetrylock ? projectDir + "/poetry.lock"
     , overrides ? [ defaultPoetryOverrides ]
-    , meta ? {}
+    , pwd ? projectDir
-    , pwd ? null
     , python ? pkgs.python3
     }:
       let
-        poetryPkg = poetry.override { inherit python; };
         py = mkPoetryPackages (
           {
-            inherit poetryPkg pyproject poetrylock overrides meta python pwd;
+            inherit pyproject poetrylock overrides python pwd;
           }
         );
       in
@@ -152,19 +144,18 @@ let
 
   /* Creates a Python application from pyproject.toml and poetry.lock */
   mkPoetryApplication =
-    { src
+    { projectDir ? null
-    , pyproject
+    , src ? poetryLib.cleanPythonSources { src = projectDir; }
-    , poetrylock
+    , pyproject ? projectDir + "/pyproject.toml"
+    , poetrylock ? projectDir + "/poetry.lock"
     , overrides ? [ defaultPoetryOverrides ]
     , meta ? {}
     , python ? pkgs.python3
-    , pwd ? null
+    , pwd ? projectDir
     , ...
     }@attrs: let
-      poetryPkg = poetry.override { inherit python; };
-
       poetryPython = mkPoetryPackages {
-        inherit poetryPkg pyproject poetrylock overrides meta python pwd;
+        inherit pyproject poetrylock overrides python pwd;
       };
       py = poetryPython.python;
 
@@ -178,11 +169,20 @@ let
       ];
       passedAttrs = builtins.removeAttrs attrs specialAttrs;
 
+      # Get dependencies and filter out depending on interpreter version
       getDeps = depAttr: let
+        compat = isCompatible py.pythonVersion;
         deps = pyProject.tool.poetry.${depAttr} or {};
         depAttrs = builtins.map (d: lib.toLower d) (builtins.attrNames deps);
       in
-        builtins.map (dep: py.pkgs."${dep}") depAttrs;
+        builtins.map (
+          dep: let
+            pkg = py.pkgs."${dep}";
+            constraints = deps.${dep}.python or "";
+            isCompat = compat constraints;
+          in
+            if isCompat then pkg else null
+        ) depAttrs;
 
       getInputs = attr: attrs.${attr} or [];
       mkInput = attr: extraInputs: getInputs attr ++ extraInputs;
@@ -198,6 +198,8 @@ let
           pname = pyProject.tool.poetry.name;
           version = pyProject.tool.poetry.version;
 
+          inherit src;
+
           format = "pyproject";
 
           buildInputs = mkInput "buildInputs" buildSystemPkgs;
@@ -211,6 +213,7 @@ let
 
           meta = meta // {
             inherit (pyProject.tool.poetry) description homepage;
+            inherit (py.meta) platforms;
             license = getLicenseBySpdxId (pyProject.tool.poetry.license or "unknown");
           };
 
@@ -220,34 +223,11 @@ let
   /* Poetry2nix CLI used to supplement SHA-256 hashes for git dependencies  */
   cli = import ./cli.nix { inherit pkgs lib version; };
 
-  /* Poetry2nix documentation  */
-  doc = pkgs.stdenv.mkDerivation {
-    pname = "poetry2nix-docs";
-    inherit version;
-
-    src = pkgs.runCommandNoCC "poetry2nix-docs-src" {} ''
-      mkdir -p $out
-      cp ${./default.nix} $out/default.nix
-    '';
-
-    buildInputs = [
-      pkgs.nixdoc
-    ];
-
-    buildPhase = ''
-      nixdoc --category poetry2nix --description "Poetry2nix functions" --file ./default.nix > poetry2nix.xml
-    '';
-
-    installPhase = ''
-      mkdir -p $out
-      cp poetry2nix.xml $out/
-    '';
-
-  };
-
 in
 {
-  inherit mkPoetryEnv mkPoetryApplication mkPoetryPackages cli doc;
+  inherit mkPoetryEnv mkPoetryApplication mkPoetryPackages cli version;
+
+  inherit (poetryLib) cleanPythonSources;
 
   /*
   The default list of poetry2nix override overlays
@@ -262,4 +242,25 @@ in
     in
       defaultSet // customSet;
   };
+
+  /*
+  Convenience functions for specifying overlays with or without the poerty2nix default overrides
+  */
+  overrides = {
+    /*
+    Returns the specified overlay in a list
+    */
+    withoutDefaults = overlay: [
+      overlay
+    ];
+
+    /*
+    Returns the specified overlay and returns a list
+    combining it with poetry2nix default overrides
+    */
+    withDefaults = overlay: [
+      defaultPoetryOverrides
+      overlay
+    ];
+  };
 }

pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix

@@ -22,4 +22,12 @@ in
       } ./remove-path-dependencies.sh
   ) {};
 
+  poetry2nixFixupHook = callPackage (
+    {}:
+      makeSetupHook {
+        name = "fixup-hook.sh";
+        deps = [];
+      } ./fixup-hook.sh
+  ) {};
+
 }

pkgs/development/tools/poetry2nix/poetry2nix/hooks/fixup-hook.sh

@@ -0,0 +1,8 @@
+poetry2nix-fixup-hook() {
+    # Including tests in the output is a common mistake
+    if [ -z "${dontFixupTests-}" ]; then
+        rm -rf $out/lib/python3.7/site-packages/tests
+    fi
+}
+
+postFixupHooks+=(poetry2nix-fixup-hook)

pkgs/development/tools/poetry2nix/poetry2nix/hooks/remove-path-dependencies.sh

@@ -1,4 +1,8 @@
 remove-path-dependencies-hook() {
+    if ! test -f pyproject.toml; then
+        return
+    fi
+
     # Tell poetry not to resolve the path dependencies. Any version is fine!
     @yj@ -tj < pyproject.toml | @pythonInterpreter@ @pyprojectPatchScript@ > pyproject.json
     @yj@ -jt < pyproject.json > pyproject.toml

pkgs/development/tools/poetry2nix/poetry2nix/lib.nix

@@ -96,6 +96,38 @@ let
         [ pythonPackages.${drvAttr} or (throw "unsupported build system ${buildSystem}") ]
       );
 
+  # Find gitignore files recursively in parent directory stopping with .git
+  findGitIgnores = path: let
+    parent = path + "/..";
+    gitIgnore = path + "/.gitignore";
+    isGitRoot = builtins.pathExists (path + "/.git");
+    hasGitIgnore = builtins.pathExists gitIgnore;
+    gitIgnores = if hasGitIgnore then [ gitIgnore ] else [];
+  in
+    lib.optionals (builtins.toString path != "/" && ! isGitRoot) (findGitIgnores parent) ++ gitIgnores;
+
+  /*
+  Provides a source filtering mechanism that:
+
+  - Filters gitignore's
+  - Filters pycache/pyc files
+  - Uses cleanSourceFilter to filter out .git/.hg, .o/.so, editor backup files & nix result symlinks
+  */
+  cleanPythonSources = { src }: let
+    gitIgnores = findGitIgnores src;
+    pycacheFilter = name: type:
+      (type == "directory" && ! lib.strings.hasInfix "__pycache__" name)
+      || (type == "regular" && ! lib.strings.hasSuffix ".pyc" name)
+    ;
+  in
+    lib.cleanSourceWith {
+      filter = lib.cleanSourceFilter;
+      src = lib.cleanSourceWith {
+        filter = pkgs.nix-gitignore.gitignoreFilterPure pycacheFilter gitIgnores src;
+        inherit src;
+      };
+    };
+
 in
 {
   inherit
@@ -105,5 +137,6 @@ in
     readTOML
     getBuildSystemPkgs
     satisfiesSemver
+    cleanPythonSources
     ;
 }

pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix

@@ -106,8 +106,10 @@ pythonPackages.callPackage (
         # Stripping pre-built wheels lead to `ELF load command address/offset not properly aligned`
         dontStrip = format == "wheel";
 
-        nativeBuildInputs = (if (!isSource && (getManyLinuxDeps fileInfo.name).str != null) then [ autoPatchelfHook ] else [])
+        nativeBuildInputs = [
-        ++ lib.optional (isLocal) pkgs.yj
+          pythonPackages.poetry2nixFixupHook
+        ]
+        ++ lib.optional (!isSource && (getManyLinuxDeps fileInfo.name).str != null) autoPatchelfHook
         ++ lib.optional (format == "pyproject") pythonPackages.removePathDependenciesHook
         ;
 
@@ -117,14 +119,24 @@ pythonPackages.callPackage (
           ++ lib.optional isLocal buildSystemPkgs
         );
 
-        propagatedBuildInputs =
+        propagatedBuildInputs = let
-          # Some dependencies like django get the attribute name django
+          compat = isCompatible python.pythonVersion;
-          # but dependencies try to access Django
+          deps = lib.filterAttrs (n: v: v) (
-          builtins.map (n: pythonPackages.${lib.toLower n}) (builtins.attrNames dependencies);
+            lib.mapAttrs (
+              n: v: let
+                constraints = v.python or "";
+              in
+                compat constraints
+            ) dependencies
+          );
+          depAttrs = lib.attrNames deps;
+        in
+          builtins.map (n: pythonPackages.${lib.toLower n}) depAttrs;
 
         meta = {
           broken = ! isCompatible python.pythonVersion python-versions;
           license = [];
+          inherit (python.meta) platforms;
         };
 
         passthru = {
@@ -139,7 +151,7 @@ pythonPackages.callPackage (
             inherit (source) url;
             rev = source.reference;
           }
-        ) else if isLocal then (localDepPath) else fetchFromPypi {
+        ) else if isLocal then (poetryLib.cleanPythonSources { src = localDepPath; }) else fetchFromPypi {
           pname = name;
           inherit (fileInfo) file hash kind;
         };

pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix

@@ -6,6 +6,13 @@
 self: super:
 
 {
+  astroid = super.astroid.overrideAttrs (
+    old: rec {
+      buildInputs = old.buildInputs ++ [ self.pytest-runner ];
+      doCheck = false;
+    }
+  );
+
   av = super.av.overrideAttrs (
     old: {
       nativeBuildInputs = old.nativeBuildInputs ++ [
@@ -81,17 +88,36 @@ self: super:
   enum34 = if self.pythonAtLeast "3.4" then null else super.enum34;
 
   faker = super.faker.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [ self.pytest-runner ];
+      doCheck = false;
+    }
+  );
+
+  fancycompleter = super.fancycompleter.overrideAttrs (
     old: {
       postPatch = ''
-        substituteInPlace setup.py --replace 'setup_requires=["pytest-runner"],' 'setup_requires=[],' || true
+        substituteInPlace setup.py \
+          --replace 'setup_requires="setupmeta"' 'setup_requires=[]' \
+          --replace 'versioning="devcommit"' 'version="${old.version}"'
       '';
     }
   );
 
   grandalf = super.grandalf.overrideAttrs (
     old: {
-      postPatch = ''
+      buildInputs = old.buildInputs ++ [ self.pytest-runner ];
-        substituteInPlace setup.py --replace "setup_requires=['pytest-runner',]," "setup_requires=[]," || true
+      doCheck = false;
+    }
+  );
+
+  h5py = super.h5py.overrideAttrs (
+    old: rec {
+      nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ];
+      buildInputs = old.buildInputs ++ [ pkgs.hdf5 self.pkgconfig self.cython ];
+      configure_flags = "--hdf5=${pkgs.hdf5}";
+      postConfigure = ''
+        ${self.python.executable} setup.py configure ${configure_flags}
       '';
     }
   );
@@ -103,12 +129,21 @@ self: super:
   );
 
   # importlib-metadata has an incomplete dependency specification
-  importlib-metadata = if super.importlib-metadata == null then null else super.importlib-metadata.overrideAttrs (
+  importlib-metadata = super.importlib-metadata.overrideAttrs (
     old: {
       propagatedBuildInputs = old.propagatedBuildInputs ++ lib.optional self.python.isPy2 self.pathlib2;
     }
   );
 
+  jupyter = super.jupyter.overrideAttrs (
+    old: rec {
+      # jupyter is a meta-package. Everything relevant comes from the
+      # dependencies. It does however have a jupyter.py file that conflicts
+      # with jupyter-core so this meta solves this conflict.
+      meta.priority = 100;
+    }
+  );
+
   lap = super.lap.overrideAttrs (
     old: {
       propagatedBuildInputs = old.propagatedBuildInputs ++ [
@@ -202,9 +237,8 @@ self: super:
 
   mccabe = super.mccabe.overrideAttrs (
     old: {
-      postPatch = ''
+      buildInputs = old.buildInputs ++ [ self.pytest-runner ];
-        substituteInPlace setup.py --replace "setup_requires=['pytest-runner']," "setup_requires=[]," || true
+      doCheck = false;
-      '';
     }
   );
 
@@ -264,6 +298,13 @@ self: super:
       }
   );
 
+  openexr = super.openexr.overrideAttrs (
+    old: rec {
+      buildInputs = old.buildInputs ++ [ pkgs.openexr pkgs.ilmbase ];
+      NIX_CFLAGS_COMPILE = [ "-I${pkgs.openexr.dev}/include/OpenEXR" "-I${pkgs.ilmbase.dev}/include/OpenEXR" ];
+    }
+  );
+
   peewee = super.peewee.overridePythonAttrs (
     old: let
       withPostgres = old.passthru.withPostgres or false;
@@ -346,6 +387,13 @@ self: super:
     }
   );
 
+  pylint = super.pylint.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [ self.pytest-runner ];
+      doCheck = false;
+    }
+  );
+
   pyopenssl = super.pyopenssl.overrideAttrs (
     old: {
       buildInputs = old.buildInputs ++ [ pkgs.openssl ];
@@ -461,6 +509,14 @@ self: super:
     }
   );
 
+  pytest = super.pytest.overridePythonAttrs (
+    old: {
+      doCheck = false;
+    }
+  );
+
+  pytest-runner = super.pytest-runner or super.pytestrunner;
+
   python-prctl = super.python-prctl.overrideAttrs (
     old: {
       buildInputs = old.buildInputs ++ [
@@ -469,6 +525,21 @@ self: super:
     }
   );
 
+  pyzmq = super.pyzmq.overrideAttrs (
+    old: {
+      nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ];
+      propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.zeromq ];
+    }
+  );
+
+  rockset = super.rockset.overrideAttrs (
+    old: rec {
+      postPatch = ''
+        cp ./setup_rockset.py ./setup.py
+      '';
+    }
+  );
+
   scaleapi = super.scaleapi.overrideAttrs (
     old: {
       postPatch = ''
@@ -477,6 +548,12 @@ self: super:
     }
   );
 
+  pandas = super.pandas.overrideAttrs (
+    old: {
+      nativeBuildInputs = old.nativeBuildInputs ++ [ self.cython ];
+    }
+  );
+
   # Pybind11 is an undeclared dependency of scipy that we need to pick from nixpkgs
   # Make it not fail with infinite recursion
   pybind11 = super.pybind11.overridePythonAttrs (
@@ -529,6 +606,30 @@ self: super:
     }
   );
 
+  shellingham = if lib.versionAtLeast super.shellingham.version "1.3.2" then (
+    super.shellingham.overridePythonAttrs (
+      old: {
+        format = "pyproject";
+      }
+    )
+  ) else super.shellingham;
+
+  tables = super.tables.overrideAttrs (
+    old: {
+      HDF5_DIR = "${pkgs.hdf5}";
+      nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ];
+      propagatedBuildInputs = old.nativeBuildInputs ++ [ pkgs.hdf5 self.numpy self.numexpr ];
+    }
+  );
+
+  tensorpack = super.tensorpack.overrideAttrs (
+    old: {
+      postPatch = ''
+        substituteInPlace setup.cfg --replace "# will call find_packages()" ""
+      '';
+    }
+  );
+
   urwidtrees = super.urwidtrees.overrideAttrs (
     old: {
       propagatedBuildInputs = old.propagatedBuildInputs ++ [