Merge pull request #1467 from iElectric/buildPythonPackage-refactor

buildPythonPackage refactoring

pkgs/development/python-modules/distutils-cfg/default.nix

@@ -0,0 +1,31 @@
+# global distutils configuration, see http://docs.python.org/2/install/index.html#distutils-configuration-files
+
+{ stdenv, python, writeText, extraCfg ? "" }:
+
+
+let
+  distutilsCfg = writeText "distutils.cfg" ''
+    [easy_install]
+
+    # don't allow network connections during build to ensure purity
+    allow-hosts = None
+
+    # make sure we always unzip installed packages otherwise setup hooks won't work
+    zip_ok = 0
+
+    ${extraCfg}
+  '';
+in stdenv.mkDerivation {
+  name = "${python.libPrefix}-distutils.cfg";
+
+  buildInputs = [ python ];
+
+  unpackPhase = "true";
+
+  installPhase = ''
+    dest="$out/lib/${python.libPrefix}/site-packages/distutils"
+    mkdir -p $dest
+    ln -s ${python}/lib/${python.libPrefix}/distutils/* $dest
+    ln -s ${distutilsCfg} $dest/distutils.cfg
+  '';
+}

pkgs/development/python-modules/generic/default.nix

@@ -1,84 +1,112 @@
 /* This function provides a generic Python package builder.  It is
-   intended to work with packages that use `setuptools'
+   intended to work with packages that use `distutils/setuptools'
    (http://pypi.python.org/pypi/setuptools/), which represents a large
    number of Python packages nowadays.  */
 
-{ python, setuptools, wrapPython, lib, offlineDistutils, recursivePthLoader }:
+{ python, setuptools, wrapPython, lib, recursivePthLoader, distutils-cfg }:
 
-{ name, namePrefix ? python.libPrefix + "-"
+{ name
+
+# by default prefix `name` e.g. "python3.3-${name}"
+, namePrefix ? python.libPrefix + "-"
 
 , buildInputs ? []
 
+# pass extra information to the distutils global configuration (think as global setup.cfg)
+, distutilsExtraCfg ? ""
+
+# propagate build dependencies so in case we have A -> B -> C,
+# C can import propagated packages by A
 , propagatedBuildInputs ? []
 
-, # List of packages that should be added to the PYTHONPATH
-  # environment variable in programs built by this function.  Packages
-  # in the standard `propagatedBuildInputs' variable are also added.
-  # The difference is that `pythonPath' is not propagated to the user
-  # environment.  This is preferrable for programs because it doesn't
-  # pollute the user environment.
-  pythonPath ? []
+# passed to "python setup.py install"
+, setupPyInstallFlags ? []
 
-, installCommand ?
-    ''
-      easy_install --always-unzip --prefix="$out" .
-    ''
-
-, preConfigure ? "true"
-
-, buildPhase ? "true"
+# passed to "python setup.py build"
+, setupPyBuildFlags ? []
 
+# enable tests by default
 , doCheck ? true
 
-, checkPhase ?
-    ''
-      runHook preCheck
-      ${python}/bin/${python.executable} setup.py test
-      runHook postCheck
-    ''
-
-, preInstall ? ""
-, postInstall ? ""
+# List of packages that should be added to the PYTHONPATH
+# environment variable in programs built by this function.  Packages
+# in the standard `propagatedBuildInputs' variable are also added.
+# The difference is that `pythonPath' is not propagated to the user
+# environment.  This is preferrable for programs because it doesn't
+# pollute the user environment.
+,  pythonPath ? []
 
 , meta ? {}
 
 , ... } @ attrs:
 
-# Keep extra attributes from ATTR, e.g., `patchPhase', etc.
+# Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
 python.stdenv.mkDerivation (attrs // {
-  inherit doCheck buildPhase checkPhase;
+  inherit doCheck;
 
   name = namePrefix + name;
 
-  # default to python's platforms and add maintainer(s) to every
-  # package
-  meta = {
-    platforms = python.meta.platforms;
-  } // meta // {
-    maintainers = (meta.maintainers or []) ++ [ lib.maintainers.chaoflow lib.maintainers.iElectric ];
-  };
-
-  # checkPhase after installPhase to run tests on installed packages
-  phases = "unpackPhase patchPhase configurePhase buildPhase installPhase checkPhase fixupPhase distPhase";
-
-  buildInputs = [ python wrapPython setuptools ] ++ buildInputs ++ pythonPath;
+  buildInputs = [ python wrapPython setuptools (distutils-cfg.override { extraCfg = distutilsExtraCfg; }) ] ++ buildInputs ++ pythonPath;
 
   propagatedBuildInputs = propagatedBuildInputs ++ [ recursivePthLoader ];
 
   pythonPath = [ setuptools ] ++ pythonPath;
 
-  preConfigure = ''
+  configurePhase = attrs.configurePhase or ''
+    runHook preConfigure
+
+    # patch python interpreter to write null timestamps when compiling python files
+    # with following var we tell python to activate the patch so that python doesn't
+    # try to update them when we freeze timestamps in nix store
     export DETERMINISTIC_BUILD=1
-    PYTHONPATH="${offlineDistutils}/lib/${python.libPrefix}/site-packages:$PYTHONPATH"
-    ${preConfigure}
+
+    # prepend following line to import setuptools before distutils
+    # this way we make sure setuptools monkeypatches distutils commands
+    # this way setuptools provides extra helpers such as "python setup.py test"
+    sed -i '0,/import distutils/s//import setuptools;import distutils/' setup.py
+    sed -i '0,/from distutils/s//import setuptools;from distutils/' setup.py
+
+    runHook postConfigure
   '';
 
-  installPhase = preInstall + ''
+  checkPhase = attrs.checkPhase or ''
+      runHook preCheck
+
+      ${python}/bin/${python.executable} setup.py test
+
+      runHook postCheck
+  '';
+
+  buildPhase = attrs.buildPhase or ''
+    runHook preBuild
+
+    ${python}/bin/${python.executable} setup.py build ${lib.concatStringsSep " " setupPyBuildFlags}
+
+    runHook postBuild
+  '';
+
+  installPhase = attrs.installPhase or ''
+    runHook preInstall
+
     mkdir -p "$out/lib/${python.libPrefix}/site-packages"
 
-    echo "installing \`${name}' with \`easy_install'..."
     export PYTHONPATH="$out/lib/${python.libPrefix}/site-packages:$PYTHONPATH"
-    ${installCommand}
+
+    ${python}/bin/${python.executable} setup.py install \
+      --install-lib=$out/lib/${python.libPrefix}/site-packages \
+      --old-and-unmanageable \
+      --prefix="$out" ${lib.concatStringsSep " " setupPyInstallFlags}
+
+    # --install-lib:
+    # sometimes packages specify where files should be installed outside the usual
+    # python lib prefix, we override that back so all infrastructure (setup hooks)
+    # work as expected
+
+    # --old-and-unmanagable:
+    # instruct setuptools not to use eggs but fallback to plan package install 
+    # this also reduces one .pth file in the chain, but the main reason is to
+    # force install process to install only scripts for the package we are
+    # installing (otherwise it will install scripts also for dependencies)
 
     # A pth file might have been generated to load the package from
     # within its own site-packages, rename this package not to
@@ -94,21 +122,22 @@ python.stdenv.mkDerivation (attrs // {
     # corresponding site.py needs to be included in the PYTHONPATH.
     rm -f "$out/lib/${python.libPrefix}"/site-packages/site.py*
 
-    ${postInstall}
+    runHook postInstall
   '';
 
   postFixup =
     ''
       wrapPythonPrograms
 
-      # If a user installs a Python package, she probably also wants its
-      # dependencies in the user environment (since Python modules don't
-      # have something like an RPATH, so the only way to find the
+      # If a user installs a Python package, they probably also wants its
+      # dependencies in the user environment profile (only way to find the
       # dependencies is to have them in the PYTHONPATH variable).
+      # Allows you to do: $ PYTHONPATH=~/.nix-profile/lib/python2.7/site-packages python
       if test -e $out/nix-support/propagated-build-inputs; then
           ln -s $out/nix-support/propagated-build-inputs $out/nix-support/propagated-user-env-packages
       fi
 
+      # TODO: document
       createBuildInputsPth build-inputs "$buildInputStrings"
       for inputsfile in propagated-build-inputs propagated-native-build-inputs; do
         if test -e $out/nix-support/$inputsfile; then
@@ -116,4 +145,13 @@ python.stdenv.mkDerivation (attrs // {
         fi
       done
     '';
+
+  meta = with lib.maintainers; {
+    # default to python's platforms
+    platforms = python.meta.platforms;
+  } // meta // {
+    # add extra maintainer(s) to every package
+    maintainers = (meta.maintainers or []) ++ [ chaoflow iElectric ];
+  };
+
 })

pkgs/development/python-modules/offline-distutils/default.nix

@@ -1,21 +0,0 @@
-# Used during module installation to prevent easy_install and python
-# setup.py install/test from downloading
-
-{ stdenv, python }:
-
-stdenv.mkDerivation {
-  name = "python-offline-distutils-${python.version}";
-
-  buildInputs = [ python ];
-
-  unpackPhase = "true";
-  installPhase = ''
-    dst="$out/lib/${python.libPrefix}/site-packages"
-    ensureDir $dst/distutils
-    ln -s ${python}/lib/${python.libPrefix}/distutils/* $dst/distutils/
-    cat <<EOF > $dst/distutils/distutils.cfg
-[easy_install]
-allow-hosts = None
-EOF
-  '';
-}

pkgs/development/python-modules/pil/default.nix

@@ -14,16 +14,15 @@ buildPythonPackage {
 
   doCheck = true;
 
-  configurePhase = ''
+  preConfigure = ''
     sed -i "setup.py" \
         -e 's|^FREETYPE_ROOT =.*$|FREETYPE_ROOT = libinclude("${freetype}")|g ;
             s|^JPEG_ROOT =.*$|JPEG_ROOT = libinclude("${libjpeg}")|g ;
             s|^ZLIB_ROOT =.*$|ZLIB_ROOT = libinclude("${zlib}")|g ;'
   '';
 
-  buildPhase   = "python setup.py build_ext -i";
   checkPhase   = "python selftest.py";
-  #installPhase = "python setup.py install --prefix=$out";
+  buildPhase   = "python setup.py build_ext -i";
 
   meta = {
     homepage = http://www.pythonware.com/products/pil/;

pkgs/development/python-modules/pycrypto/default.nix

@@ -9,17 +9,7 @@ buildPythonPackage rec {
     sha256 = "0g0ayql5b9mkjam8hym6zyg6bv77lbh66rv1fyvgqb17kfc1xkpj";
   };
 
-  buildInputs = [ python gmp ];
-
-  buildPhase =
-    ''
-      python ./setup.py build_ext --library-dirs=${gmp}/lib
-    '';
-
-#  installPhase =
-#    ''
-#      python ./setup.py install --prefix=$out
-#    '';
+  buildInputs = [ gmp ];
 
   meta = {
     homepage = "http://www.pycrypto.org/";

pkgs/development/python-modules/pygtk/default.nix

@@ -14,7 +14,12 @@ buildPythonPackage rec {
 
   propagatedBuildInputs = [ gtk pygobject pycairo ];
 
-  installCommand = "make install";
+  configurePhase = "configurePhase";
+
+  buildPhase = "buildPhase";
+
+  installPhase = "installPhase";
+
   checkPhase = stdenv.lib.optionalString (libglade == null)
     ''
       sed -i -e "s/glade = importModule('gtk.glade', buildDir)//" \

pkgs/development/python-modules/setuptools/default.nix

@@ -1,19 +1,19 @@
-{ stdenv, fetchurl, python, wrapPython }:
+{ stdenv, fetchurl, python, wrapPython, distutils-cfg }:
 
 stdenv.mkDerivation rec {
   shortName = "setuptools-${version}";
   name = "${python.executable}-${shortName}";
 
-  version = "2.0.2";
+  version = "2.1";
 
   src = fetchurl {
     url = "http://pypi.python.org/packages/source/s/setuptools/${shortName}.tar.gz";
-    sha256 = "09nv5x45y8fgc0kjmmw4gig3hr0is9xlc5rq053vnbmkxr5q5xmi";
+    sha256 = "1m8qjvj5bfbphdags5s6pgmvk3xnw509lgdlq9whkq5a9mgxf8m7";
   };
 
-  buildInputs = [ python wrapPython ];
+  buildInputs = [ python wrapPython distutils-cfg ];
 
-  buildPhase = "${python}/bin/${python.executable} setup.py build --build-base $out";
+  buildPhase = "${python}/bin/${python.executable} setup.py build";
 
   installPhase =
     ''

pkgs/development/python-modules/setuptools/distribute-skip-sdist_with_utf8_encoded_filename.patch

@@ -1,28 +0,0 @@
-diff -r f5ac515f062a setuptools/tests/test_sdist.py
---- a/setuptools/tests/test_sdist.py	Fri Jul 26 09:52:26 2013 +0200
-+++ b/setuptools/tests/test_sdist.py	Sat Jul 27 20:22:17 2013 +0200
-@@ -3,12 +3,14 @@
- 
- 
- import os
-+import locale
- import shutil
- import sys
- import tempfile
- import unittest
- import unicodedata
- 
-+from setuptools.tests.py26compat import skipIf
- from setuptools.compat import StringIO, unicode
- from setuptools.command.sdist import sdist
- from setuptools.command.egg_info import manifest_maker
-@@ -318,6 +320,9 @@
-             filename = filename.decode('latin-1')
-             self.assertFalse(filename in cmd.filelist.files)
- 
-+
-+    @skipIf(sys.version_info >= (3,) and locale.getpreferredencoding() != 'UTF-8',
-+            'Unittest fails if locale is not utf-8 but the manifests is recorded correctly')
-     def test_sdist_with_utf8_encoded_filename(self):
-         # Test for #303.
-         dist = Distribution(SETUP_ATTRS)

pkgs/development/python-modules/setuptools/site.nix

@@ -1,16 +0,0 @@
-# Propagated by buildPythonPackge to process pth files
-
-{ stdenv, python, setuptools }:
-
-stdenv.mkDerivation {
-  name = "python-setuptools-site-${setuptools.version}";
-
-  buildInputs = [ python setuptools ];
-
-  unpackPhase = "true";
-  installPhase = ''
-    dst="$out/lib/${python.libPrefix}/site-packages"
-    ensureDir $dst
-    ln -s ${setuptools}/lib/${python.libPrefix}/site-packages/site.* $dst/
-  '';
-}