Merge remote-tracking branch 'origin/master' into staging

Conflicts:
	pkgs/development/libraries/poppler/default.nix

pkgs/development/interpreters/python/2.6/default.nix

@@ -1,13 +1,11 @@
-{ stdenv, fetchurl, zlib ? null, zlibSupport ? true, bzip2
-, sqlite, tcl, tk, x11, openssl, readline, db, ncurses, gdbm
-}:
+{ stdenv, fetchurl, zlib ? null, zlibSupport ? true, bzip2, includeModules ? false
+, sqlite, tcl, tk, x11, openssl, readline, db, ncurses, gdbm}:
 
 assert zlibSupport -> zlib != null;
 
 with stdenv.lib;
 
 let
-
   majorVersion = "2.6";
   version = "${majorVersion}.9";
 
@@ -27,37 +25,42 @@ let
       # the Nix store to 1.  So treat that as a special case.
       ./nix-store-mtime.patch
     ];
+    
+  preConfigure = ''
+      # Purity.
+      for i in /usr /sw /opt /pkg; do
+        substituteInPlace ./setup.py --replace $i /no-such-path
+      done
+    '' + optionalString (stdenv ? gcc && stdenv.gcc.libc != null) ''
+      for i in Lib/plat-*/regen; do
+        substituteInPlace $i --replace /usr/include/ ${stdenv.gcc.libc}/include/
+      done
+    '' + optionalString stdenv.isCygwin ''
+      # On Cygwin, `make install' tries to read this Makefile.
+      mkdir -p $out/lib/python${majorVersion}/config
+      touch $out/lib/python${majorVersion}/config/Makefile
+      mkdir -p $out/include/python${majorVersion}
+      touch $out/include/python${majorVersion}/pyconfig.h
+    '';
 
   buildInputs =
     optional (stdenv ? gcc && stdenv.gcc.libc != null) stdenv.gcc.libc ++
-    [ bzip2 openssl ]
+    [ bzip2 openssl ]++ optionals includeModules [ db openssl ncurses gdbm readline x11 tcl tk sqlite ]
     ++ optional zlibSupport zlib;
 
 
   # Build the basic Python interpreter without modules that have
   # external dependencies.
   python = stdenv.mkDerivation {
-    name = "python-${version}";
+    name = "python${if includeModules then "" else "-minimal"}-${version}";
 
-    inherit majorVersion version src patches buildInputs;
+    inherit majorVersion version src patches buildInputs preConfigure;
 
     C_INCLUDE_PATH = concatStringsSep ":" (map (p: "${p}/include") buildInputs);
     LIBRARY_PATH = concatStringsSep ":" (map (p: "${p}/lib") buildInputs);
 
     configureFlags = "--enable-shared --with-threads --enable-unicode";
 
-    preConfigure =
-      ''
-        # Purity.
-        for i in /usr /sw /opt /pkg; do
-          substituteInPlace ./setup.py --replace $i /no-such-path
-        done
-      '' + optionalString (stdenv ? gcc && stdenv.gcc.libc != null) ''
-        for i in Lib/plat-*/regen; do
-          substituteInPlace $i --replace /usr/include/ ${stdenv.gcc.libc}/include/
-        done
-      '';
-
     NIX_CFLAGS_COMPILE = optionalString stdenv.isDarwin "-msse2";
 
     setupHook = ./setup-hook.sh;
@@ -69,6 +72,10 @@ let
         ln -s $out/lib/python${majorVersion}/pdb.py $out/bin/pdb${majorVersion}
         mv $out/share/man/man1/{python.1,python2.6.1}
         ln -s $out/share/man/man1/{python2.6.1,python.1}
+        
+        paxmark E $out/bin/python${majorVersion}
+        
+        ${ optionalString includeModules "$out/bin/python ./setup.py build_ext"}
       '';
 
     passthru = rec {
@@ -96,7 +103,7 @@ let
       '';
       license = stdenv.lib.licenses.psfl;
       platforms = stdenv.lib.platforms.all;
-      maintainers = with stdenv.lib.maintainers; [ simons chaoflow ];
+      maintainers = with stdenv.lib.maintainers; [ simons chaoflow iElectric ];
     };
   };
 
@@ -108,18 +115,16 @@ let
     , internalName ? "_" + moduleName
     , deps
     }:
-    stdenv.mkDerivation rec {
+    if includeModules then null else stdenv.mkDerivation rec {
       name = "python-${moduleName}-${python.version}";
 
-      inherit src patches;
+      inherit src patches preConfigure;
 
       buildInputs = [ python ] ++ deps;
 
       C_INCLUDE_PATH = concatStringsSep ":" (map (p: "${p}/include") buildInputs);
       LIBRARY_PATH = concatStringsSep ":" (map (p: "${p}/lib") buildInputs);
 
-      configurePhase = "true";
-
       buildPhase =
         ''
           # Fake the build environment that setup.py expects.
@@ -178,8 +183,6 @@ let
       deps = [ sqlite ];
     };
 
-    ssl = null;
-
     tkinter = buildInternalPythonModule {
       moduleName = "tkinter";
       deps = [ tcl tk x11 ];

pkgs/development/interpreters/python/2.7/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, fetchurl, zlib ? null, zlibSupport ? true, bzip2
+{ stdenv, fetchurl, zlib ? null, zlibSupport ? true, bzip2, includeModules ? false
 , sqlite, tcl, tk, x11, openssl, readline, db, ncurses, gdbm, libX11 }:
 
 assert zlibSupport -> zlib != null;
@@ -6,7 +6,6 @@ assert zlibSupport -> zlib != null;
 with stdenv.lib;
 
 let
-
   majorVersion = "2.7";
   version = "${majorVersion}.8";
 
@@ -28,33 +27,39 @@ let
       # patch python to put zero timestamp into pyc
       # if DETERMINISTIC_BUILD env var is set
       ./deterministic-build.patch
+
+      # http://bugs.python.org/issue21963
+      ./remove-avoid-daemon-thread-shutdown.patch
     ];
-
-  postPatch = stdenv.lib.optionalString (stdenv.gcc.libc != null) ''
-    substituteInPlace ./Lib/plat-generic/regen \
-                      --replace /usr/include/netinet/in.h \
-                                ${stdenv.gcc.libc}/include/netinet/in.h
-  '';
-
-  buildInputs =
-    optional (stdenv ? gcc && stdenv.gcc.libc != null) stdenv.gcc.libc ++
-    [ bzip2 openssl ]
-    ++ optional zlibSupport zlib;
-
-  ensurePurity =
-    ''
+    
+  preConfigure = ''
       # Purity.
       for i in /usr /sw /opt /pkg; do
         substituteInPlace ./setup.py --replace $i /no-such-path
       done
+    '' + optionalString (stdenv ? gcc && stdenv.gcc.libc != null) ''
+      for i in Lib/plat-*/regen; do
+        substituteInPlace $i --replace /usr/include/ ${stdenv.gcc.libc}/include/
+      done
+    '' + optionalString stdenv.isCygwin ''
+      # On Cygwin, `make install' tries to read this Makefile.
+      mkdir -p $out/lib/python${majorVersion}/config
+      touch $out/lib/python${majorVersion}/config/Makefile
+      mkdir -p $out/include/python${majorVersion}
+      touch $out/include/python${majorVersion}/pyconfig.h
     '';
 
+  buildInputs =
+    optional (stdenv ? gcc && stdenv.gcc.libc != null) stdenv.gcc.libc ++
+    [ bzip2 openssl ] ++ optionals includeModules [ db openssl ncurses gdbm libX11 readline x11 tcl tk sqlite ]
+    ++ optional zlibSupport zlib;
+
   # Build the basic Python interpreter without modules that have
   # external dependencies.
   python = stdenv.mkDerivation {
     name = "python-${version}";
 
-    inherit majorVersion version src patches postPatch buildInputs;
+    inherit majorVersion version src patches buildInputs preConfigure;
 
     LDFLAGS = stdenv.lib.optionalString (!stdenv.isDarwin) "-lgcc_s";
     C_INCLUDE_PATH = concatStringsSep ":" (map (p: "${p}/include") buildInputs);
@@ -62,15 +67,6 @@ let
 
     configureFlags = "--enable-shared --with-threads --enable-unicode";
 
-    preConfigure = "${ensurePurity}" + optionalString stdenv.isCygwin
-      ''
-        # On Cygwin, `make install' tries to read this Makefile.
-        mkdir -p $out/lib/python${majorVersion}/config
-        touch $out/lib/python${majorVersion}/config/Makefile
-        mkdir -p $out/include/python${majorVersion}
-        touch $out/include/python${majorVersion}/pyconfig.h
-      '';
-
     NIX_CFLAGS_COMPILE = optionalString stdenv.isDarwin "-msse2";
     DETERMINISTIC_BUILD = 1;
 
@@ -84,6 +80,8 @@ let
         ln -s $out/share/man/man1/{python2.7.1.gz,python.1.gz}
 
         paxmark E $out/bin/python${majorVersion}
+        
+        ${ optionalString includeModules "$out/bin/python ./setup.py build_ext"}
       '';
 
     passthru = rec {
@@ -111,7 +109,7 @@ let
       '';
       license = stdenv.lib.licenses.psfl;
       platforms = stdenv.lib.platforms.all;
-      maintainers = with stdenv.lib.maintainers; [ simons chaoflow ];
+      maintainers = with stdenv.lib.maintainers; [ simons chaoflow iElectric ];
     };
   };
 
@@ -123,25 +121,17 @@ let
     , internalName ? "_" + moduleName
     , deps
     }:
-    stdenv.mkDerivation rec {
+    if includeModules then null else stdenv.mkDerivation rec {
       name = "python-${moduleName}-${python.version}";
 
-      inherit src patches postPatch;
+      inherit src patches preConfigure;
 
       buildInputs = [ python ] ++ deps;
 
       C_INCLUDE_PATH = concatStringsSep ":" (map (p: "${p}/include") buildInputs);
       LIBRARY_PATH = concatStringsSep ":" (map (p: "${p}/lib") buildInputs);
 
-      configurePhase = "${ensurePurity}";
-
-      buildPhase =
-        ''
-          # Fake the build environment that setup.py expects.
-          ln -s ${python}/include/python*/pyconfig.h .
-          ln -s ${python}/lib/python*/config/Setup Modules/
-          ln -s ${python}/lib/python*/config/Setup.local Modules/
-
+      buildPhase = ''
           substituteInPlace setup.py --replace 'self.extensions = extensions' \
             'self.extensions = [ext for ext in self.extensions if ext.name in ["${internalName}"]]'
 
@@ -193,8 +183,6 @@ let
       deps = [ sqlite ];
     };
 
-    ssl = null;
-
     tkinter = buildInternalPythonModule {
       moduleName = "tkinter";
       deps = [ tcl tk x11 libX11 ];

pkgs/development/interpreters/python/2.7/remove-avoid-daemon-thread-shutdown.patch

@@ -0,0 +1,170 @@
+changeset:   93046:61ad2208a5ce
+branch:      2.7
+tag:         tip
+user:        William A. Kennington III <william@wkennington.com>
+date:        Mon Oct 13 13:57:12 2014 -0700
+summary:     Revert: 91229:7741d0dd66ca to fix i21963
+
+diff -r ed4098380799 -r 61ad2208a5ce Include/pythonrun.h
+--- a/Include/pythonrun.h	Mon Oct 13 12:58:03 2014 -0700
++++ b/Include/pythonrun.h	Mon Oct 13 13:57:12 2014 -0700
+@@ -147,8 +147,6 @@
+ PyAPI_FUNC(void) PyByteArray_Fini(void);
+ PyAPI_FUNC(void) _PyRandom_Fini(void);
+ 
+-PyAPI_DATA(PyThreadState *) _Py_Finalizing;
+-
+ /* Stuff with no proper home (yet) */
+ PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, char *);
+ PyAPI_DATA(int) (*PyOS_InputHook)(void);
+diff -r ed4098380799 -r 61ad2208a5ce Lib/test/test_threading.py
+--- a/Lib/test/test_threading.py	Mon Oct 13 12:58:03 2014 -0700
++++ b/Lib/test/test_threading.py	Mon Oct 13 13:57:12 2014 -0700
+@@ -700,49 +700,6 @@
+         output = "end of worker thread\nend of main thread\n"
+         self.assertScriptHasOutput(script, output)
+ 
+-    @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug")
+-    def test_6_daemon_threads(self):
+-        # Check that a daemon thread cannot crash the interpreter on shutdown
+-        # by manipulating internal structures that are being disposed of in
+-        # the main thread.
+-        script = """if True:
+-            import os
+-            import random
+-            import sys
+-            import time
+-            import threading
+-
+-            thread_has_run = set()
+-
+-            def random_io():
+-                '''Loop for a while sleeping random tiny amounts and doing some I/O.'''
+-                while True:
+-                    in_f = open(os.__file__, 'rb')
+-                    stuff = in_f.read(200)
+-                    null_f = open(os.devnull, 'wb')
+-                    null_f.write(stuff)
+-                    time.sleep(random.random() / 1995)
+-                    null_f.close()
+-                    in_f.close()
+-                    thread_has_run.add(threading.current_thread())
+-
+-            def main():
+-                count = 0
+-                for _ in range(40):
+-                    new_thread = threading.Thread(target=random_io)
+-                    new_thread.daemon = True
+-                    new_thread.start()
+-                    count += 1
+-                while len(thread_has_run) < count:
+-                    time.sleep(0.001)
+-                # Trigger process shutdown
+-                sys.exit(0)
+-
+-            main()
+-            """
+-        rc, out, err = assert_python_ok('-c', script)
+-        self.assertFalse(err)
+-
+     @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()")
+     @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug")
+     def test_reinit_tls_after_fork(self):
+diff -r ed4098380799 -r 61ad2208a5ce Misc/NEWS
+--- a/Misc/NEWS	Mon Oct 13 12:58:03 2014 -0700
++++ b/Misc/NEWS	Mon Oct 13 13:57:12 2014 -0700
+@@ -96,10 +96,6 @@
+ - Issue #21831: Avoid integer overflow when large sizes and offsets are given to
+   the buffer type.
+ 
+-- Issue #1856: Avoid crashes and lockups when daemon threads run while the
+-  interpreter is shutting down; instead, these threads are now killed when they
+-  try to take the GIL.
+-
+ - Issue #19656: Running Python with the -3 option now also warns about
+   non-ascii bytes literals.
+ 
+diff -r ed4098380799 -r 61ad2208a5ce Python/ceval.c
+--- a/Python/ceval.c	Mon Oct 13 12:58:03 2014 -0700
++++ b/Python/ceval.c	Mon Oct 13 13:57:12 2014 -0700
+@@ -355,12 +355,6 @@
+     if (interpreter_lock) {
+         int err = errno;
+         PyThread_acquire_lock(interpreter_lock, 1);
+-        /* _Py_Finalizing is protected by the GIL */
+-        if (_Py_Finalizing && tstate != _Py_Finalizing) {
+-            PyThread_release_lock(interpreter_lock);
+-            PyThread_exit_thread();
+-            assert(0);  /* unreachable */
+-        }
+         errno = err;
+     }
+ #endif
+@@ -1024,13 +1018,6 @@
+                 /* Other threads may run now */
+ 
+                 PyThread_acquire_lock(interpreter_lock, 1);
+-
+-                /* Check if we should make a quick exit. */
+-                if (_Py_Finalizing && _Py_Finalizing != tstate) {
+-                    PyThread_release_lock(interpreter_lock);
+-                    PyThread_exit_thread();
+-                }
+-
+                 if (PyThreadState_Swap(tstate) != NULL)
+                     Py_FatalError("ceval: orphan tstate");
+ 
+diff -r ed4098380799 -r 61ad2208a5ce Python/pythonrun.c
+--- a/Python/pythonrun.c	Mon Oct 13 12:58:03 2014 -0700
++++ b/Python/pythonrun.c	Mon Oct 13 13:57:12 2014 -0700
+@@ -91,8 +91,6 @@
+ int Py_NoUserSiteDirectory = 0; /* for -s and site.py */
+ int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */
+ 
+-PyThreadState *_Py_Finalizing = NULL;
+-
+ 
+ /* Hack to force loading of object files */
+ int (*_PyOS_mystrnicmp_hack)(const char *, const char *, Py_ssize_t) = \
+@@ -165,7 +163,6 @@
+     if (initialized)
+         return;
+     initialized = 1;
+-    _Py_Finalizing = NULL;
+ 
+     if ((p = Py_GETENV("PYTHONDEBUG")) && *p != '\0')
+         Py_DebugFlag = add_flag(Py_DebugFlag, p);
+@@ -425,16 +422,12 @@
+      * the threads created via Threading.
+      */
+     call_sys_exitfunc();
++    initialized = 0;
+ 
+     /* Get current thread state and interpreter pointer */
+     tstate = PyThreadState_GET();
+     interp = tstate->interp;
+ 
+-    /* Remaining threads (e.g. daemon threads) will automatically exit
+-       after taking the GIL (in PyEval_RestoreThread()). */
+-    _Py_Finalizing = tstate;
+-    initialized = 0;
+-
+     /* Disable signal handling */
+     PyOS_FiniInterrupts();
+ 
+diff -r ed4098380799 -r 61ad2208a5ce Python/thread_pthread.h
+--- a/Python/thread_pthread.h	Mon Oct 13 12:58:03 2014 -0700
++++ b/Python/thread_pthread.h	Mon Oct 13 13:57:12 2014 -0700
+@@ -242,9 +242,9 @@
+ PyThread_exit_thread(void)
+ {
+     dprintf(("PyThread_exit_thread called\n"));
+-    if (!initialized)
++    if (!initialized) {
+         exit(0);
+-    pthread_exit(0);
++    }
+ }
+ 
+ #ifdef USE_SEMAPHORES
+

pkgs/development/interpreters/python/3.4/default.nix

@@ -18,7 +18,7 @@ with stdenv.lib;
 
 let
   majorVersion = "3.4";
-  version = "${majorVersion}.1";
+  version = "${majorVersion}.2";
   fullVersion = "${version}";
 
   buildInputs = filter (p: p != null) [
@@ -31,13 +31,11 @@ stdenv.mkDerivation {
 
   src = fetchurl {
     url = "http://www.python.org/ftp/python/${version}/Python-${fullVersion}.tar.xz";
-    sha256 = "1i7dgbzyvj24i6gfhb5q2zwr9nn1ni6w1ig1rcgh96a321is35f5";
+    sha256 = "1vrd9gqdqw7rw0kiiprqvng7ywnfc2hbyys7gr9mdh25s619cv8w";
   };
 
   NIX_LDFLAGS = stdenv.lib.optionalString stdenv.isLinux "-lgcc_s";
 
-  patches = [ ./issue21121-3.patch ];
-
   preConfigure = ''
     for i in /usr /sw /opt /pkg; do	# improve purity
       substituteInPlace ./setup.py --replace $i /no-such-path

pkgs/development/interpreters/python/3.4/issue21121-3.patch

@@ -1,86 +0,0 @@
-diff --git a/Makefile.pre.in b/Makefile.pre.in
---- a/Makefile.pre.in
-+++ b/Makefile.pre.in
-@@ -71,12 +71,17 @@
- BASECFLAGS=	@BASECFLAGS@
- BASECPPFLAGS=	@BASECPPFLAGS@
- CONFIGURE_CFLAGS=	@CFLAGS@
-+# CFLAGS_NODIST is used for building the interpreter and stdlib C extensions.
-+# Use it when a compiler flag should _not_ be part of the distutils CFLAGS
-+# once Python is installed (Issue #21121).
-+CONFIGURE_CFLAGS_NODIST=@CFLAGS_NODIST@
- CONFIGURE_CPPFLAGS=	@CPPFLAGS@
- CONFIGURE_LDFLAGS=	@LDFLAGS@
- # Avoid assigning CFLAGS, LDFLAGS, etc. so users can use them on the
- # command line to append to these values without stomping the pre-set
- # values.
- PY_CFLAGS=	$(BASECFLAGS) $(OPT) $(CONFIGURE_CFLAGS) $(CFLAGS) $(EXTRA_CFLAGS)
-+PY_CFLAGS_NODIST=$(CONFIGURE_CFLAGS_NODIST) $(CFLAGS_NODIST)
- # Both CPPFLAGS and LDFLAGS need to contain the shell's value for setup.py to
- # be able to build extension modules using the directories specified in the
- # environment variables
-@@ -91,7 +96,7 @@
- # Extra C flags added for building the interpreter object files.
- CFLAGSFORSHARED=@CFLAGSFORSHARED@
- # C flags used for building the interpreter object files
--PY_CORE_CFLAGS=	$(PY_CFLAGS) $(PY_CPPFLAGS) $(CFLAGSFORSHARED) -DPy_BUILD_CORE
-+PY_CORE_CFLAGS=	$(PY_CFLAGS) $(PY_CFLAGS_NODIST) $(PY_CPPFLAGS) $(CFLAGSFORSHARED) -DPy_BUILD_CORE
- 
- 
- # Machine-dependent subdirectories
-diff --git a/configure b/configure
---- a/configure
-+++ b/configure
-@@ -662,6 +662,7 @@
- LIBTOOL_CRUFT
- OTHER_LIBTOOL_OPT
- UNIVERSAL_ARCH_FLAGS
-+CFLAGS_NODIST
- BASECFLAGS
- OPT
- ABIFLAGS
-@@ -6504,7 +6505,7 @@
- 
-     if test $ac_cv_declaration_after_statement_warning = yes
-     then
--      BASECFLAGS="$BASECFLAGS -Werror=declaration-after-statement"
-+      CFLAGS_NODIST="$CFLAGS_NODIST -Werror=declaration-after-statement"
-     fi
- 
-     # if using gcc on alpha, use -mieee to get (near) full IEEE 754
-diff --git a/configure.ac b/configure.ac
---- a/configure.ac
-+++ b/configure.ac
-@@ -1147,6 +1147,7 @@
- fi
- 
- AC_SUBST(BASECFLAGS)
-+AC_SUBST(CFLAGS_NODIST)
- 
- # The -arch flags for universal builds on OSX
- UNIVERSAL_ARCH_FLAGS=
-@@ -1231,7 +1232,7 @@
- 
-     if test $ac_cv_declaration_after_statement_warning = yes
-     then
--      BASECFLAGS="$BASECFLAGS -Werror=declaration-after-statement"
-+      CFLAGS_NODIST="$CFLAGS_NODIST -Werror=declaration-after-statement"
-     fi
- 
-     # if using gcc on alpha, use -mieee to get (near) full IEEE 754
-diff --git a/setup.py b/setup.py
---- a/setup.py
-+++ b/setup.py
-@@ -19,6 +19,12 @@
- 
- cross_compiling = "_PYTHON_HOST_PLATFORM" in os.environ
- 
-+# Add special CFLAGS reserved for building the interpreter and the stdlib
-+# modules (Issue #21121).
-+cflags = sysconfig.get_config_var('CFLAGS')
-+py_cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST')
-+sysconfig.get_config_vars()['CFLAGS'] = cflags + ' ' + py_cflags_nodist
-+
- def get_platform():
-     # cross build
-     if "_PYTHON_HOST_PLATFORM" in os.environ:

pkgs/development/interpreters/python/wrapper.nix

@@ -1,12 +1,11 @@
 { stdenv, python, buildEnv, makeWrapper, recursivePthLoader, extraLibs ? [], postBuild ? ""
-, stdLibs ? stdenv.lib.attrValues python.modules, ignoreCollisions ? false
-}:
+, ignoreCollisions ? false }:
 
 # Create a python executable that knows about additional packages.
 
 (buildEnv {
-  name = "python-${python.version}-wrapper";
-  paths = stdenv.lib.filter (x : x ? pythonPath) (stdenv.lib.closePropagation extraLibs) ++ stdLibs ++ [ python recursivePthLoader ];
+  name = "python-${python.version}-env";
+  paths = stdenv.lib.filter (x : x ? pythonPath) (stdenv.lib.closePropagation extraLibs) ++ [ python recursivePthLoader ];
 
   inherit ignoreCollisions;