diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix index ff1497a24e4..23b88b168b5 100644 --- a/pkgs/development/interpreters/python/cpython/2.7/default.nix +++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix @@ -1,7 +1,9 @@ -{ stdenv, buildPackages, fetchurl +{ stdenv, fetchurl, fetchpatch , bzip2 +, expat +, libffi , gdbm -, fetchpatch +, db , ncurses , openssl , readline @@ -10,15 +12,16 @@ , zlib , callPackage , self -, db -, expat -, libffi , CF, configd, coreutils , python-setup-hook # Some proprietary libs assume UCS2 unicode, especially on darwin :( , ucsEncoding ? 4 # For the Python package set , packageOverrides ? (self: super: {}) +, buildPackages +, sourceVersion +, sha256 +, passthruFun }: assert x11Support -> tcl != null @@ -29,16 +32,23 @@ assert x11Support -> tcl != null with stdenv.lib; let - majorVersion = "2.7"; - minorVersion = "15"; - minorVersionSuffix = ""; - version = "${majorVersion}.${minorVersion}${minorVersionSuffix}"; - libPrefix = "python${majorVersion}"; - sitePackages = "lib/${libPrefix}/site-packages"; + passthru = passthruFun rec { + inherit self sourceVersion packageOverrides; + implementation = "cpython"; + libPrefix = "python${pythonVersion}"; + executable = libPrefix; + pythonVersion = with sourceVersion; "${major}.${minor}"; + sitePackages = "lib/${libPrefix}/site-packages"; + inherit pythonForBuild; + } // { + inherit ucsEncoding; + }; + + version = with sourceVersion; "${major}.${minor}.${patch}${suffix}"; src = fetchurl { - url = "https://www.python.org/ftp/python/${majorVersion}.${minorVersion}/Python-${version}.tar.xz"; - sha256 = "0x2mvz9dp11wj7p5ccvmk9s0hzjk2fa1m462p395l4r6bfnb3n92"; + url = with sourceVersion; "https://www.python.org/ftp/python/${major}.${minor}.${patch}/Python-${version}.tar.xz"; + inherit sha256; }; hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false); @@ -191,12 +201,11 @@ let # Build the basic Python interpreter without modules that have # external dependencies. -in stdenv.mkDerivation ({ - name = "python-${version}"; - pythonVersion = majorVersion; +in with passthru; stdenv.mkDerivation ({ + pname = "python"; + inherit version; - inherit majorVersion version src patches buildInputs nativeBuildInputs - preConfigure configureFlags; + inherit src patches buildInputs nativeBuildInputs preConfigure configureFlags; LDFLAGS = stdenv.lib.optionalString (!stdenv.isDarwin) "-lgcc_s"; inherit (mkPaths buildInputs) C_INCLUDE_PATH LIBRARY_PATH; @@ -215,7 +224,7 @@ in stdenv.mkDerivation ({ '' # needed for some packages, especially packages that backport # functionality to 2.x from 3.x - for item in $out/lib/python${majorVersion}/test/*; do + for item in $out/lib/${libPrefix}/test/*; do if [[ "$item" != */test_support.py* && "$item" != */test/support && "$item" != */test/regrtest.py* ]]; then @@ -224,9 +233,9 @@ in stdenv.mkDerivation ({ echo $item fi done - touch $out/lib/python${majorVersion}/test/__init__.py - ln -s $out/lib/python${majorVersion}/pdb.py $out/bin/pdb - ln -s $out/lib/python${majorVersion}/pdb.py $out/bin/pdb${majorVersion} + touch $out/lib/${libPrefix}/test/__init__.py + ln -s $out/lib/${libPrefix}/pdb.py $out/bin/pdb + ln -s $out/lib/${libPrefix}/pdb.py $out/bin/pdb${sourceVersion.major}.${sourceVersion.minor} ln -s $out/share/man/man1/{python2.7.1.gz,python.1.gz} # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484 @@ -249,21 +258,7 @@ in stdenv.mkDerivation ({ cp libpython2.7.dll.a $out/lib ''; - passthru = let - pythonPackages = callPackage ../../../../../top-level/python-packages.nix { - python = self; - overrides = packageOverrides; - }; - in rec { - inherit libPrefix sitePackages x11Support hasDistutilsCxxPatch ucsEncoding; - executable = libPrefix; - buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; }; - withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; - pkgs = pythonPackages; - isPy2 = true; - isPy27 = true; - interpreter = "${self}/bin/${executable}"; - }; + inherit passthru; enableParallelBuilding = true; diff --git a/pkgs/development/interpreters/python/cpython/3.5/default.nix b/pkgs/development/interpreters/python/cpython/3.5/default.nix deleted file mode 100644 index 19e9f3169c7..00000000000 --- a/pkgs/development/interpreters/python/cpython/3.5/default.nix +++ /dev/null @@ -1,212 +0,0 @@ -{ stdenv, fetchurl, fetchpatch -, bzip2 -, expat -, libffi -, gdbm -, lzma -, ncurses -, openssl -, readline -, sqlite -, tcl ? null, tk ? null, tix ? null, libX11 ? null, xproto ? null, x11Support ? false -, zlib -, callPackage -, self -, CF, configd -, python-setup-hook -# For the Python package set -, packageOverrides ? (self: super: {}) -}: - -assert x11Support -> tcl != null - && tk != null - && xproto != null - && libX11 != null; - -with stdenv.lib; - -let - majorVersion = "3.5"; - minorVersion = "6"; - minorVersionSuffix = ""; - version = "${majorVersion}.${minorVersion}${minorVersionSuffix}"; - libPrefix = "python${majorVersion}"; - sitePackages = "lib/${libPrefix}/site-packages"; - - buildInputs = filter (p: p != null) [ - zlib bzip2 expat lzma libffi gdbm sqlite readline ncurses openssl ] - ++ optionals x11Support [ tcl tk libX11 xproto ] - ++ optionals stdenv.isDarwin [ CF configd ]; - - hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false); - -in stdenv.mkDerivation { - name = "python3-${version}"; - pythonVersion = majorVersion; - inherit majorVersion version; - - inherit buildInputs; - - src = fetchurl { - url = "https://www.python.org/ftp/python/${majorVersion}.${minorVersion}/Python-${version}.tar.xz"; - sha256 = "0pqmf51zy2lzhbaj4yya2py2qr653j9152d0rg3p7wi1yl2dwp7m"; - }; - - NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s"; - - # Determinism: The interpreter is patched to write null timestamps when compiling python files. - # This way python doesn't try to update them when we freeze timestamps in nix store. - DETERMINISTIC_BUILD=1; - # Determinism: We fix the hashes of str, bytes and datetime objects. - PYTHONHASHSEED=0; - - prePatch = optionalString stdenv.isDarwin '' - substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"' - substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' ' - ''; - - patches = [ - ./no-ldconfig.patch - ./ld_library_path.patch - ] ++ optionals stdenv.isDarwin [ - # Fix for https://bugs.python.org/issue24658 - (fetchpatch { - url = "https://bugs.python.org/file45178/issue24658-3-3.6.diff"; - sha256 = "1x060hs80nl34mcl2ji2i7l4shxkmxwgq8h8lcmav8rjqqz1nb4a"; - }) - ] ++ optionals (x11Support && stdenv.isDarwin) [ - ./use-correct-tcl-tk-on-darwin.patch - ] ++ optionals hasDistutilsCxxPatch [ - # Fix for http://bugs.python.org/issue1222585 - # Upstream distutils is calling C compiler to compile C++ code, which - # only works for GCC and Apple Clang. This makes distutils to call C++ - # compiler when needed. - (fetchpatch { - url = "https://bugs.python.org/file47046/python-3.x-distutils-C++.patch"; - sha256 = "0dgdn9k2kmw4wh90vdnjcrnn97ylxgx7mbn9l87fwz6j501jqvk8"; - extraPrefix = ""; - }) - ]; - - postPatch = '' - # Determinism - substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])" - # Determinism. This is done unconditionally - substituteInPlace "Lib/importlib/_bootstrap_external.py" --replace "source_mtime = int(st['mtime'])" "source_mtime = 1" - '' + optionalString (x11Support && (tix != null)) '' - substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'" - ''; - - CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}"; - LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}"; - LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}"; - - configureFlags = [ - "--enable-shared" - "--with-threads" - "--without-ensurepip" - "--with-system-expat" - "--with-system-ffi" - ] - # Never even try to use lchmod on linux, - # don't rely on detecting glibc-isms. - ++ optional stdenv.hostPlatform.isLinux "ac_cv_func_lchmod=no"; - - preConfigure = '' - for i in /usr /sw /opt /pkg; do # improve purity - substituteInPlace ./setup.py --replace $i /no-such-path - done - ${optionalString stdenv.isDarwin '' - export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2" - export MACOSX_DEPLOYMENT_TARGET=10.6 - '' - + optionalString stdenv.hostPlatform.isMusl '' - export NIX_CFLAGS_COMPILE+=" -DTHREAD_STACK_SIZE=0x100000" - ''} - ''; - - setupHook = python-setup-hook sitePackages; - - postInstall = '' - # needed for some packages, especially packages that backport functionality - # to 2.x from 3.x - for item in $out/lib/python${majorVersion}/test/*; do - if [[ "$item" != */test_support.py* - && "$item" != */test/support - && "$item" != */test/libregrtest - && "$item" != */test/regrtest.py* ]]; then - rm -rf "$item" - else - echo $item - fi - done - touch $out/lib/python${majorVersion}/test/__init__.py - - ln -s "$out/include/python${majorVersion}m" "$out/include/python${majorVersion}" - - # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484 - echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py - - # Determinism: Windows installers were not deterministic. - # We're also not interested in building Windows installers. - find "$out" -name 'wininst*.exe' | xargs -r rm -f - - # Use Python3 as default python - ln -s "$out/bin/idle3" "$out/bin/idle" - ln -s "$out/bin/pydoc3" "$out/bin/pydoc" - ln -s "$out/bin/python3" "$out/bin/python" - ln -s "$out/bin/python3-config" "$out/bin/python-config" - ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc" - - # Get rid of retained dependencies on -dev packages, and remove - # some $TMPDIR references to improve binary reproducibility. - # Note that the .pyc file of _sysconfigdata.py should be regenerated! - for i in $out/lib/python${majorVersion}/_sysconfigdata.py $out/lib/python${majorVersion}/config-${majorVersion}m/Makefile; do - sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g" - done - - # Determinism: rebuild all bytecode - # We exclude lib2to3 because that's Python 2 code which fails - # We rebuild three times, once for each optimization level - find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i - - find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i - - find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i - - ''; - - passthru = let - pythonPackages = callPackage ../../../../../top-level/python-packages.nix { - python = self; - overrides = packageOverrides; - }; - in rec { - inherit libPrefix sitePackages x11Support hasDistutilsCxxPatch; - executable = "${libPrefix}m"; - buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; }; - withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; - pkgs = pythonPackages; - isPy3 = true; - isPy35 = true; - interpreter = "${self}/bin/${executable}"; - }; - - enableParallelBuilding = true; - - doCheck = false; # expensive, and fails - - meta = { - homepage = http://python.org; - description = "A high-level dynamically-typed programming language"; - longDescription = '' - Python is a remarkably powerful dynamic programming language that - is used in a wide variety of application domains. Some of its key - distinguishing features include: clear, readable syntax; strong - introspection capabilities; intuitive object orientation; natural - expression of procedural code; full modularity, supporting - hierarchical packages; exception-based error handling; and very - high level dynamic data types. - ''; - license = licenses.psfl; - platforms = with platforms; linux ++ darwin; - maintainers = with maintainers; [ fridh ]; - }; -} diff --git a/pkgs/development/interpreters/python/cpython/3.6/default.nix b/pkgs/development/interpreters/python/cpython/3.6/default.nix deleted file mode 100644 index a36965a7801..00000000000 --- a/pkgs/development/interpreters/python/cpython/3.6/default.nix +++ /dev/null @@ -1,228 +0,0 @@ -{ stdenv, fetchurl, fetchpatch, buildPackages -, bzip2 -, expat -, libffi -, gdbm -, lzma -, ncurses -, openssl -, readline -, sqlite -, tcl ? null, tk ? null, tix ? null, libX11 ? null, xproto ? null, x11Support ? false -, zlib -, callPackage -, self -, CF, configd -, python-setup-hook -# For the Python package set -, packageOverrides ? (self: super: {}) -}: - -assert x11Support -> tcl != null - && tk != null - && xproto != null - && libX11 != null; -with stdenv.lib; - -let - majorVersion = "3.6"; - minorVersion = "8"; - minorVersionSuffix = ""; - version = "${majorVersion}.${minorVersion}${minorVersionSuffix}"; - libPrefix = "python${majorVersion}"; - sitePackages = "lib/${libPrefix}/site-packages"; - - buildInputs = filter (p: p != null) [ - zlib bzip2 expat lzma libffi gdbm sqlite readline ncurses openssl ] - ++ optionals x11Support [ tcl tk libX11 xproto ] - ++ optionals stdenv.isDarwin [ CF configd ]; - - nativeBuildInputs = - optional (stdenv.hostPlatform != stdenv.buildPlatform) buildPackages.python3; - - hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false); - -in stdenv.mkDerivation { - name = "python3-${version}"; - pythonVersion = majorVersion; - inherit majorVersion version; - - inherit buildInputs nativeBuildInputs; - - src = fetchurl { - url = "https://www.python.org/ftp/python/${majorVersion}.${minorVersion}/Python-${version}.tar.xz"; - sha256 = "14qi6n5gpcjnwy165wi9hkfcmbadc95ny6bxxldknxwmx50n4i1m"; - }; - - NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s"; - - # Determinism: The interpreter is patched to write null timestamps when compiling python files. - # This way python doesn't try to update them when we freeze timestamps in nix store. - DETERMINISTIC_BUILD=1; - # Determinism: We fix the hashes of str, bytes and datetime objects. - PYTHONHASHSEED=0; - - prePatch = optionalString stdenv.isDarwin '' - substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"' - substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' ' - ''; - - patches = [ - ./no-ldconfig.patch - ] ++ optionals (x11Support && stdenv.isDarwin) [ - ./use-correct-tcl-tk-on-darwin.patch - ] ++ optionals hasDistutilsCxxPatch [ - # Fix for http://bugs.python.org/issue1222585 - # Upstream distutils is calling C compiler to compile C++ code, which - # only works for GCC and Apple Clang. This makes distutils to call C++ - # compiler when needed. - (fetchpatch { - url = "https://bugs.python.org/file48016/python-3.x-distutils-C++.patch"; - sha256 = "1h18lnpx539h5lfxyk379dxwr8m2raigcjixkf133l4xy3f4bzi2"; - }) - ]; - - postPatch = '' - # Determinism - substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])" - # Determinism. This is done unconditionally - substituteInPlace "Lib/importlib/_bootstrap_external.py" --replace "source_mtime = int(st['mtime'])" "source_mtime = 1" - '' + optionalString (x11Support && (tix != null)) '' - substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'" - ''; - - CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}"; - LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}"; - LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}"; - - configureFlags = [ - "--enable-shared" - "--with-threads" - "--without-ensurepip" - "--with-system-expat" - "--with-system-ffi" - ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ - "ac_cv_buggy_getaddrinfo=no" - # Assume little-endian IEEE 754 floating point when cross compiling - "ac_cv_little_endian_double=yes" - "ac_cv_big_endian_double=no" - "ac_cv_mixed_endian_double=no" - "ac_cv_x87_double_rounding=yes" - "ac_cv_tanh_preserves_zero_sign=yes" - # Generally assume that things are present and work - "ac_cv_posix_semaphores_enabled=yes" - "ac_cv_broken_sem_getvalue=no" - "ac_cv_wchar_t_signed=yes" - "ac_cv_rshift_extends_sign=yes" - "ac_cv_broken_nice=no" - "ac_cv_broken_poll=no" - "ac_cv_working_tzset=yes" - "ac_cv_have_long_long_format=yes" - "ac_cv_have_size_t_format=yes" - "ac_cv_computed_gotos=yes" - "ac_cv_file__dev_ptmx=yes" - "ac_cv_file__dev_ptc=yes" - ] - # Never even try to use lchmod on linux, - # don't rely on detecting glibc-isms. - ++ optional stdenv.hostPlatform.isLinux "ac_cv_func_lchmod=no"; - - preConfigure = '' - for i in /usr /sw /opt /pkg; do # improve purity - substituteInPlace ./setup.py --replace $i /no-such-path - done - ${optionalString stdenv.isDarwin '' - export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2" - export MACOSX_DEPLOYMENT_TARGET=10.6 - '' - + optionalString stdenv.hostPlatform.isMusl '' - export NIX_CFLAGS_COMPILE+=" -DTHREAD_STACK_SIZE=0x100000" - ''} - ''; - - setupHook = python-setup-hook sitePackages; - - postInstall = '' - # needed for some packages, especially packages that backport functionality - # to 2.x from 3.x - for item in $out/lib/python${majorVersion}/test/*; do - if [[ "$item" != */test_support.py* - && "$item" != */test/support - && "$item" != */test/libregrtest - && "$item" != */test/regrtest.py* ]]; then - rm -rf "$item" - else - echo $item - fi - done - touch $out/lib/python${majorVersion}/test/__init__.py - - ln -s "$out/include/python${majorVersion}m" "$out/include/python${majorVersion}" - - # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484 - echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py - - # Determinism: Windows installers were not deterministic. - # We're also not interested in building Windows installers. - find "$out" -name 'wininst*.exe' | xargs -r rm -f - - # Use Python3 as default python - ln -s "$out/bin/idle3" "$out/bin/idle" - ln -s "$out/bin/pydoc3" "$out/bin/pydoc" - ln -s "$out/bin/python3" "$out/bin/python" - ln -s "$out/bin/python3-config" "$out/bin/python-config" - ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc" - - # Get rid of retained dependencies on -dev packages, and remove - # some $TMPDIR references to improve binary reproducibility. - # Note that the .pyc file of _sysconfigdata.py should be regenerated! - for i in $out/lib/python${majorVersion}/_sysconfigdata*.py $out/lib/python${majorVersion}/config-${majorVersion}m*/Makefile; do - sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g" - done - '' + optionalString (stdenv.hostPlatform == stdenv.buildPlatform) '' - # Determinism: rebuild all bytecode - # We exclude lib2to3 because that's Python 2 code which fails - # We rebuild three times, once for each optimization level - find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i - - find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i - - find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i - - ''; - - passthru = let - pythonPackages = callPackage ../../../../../top-level/python-packages.nix { - python = self; - overrides = packageOverrides; - }; - in rec { - inherit libPrefix sitePackages x11Support hasDistutilsCxxPatch; - executable = "${libPrefix}m"; - buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; }; - withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; - pkgs = pythonPackages; - isPy3 = true; - isPy36 = true; - is_py3k = true; # deprecated - interpreter = "${self}/bin/${executable}"; - }; - - enableParallelBuilding = true; - - doCheck = false; # expensive, and fails - - meta = { - homepage = http://python.org; - description = "A high-level dynamically-typed programming language"; - longDescription = '' - Python is a remarkably powerful dynamic programming language that - is used in a wide variety of application domains. Some of its key - distinguishing features include: clear, readable syntax; strong - introspection capabilities; intuitive object orientation; natural - expression of procedural code; full modularity, supporting - hierarchical packages; exception-based error handling; and very - high level dynamic data types. - ''; - license = licenses.psfl; - platforms = with platforms; linux ++ darwin; - maintainers = with maintainers; [ fridh kragniz ]; - }; -} diff --git a/pkgs/development/interpreters/python/cpython/3.7/default.nix b/pkgs/development/interpreters/python/cpython/default.nix similarity index 66% rename from pkgs/development/interpreters/python/cpython/3.7/default.nix rename to pkgs/development/interpreters/python/cpython/default.nix index 61ffa8ec5a4..0d1794f04fa 100644 --- a/pkgs/development/interpreters/python/cpython/3.7/default.nix +++ b/pkgs/development/interpreters/python/cpython/default.nix @@ -18,6 +18,9 @@ # For the Python package set , packageOverrides ? (self: super: {}) , buildPackages +, sourceVersion +, sha256 +, passthruFun }: assert x11Support -> tcl != null @@ -27,12 +30,24 @@ assert x11Support -> tcl != null with stdenv.lib; let - majorVersion = "3.7"; - minorVersion = "2"; - minorVersionSuffix = ""; - version = "${majorVersion}.${minorVersion}${minorVersionSuffix}"; - libPrefix = "python${majorVersion}"; - sitePackages = "lib/${libPrefix}/site-packages"; + + passthru = passthruFun rec { + inherit self sourceVersion packageOverrides; + implementation = "cpython"; + libPrefix = "python${pythonVersion}"; + executable = libPrefix; + pythonVersion = with sourceVersion; "${major}.${minor}"; + sitePackages = "lib/${libPrefix}/site-packages"; + inherit pythonForBuild; + }; + + version = with sourceVersion; "${major}.${minor}.${patch}${suffix}"; + + nativeBuildInputs = [ + nukeReferences + ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + buildPackages.stdenv.cc crossPython + ]; buildInputs = filter (p: p != null) [ zlib bzip2 expat lzma libffi gdbm sqlite readline ncurses openssl ] @@ -40,44 +55,46 @@ let ++ optionals stdenv.isDarwin [ CF configd ]; hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false); + + crossPython = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"}; + pythonForBuild = if stdenv.hostPlatform == stdenv.buildPlatform then "$out/bin/python" - else - buildPackages.python37.interpreter; -in stdenv.mkDerivation { - name = "python3-${version}"; - pythonVersion = majorVersion; - inherit majorVersion version; + else crossPython.interpreter; - inherit buildInputs; +in with passthru; stdenv.mkDerivation { + pname = "python3"; + inherit version; - nativeBuildInputs = [ nukeReferences ] ++ - optionals (stdenv.hostPlatform != stdenv.buildPlatform) - [ buildPackages.stdenv.cc buildPackages.python37 ]; + inherit buildInputs nativeBuildInputs; src = fetchurl { - url = "https://www.python.org/ftp/python/${majorVersion}.${minorVersion}/Python-${version}.tar.xz"; - sha256 = "1fzi9d2gibh0wzwidyckzbywsxcsbckgsl05ryxlifxia77fhgyq"; + url = with sourceVersion; "https://www.python.org/ftp/python/${major}.${minor}.${patch}/Python-${version}.tar.xz"; + inherit sha256; }; - NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s"; - - # Determinism: We fix the hashes of str, bytes and datetime objects. - PYTHONHASHSEED=0; - prePatch = optionalString stdenv.isDarwin '' substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"' substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' ' ''; patches = [ - ./no-ldconfig.patch + # Disable the use of ldconfig in ctypes.util.find_library (since + # ldconfig doesn't work on NixOS), and don't use + # ctypes.util.find_library during the loading of the uuid module + # (since it will do a futile invocation of gcc (!) to find + # libuuid, slowing down program startup a lot). + (./. + "/${sourceVersion.major}.${sourceVersion.minor}/no-ldconfig.patch") + ] ++ optionals isPy35 [ + # Backports support for LD_LIBRARY_PATH from 3.6 + ./3.5/ld_library_path.patch + ] ++ optionals isPy37 [ # Fix darwin build https://bugs.python.org/issue34027 (fetchpatch { url = https://bugs.python.org/file47666/darwin-libutil.patch; sha256 = "0242gihnw3wfskl4fydp2xanpl8k5q7fj4dp7dbbqf46a4iwdzpa"; }) - ] ++ optionals hasDistutilsCxxPatch [ + ] ++ optionals (isPy3k && hasDistutilsCxxPatch) [ # Fix for http://bugs.python.org/issue1222585 # Upstream distutils is calling C compiler to compile C++ code, which # only works for GCC and Apple Clang. This makes distutils to call C++ @@ -93,9 +110,12 @@ in stdenv.mkDerivation { substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'" ''; - CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}"; - LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}"; - LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}"; + CPPFLAGS = "${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}"; + LDFLAGS = "${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}"; + LIBS = "${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}"; + NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s"; + # Determinism: We fix the hashes of str, bytes and datetime objects. + PYTHONHASHSEED=0; configureFlags = [ "--enable-shared" @@ -125,16 +145,25 @@ in stdenv.mkDerivation { "ac_cv_computed_gotos=yes" "ac_cv_file__dev_ptmx=yes" "ac_cv_file__dev_ptc=yes" + ] ++ optionals stdenv.hostPlatform.isLinux [ + # Never even try to use lchmod on linux, + # don't rely on detecting glibc-isms. + "ac_cv_func_lchmod=no" ]; preConfigure = '' for i in /usr /sw /opt /pkg; do # improve purity substituteInPlace ./setup.py --replace $i /no-such-path done - ${optionalString stdenv.isDarwin '' - export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2" - export MACOSX_DEPLOYMENT_TARGET=10.6 - ''} + '' + optionalString stdenv.isDarwin '' + export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2" + export MACOSX_DEPLOYMENT_TARGET=10.6 + '' + optionalString (isPy3k && pythonOlder "3.7") '' + # Determinism: The interpreter is patched to write null timestamps when compiling python files. + # This way python does not try to update them when we freeze timestamps in nix store. + export DETERMINISTIC_BUILD=1; + '' + optionalString stdenv.hostPlatform.isMusl '' + export NIX_CFLAGS_COMPILE+=" -DTHREAD_STACK_SIZE=0x100000" ''; setupHook = python-setup-hook sitePackages; @@ -142,7 +171,7 @@ in stdenv.mkDerivation { postInstall = '' # needed for some packages, especially packages that backport functionality # to 2.x from 3.x - for item in $out/lib/python${majorVersion}/test/*; do + for item in $out/lib/${libPrefix}/test/*; do if [[ "$item" != */test_support.py* && "$item" != */test/support && "$item" != */test/libregrtest @@ -152,9 +181,9 @@ in stdenv.mkDerivation { echo $item fi done - touch $out/lib/python${majorVersion}/test/__init__.py + touch $out/lib/${libPrefix}/test/__init__.py - ln -s "$out/include/python${majorVersion}m" "$out/include/python${majorVersion}" + ln -s "$out/include/${executable}m" "$out/include/${executable}" # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484 echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py @@ -173,14 +202,13 @@ in stdenv.mkDerivation { # Get rid of retained dependencies on -dev packages, and remove # some $TMPDIR references to improve binary reproducibility. # Note that the .pyc file of _sysconfigdata.py should be regenerated! - for i in $out/lib/python${majorVersion}/_sysconfigdata*.py $out/lib/python${majorVersion}/config-${majorVersion}m*/Makefile; do - sed -i $i -e "s|$TMPDIR|/no-such-path|g" - nuke-refs $i + for i in $out/lib/${libPrefix}/_sysconfigdata*.py $out/lib/${libPrefix}/config-${sourceVersion.major}${sourceVersion.minor}*/Makefile; do + sed -i $i -e "s|$TMPDIR|/no-such-path|g" done # Further get rid of references. https://github.com/NixOS/nixpkgs/issues/51668 - find $out/lib/python*/config-*-* -type f -print -exec nuke-refs '{}' + - find $out/lib -name '_sysconfigdata_m*.py*' -print -exec nuke-refs '{}' + + find $out/lib/python*/config-* -type f -print -exec nuke-refs '{}' + + find $out/lib -name '_sysconfigdata*.py*' -print -exec nuke-refs '{}' + # Determinism: rebuild all bytecode # We exclude lib2to3 because that's Python 2 code which fails @@ -196,22 +224,7 @@ in stdenv.mkDerivation { # explicitly specify in our configure flags above. disallowedReferences = [ openssl.dev ]; - passthru = let - pythonPackages = callPackage ../../../../../top-level/python-packages.nix { - python = self; - overrides = packageOverrides; - }; - in rec { - inherit libPrefix sitePackages x11Support hasDistutilsCxxPatch; - executable = "${libPrefix}m"; - buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; }; - withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;}; - pkgs = pythonPackages; - isPy3 = true; - isPy37 = true; - is_py3k = true; # deprecated - interpreter = "${self}/bin/${executable}"; - }; + inherit passthru; enableParallelBuilding = true; @@ -229,6 +242,6 @@ in stdenv.mkDerivation { ''; license = licenses.psfl; platforms = with platforms; linux ++ darwin; - maintainers = with maintainers; [ fridh kragniz ]; + maintainers = with maintainers; [ fridh ]; }; } diff --git a/pkgs/development/interpreters/python/default.nix b/pkgs/development/interpreters/python/default.nix new file mode 100644 index 00000000000..75226ef9a73 --- /dev/null +++ b/pkgs/development/interpreters/python/default.nix @@ -0,0 +1,106 @@ +{ pkgs, lib }: + +with pkgs; + +(let + + # Common passthru for all Python interpreters. + passthruFun = + { implementation + , libPrefix + , executable + , sourceVersion + , pythonVersion + , packageOverrides + , sitePackages + , pythonForBuild + , self + }: let + pythonPackages = callPackage ../../../top-level/python-packages.nix { + python = self; + overrides = packageOverrides; + }; + in rec { + sitePackages = "lib/${libPrefix}/site-packages"; + isPy27 = pythonVersion == "2.7"; + isPy33 = pythonVersion == "3.3"; # TODO: remove + isPy34 = pythonVersion == "3.4"; # TODO: remove + isPy35 = pythonVersion == "3.5"; + isPy36 = pythonVersion == "3.6"; + isPy37 = pythonVersion == "3.7"; + isPy3k = lib.strings.substring 0 1 pythonVersion == "3"; + isPyPy = interpreter == "pypy"; + + buildEnv = callPackage ./wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; }; + withPackages = import ./with-packages.nix { inherit buildEnv pythonPackages;}; + pkgs = pythonPackages; + interpreter = "${self}/bin/${executable}"; + inherit executable implementation libPrefix pythonVersion; + inherit sourceVersion; + pythonAtLeast = lib.versionAtLeast pythonVersion; + pythonOlder = lib.versionOlder pythonVersion; + inherit pythonForBuild; + + majorVersion = pythonVersion; # Backwards compatibility. Should be dropped. + }; + +in { + + python27 = callPackage ./cpython/2.7 { + self = python27; + sourceVersion = { + major = "2"; + minor = "7"; + patch = "15"; + suffix = ""; + }; + sha256 = "0x2mvz9dp11wj7p5ccvmk9s0hzjk2fa1m462p395l4r6bfnb3n92"; + inherit (darwin) CF configd; + inherit passthruFun; + }; + + python35 = callPackage ./cpython { + self = python35; + sourceVersion = { + major = "3"; + minor = "5"; + patch = "6"; + suffix = ""; + }; + sha256 = "0pqmf51zy2lzhbaj4yya2py2qr653j9152d0rg3p7wi1yl2dwp7m"; + inherit (darwin) CF configd; + inherit passthruFun; + }; + + python36 = callPackage ./cpython { + self = python36; + sourceVersion = { + major = "3"; + minor = "6"; + patch = "8"; + suffix = ""; + }; + sha256 = "14qi6n5gpcjnwy165wi9hkfcmbadc95ny6bxxldknxwmx50n4i1m"; + inherit (darwin) CF configd; + inherit passthruFun; + }; + + python37 = callPackage ./cpython { + self = python37; + sourceVersion = { + major = "3"; + minor = "7"; + patch = "2"; + suffix = ""; + }; + sha256 = "1fzi9d2gibh0wzwidyckzbywsxcsbckgsl05ryxlifxia77fhgyq"; + inherit (darwin) CF configd; + inherit passthruFun; + }; + +# pypy27 = callPackage ./pypy { +# self = pypy27; +# implementation = "pypy"; +# }; + +}) \ No newline at end of file diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 3e7a991621c..34507b67b57 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -7953,22 +7953,8 @@ in python2Packages = python2.pkgs; python3Packages = python3.pkgs; - python27 = callPackage ../development/interpreters/python/cpython/2.7 { - self = python27; - inherit (darwin) CF configd; - }; - python35 = callPackage ../development/interpreters/python/cpython/3.5 { - inherit (darwin) CF configd; - self = python35; - }; - python36 = callPackage ../development/interpreters/python/cpython/3.6 { - inherit (darwin) CF configd; - self = python36; - }; - python37 = callPackage ../development/interpreters/python/cpython/3.7 { - inherit (darwin) CF configd; - self = python37; - }; + pythonInterpreters = callPackage ./../development/interpreters/python {}; + inherit (pythonInterpreters) python27 python35 python36 python37; pypy27 = callPackage ../development/interpreters/python/pypy/2.7 { self = pypy27; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 8403b398ccd..e0e45ea9ecd 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -18,16 +18,7 @@ let packages = ( self: let - pythonAtLeast = versionAtLeast python.pythonVersion; - pythonOlder = versionOlder python.pythonVersion; - isPy27 = python.pythonVersion == "2.7"; - isPy33 = python.pythonVersion == "3.3"; - isPy34 = python.pythonVersion == "3.4"; - isPy35 = python.pythonVersion == "3.5"; - isPy36 = python.pythonVersion == "3.6"; - isPy37 = python.pythonVersion == "3.7"; - isPyPy = strings.substring 0 4 python.executable == "pypy"; - isPy3k = strings.substring 0 1 python.pythonVersion == "3"; + inherit (python.passthru) isPy27 isPy33 isPy34 isPy35 isPy36 isPy37 isPy3k isPyPy pythonAtLeast pythonOlder; callPackage = pkgs.newScope self; @@ -130,7 +121,8 @@ let in { - inherit python bootstrapped-pip pythonAtLeast pythonOlder isPy27 isPy33 isPy34 isPy35 isPy36 isPy37 isPyPy isPy3k buildPythonPackage buildPythonApplication; + inherit (python.passthru) isPy27 isPy33 isPy34 isPy35 isPy36 isPy37 isPy3k isPyPy pythonAtLeast pythonOlder; + inherit python bootstrapped-pip buildPythonPackage buildPythonApplication; inherit fetchPypi callPackage; inherit hasPythonModule requiredPythonModules makePythonPath disabledIf; inherit toPythonModule toPythonApplication;