From acd32a4caf533ca6202afad07c085450649dbe4a Mon Sep 17 00:00:00 2001
From: Frederik Rietdijk <fridh@fridh.nl>
Date: Fri, 19 May 2017 14:21:58 +0200
Subject: [PATCH] Python: set DETERMINISTIC_BUILD and PYTHONHASHSEED in
setupHook
The Python interpreters are patched so they can build .pyc bytecode free
of certain indeterminism.
When building Python packages we currently set
```
compiling python files.
in nix store.
DETERMINISTIC_BUILD=1;
PYTHONHASHSEED = 0;
```
Instead if setting these environment variables in the function that
builds the package, this commit sets the variables instead in the Python
setup hook. That way, whenever Python is included in a derivation, these
variables are set.
See also the issue https://github.com/NixOS/nixpkgs/issues/25707.
---
.../python/cpython/2.7/default.nix | 3 ++-
.../python/cpython/2.7/setup-hook.sh | 15 -------------
.../python/cpython/3.3/default.nix | 2 +-
.../python/cpython/3.3/setup-hook.sh | 15 -------------
.../python/cpython/3.4/default.nix | 3 ++-
.../python/cpython/3.4/setup-hook.sh | 15 -------------
.../python/cpython/3.5/default.nix | 3 ++-
.../python/cpython/3.5/setup-hook.sh | 15 -------------
.../python/cpython/3.6/default.nix | 3 ++-
.../python/cpython/3.6/setup-hook.sh | 15 -------------
.../python/mk-python-derivation.nix | 7 -------
.../interpreters/python/pypy/2.7/default.nix | 7 ++++---
.../python/pypy/2.7/setup-hook.sh | 15 -------------
.../interpreters/python/setup-hook.nix | 13 ++++++++++++
.../interpreters/python/setup-hook.sh | 21 +++++++++++++++++++
pkgs/top-level/all-packages.nix | 3 +++
16 files changed, 50 insertions(+), 105 deletions(-)
delete mode 100644 pkgs/development/interpreters/python/cpython/2.7/setup-hook.sh
delete mode 100644 pkgs/development/interpreters/python/cpython/3.3/setup-hook.sh
delete mode 100644 pkgs/development/interpreters/python/cpython/3.4/setup-hook.sh
delete mode 100644 pkgs/development/interpreters/python/cpython/3.5/setup-hook.sh
delete mode 100644 pkgs/development/interpreters/python/cpython/3.6/setup-hook.sh
delete mode 100644 pkgs/development/interpreters/python/pypy/2.7/setup-hook.sh
create mode 100644 pkgs/development/interpreters/python/setup-hook.nix
create mode 100644 pkgs/development/interpreters/python/setup-hook.sh
diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix
index 8426902414a..c5fa05651bb 100644
--- a/pkgs/development/interpreters/python/cpython/2.7/default.nix
+++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix
@@ -15,6 +15,7 @@
, expat
, libffi
, CF, configd, coreutils
+, python-setup-hook
# For the Python package set
, pkgs, packageOverrides ? (self: super: {})
}:
@@ -150,7 +151,7 @@ in stdenv.mkDerivation {
NIX_CFLAGS_COMPILE = optionalString stdenv.isDarwin "-msse2";
DETERMINISTIC_BUILD = 1;
- setupHook = ./setup-hook.sh;
+ setupHook = python-setup-hook sitePackages;
postPatch = optionalString (x11Support && (tix != null)) ''
substituteInPlace "Lib/lib-tk/Tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
diff --git a/pkgs/development/interpreters/python/cpython/2.7/setup-hook.sh b/pkgs/development/interpreters/python/cpython/2.7/setup-hook.sh
deleted file mode 100644
index 4770eea886f..00000000000
--- a/pkgs/development/interpreters/python/cpython/2.7/setup-hook.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-addPythonPath() {
- addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python2.7/site-packages
-}
-
-toPythonPath() {
- local paths="$1"
- local result=
- for i in $paths; do
- p="$i/lib/python2.7/site-packages"
- result="${result}${result:+:}$p"
- done
- echo $result
-}
-
-envHooks+=(addPythonPath)
diff --git a/pkgs/development/interpreters/python/cpython/3.3/default.nix b/pkgs/development/interpreters/python/cpython/3.3/default.nix
index c561a1ed750..061176335c4 100644
--- a/pkgs/development/interpreters/python/cpython/3.3/default.nix
+++ b/pkgs/development/interpreters/python/cpython/3.3/default.nix
@@ -77,7 +77,7 @@ in stdenv.mkDerivation {
)
'';
- setupHook = ./setup-hook.sh;
+ setupHook = python-setup-hook sitePackages;
postInstall = ''
# needed for some packages, especially packages that backport functionality
diff --git a/pkgs/development/interpreters/python/cpython/3.3/setup-hook.sh b/pkgs/development/interpreters/python/cpython/3.3/setup-hook.sh
deleted file mode 100644
index 82a8c0abd32..00000000000
--- a/pkgs/development/interpreters/python/cpython/3.3/setup-hook.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-addPythonPath() {
- addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.3/site-packages
-}
-
-toPythonPath() {
- local paths="$1"
- local result=
- for i in $paths; do
- p="$i/lib/python3.3/site-packages"
- result="${result}${result:+:}$p"
- done
- echo $result
-}
-
-envHooks+=(addPythonPath)
diff --git a/pkgs/development/interpreters/python/cpython/3.4/default.nix b/pkgs/development/interpreters/python/cpython/3.4/default.nix
index b2a4d849c94..4d20a21a4d1 100644
--- a/pkgs/development/interpreters/python/cpython/3.4/default.nix
+++ b/pkgs/development/interpreters/python/cpython/3.4/default.nix
@@ -13,6 +13,7 @@
, callPackage
, self
, CF, configd
+, python-setup-hook
# For the Python package set
, pkgs, packageOverrides ? (self: super: {})
}:
@@ -100,7 +101,7 @@ in stdenv.mkDerivation {
''}
'';
- setupHook = ./setup-hook.sh;
+ setupHook = python-setup-hook sitePackages;
postInstall = ''
# needed for some packages, especially packages that backport functionality
diff --git a/pkgs/development/interpreters/python/cpython/3.4/setup-hook.sh b/pkgs/development/interpreters/python/cpython/3.4/setup-hook.sh
deleted file mode 100644
index fddcc0b73fe..00000000000
--- a/pkgs/development/interpreters/python/cpython/3.4/setup-hook.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-addPythonPath() {
- addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.4/site-packages
-}
-
-toPythonPath() {
- local paths="$1"
- local result=
- for i in $paths; do
- p="$i/lib/python3.4/site-packages"
- result="${result}${result:+:}$p"
- done
- echo $result
-}
-
-envHooks+=(addPythonPath)
diff --git a/pkgs/development/interpreters/python/cpython/3.5/default.nix b/pkgs/development/interpreters/python/cpython/3.5/default.nix
index 76f445f7a50..a4aec241a1a 100644
--- a/pkgs/development/interpreters/python/cpython/3.5/default.nix
+++ b/pkgs/development/interpreters/python/cpython/3.5/default.nix
@@ -13,6 +13,7 @@
, callPackage
, self
, CF, configd
+, python-setup-hook
# For the Python package set
, pkgs, packageOverrides ? (self: super: {})
}:
@@ -102,7 +103,7 @@ in stdenv.mkDerivation {
''}
'';
- setupHook = ./setup-hook.sh;
+ setupHook = python-setup-hook sitePackages;
postInstall = ''
# needed for some packages, especially packages that backport functionality
diff --git a/pkgs/development/interpreters/python/cpython/3.5/setup-hook.sh b/pkgs/development/interpreters/python/cpython/3.5/setup-hook.sh
deleted file mode 100644
index 2836ad7e8f5..00000000000
--- a/pkgs/development/interpreters/python/cpython/3.5/setup-hook.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-addPythonPath() {
- addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.5/site-packages
-}
-
-toPythonPath() {
- local paths="$1"
- local result=
- for i in $paths; do
- p="$i/lib/python3.5/site-packages"
- result="${result}${result:+:}$p"
- done
- echo $result
-}
-
-envHooks+=(addPythonPath)
diff --git a/pkgs/development/interpreters/python/cpython/3.6/default.nix b/pkgs/development/interpreters/python/cpython/3.6/default.nix
index 15078619166..d2d922ce495 100644
--- a/pkgs/development/interpreters/python/cpython/3.6/default.nix
+++ b/pkgs/development/interpreters/python/cpython/3.6/default.nix
@@ -14,6 +14,7 @@
, callPackage
, self
, CF, configd
+, python-setup-hook
# For the Python package set
, pkgs, packageOverrides ? (self: super: {})
}:
@@ -94,7 +95,7 @@ in stdenv.mkDerivation {
''}
'';
- setupHook = ./setup-hook.sh;
+ setupHook = python-setup-hook sitePackages;
postInstall = ''
# needed for some packages, especially packages that backport functionality
diff --git a/pkgs/development/interpreters/python/cpython/3.6/setup-hook.sh b/pkgs/development/interpreters/python/cpython/3.6/setup-hook.sh
deleted file mode 100644
index 26a0d57bc87..00000000000
--- a/pkgs/development/interpreters/python/cpython/3.6/setup-hook.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-addPythonPath() {
- addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.6/site-packages
-}
-
-toPythonPath() {
- local paths="$1"
- local result=
- for i in $paths; do
- p="$i/lib/python3.6/site-packages"
- result="${result}${result:+:}$p"
- done
- echo $result
-}
-
-envHooks+=(addPythonPath)
diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix
index 5d710fcad88..1a388bfe4af 100644
--- a/pkgs/development/interpreters/python/mk-python-derivation.nix
+++ b/pkgs/development/interpreters/python/mk-python-derivation.nix
@@ -57,13 +57,6 @@ python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled"] // {
inherit pythonPath;
-
- # Determinism: The interpreter is patched to write null timestamps when compiling python files.
- # This way python doesn't try to update them when we freeze timestamps in nix store.
- DETERMINISTIC_BUILD=1;
- # Determinism: We fix the hashes of str, bytes and datetime objects.
- PYTHONHASHSEED = 0;
-
buildInputs = [ wrapPython ] ++ buildInputs ++ pythonPath
++ [ (ensureNewerSourcesHook { year = "1980"; }) ]
++ (lib.optional (lib.hasSuffix "zip" attrs.src.name or "") unzip)
diff --git a/pkgs/development/interpreters/python/pypy/2.7/default.nix b/pkgs/development/interpreters/python/pypy/2.7/default.nix
index 456a078874c..7552c6cd285 100644
--- a/pkgs/development/interpreters/python/pypy/2.7/default.nix
+++ b/pkgs/development/interpreters/python/pypy/2.7/default.nix
@@ -1,6 +1,7 @@
{ stdenv, fetchurl, zlib ? null, zlibSupport ? true, bzip2, pkgconfig, libffi
, sqlite, openssl, ncurses, python, expat, tcl, tk, tix, xlibsWrapper, libX11
, makeWrapper, callPackage, self, gdbm, db
+, python-setup-hook
# For the Python package set
, pkgs, packageOverrides ? (self: super: {})
}:
@@ -14,6 +15,7 @@ let
pythonVersion = "2.7";
version = "${majorVersion}.${minorVersion}${minorVersionSuffix}";
libPrefix = "pypy${majorVersion}";
+ sitePackages = "site-packages";
in stdenv.mkDerivation rec {
name = "pypy-${version}";
@@ -67,7 +69,7 @@ in stdenv.mkDerivation rec {
${python.interpreter} rpython/bin/rpython --make-jobs="$NIX_BUILD_CORES" -Ojit --batch pypy/goal/targetpypystandalone.py --withmod-_minimal_curses --withmod-unicodedata --withmod-thread --withmod-bz2 --withmod-_multiprocessing
'';
- setupHook = ./setup-hook.sh;
+ setupHook = python-setup-hook sitePackages;
postBuild = ''
cd ./lib_pypy
@@ -125,12 +127,11 @@ in stdenv.mkDerivation rec {
passthru = let
pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;};
in rec {
- inherit zlibSupport libPrefix;
+ inherit zlibSupport libPrefix sitePackages;
executable = "pypy";
isPypy = true;
buildEnv = callPackage ../../wrapper.nix { python = self; };
interpreter = "${self}/bin/${executable}";
- sitePackages = "site-packages";
withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
pkgs = pythonPackages;
};
diff --git a/pkgs/development/interpreters/python/pypy/2.7/setup-hook.sh b/pkgs/development/interpreters/python/pypy/2.7/setup-hook.sh
deleted file mode 100644
index e9081d1eaa5..00000000000
--- a/pkgs/development/interpreters/python/pypy/2.7/setup-hook.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-addPythonPath() {
- addToSearchPathWithCustomDelimiter : PYTHONPATH $1/site-packages
-}
-
-toPythonPath() {
- local paths="$1"
- local result=
- for i in $paths; do
- p="$i/site-packages"
- result="${result}${result:+:}$p"
- done
- echo $result
-}
-
-envHooks+=(addPythonPath)
diff --git a/pkgs/development/interpreters/python/setup-hook.nix b/pkgs/development/interpreters/python/setup-hook.nix
new file mode 100644
index 00000000000..b66bd1cc5f6
--- /dev/null
+++ b/pkgs/development/interpreters/python/setup-hook.nix
@@ -0,0 +1,13 @@
+{ runCommand }:
+
+sitePackages:
+
+let
+ hook = ./setup-hook.sh;
+in runCommand "python-setup-hook.sh" {
+ inherit sitePackages;
+} ''
+ cp ${hook} hook.sh
+ substituteAllInPlace hook.sh
+ mv hook.sh $out
+''
diff --git a/pkgs/development/interpreters/python/setup-hook.sh b/pkgs/development/interpreters/python/setup-hook.sh
new file mode 100644
index 00000000000..dda9bed39f8
--- /dev/null
+++ b/pkgs/development/interpreters/python/setup-hook.sh
@@ -0,0 +1,21 @@
+addPythonPath() {
+ addToSearchPathWithCustomDelimiter : PYTHONPATH $1/@sitePackages@
+}
+
+toPythonPath() {
+ local paths="$1"
+ local result=
+ for i in $paths; do
+ p="$i/@sitePackages@"
+ result="${result}${result:+:}$p"
+ done
+ echo $result
+}
+
+envHooks+=(addPythonPath)
+
+# Determinism: The interpreter is patched to write null timestamps when compiling python files.
+# This way python doesn't try to update them when we freeze timestamps in nix store.
+export DETERMINISTIC_BUILD=1;
+# Determinism: We fix the hashes of str, bytes and datetime objects.
+export PYTHONHASHSEED=0;
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index ea47d41fc34..ae7e30d0ff8 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -6162,6 +6162,9 @@ with pkgs;
self = python36;
};
+ # Should eventually be moved inside Python interpreters.
+ python-setup-hook = callPackage ../development/interpreters/python/setup-hook.nix { };
+
pypy = pypy27;
pypy27 = callPackage ../development/interpreters/python/pypy/2.7 {