python: reproducible builds

Achieve reproducible builds of the interpreter. Note this meant
disabling optimizations again.

doc/languages-frameworks/python.section.md

@@ -788,6 +788,23 @@ Each interpreter has the following attributes:
 - `executable`. Name of the interpreter executable, e.g. `python3.8`.
 - `pkgs`. Set of Python packages for that specific interpreter. The package set can be modified by overriding the interpreter and passing `packageOverrides`.
 
+### Optimizations
+
+The Python interpreters are by default not build with optimizations enabled, because
+the builds are in that case not reproducible. To enable optimizations, override the
+interpreter of interest, e.g using
+
+```
+let
+  pkgs = import ./. {};
+  mypython = pkgs.python3.override {
+    enableOptimizations = true;
+    reproducibleBuild = false;
+    self = mypython;
+  };
+in mypython
+```
+
 ### Building packages and applications
 
 Python libraries and applications that use `setuptools` or

nixos/doc/manual/release-notes/rl-2105.xml

@@ -57,6 +57,12 @@
      for the motivation).
     </para>
    </listitem>
+   <listitem>
+    </para>
+    Python optimizations were disabled again. Builds with optimizations enabled
+    are not reproducible. Optimizations can now be enabled with an option.
+    <para>
+   </listitem>
   </itemizedlist>
  </section>
 

pkgs/development/interpreters/python/cpython/2.7/default.nix

@@ -27,7 +27,10 @@
 , sha256
 , passthruFun
 , static ? false
-, enableOptimizations ? (!stdenv.isDarwin)
+, stripBytecode ? reproducibleBuild
+, rebuildBytecode ? true
+, reproducibleBuild ? true
+, enableOptimizations ? false
 , pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
 }:
 
@@ -36,12 +39,26 @@ assert x11Support -> tcl != null
                   && xlibsWrapper != null
                   && libX11 != null;
 
+assert lib.assertMsg (enableOptimizations -> (!stdenv.cc.isClang))
+  "Optimizations with clang are not supported. configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.";
+
+assert lib.assertMsg (reproducibleBuild -> stripBytecode)
+  "Deterministic builds require stripping bytecode.";
+
+assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
+  "Deterministic builds are not achieved when optimizations are enabled.";
+
+
 with lib;
 
 let
   buildPackages = pkgsBuildHost;
   inherit (passthru) pythonForBuild;
 
+  pythonForBuildInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then
+    "$out/bin/python"
+  else pythonForBuild.interpreter;
+
   passthru = passthruFun rec {
     inherit self sourceVersion packageOverrides;
     implementation = "cpython";
@@ -272,14 +289,15 @@ in with passthru; stdenv.mkDerivation ({
         # Determinism: Windows installers were not deterministic.
         # We're also not interested in building Windows installers.
         find "$out" -name 'wininst*.exe' | xargs -r rm -f
-      '' + optionalString (stdenv.hostPlatform == stdenv.buildPlatform)
+      '' + optionalString stripBytecode ''
-      ''
+        # Determinism: deterministic bytecode
-        # Determinism: rebuild all bytecode
+        # First we delete all old bytecode.
-        # We exclude lib2to3 because that's Python 2 code which fails
+        find $out -name "*.pyc" -delete
-        # We rebuild three times, once for each optimization level
+        '' + optionalString rebuildBytecode ''
-        find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
+        # Then, we build for the two optimization levels.
-        find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
+        # We do not build unoptimized bytecode, because its not entirely deterministic yet.
-        find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
+        find $out -name "*.py" | ${pythonForBuildInterpreter} -O  -m compileall -q -f -x "lib2to3" -i -
+        find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
       '' + optionalString stdenv.hostPlatform.isCygwin ''
         cp libpython2.7.dll.a $out/lib
       '';

pkgs/development/interpreters/python/cpython/default.nix

@@ -33,12 +33,11 @@
 , stripTests ? false
 , stripTkinter ? false
 , rebuildBytecode ? true
-, stripBytecode ? false
+, stripBytecode ? reproducibleBuild
 , includeSiteCustomize ? true
 , static ? stdenv.hostPlatform.isStatic
-# Not using optimizations on Darwin
+, enableOptimizations ? false
-# configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.
+, reproducibleBuild ? true
-, enableOptimizations ? (!stdenv.isDarwin)
 , pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
 }:
 
@@ -54,6 +53,15 @@ assert x11Support -> tcl != null
 
 assert bluezSupport -> bluez != null;
 
+assert lib.assertMsg (enableOptimizations -> (!stdenv.cc.isClang))
+  "Optimizations with clang are not supported. configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.";
+
+assert lib.assertMsg (reproducibleBuild -> stripBytecode)
+  "Deterministic builds require stripping bytecode.";
+
+assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
+  "Deterministic builds are not achieved when optimizations are enabled.";
+
 with lib;
 
 let
@@ -360,18 +368,19 @@ in with passthru; stdenv.mkDerivation {
     '' + optionalString includeSiteCustomize ''
     # Include a sitecustomize.py file
     cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
-    '' + optionalString rebuildBytecode ''
 
-    # Determinism: rebuild all bytecode
+    '' + optionalString stripBytecode ''
-    # We exclude lib2to3 because that's Python 2 code which fails
+    # Determinism: deterministic bytecode
-    # We rebuild three times, once for each optimization level
+    # First we delete all old bytecode.
+    find $out -type d -name __pycache__ -print0 | xargs -0 -I {} rm -rf "{}"
+    '' + optionalString rebuildBytecode ''
+    # Then, we build for the two optimization levels.
+    # We do not build unoptimized bytecode, because its not entirely deterministic yet.
     # Python 3.7 implements PEP 552, introducing support for deterministic bytecode.
-    # This is automatically used when `SOURCE_DATE_EPOCH` is set.
+    # compileall uses this checked-hash method by default when `SOURCE_DATE_EPOCH` is set.
-    find $out -name "*.py" | ${pythonForBuildInterpreter}     -m compileall -q -f -x "lib2to3" -i -
+    # We exclude lib2to3 because that's Python 2 code which fails
     find $out -name "*.py" | ${pythonForBuildInterpreter} -O  -m compileall -q -f -x "lib2to3" -i -
     find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
-    '' + optionalString stripBytecode ''
-    find $out -type d -name __pycache__ -print0 | xargs -0 -I {} rm -rf "{}"
   '';
 
   preFixup = lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''