public
/
nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

python2 and python3: build unoptimized bytecode again 

In 9d03ff5222 I made the CPython builds
reproducible. This required not generating default unoptimized bytecode.
I was under the impression the optimized bytecode would be used then,
but you need to opt-in on that. Not having the default bytecode resulted
in a significant performance hit. Therefore, bytecode is generated again
in this commit, and thereby the builds are no longer reproducible.

https://bugs.python.org/issue29708
(cherry picked from commit 23e348bfe2b161e761dd37e3be86a807f3cd6f94)
This commit is contained in:
Frederik Rietdijk 2021-04-27 11:42:10 +02:00
parent 1ea742607e
commit c32a9d4158
2 changed files with 19 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pkgs/development/interpreters/python/cpython/2.7/default.nix
View File

@ -27,9 +27,9 @@
, sha256
, passthruFun
, static ? false
, stripBytecode ? reproducibleBuild
, stripBytecode ? true
, rebuildBytecode ? true
, reproducibleBuild ? true
, reproducibleBuild ? false
, enableOptimizations ? false
, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
}:
@ -48,6 +48,8 @@ assert lib.assertMsg (reproducibleBuild -> stripBytecode)
assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
"Deterministic builds are not achieved when optimizations are enabled.";
assert lib.assertMsg (reproducibleBuild -> (!rebuildBytecode))
"Deterministic builds are not achieved when (default unoptimized) bytecode is created.";
with lib;
@ -296,8 +298,10 @@ in with passthru; stdenv.mkDerivation ({
# First we delete all old bytecode.
find $out -name "*.pyc" -delete
'' + optionalString rebuildBytecode ''
# Then, we build for the two optimization levels.
# We do not build unoptimized bytecode, because its not entirely deterministic yet.
# We build 3 levels of optimized bytecode. Note the default level, without optimizations,
# is not reproducible yet. https://bugs.python.org/issue29708
# Not creating bytecode will result in a large performance loss however, so we do build it.
find $out -name "*.py" | ${pythonForBuildInterpreter} -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
'' + optionalString stdenv.hostPlatform.isCygwin ''

16
pkgs/development/interpreters/python/cpython/default.nix
View File

@ -35,11 +35,11 @@
, stripTests ? false
, stripTkinter ? false
, rebuildBytecode ? true
, stripBytecode ? reproducibleBuild
, stripBytecode ? true
, includeSiteCustomize ? true
, static ? stdenv.hostPlatform.isStatic
, enableOptimizations ? false
, reproducibleBuild ? true
, reproducibleBuild ? false
, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
}:
@ -66,6 +66,9 @@ assert lib.assertMsg (reproducibleBuild -> stripBytecode)
assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
"Deterministic builds are not achieved when optimizations are enabled.";
assert lib.assertMsg (reproducibleBuild -> (!rebuildBytecode))
"Deterministic builds are not achieved when (default unoptimized) bytecode is created.";
with lib;
let
@ -396,11 +399,14 @@ in with passthru; stdenv.mkDerivation {
# First we delete all old bytecode.
find $out -type d -name __pycache__ -print0 | xargs -0 -I {} rm -rf "{}"
'' + optionalString rebuildBytecode ''
# Then, we build for the two optimization levels.
# We do not build unoptimized bytecode, because its not entirely deterministic yet.
# Python 3.7 implements PEP 552, introducing support for deterministic bytecode.
# compileall uses this checked-hash method by default when `SOURCE_DATE_EPOCH` is set.
# compileall uses the therein introduced checked-hash method by default when
# `SOURCE_DATE_EPOCH` is set.
# We exclude lib2to3 because that's Python 2 code which fails
# We build 3 levels of optimized bytecode. Note the default level, without optimizations,
# is not reproducible yet. https://bugs.python.org/issue29708
# Not creating bytecode will result in a large performance loss however, so we do build it.
find $out -name "*.py" | ${pythonForBuildInterpreter} -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
'';