Merge pull request #90208 from FRidh/bytecode

buildPythonPackage: recompile bytecode for reproducibility

pkgs/development/interpreters/python/hooks/default.nix

@@ -1,5 +1,6 @@
 # Hooks for building Python packages.
 { python
+, lib
 , callPackage
 , makeSetupHook
 , disabledIf
@@ -97,6 +98,16 @@ in rec {
       };
     } ./python-namespaces-hook.sh) {};
 
+  pythonRecompileBytecodeHook = callPackage ({ }:
+    makeSetupHook {
+      name = "python-recompile-bytecode-hook";
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages;
+        compileArgs = lib.concatStringsSep " " (["-q" "-f" "-i -"] ++ lib.optionals isPy3k ["-j $NIX_BUILD_CORES"]);
+        bytecodeName = if isPy3k then "__pycache__" else "*.pyc";
+      };
+    } ./python-recompile-bytecode-hook.sh ) {};
+
   pythonRemoveBinBytecodeHook = callPackage ({ }:
     makeSetupHook {
       name = "python-remove-bin-bytecode-hook";

pkgs/development/interpreters/python/hooks/python-recompile-bytecode-hook.sh

@@ -0,0 +1,24 @@
+# Setup hook for recompiling bytecode.
+# https://github.com/NixOS/nixpkgs/issues/81441
+echo "Sourcing python-recompile-bytecode-hook.sh"
+
+# Remove all bytecode from the $out output. Then, recompile only site packages folder
+# Note this effectively duplicates `python-remove-bin-bytecode`, but long-term
+# this hook should be removed again.
+
+pythonRecompileBytecodePhase () {
+    # TODO: consider other outputs than $out
+
+    items="$(find "$out" -name "@bytecodeName@")"
+    if [[ -n $items ]]; then
+        for pycache in $items; do
+            rm -rf "$pycache"
+        done
+    fi
+
+    find "$out"/@pythonSitePackages@ -name "*.py" -exec @pythonInterpreter@ -OO -m compileall @compileArgs@ {} +
+}
+
+if [ -z "${dontUsePythonRecompileBytecode-}" ]; then
+    postPhases+=" pythonRecompileBytecodePhase"
+fi

pkgs/development/interpreters/python/hooks/python-remove-bin-bytecode-hook.sh

@@ -1,9 +1,9 @@
-# Setup hook for detecting conflicts in Python packages
+# Setup hook for removing bytecode from the bin folder
 echo "Sourcing python-remove-bin-bytecode-hook.sh"
 
-# Check if we have two packages with the same name in the closure and fail.
-# If this happens, something went wrong with the dependencies specs.
-# Intentionally kept in a subdirectory, see catch_conflicts/README.md.
+# The bin folder is added to $PATH and should only contain executables.
+# It may happen there are executables with a .py extension for which
+# bytecode is generated. This hook removes that bytecode.
 
 pythonRemoveBinBytecodePhase () {
     if [ -d "$out/bin" ]; then

pkgs/development/interpreters/python/mk-python-derivation.nix

@@ -17,6 +17,7 @@
 , pythonCatchConflictsHook
 , pythonImportsCheckHook
 , pythonNamespacesHook
+, pythonRecompileBytecodeHook
 , pythonRemoveBinBytecodeHook
 , pythonRemoveTestsDirHook
 , setuptoolsBuildHook
@@ -110,6 +111,7 @@ let
     python
     wrapPython
     ensureNewerSourcesForZipFilesHook  # move to wheel installer (pip) or builder (setuptools, flit, ...)?
+    pythonRecompileBytecodeHook  # Remove when solved https://github.com/NixOS/nixpkgs/issues/81441
     pythonRemoveTestsDirHook
   ] ++ lib.optionals catchConflicts [
     setuptools pythonCatchConflictsHook

pkgs/top-level/python-packages.nix

@@ -118,6 +118,7 @@ in {
     pythonCatchConflictsHook
     pythonImportsCheckHook
     pythonNamespacesHook
+    pythonRecompileBytecodeHook
     pythonRemoveBinBytecodeHook
     pythonRemoveTestsDirHook
     setuptoolsBuildHook