Merge pull request #49565 from obsidiansystems/release-lib-cleanup

release-lib: Cache cross nixpkgs evals too, and other fixes

pkgs/stdenv/linux/make-bootstrap-tools.nix

@@ -19,7 +19,7 @@ in with pkgs; rec {
   tarMinimal = gnutar.override { acl = null; };
 
   busyboxMinimal = busybox.override {
-    useMusl = !targetPlatform.isRiscV;
+    useMusl = !stdenv.targetPlatform.isRiscV;
     enableStatic = true;
     enableMinimal = true;
     extraConfig = ''
@@ -44,7 +44,7 @@ in with pkgs; rec {
         set -x
         mkdir -p $out/bin $out/lib $out/libexec
 
-      '' + (if (hostPlatform.libc == "glibc") then ''
+      '' + (if (stdenv.hostPlatform.libc == "glibc") then ''
         # Copy what we need of Glibc.
         cp -d ${libc.out}/lib/ld*.so* $out/lib
         cp -d ${libc.out}/lib/libc*.so* $out/lib
@@ -75,7 +75,7 @@ in with pkgs; rec {
         find $out/include -name .install -exec rm {} \;
         find $out/include -name ..install.cmd -exec rm {} \;
         mv $out/include $out/include-glibc
-    '' else if (hostPlatform.libc == "musl") then ''
+    '' else if (stdenv.hostPlatform.libc == "musl") then ''
         # Copy what we need from musl
         cp ${libc.out}/lib/* $out/lib
         cp -rL ${libc.dev}/include $out
@@ -137,7 +137,7 @@ in with pkgs; rec {
         cp -d ${zlib.out}/lib/libz.so* $out/lib
         cp -d ${libelf}/lib/libelf.so* $out/lib
 
-      '' + lib.optionalString (hostPlatform != buildPlatform) ''
+      '' + lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
         # These needed for cross but not native tools because the stdenv
         # GCC has certain things built in statically. See
         # pkgs/stdenv/linux/default.nix for the details.
@@ -199,21 +199,21 @@ in with pkgs; rec {
     bootstrapTools = runCommand "bootstrap-tools.tar.xz" {} "cp ${build}/on-server/bootstrap-tools.tar.xz $out";
   };
 
-  bootstrapTools = if (hostPlatform.libc == "glibc") then
+  bootstrapTools = if (stdenv.hostPlatform.libc == "glibc") then
     import ./bootstrap-tools {
-      inherit (hostPlatform) system;
+      inherit (stdenv.buildPlatform) system; # Used to determine where to build
       inherit bootstrapFiles;
     }
-    else if (hostPlatform.libc == "musl") then
+    else if (stdenv.hostPlatform.libc == "musl") then
     import ./bootstrap-tools-musl {
-      inherit (hostPlatform) system;
+      inherit (stdenv.buildPlatform) system; # Used to determine where to build
       inherit bootstrapFiles;
     }
     else throw "unsupported libc";
 
   test = derivation {
     name = "test-bootstrap-tools";
-    inherit (hostPlatform) system;
+    inherit (stdenv.hostPlatform) system; # We cannot "cross test"
     builder = bootstrapFiles.busybox;
     args = [ "ash" "-e" "-c" "eval \"$buildCommand\"" ];
 
@@ -232,12 +232,12 @@ in with pkgs; rec {
       grep --version
       gcc --version
 
-    '' + lib.optionalString (hostPlatform.libc == "glibc") ''
+    '' + lib.optionalString (stdenv.hostPlatform.libc == "glibc") ''
       ldlinux=$(echo ${bootstrapTools}/lib/ld-linux*.so.?)
       export CPP="cpp -idirafter ${bootstrapTools}/include-glibc -B${bootstrapTools}"
       export CC="gcc -idirafter ${bootstrapTools}/include-glibc -B${bootstrapTools} -Wl,-dynamic-linker,$ldlinux -Wl,-rpath,${bootstrapTools}/lib"
       export CXX="g++ -idirafter ${bootstrapTools}/include-glibc -B${bootstrapTools} -Wl,-dynamic-linker,$ldlinux -Wl,-rpath,${bootstrapTools}/lib"
-    '' + lib.optionalString (hostPlatform.libc == "musl") ''
+    '' + lib.optionalString (stdenv.hostPlatform.libc == "musl") ''
       ldmusl=$(echo ${bootstrapTools}/lib/ld-musl*.so.?)
       export CPP="cpp -idirafter ${bootstrapTools}/include-libc -B${bootstrapTools}"
       export CC="gcc -idirafter ${bootstrapTools}/include-libc -B${bootstrapTools} -Wl,-dynamic-linker,$ldmusl -Wl,-rpath,${bootstrapTools}/lib"

pkgs/top-level/all-packages.nix

@@ -9719,13 +9719,13 @@ with pkgs;
     # libc is hackily often used from the previous stage. This `or`
     # hack fixes the hack, *sigh*.
     /**/ if name == "glibc" then targetPackages.glibcCross or glibcCross
-    else if name == "bionic" then targetPackages.bionic
+    else if name == "bionic" then targetPackages.bionic or bionic
-    else if name == "uclibc" then targetPackages.uclibcCross
+    else if name == "uclibc" then targetPackages.uclibcCross or uclibcCross
     else if name == "avrlibc" then targetPackages.avrlibcCross or avrlibcCross
     else if name == "newlib" then targetPackages.newlibCross or newlibCross
     else if name == "musl" then targetPackages.muslCross or muslCross
     else if name == "msvcrt" then targetPackages.windows.mingw_w64 or windows.mingw_w64
-    else if stdenv.targetPlatform.useiOSPrebuilt then targetPackages.darwin.iosSdkPkgs.libraries
+    else if stdenv.targetPlatform.useiOSPrebuilt then targetPackages.darwin.iosSdkPkgs.libraries or darwin.iosSdkPkgs.libraries
     else if name == "libSystem" then targetPackages.darwin.xcode
     else throw "Unknown libc";
 

pkgs/top-level/release-cross.nix

@@ -87,11 +87,11 @@ in
     # good idea lest there be some irrelevant pass-through debug attrs that
     # cause false negatives.
     testEqualOne = path: system: let
-      f = path: attrs: builtins.toString (lib.getAttrFromPath path (allPackages attrs));
+      f = path: crossSystem: system: builtins.toString (lib.getAttrFromPath path (pkgsForCross crossSystem system));
     in assertTrue (
-        f path { inherit system; }
+        f path null system
         ==
-        f (["buildPackages"] ++ path) { inherit system crossSystem; }
+        f (["buildPackages"] ++ path) crossSystem system
       );
 
     testEqual = path: systems: forMatchingSystems systems (testEqualOne path);
@@ -152,7 +152,7 @@ in
     tools = import ../stdenv/linux/make-bootstrap-tools-cross.nix { system = "x86_64-linux"; };
     maintainers = [ lib.maintainers.dezgeg ];
     mkBootstrapToolsJob = drv:
-      assert lib.elem drv.system (supportedSystems ++ [ "aarch64-linux" ]);
+      assert lib.elem drv.system supportedSystems;
       hydraJob' (lib.addMetaAttrs { inherit maintainers; } drv);
   in lib.mapAttrsRecursiveCond (as: !lib.isDerivation as) (name: mkBootstrapToolsJob) tools;
 }

pkgs/top-level/release-lib.nix

@@ -11,8 +11,6 @@ in with lib;
 
 rec {
 
-  allPackages = args: packageSet (args // nixpkgsArgs);
-
   pkgs = packageSet (lib.recursiveUpdate { system = "x86_64-linux"; config.allowUnsupportedSystem = true; } nixpkgsArgs);
   inherit lib;
 
@@ -23,7 +21,21 @@ rec {
   /* !!! Hack: poor man's memoisation function.  Necessary to prevent
      Nixpkgs from being evaluated again and again for every
      job/platform pair. */
-  pkgsFor = system:
+  mkPkgsFor = crossSystem: let
+    packageSet' = args: packageSet (args // { inherit crossSystem; } // nixpkgsArgs);
+
+    pkgs_x86_64_linux = packageSet' { system = "x86_64-linux"; };
+    pkgs_i686_linux = packageSet' { system = "i686-linux"; };
+    pkgs_aarch64_linux = packageSet' { system = "aarch64-linux"; };
+    pkgs_armv6l_linux = packageSet' { system = "armv6l-linux"; };
+    pkgs_armv7l_linux = packageSet' { system = "armv7l-linux"; };
+    pkgs_x86_64_darwin = packageSet' { system = "x86_64-darwin"; };
+    pkgs_x86_64_freebsd = packageSet' { system = "x86_64-freebsd"; };
+    pkgs_i686_freebsd = packageSet' { system = "i686-freebsd"; };
+    pkgs_i686_cygwin = packageSet' { system = "i686-cygwin"; };
+    pkgs_x86_64_cygwin = packageSet' { system = "x86_64-cygwin"; };
+
+    in system:
       if system == "x86_64-linux" then pkgs_x86_64_linux
       else if system == "i686-linux" then pkgs_i686_linux
       else if system == "aarch64-linux" then pkgs_aarch64_linux
@@ -36,16 +48,25 @@ rec {
       else if system == "x86_64-cygwin" then pkgs_x86_64_cygwin
       else abort "unsupported system type: ${system}";
 
-  pkgs_x86_64_linux = allPackages { system = "x86_64-linux"; };
+  inherit (pkgsForCross null) pkgsFor;
-  pkgs_i686_linux = allPackages { system = "i686-linux"; };
+
-  pkgs_aarch64_linux = allPackages { system = "aarch64-linux"; };
+
-  pkgs_armv6l_linux = allPackages { system = "armv6l-linux"; };
+  # More poor man's memoisation
-  pkgs_armv7l_linux = allPackages { system = "armv7l-linux"; };
+  pkgsForCross = let
-  pkgs_x86_64_darwin = allPackages { system = "x86_64-darwin"; };
+    examplesByConfig = lib.flip lib.mapAttrs'
-  pkgs_x86_64_freebsd = allPackages { system = "x86_64-freebsd"; };
+      (builtins.removeAttrs lib.systems.examples [ "riscv" ])
-  pkgs_i686_freebsd = allPackages { system = "i686-freebsd"; };
+      (_: crossSystem: nameValuePair crossSystem.config {
-  pkgs_i686_cygwin = allPackages { system = "i686-cygwin"; };
+        inherit crossSystem;
-  pkgs_x86_64_cygwin = allPackages { system = "x86_64-cygwin"; };
+        pkgsFor = mkPkgsFor crossSystem;
+      });
+    native = mkPkgsFor null;
+  in crossSystem: let
+    candidate = examplesByConfig.${crossSystem.config} or null;
+  in if crossSystem == null
+      then native
+    else if candidate != null && lib.matchAttrs crossSystem candidate.crossSystem
+      then candidate.pkgsFor
+    else mkPkgsFor crossSystem; # uncached fallback
 
 
   # Given a list of 'meta.platforms'-style patterns, return the sublist of
@@ -90,30 +111,32 @@ rec {
      platform as an argument .  We return an attribute set containing
      a derivation for each supported platform, i.e. ‘{ x86_64-linux =
      f pkgs_x86_64_linux; i686-linux = f pkgs_i686_linux; ... }’. */
-  testOn = metaPatterns: f: forMatchingSystems metaPatterns
+  testOn = testOnCross null;
-    (system: hydraJob' (f (pkgsFor system)));
 
 
   /* Similar to the testOn function, but with an additional
-     'crossSystem' parameter for allPackages, defining the target
+     'crossSystem' parameter for packageSet', defining the target
      platform for cross builds. */
   testOnCross = crossSystem: metaPatterns: f: forMatchingSystems metaPatterns
-    (system: hydraJob' (f (allPackages { inherit system crossSystem; })));
+    (system: hydraJob' (f (pkgsForCross crossSystem system)));
 
 
   /* Given a nested set where the leaf nodes are lists of platforms,
      map each leaf node to `testOn [platforms...] (pkgs:
      pkgs.<attrPath>)'. */
-  mapTestOn = mapAttrsRecursive
+  mapTestOn = _mapTestOnHelper id null;
-    (path: metaPatterns: testOn metaPatterns (pkgs: getAttrFromPath path pkgs));
+
+
+  _mapTestOnHelper = f: crossSystem: mapAttrsRecursive
+    (path: metaPatterns: testOnCross crossSystem metaPatterns
+      (pkgs: f (getAttrFromPath path pkgs)));
 
 
   /* Similar to the testOn function, but with an additional 'crossSystem'
-   * parameter for allPackages, defining the target platform for cross builds,
+   * parameter for packageSet', defining the target platform for cross builds,
-   * and triggering the build of the host derivation (cross built - crossDrv). */
+   * and triggering the build of the host derivation. */
-  mapTestOnCross = crossSystem: mapAttrsRecursive
+  mapTestOnCross = _mapTestOnHelper
-    (path: metaPatterns: testOnCross crossSystem metaPatterns
+    (addMetaAttrs { maintainers = crossMaintainers; });
-      (pkgs: addMetaAttrs { maintainers = crossMaintainers; } (getAttrFromPath path pkgs)));
 
 
   /* Recursively map a (nested) set of derivations to an isomorphic