From 79c713bc1498ccceed9f8d2fc0b8d29b1b45a209 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 6 Sep 2018 14:28:44 -0400 Subject: [PATCH 1/5] release-lib: Cache cross nixpkgs evals too This will help with release-cross.nix eval time. It also allowed me to share code between the cross and native helpers. --- pkgs/top-level/release-cross.nix | 6 +- pkgs/top-level/release-lib.nix | 94 ++++++++++++++++++++------------ 2 files changed, 62 insertions(+), 38 deletions(-) diff --git a/pkgs/top-level/release-cross.nix b/pkgs/top-level/release-cross.nix index 0d15d817a66..ee69dd5a43c 100644 --- a/pkgs/top-level/release-cross.nix +++ b/pkgs/top-level/release-cross.nix @@ -81,11 +81,11 @@ in # good idea lest there be some irrelevant pass-through debug attrs that # cause false negatives. testEqualOne = path: system: let - f = path: attrs: builtins.toString (lib.getAttrFromPath path (allPackages attrs)); + f = path: crossSystem: system: builtins.toString (lib.getAttrFromPath path (pkgsForCross crossSystem system)); in assertTrue ( - f path { inherit system; } + f path null system == - f (["buildPackages"] ++ path) { inherit system crossSystem; } + f (["buildPackages"] ++ path) crossSystem system ); testEqual = path: systems: forMatchingSystems systems (testEqualOne path); diff --git a/pkgs/top-level/release-lib.nix b/pkgs/top-level/release-lib.nix index 623c9d9cb00..2b72de7c7ef 100644 --- a/pkgs/top-level/release-lib.nix +++ b/pkgs/top-level/release-lib.nix @@ -11,8 +11,6 @@ in with lib; rec { - allPackages = args: packageSet (args // nixpkgsArgs); - pkgs = packageSet (lib.recursiveUpdate { system = "x86_64-linux"; config.allowUnsupportedSystem = true; } nixpkgsArgs); inherit lib; @@ -23,29 +21,53 @@ rec { /* !!! Hack: poor man's memoisation function. Necessary to prevent Nixpkgs from being evaluated again and again for every job/platform pair. */ - pkgsFor = system: - if system == "x86_64-linux" then pkgs_x86_64_linux - else if system == "i686-linux" then pkgs_i686_linux - else if system == "aarch64-linux" then pkgs_aarch64_linux - else if system == "armv6l-linux" then pkgs_armv6l_linux - else if system == "armv7l-linux" then pkgs_armv7l_linux - else if system == "x86_64-darwin" then pkgs_x86_64_darwin - else if system == "x86_64-freebsd" then pkgs_x86_64_freebsd - else if system == "i686-freebsd" then pkgs_i686_freebsd - else if system == "i686-cygwin" then pkgs_i686_cygwin - else if system == "x86_64-cygwin" then pkgs_x86_64_cygwin - else abort "unsupported system type: ${system}"; + mkPkgsFor = crossSystem: let + packageSet' = args: packageSet (args // { inherit crossSystem; } // nixpkgsArgs); - pkgs_x86_64_linux = allPackages { system = "x86_64-linux"; }; - pkgs_i686_linux = allPackages { system = "i686-linux"; }; - pkgs_aarch64_linux = allPackages { system = "aarch64-linux"; }; - pkgs_armv6l_linux = allPackages { system = "armv6l-linux"; }; - pkgs_armv7l_linux = allPackages { system = "armv7l-linux"; }; - pkgs_x86_64_darwin = allPackages { system = "x86_64-darwin"; }; - pkgs_x86_64_freebsd = allPackages { system = "x86_64-freebsd"; }; - pkgs_i686_freebsd = allPackages { system = "i686-freebsd"; }; - pkgs_i686_cygwin = allPackages { system = "i686-cygwin"; }; - pkgs_x86_64_cygwin = allPackages { system = "x86_64-cygwin"; }; + pkgs_x86_64_linux = packageSet' { system = "x86_64-linux"; }; + pkgs_i686_linux = packageSet' { system = "i686-linux"; }; + pkgs_aarch64_linux = packageSet' { system = "aarch64-linux"; }; + pkgs_armv6l_linux = packageSet' { system = "armv6l-linux"; }; + pkgs_armv7l_linux = packageSet' { system = "armv7l-linux"; }; + pkgs_x86_64_darwin = packageSet' { system = "x86_64-darwin"; }; + pkgs_x86_64_freebsd = packageSet' { system = "x86_64-freebsd"; }; + pkgs_i686_freebsd = packageSet' { system = "i686-freebsd"; }; + pkgs_i686_cygwin = packageSet' { system = "i686-cygwin"; }; + pkgs_x86_64_cygwin = packageSet' { system = "x86_64-cygwin"; }; + + in system: + if system == "x86_64-linux" then pkgs_x86_64_linux + else if system == "i686-linux" then pkgs_i686_linux + else if system == "aarch64-linux" then pkgs_aarch64_linux + else if system == "armv6l-linux" then pkgs_armv6l_linux + else if system == "armv7l-linux" then pkgs_armv7l_linux + else if system == "x86_64-darwin" then pkgs_x86_64_darwin + else if system == "x86_64-freebsd" then pkgs_x86_64_freebsd + else if system == "i686-freebsd" then pkgs_i686_freebsd + else if system == "i686-cygwin" then pkgs_i686_cygwin + else if system == "x86_64-cygwin" then pkgs_x86_64_cygwin + else abort "unsupported system type: ${system}"; + + inherit (pkgsForCross null) pkgsFor; + + + # More poor man's memoisation + pkgsForCross = let + examplesByConfig = lib.flip lib.mapAttrs' + (builtins.removeAttrs lib.systems.examples [ "riscv" ]) + (_: crossSystem: nameValuePair crossSystem.config { + inherit crossSystem; + pkgsFor = mkPkgsFor crossSystem; + }); + native = mkPkgsFor null; + in crossSystem: let + errorMsg = "unsupported crossSystem: ${toString crossSystem.config or ""}"; + candidate = examplesByConfig.${crossSystem.config} or (throw errorMsg); + in if crossSystem == null + then native + else if lib.matchAttrs crossSystem candidate.crossSystem + then candidate.pkgsFor + else throw errorMsg; # Given a list of 'meta.platforms'-style patterns, return the sublist of @@ -90,30 +112,32 @@ rec { platform as an argument . We return an attribute set containing a derivation for each supported platform, i.e. ‘{ x86_64-linux = f pkgs_x86_64_linux; i686-linux = f pkgs_i686_linux; ... }’. */ - testOn = metaPatterns: f: forMatchingSystems metaPatterns - (system: hydraJob' (f (pkgsFor system))); + testOn = testOnCross null; /* Similar to the testOn function, but with an additional - 'crossSystem' parameter for allPackages, defining the target + 'crossSystem' parameter for packageSet', defining the target platform for cross builds. */ testOnCross = crossSystem: metaPatterns: f: forMatchingSystems metaPatterns - (system: hydraJob' (f (allPackages { inherit system crossSystem; }))); + (system: hydraJob' (f (pkgsForCross crossSystem system))); /* Given a nested set where the leaf nodes are lists of platforms, map each leaf node to `testOn [platforms...] (pkgs: pkgs.)'. */ - mapTestOn = mapAttrsRecursive - (path: metaPatterns: testOn metaPatterns (pkgs: getAttrFromPath path pkgs)); + mapTestOn = _mapTestOnHelper id null; + + + _mapTestOnHelper = f: crossSystem: mapAttrsRecursive + (path: metaPatterns: testOnCross crossSystem metaPatterns + (pkgs: f (getAttrFromPath path pkgs))); /* Similar to the testOn function, but with an additional 'crossSystem' - * parameter for allPackages, defining the target platform for cross builds, - * and triggering the build of the host derivation (cross built - crossDrv). */ - mapTestOnCross = crossSystem: mapAttrsRecursive - (path: metaPatterns: testOnCross crossSystem metaPatterns - (pkgs: addMetaAttrs { maintainers = crossMaintainers; } (getAttrFromPath path pkgs))); + * parameter for packageSet', defining the target platform for cross builds, + * and triggering the build of the host derivation. */ + mapTestOnCross = _mapTestOnHelper + (addMetaAttrs { maintainers = crossMaintainers; }); /* Recursively map a (nested) set of derivations to an isomorphic From 24e2bc18b6cd15a5d761048f5d3ec65b61d8fbd0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 1 Nov 2018 16:14:55 -0400 Subject: [PATCH 2/5] release-lib: Fallback on uncached rather than error for unknown crossSystem The `ensureUnaffected` the tests purposefully use an absurd crossSystem. Also sheevaplug and pogoplug share the same config. --- pkgs/top-level/release-lib.nix | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkgs/top-level/release-lib.nix b/pkgs/top-level/release-lib.nix index 2b72de7c7ef..ac11de698b5 100644 --- a/pkgs/top-level/release-lib.nix +++ b/pkgs/top-level/release-lib.nix @@ -61,13 +61,12 @@ rec { }); native = mkPkgsFor null; in crossSystem: let - errorMsg = "unsupported crossSystem: ${toString crossSystem.config or ""}"; - candidate = examplesByConfig.${crossSystem.config} or (throw errorMsg); + candidate = examplesByConfig.${crossSystem.config} or null; in if crossSystem == null then native - else if lib.matchAttrs crossSystem candidate.crossSystem + else if candidate != null && lib.matchAttrs crossSystem candidate.crossSystem then candidate.pkgsFor - else throw errorMsg; + else mkPkgsFor crossSystem; # uncached fallback # Given a list of 'meta.platforms'-style patterns, return the sublist of From 984df7ca0605554bd760efc1a53a8b94d8e04c15 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 1 Nov 2018 16:16:58 -0400 Subject: [PATCH 3/5] top-level: libcCrossChooser: More `or` fallback hack This fixes some eval errors in release-cross.nix --- pkgs/top-level/all-packages.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index abd7db6263d..a2e93f038c3 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -9536,11 +9536,11 @@ with pkgs; # libc is hackily often used from the previous stage. This `or` # hack fixes the hack, *sigh*. /**/ if name == "glibc" then targetPackages.glibcCross or glibcCross - else if name == "bionic" then targetPackages.bionic - else if name == "uclibc" then targetPackages.uclibcCross + else if name == "bionic" then targetPackages.bionic or bionic + else if name == "uclibc" then targetPackages.uclibcCross or uclibcCross else if name == "musl" then targetPackages.muslCross or muslCross else if name == "msvcrt" then targetPackages.windows.mingw_w64 or windows.mingw_w64 - else if stdenv.targetPlatform.useiOSPrebuilt then targetPackages.darwin.iosSdkPkgs.libraries + else if stdenv.targetPlatform.useiOSPrebuilt then targetPackages.darwin.iosSdkPkgs.libraries or darwin.iosSdkPkgs.libraries else if name == "libSystem" then targetPackages.darwin.xcode else throw "Unknown libc"; From ffaffb36d152d2cba1da51332d4079a96cd5288d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 1 Nov 2018 16:18:51 -0400 Subject: [PATCH 4/5] linux bootstrap-tools: use `stdenv.*Platform` to avoid deprecation warning --- pkgs/stdenv/linux/make-bootstrap-tools.nix | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkgs/stdenv/linux/make-bootstrap-tools.nix b/pkgs/stdenv/linux/make-bootstrap-tools.nix index 4fc9999b538..3ebb4e634e2 100644 --- a/pkgs/stdenv/linux/make-bootstrap-tools.nix +++ b/pkgs/stdenv/linux/make-bootstrap-tools.nix @@ -19,7 +19,7 @@ in with pkgs; rec { tarMinimal = gnutar.override { acl = null; }; busyboxMinimal = busybox.override { - useMusl = !targetPlatform.isRiscV; + useMusl = !stdenv.targetPlatform.isRiscV; enableStatic = true; enableMinimal = true; extraConfig = '' @@ -44,7 +44,7 @@ in with pkgs; rec { set -x mkdir -p $out/bin $out/lib $out/libexec - '' + (if (hostPlatform.libc == "glibc") then '' + '' + (if (stdenv.hostPlatform.libc == "glibc") then '' # Copy what we need of Glibc. cp -d ${libc.out}/lib/ld*.so* $out/lib cp -d ${libc.out}/lib/libc*.so* $out/lib @@ -75,7 +75,7 @@ in with pkgs; rec { find $out/include -name .install -exec rm {} \; find $out/include -name ..install.cmd -exec rm {} \; mv $out/include $out/include-glibc - '' else if (hostPlatform.libc == "musl") then '' + '' else if (stdenv.hostPlatform.libc == "musl") then '' # Copy what we need from musl cp ${libc.out}/lib/* $out/lib cp -rL ${libc.dev}/include $out @@ -137,7 +137,7 @@ in with pkgs; rec { cp -d ${zlib.out}/lib/libz.so* $out/lib cp -d ${libelf}/lib/libelf.so* $out/lib - '' + lib.optionalString (hostPlatform != buildPlatform) '' + '' + lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) '' # These needed for cross but not native tools because the stdenv # GCC has certain things built in statically. See # pkgs/stdenv/linux/default.nix for the details. @@ -199,21 +199,21 @@ in with pkgs; rec { bootstrapTools = runCommand "bootstrap-tools.tar.xz" {} "cp ${build}/on-server/bootstrap-tools.tar.xz $out"; }; - bootstrapTools = if (hostPlatform.libc == "glibc") then + bootstrapTools = if (stdenv.hostPlatform.libc == "glibc") then import ./bootstrap-tools { - inherit (hostPlatform) system; + inherit (stdenv.hostPlatform) system; inherit bootstrapFiles; } - else if (hostPlatform.libc == "musl") then + else if (stdenv.hostPlatform.libc == "musl") then import ./bootstrap-tools-musl { - inherit (hostPlatform) system; + inherit (stdenv.hostPlatform) system; inherit bootstrapFiles; } else throw "unsupported libc"; test = derivation { name = "test-bootstrap-tools"; - inherit (hostPlatform) system; + inherit (stdenv.hostPlatform) system; builder = bootstrapFiles.busybox; args = [ "ash" "-e" "-c" "eval \"$buildCommand\"" ]; @@ -232,12 +232,12 @@ in with pkgs; rec { grep --version gcc --version - '' + lib.optionalString (hostPlatform.libc == "glibc") '' + '' + lib.optionalString (stdenv.hostPlatform.libc == "glibc") '' ldlinux=$(echo ${bootstrapTools}/lib/ld-linux*.so.?) export CPP="cpp -idirafter ${bootstrapTools}/include-glibc -B${bootstrapTools}" export CC="gcc -idirafter ${bootstrapTools}/include-glibc -B${bootstrapTools} -Wl,-dynamic-linker,$ldlinux -Wl,-rpath,${bootstrapTools}/lib" export CXX="g++ -idirafter ${bootstrapTools}/include-glibc -B${bootstrapTools} -Wl,-dynamic-linker,$ldlinux -Wl,-rpath,${bootstrapTools}/lib" - '' + lib.optionalString (hostPlatform.libc == "musl") '' + '' + lib.optionalString (stdenv.hostPlatform.libc == "musl") '' ldmusl=$(echo ${bootstrapTools}/lib/ld-musl*.so.?) export CPP="cpp -idirafter ${bootstrapTools}/include-libc -B${bootstrapTools}" export CC="gcc -idirafter ${bootstrapTools}/include-libc -B${bootstrapTools} -Wl,-dynamic-linker,$ldmusl -Wl,-rpath,${bootstrapTools}/lib" From f2ed7c7af9f5b2db5e88b78e192b470c47f4c4fd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 1 Nov 2018 16:19:17 -0400 Subject: [PATCH 5/5] linux bootstrap tools: Use right system for some raw derivations This allows cross builds to work. Evidentallyy this has been done wrong since I combined the bootstrap tool creation files in ab651d2c9bab620ebe5e515476fbd70d2c5b0c61. Oops! --- pkgs/stdenv/linux/make-bootstrap-tools.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/stdenv/linux/make-bootstrap-tools.nix b/pkgs/stdenv/linux/make-bootstrap-tools.nix index 3ebb4e634e2..58104e6ce26 100644 --- a/pkgs/stdenv/linux/make-bootstrap-tools.nix +++ b/pkgs/stdenv/linux/make-bootstrap-tools.nix @@ -201,19 +201,19 @@ in with pkgs; rec { bootstrapTools = if (stdenv.hostPlatform.libc == "glibc") then import ./bootstrap-tools { - inherit (stdenv.hostPlatform) system; + inherit (stdenv.buildPlatform) system; # Used to determine where to build inherit bootstrapFiles; } else if (stdenv.hostPlatform.libc == "musl") then import ./bootstrap-tools-musl { - inherit (stdenv.hostPlatform) system; + inherit (stdenv.buildPlatform) system; # Used to determine where to build inherit bootstrapFiles; } else throw "unsupported libc"; test = derivation { name = "test-bootstrap-tools"; - inherit (stdenv.hostPlatform) system; + inherit (stdenv.hostPlatform) system; # We cannot "cross test" builder = bootstrapFiles.busybox; args = [ "ash" "-e" "-c" "eval \"$buildCommand\"" ];