diff --git a/nixos/doc/manual/release-notes/rl-1703.xml b/nixos/doc/manual/release-notes/rl-1703.xml index d8b0ae01e33..6997155d94d 100644 --- a/nixos/doc/manual/release-notes/rl-1703.xml +++ b/nixos/doc/manual/release-notes/rl-1703.xml @@ -28,6 +28,14 @@ has the following highlights: following incompatible changes: + + + stdenv.overrides is now expected to take self + and super arguments. See lib.trivial.extends + for what those parameters represent. + + + gnome alias has been removed along with diff --git a/pkgs/stdenv/booter.nix b/pkgs/stdenv/booter.nix new file mode 100644 index 00000000000..11ca8e1440e --- /dev/null +++ b/pkgs/stdenv/booter.nix @@ -0,0 +1,68 @@ +# This file defines a single function for booting a package set from a list of +# stages. The exact mechanics of that function are defined below; here I +# (@Ericson2314) wish to describe the purpose of the abstraction. +# +# The first goal is consistency across stdenvs. Regardless of what this function +# does, by making every stdenv use it for bootstrapping we ensure that they all +# work in a similar way. [Before this abstraction, each stdenv was its own +# special snowflake due to different authors writing in different times.] +# +# The second goal is consistency across each stdenv's stage functions. By +# writing each stage it terms of the previous stage, commonalities between them +# are more easily observable. [Before, there usually was a big attribute set +# with each stage, and stages would access the previous stage by name.] +# +# The third goal is composition. Because each stage is written in terms of the +# previous, the list can be reordered or, more practically, extended with new +# stages. The latter is used for cross compiling and custom +# stdenvs. Additionally, certain options should by default apply only to the +# last stage, whatever it may be. By delaying the creation of stage package sets +# until the final fold, we prevent these options from inhibiting composition. +# +# The fourth and final goal is debugging. Normal packages should only source +# their dependencies from the current stage. But for the sake of debugging, it +# is nice that all packages still remain accessible. We make sure previous +# stages are kept around with a `stdenv.__bootPackges` attribute referring the +# previous stage. It is idiomatic that attributes prefixed with `__` come with +# special restrictions and should not be used under normal circumstances. +{ lib, allPackages }: + +# Type: +# [ pkgset -> (args to stage/default.nix) or ({ __raw = true; } // pkgs) ] +# -> pkgset +# +# In english: This takes a list of function from the previous stage pkgset and +# returns the final pkgset. Each of those functions returns, if `__raw` is +# undefined or false, args for this stage's pkgset (the most complex and +# important arg is the stdenv), or, if `__raw = true`, simply this stage's +# pkgset itself. +# +# The list takes stages in order, so the final stage is last in the list. In +# other words, this does a foldr not foldl. +stageFuns: let + + # Take the list and disallow custom overrides in all but the final stage, + # and allow it in the final flag. Only defaults this boolean field if it + # isn't already set. + withAllowCustomOverrides = lib.lists.imap + (index: stageFun: prevStage: + # So true by default for only the first element because one + # 1-indexing. Since we reverse the list, this means this is true + # for the final stage. + { allowCustomOverrides = index == 1; } + // (stageFun prevStage)) + (lib.lists.reverseList stageFuns); + + # Adds the stdenv to the arguments, and sticks in it the previous stage for + # debugging purposes. + folder = stageFun: finalSoFar: let + args = stageFun finalSoFar; + stdenv = args.stdenv // { + # For debugging + __bootPackages = finalSoFar; + }; + args' = args // { inherit stdenv; }; + in + (if args.__raw or false then lib.id else allPackages) args'; + +in lib.lists.fold folder {} withAllowCustomOverrides diff --git a/pkgs/stdenv/cross/default.nix b/pkgs/stdenv/cross/default.nix index 10e2a766356..359d45b78b9 100644 --- a/pkgs/stdenv/cross/default.nix +++ b/pkgs/stdenv/cross/default.nix @@ -1,35 +1,49 @@ -{ lib, allPackages +{ lib , system, platform, crossSystem, config }: -rec { - vanillaStdenv = (import ../. { - inherit lib allPackages system platform; +let + bootStages = import ../. { + inherit lib system platform; crossSystem = null; # Ignore custom stdenvs when cross compiling for compatability config = builtins.removeAttrs config [ "replaceStdenv" ]; - }) // { - # Needed elsewhere as a hacky way to pass the target - cross = crossSystem; }; - # For now, this is just used to build the native stdenv. Eventually, it should - # be used to build compilers and other such tools targeting the cross +in bootStages ++ [ + + # Build Packages. + # + # For now, this is just used to build the native stdenv. Eventually, it + # should be used to build compilers and other such tools targeting the cross # platform. Then, `forceNativeDrv` can be removed. - buildPackages = allPackages { + (vanillaPackages: { inherit system platform crossSystem config; # It's OK to change the built-time dependencies allowCustomOverrides = true; - stdenv = vanillaStdenv; - }; + stdenv = vanillaPackages.stdenv // { + # Needed elsewhere as a hacky way to pass the target + cross = crossSystem; + overrides = _: _: {}; + }; + }) - stdenvCross = buildPackages.makeStdenvCross - buildPackages.stdenv crossSystem - buildPackages.binutilsCross buildPackages.gccCrossStageFinal; + # Run packages + (buildPackages: { + inherit system platform crossSystem config; + stdenv = if crossSystem.useiOSCross or false + then let + inherit (buildPackages.darwin.ios-cross { + prefix = crossSystem.config; + inherit (crossSystem) arch; + simulator = crossSystem.isiPhoneSimulator or false; }) + cc binutils; + in buildPackages.makeStdenvCross + buildPackages.stdenv crossSystem + binutils cc + else buildPackages.makeStdenvCross + buildPackages.stdenv crossSystem + buildPackages.binutilsCross buildPackages.gccCrossStageFinal; + }) - stdenvCrossiOS = let - inherit (buildPackages.darwin.ios-cross { prefix = crossSystem.config; inherit (crossSystem) arch; simulator = crossSystem.isiPhoneSimulator or false; }) cc binutils; - in buildPackages.makeStdenvCross - buildPackages.stdenv crossSystem - binutils cc; -} +] diff --git a/pkgs/stdenv/custom/default.nix b/pkgs/stdenv/custom/default.nix index 188d0027773..59a6e871cfd 100644 --- a/pkgs/stdenv/custom/default.nix +++ b/pkgs/stdenv/custom/default.nix @@ -1,22 +1,22 @@ -{ lib, allPackages +{ lib , system, platform, crossSystem, config }: assert crossSystem == null; -rec { - vanillaStdenv = import ../. { - inherit lib allPackages system platform crossSystem; +let + bootStages = import ../. { + inherit lib system platform crossSystem; # Remove config.replaceStdenv to ensure termination. config = builtins.removeAttrs config [ "replaceStdenv" ]; }; - buildPackages = allPackages { - inherit system platform crossSystem config; - # It's OK to change the built-time dependencies - allowCustomOverrides = true; - stdenv = vanillaStdenv; - }; +in bootStages ++ [ - stdenvCustom = config.replaceStdenv { pkgs = buildPackages; }; -} + # Additional stage, built using custom stdenv + (vanillaPackages: { + inherit system platform crossSystem config; + stdenv = config.replaceStdenv { pkgs = vanillaPackages; }; + }) + +] diff --git a/pkgs/stdenv/darwin/default.nix b/pkgs/stdenv/darwin/default.nix index b9044f25cd7..9c860bfb0f9 100644 --- a/pkgs/stdenv/darwin/default.nix +++ b/pkgs/stdenv/darwin/default.nix @@ -1,4 +1,4 @@ -{ lib, allPackages +{ lib , system, platform, crossSystem, config # Allow passing in bootstrap files directly so we can test the stdenv bootstrap process when changing the bootstrap tools @@ -22,8 +22,6 @@ let (import "${./standard-sandbox.sb}") ''; in rec { - inherit allPackages; - commonPreHook = '' export NIX_ENFORCE_PURITY="''${NIX_ENFORCE_PURITY-1}" export NIX_ENFORCE_NO_NATIVE="''${NIX_ENFORCE_NO_NATIVE-1}" @@ -54,7 +52,7 @@ in rec { }; stageFun = step: last: {shell ? "${bootstrapTools}/bin/sh", - overrides ? (pkgs: {}), + overrides ? (self: super: {}), extraPreHook ? "", extraBuildInputs, allowedRequisites ? null}: @@ -96,19 +94,17 @@ in rec { extraSandboxProfile = binShClosure + libSystemProfile; extraAttrs = { inherit platform; parent = last; }; - overrides = pkgs: (overrides pkgs) // { fetchurl = thisStdenv.fetchurlBoot; }; + overrides = self: super: (overrides self super) // { fetchurl = thisStdenv.fetchurlBoot; }; }; - thisPkgs = allPackages { - inherit system platform crossSystem config; - allowCustomOverrides = false; - stdenv = thisStdenv; - }; - in { stdenv = thisStdenv; pkgs = thisPkgs; }; + in { + inherit system platform crossSystem config; + stdenv = thisStdenv; + }; stage0 = stageFun 0 null { - overrides = orig: with stage0; rec { - darwin = orig.darwin // { + overrides = self: super: with stage0; rec { + darwin = super.darwin // { Libsystem = stdenv.mkDerivation { name = "bootstrap-Libsystem"; buildCommand = '' @@ -145,32 +141,32 @@ in rec { extraBuildInputs = []; }; - persistent0 = _: {}; + persistent0 = _: _: _: {}; - stage1 = with stage0; stageFun 1 stage0 { + stage1 = prevStage: with prevStage; stageFun 1 prevStage { extraPreHook = "export NIX_CFLAGS_COMPILE+=\" -F${bootstrapTools}/Library/Frameworks\""; extraBuildInputs = [ pkgs.libcxx ]; allowedRequisites = [ bootstrapTools ] ++ (with pkgs; [ libcxx libcxxabi ]) ++ [ pkgs.darwin.Libsystem ]; - overrides = persistent0; + overrides = persistent0 prevStage; }; - persistent1 = orig: with stage1.pkgs; { + persistent1 = prevStage: self: super: with prevStage; { inherit zlib patchutils m4 scons flex perl bison unifdef unzip openssl icu python libxml2 gettext sharutils gmp libarchive ncurses pkg-config libedit groff openssh sqlite sed serf openldap db cyrus-sasl expat apr-util subversion xz findfreetype libssh curl cmake autoconf automake libtool ed cpio coreutils; - darwin = orig.darwin // { + darwin = super.darwin // { inherit (darwin) dyld Libsystem xnu configd libdispatch libclosure launchd; }; }; - stage2 = with stage1; stageFun 2 stage1 { + stage2 = prevStage: with prevStage; stageFun 2 prevStage { extraPreHook = '' export PATH_LOCALE=${pkgs.darwin.locale}/share/locale ''; @@ -182,10 +178,10 @@ in rec { (with pkgs; [ xz.bin xz.out libcxx libcxxabi icu.out ]) ++ (with pkgs.darwin; [ dyld Libsystem CF locale ]); - overrides = persistent1; + overrides = persistent1 prevStage; }; - persistent2 = orig: with stage2.pkgs; { + persistent2 = prevStage: self: super: with prevStage; { inherit patchutils m4 scons flex perl bison unifdef unzip openssl python gettext sharutils libarchive pkg-config groff bash subversion @@ -193,13 +189,13 @@ in rec { findfreetype libssh curl cmake autoconf automake libtool cpio libcxx libcxxabi; - darwin = orig.darwin // { + darwin = super.darwin // { inherit (darwin) dyld Libsystem xnu configd libdispatch libclosure launchd libiconv locale; }; }; - stage3 = with stage2; stageFun 3 stage2 { + stage3 = prevStage: with prevStage; stageFun 3 prevStage { shell = "${pkgs.bash}/bin/bash"; # We have a valid shell here (this one has no bootstrap-tools runtime deps) so stageFun @@ -218,53 +214,53 @@ in rec { (with pkgs; [ xz.bin xz.out icu.out bash libcxx libcxxabi ]) ++ (with pkgs.darwin; [ dyld Libsystem locale ]); - overrides = persistent2; + overrides = persistent2 prevStage; }; - persistent3 = orig: with stage3.pkgs; { + persistent3 = prevStage: self: super: with prevStage; { inherit gnumake gzip gnused bzip2 gawk ed xz patch bash libcxxabi libcxx ncurses libffi zlib gmp pcre gnugrep coreutils findutils diffutils patchutils; llvmPackages = let llvmOverride = llvmPackages.llvm.override { inherit libcxxabi; }; - in orig.llvmPackages // { + in super.llvmPackages // { llvm = llvmOverride; clang-unwrapped = llvmPackages.clang-unwrapped.override { llvm = llvmOverride; }; }; - darwin = orig.darwin // { + darwin = super.darwin // { inherit (darwin) dyld Libsystem libiconv locale; }; }; - stage4 = with stage3; stageFun 4 stage3 { + stage4 = prevStage: with prevStage; stageFun 4 prevStage { shell = "${pkgs.bash}/bin/bash"; extraBuildInputs = with pkgs; [ xz darwin.CF libcxx pkgs.bash ]; extraPreHook = '' export PATH_LOCALE=${pkgs.darwin.locale}/share/locale ''; - overrides = persistent3; + overrides = persistent3 prevStage; }; - persistent4 = orig: with stage4.pkgs; { + persistent4 = prevStage: self: super: with prevStage; { inherit gnumake gzip gnused bzip2 gawk ed xz patch bash libcxxabi libcxx ncurses libffi zlib icu llvm gmp pcre gnugrep coreutils findutils diffutils patchutils binutils binutils-raw; - llvmPackages = orig.llvmPackages // { + llvmPackages = super.llvmPackages // { inherit (llvmPackages) llvm clang-unwrapped; }; - darwin = orig.darwin // { + darwin = super.darwin // { inherit (darwin) dyld Libsystem cctools libiconv; }; }; - stage5 = with stage4; import ../generic rec { + stdenvDarwin = prevStage: let pkgs = prevStage; in import ../generic rec { inherit system config; - inherit (stdenv) fetchurlBoot; + inherit (pkgs.stdenv) fetchurlBoot; name = "stdenv-darwin"; @@ -279,7 +275,8 @@ in rec { shell = "${pkgs.bash}/bin/bash"; cc = import ../../build-support/cc-wrapper { - inherit stdenv shell; + inherit (pkgs) stdenv; + inherit shell; nativeTools = false; nativeLibc = false; inherit (pkgs) coreutils binutils gnugrep; @@ -294,7 +291,6 @@ in rec { inherit platform bootstrapTools; libc = pkgs.darwin.Libsystem; shellPackage = pkgs.bash; - parent = stage4; }; allowedRequisites = (with pkgs; [ @@ -307,11 +303,21 @@ in rec { dyld Libsystem CF cctools libiconv locale ]); - overrides = orig: persistent4 orig // { + overrides = self: super: persistent4 prevStage self super // { clang = cc; inherit cc; }; }; - stdenvDarwin = stage5; + stagesDarwin = [ + ({}: stage0) + stage1 + stage2 + stage3 + stage4 + (prevStage: { + inherit system crossSystem platform config; + stdenv = stdenvDarwin prevStage; + }) + ]; } diff --git a/pkgs/stdenv/darwin/make-bootstrap-tools.nix b/pkgs/stdenv/darwin/make-bootstrap-tools.nix index c862eb141a8..85e4dabbbde 100644 --- a/pkgs/stdenv/darwin/make-bootstrap-tools.nix +++ b/pkgs/stdenv/darwin/make-bootstrap-tools.nix @@ -334,8 +334,8 @@ in rec { # The ultimate test: bootstrap a whole stdenv from the tools specified above and get a package set out of it test-pkgs = import test-pkgspath { inherit system; - stdenvFunc = args: let + stdenvStages = args: let args' = args // { inherit bootstrapFiles; }; - in (import (test-pkgspath + "/pkgs/stdenv/darwin") args').stdenvDarwin; + in (import (test-pkgspath + "/pkgs/stdenv/darwin") args').stagesDarwin; }; } diff --git a/pkgs/stdenv/default.nix b/pkgs/stdenv/default.nix index 3b49d0de830..0ca03ecdde1 100644 --- a/pkgs/stdenv/default.nix +++ b/pkgs/stdenv/default.nix @@ -1,13 +1,12 @@ -# This file defines the various standard build environments. +# This file chooses a sane default stdenv given the system, platform, etc. # -# On Linux systems, the standard build environment consists of -# Nix-built instances glibc and the `standard' Unix tools, i.e., the -# Posix utilities, the GNU C compiler, and so on. On other systems, -# we use the native C library. +# Rather than returning a stdenv, this returns a list of functions---one per +# each bootstrapping stage. See `./booter.nix` for exactly what this list should +# contain. { # Args just for stdenvs' usage - lib, allPackages - # Args to pass on to `allPacakges` too + lib + # Args to pass on to the pkgset builder, too , system, platform, crossSystem, config } @ args: @@ -17,50 +16,39 @@ let # i.e., the stuff in /bin, /usr/bin, etc. This environment should # be used with care, since many Nix packages will not build properly # with it (e.g., because they require GNU Make). - inherit (import ./native args) stdenvNative; - - stdenvNativePkgs = allPackages { - inherit system platform crossSystem config; - allowCustomOverrides = false; - stdenv = stdenvNative; - noSysDirs = false; - }; - + stagesNative = import ./native args; # The Nix build environment. - stdenvNix = assert crossSystem == null; import ./nix { - inherit config lib; - stdenv = stdenvNative; - pkgs = stdenvNativePkgs; - }; + stagesNix = import ./nix (args // { bootStages = stagesNative; }); - inherit (import ./freebsd args) stdenvFreeBSD; + stagesFreeBSD = import ./freebsd args; - # Linux standard environment. - inherit (import ./linux args) stdenvLinux; + # On Linux systems, the standard build environment consists of Nix-built + # instances glibc and the `standard' Unix tools, i.e., the Posix utilities, + # the GNU C compiler, and so on. + stagesLinux = import ./linux args; - inherit (import ./darwin args) stdenvDarwin; + inherit (import ./darwin args) stagesDarwin; - inherit (import ./cross args) stdenvCross stdenvCrossiOS; + stagesCross = import ./cross args; - inherit (import ./custom args) stdenvCustom; + stagesCustom = import ./custom args; - # Select the appropriate stdenv for the platform `system'. + # Select the appropriate stages for the platform `system'. in - if crossSystem != null then - if crossSystem.useiOSCross or false then stdenvCrossiOS - else stdenvCross else - if config ? replaceStdenv then stdenvCustom else - if system == "i686-linux" then stdenvLinux else - if system == "x86_64-linux" then stdenvLinux else - if system == "armv5tel-linux" then stdenvLinux else - if system == "armv6l-linux" then stdenvLinux else - if system == "armv7l-linux" then stdenvLinux else - if system == "mips64el-linux" then stdenvLinux else - if system == "powerpc-linux" then /* stdenvLinux */ stdenvNative else - if system == "x86_64-darwin" then stdenvDarwin else - if system == "x86_64-solaris" then stdenvNix else - if system == "i686-cygwin" then stdenvNative else - if system == "x86_64-cygwin" then stdenvNative else - if system == "x86_64-freebsd" then stdenvFreeBSD else - stdenvNative + if crossSystem != null then stagesCross + else if config ? replaceStdenv then stagesCustom + else { # switch + "i686-linux" = stagesLinux; + "x86_64-linux" = stagesLinux; + "armv5tel-linux" = stagesLinux; + "armv6l-linux" = stagesLinux; + "armv7l-linux" = stagesLinux; + "mips64el-linux" = stagesLinux; + "powerpc-linux" = /* stagesLinux */ stagesNative; + "x86_64-darwin" = stagesDarwin; + "x86_64-solaris" = stagesNix; + "i686-cygwin" = stagesNative; + "x86_64-cygwin" = stagesNative; + "x86_64-freebsd" = stagesFreeBSD; + }.${system} or stagesNative diff --git a/pkgs/stdenv/freebsd/default.nix b/pkgs/stdenv/freebsd/default.nix index ea2ebcc7917..e0256e26f5f 100644 --- a/pkgs/stdenv/freebsd/default.nix +++ b/pkgs/stdenv/freebsd/default.nix @@ -1,65 +1,86 @@ -{ lib, allPackages +{ lib , system, platform, crossSystem, config }: assert crossSystem == null; -rec { - inherit allPackages; - bootstrapTools = derivation { - inherit system; +[ - name = "trivial-bootstrap-tools"; - builder = "/usr/local/bin/bash"; - args = [ ./trivial-bootstrap.sh ]; + ({}: { + __raw = true; - mkdir = "/bin/mkdir"; - ln = "/bin/ln"; - }; + bootstrapTools = derivation { + inherit system; + + name = "trivial-bootstrap-tools"; + builder = "/usr/local/bin/bash"; + args = [ ./trivial-bootstrap.sh ]; + + mkdir = "/bin/mkdir"; + ln = "/bin/ln"; + }; + }) + + ({ bootstrapTools, ... }: rec { + __raw = true; + + inherit bootstrapTools; + + fetchurl = import ../../build-support/fetchurl { + inherit stdenv; + curl = bootstrapTools; + }; - fetchurl = import ../../build-support/fetchurl { stdenv = import ../generic { name = "stdenv-freebsd-boot-1"; inherit system config; - - initialPath = [ "/" "/usr" ]; - shell = "${bootstrapTools}/bin/bash"; + initialPath = [ "/" "/usr" ]; + shell = "${bootstrapTools}/bin/bash"; fetchurlBoot = null; cc = null; - }; - curl = bootstrapTools; - }; - - stdenvFreeBSD = import ../generic { - name = "stdenv-freebsd-boot-3"; - - inherit system config; - - initialPath = [ bootstrapTools ]; - shell = "${bootstrapTools}/bin/bash"; - fetchurlBoot = fetchurl; - - cc = import ../../build-support/cc-wrapper { - nativeTools = true; - nativePrefix = "/usr"; - nativeLibc = true; - stdenv = import ../generic { - inherit system config; - name = "stdenv-freebsd-boot-0"; - initialPath = [ bootstrapTools ]; - shell = stdenvFreeBSD.shell; - fetchurlBoot = fetchurl; - cc = null; + overrides = self: super: { }; - cc = { - name = "clang-9.9.9"; - cc = "/usr"; - outPath = "/usr"; - }; - isClang = true; }; + }) - preHook = ''export NIX_NO_SELF_RPATH=1''; - }; -} + (prevStage: { + __raw = true; + + stdenv = import ../generic { + name = "stdenv-freebsd-boot-0"; + inherit system config; + initialPath = [ prevStage.bootstrapTools ]; + inherit (prevStage.stdenv) shell; + fetchurlBoot = prevStage.fetchurl; + cc = null; + }; + }) + + (prevStage: { + inherit system crossSystem platform config; + stdenv = import ../generic { + name = "stdenv-freebsd-boot-3"; + inherit system config; + + inherit (prevStage.stdenv) + initialPath shell fetchurlBoot; + + cc = import ../../build-support/cc-wrapper { + nativeTools = true; + nativePrefix = "/usr"; + nativeLibc = true; + inherit (prevStage) stdenv; + cc = { + name = "clang-9.9.9"; + cc = "/usr"; + outPath = "/usr"; + }; + isClang = true; + }; + + preHook = ''export NIX_NO_SELF_RPATH=1''; + }; + }) + +] diff --git a/pkgs/stdenv/generic/default.nix b/pkgs/stdenv/generic/default.nix index bd35970e0d1..32e0d894818 100644 --- a/pkgs/stdenv/generic/default.nix +++ b/pkgs/stdenv/generic/default.nix @@ -1,7 +1,7 @@ let lib = import ../../../lib; in lib.makeOverridable ( { system, name ? "stdenv", preHook ? "", initialPath, cc, shell -, allowedRequisites ? null, extraAttrs ? {}, overrides ? (pkgs: {}), config +, allowedRequisites ? null, extraAttrs ? {}, overrides ? (self: super: {}), config , # The `fetchurl' to use for downloading curl and its dependencies # (see all-packages.nix). diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index 9900fc6dd3d..430122c36f7 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -3,7 +3,7 @@ # external (non-Nix) tools, such as /usr/bin/gcc, and it contains a C # compiler and linker that do not search in default locations, # ensuring purity of components produced by it. -{ lib, allPackages +{ lib , system, platform, crossSystem, config , bootstrapFiles ? @@ -18,7 +18,7 @@ assert crossSystem == null; -rec { +let commonPreHook = '' @@ -43,8 +43,8 @@ rec { # This function builds the various standard environments used during # the bootstrap. In all stages, we build an stdenv and the package # set that can be built with that stdenv. - stageFun = - {gccPlain, glibc, binutils, coreutils, gnugrep, name, overrides ? (pkgs: {}), extraBuildInputs ? []}: + stageFun = prevStage: + { name, overrides ? (self: super: {}), extraBuildInputs ? [] }: let @@ -65,17 +65,17 @@ rec { inherit system; }; - cc = if isNull gccPlain + cc = if isNull prevStage.gcc-unwrapped then null else lib.makeOverridable (import ../../build-support/cc-wrapper) { nativeTools = false; nativeLibc = false; - cc = gccPlain; + cc = prevStage.gcc-unwrapped; isGNU = true; - libc = glibc; - inherit binutils coreutils gnugrep; + libc = prevStage.glibc; + inherit (prevStage) binutils coreutils gnugrep; name = name; - stdenv = stage0.stdenv; + stdenv = prevStage.ccWrapperStdenv; }; extraAttrs = { @@ -85,37 +85,47 @@ rec { # stdenv.glibc is used by GCC build to figure out the system-level # /usr/include directory. - inherit glibc; + inherit (prevStage) glibc; }; - overrides = pkgs: (overrides pkgs) // { fetchurl = thisStdenv.fetchurlBoot; }; + overrides = self: super: (overrides self super) // { fetchurl = thisStdenv.fetchurlBoot; }; }; - thisPkgs = allPackages { - inherit system platform crossSystem config; - allowCustomOverrides = false; - stdenv = thisStdenv; - }; + in { + inherit system platform crossSystem config; + stdenv = thisStdenv; + }; - in { stdenv = thisStdenv; pkgs = thisPkgs; }; +in +[ - # Build a dummy stdenv with no GCC or working fetchurl. This is - # because we need a stdenv to build the GCC wrapper and fetchurl. - stage0 = stageFun { - gccPlain = null; + ({}: { + __raw = true; + + gcc-unwrapped = null; glibc = null; binutils = null; coreutils = null; gnugrep = null; + }) + + # Build a dummy stdenv with no GCC or working fetchurl. This is + # because we need a stdenv to build the GCC wrapper and fetchurl. + (prevStage: stageFun prevStage { name = null; - overrides = pkgs: { + overrides = self: super: { + # We thread stage0's stdenv through under this name so downstream stages + # can use it for wrapping gcc too. This way, downstream stages don't need + # to refer to this stage directly, which violates the principle that each + # stage should only access the stage that came before it. + ccWrapperStdenv = self.stdenv; # The Glibc include directory cannot have the same prefix as the # GCC include directory, since GCC gets confused otherwise (it # will search the Glibc headers before the GCC headers). So # create a dummy Glibc here, which will be used in the stdenv of # stage1. - glibc = stage0.stdenv.mkDerivation { + glibc = self.stdenv.mkDerivation { name = "bootstrap-glibc"; buildCommand = '' mkdir -p $out @@ -123,8 +133,12 @@ rec { ln -s ${bootstrapTools}/include-glibc $out/include ''; }; + gcc-unwrapped = bootstrapTools; + binutils = bootstrapTools; + coreutils = bootstrapTools; + gnugrep = bootstrapTools; }; - }; + }) # Create the first "real" standard environment. This one consists @@ -137,103 +151,92 @@ rec { # If we ever need to use a package from more than one stage back, we # simply re-export those packages in the middle stage(s) using the # overrides attribute and the inherit syntax. - stage1 = stageFun { - gccPlain = bootstrapTools; - inherit (stage0.pkgs) glibc; - binutils = bootstrapTools; - coreutils = bootstrapTools; - gnugrep = bootstrapTools; + (prevStage: stageFun prevStage { name = "bootstrap-gcc-wrapper"; # Rebuild binutils to use from stage2 onwards. - overrides = pkgs: { - binutils = pkgs.binutils.override { gold = false; }; - inherit (stage0.pkgs) glibc; + overrides = self: super: { + binutils = super.binutils.override { gold = false; }; + inherit (prevStage) + ccWrapperStdenv + glibc gcc-unwrapped coreutils gnugrep; # A threaded perl build needs glibc/libpthread_nonshared.a, # which is not included in bootstrapTools, so disable threading. # This is not an issue for the final stdenv, because this perl # won't be included in the final stdenv and won't be exported to # top-level pkgs as an override either. - perl = pkgs.perl.override { enableThreading = false; }; + perl = super.perl.override { enableThreading = false; }; }; - }; + }) # 2nd stdenv that contains our own rebuilt binutils and is used for # compiling our own Glibc. - stage2 = stageFun { - gccPlain = bootstrapTools; - inherit (stage1.pkgs) glibc; - binutils = stage1.pkgs.binutils; - coreutils = bootstrapTools; - gnugrep = bootstrapTools; + (prevStage: stageFun prevStage { name = "bootstrap-gcc-wrapper"; - overrides = pkgs: { - inherit (stage1.pkgs) perl binutils paxctl gnum4 bison; + overrides = self: super: { + inherit (prevStage) + ccWrapperStdenv + binutils gcc-unwrapped coreutils gnugrep + perl paxctl gnum4 bison; # This also contains the full, dynamically linked, final Glibc. }; - }; + }) # Construct a third stdenv identical to the 2nd, except that this # one uses the rebuilt Glibc from stage2. It still uses the recent # binutils and rest of the bootstrap tools, including GCC. - stage3 = stageFun { - gccPlain = bootstrapTools; - inherit (stage2.pkgs) glibc binutils; - coreutils = bootstrapTools; - gnugrep = bootstrapTools; + (prevStage: stageFun prevStage { name = "bootstrap-gcc-wrapper"; - overrides = pkgs: rec { - inherit (stage2.pkgs) binutils glibc perl patchelf linuxHeaders gnum4 bison; + overrides = self: super: rec { + inherit (prevStage) + ccWrapperStdenv + binutils glibc coreutils gnugrep + perl patchelf linuxHeaders gnum4 bison; # Link GCC statically against GMP etc. This makes sense because # these builds of the libraries are only used by GCC, so it # reduces the size of the stdenv closure. - gmp = pkgs.gmp.override { stdenv = pkgs.makeStaticLibraries pkgs.stdenv; }; - mpfr = pkgs.mpfr.override { stdenv = pkgs.makeStaticLibraries pkgs.stdenv; }; - libmpc = pkgs.libmpc.override { stdenv = pkgs.makeStaticLibraries pkgs.stdenv; }; - isl_0_14 = pkgs.isl_0_14.override { stdenv = pkgs.makeStaticLibraries pkgs.stdenv; }; - gccPlain = pkgs.gcc.cc.override { + gmp = super.gmp.override { stdenv = self.makeStaticLibraries self.stdenv; }; + mpfr = super.mpfr.override { stdenv = self.makeStaticLibraries self.stdenv; }; + libmpc = super.libmpc.override { stdenv = self.makeStaticLibraries self.stdenv; }; + isl_0_14 = super.isl_0_14.override { stdenv = self.makeStaticLibraries self.stdenv; }; + gcc-unwrapped = super.gcc-unwrapped.override { isl = isl_0_14; }; }; - extraBuildInputs = [ stage2.pkgs.patchelf stage2.pkgs.paxctl ]; - }; + extraBuildInputs = [ prevStage.patchelf prevStage.paxctl ]; + }) # Construct a fourth stdenv that uses the new GCC. But coreutils is # still from the bootstrap tools. - stage4 = stageFun { - inherit (stage3.pkgs) gccPlain glibc binutils; - gnugrep = bootstrapTools; - coreutils = bootstrapTools; + (prevStage: stageFun prevStage { name = ""; - overrides = pkgs: { + overrides = self: super: { # Zlib has to be inherited and not rebuilt in this stage, # because gcc (since JAR support) already depends on zlib, and # then if we already have a zlib we want to use that for the # other purposes (binutils and top-level pkgs) too. - inherit (stage3.pkgs) gettext gnum4 bison gmp perl glibc zlib linuxHeaders; + inherit (prevStage) gettext gnum4 bison gmp perl glibc zlib linuxHeaders; gcc = lib.makeOverridable (import ../../build-support/cc-wrapper) { nativeTools = false; nativeLibc = false; isGNU = true; - cc = stage4.stdenv.cc.cc; - libc = stage4.pkgs.glibc; - inherit (stage4.pkgs) binutils coreutils gnugrep; + cc = prevStage.gcc-unwrapped; + libc = self.glibc; + inherit (self) stdenv binutils coreutils gnugrep; name = ""; - stdenv = stage4.stdenv; - shell = stage4.pkgs.bash + "/bin/bash"; + shell = self.bash + "/bin/bash"; }; }; - extraBuildInputs = [ stage3.pkgs.patchelf stage3.pkgs.xz ]; - }; - + extraBuildInputs = [ prevStage.patchelf prevStage.xz ]; + }) # Construct the final stdenv. It uses the Glibc and GCC, and adds # in a new binutils that doesn't depend on bootstrap-tools, as well @@ -242,50 +245,52 @@ rec { # When updating stdenvLinux, make sure that the result has no # dependency (`nix-store -qR') on bootstrapTools or the first # binutils built. - stdenvLinux = import ../generic rec { - inherit system config; + (prevStage: { + inherit system crossSystem platform config; + stdenv = import ../generic rec { + inherit system config; - preHook = - '' + preHook = '' # Make "strip" produce deterministic output, by setting # timestamps etc. to a fixed value. commonStripFlags="--enable-deterministic-archives" ${commonPreHook} ''; - initialPath = - ((import ../common-path.nix) {pkgs = stage4.pkgs;}); + initialPath = + ((import ../common-path.nix) {pkgs = prevStage;}); - extraBuildInputs = [ stage4.pkgs.patchelf stage4.pkgs.paxctl ]; + extraBuildInputs = [ prevStage.patchelf prevStage.paxctl ]; - cc = stage4.pkgs.gcc; + cc = prevStage.gcc; - shell = cc.shell; + shell = cc.shell; - inherit (stage4.stdenv) fetchurlBoot; + inherit (prevStage.stdenv) fetchurlBoot; - extraAttrs = { - inherit (stage4.pkgs) glibc; - inherit platform bootstrapTools; - shellPackage = stage4.pkgs.bash; + extraAttrs = { + inherit (prevStage) glibc; + inherit platform bootstrapTools; + shellPackage = prevStage.bash; + }; + + /* outputs TODO + allowedRequisites = with prevStage; + [ gzip bzip2 xz bash binutils coreutils diffutils findutils gawk + glibc gnumake gnused gnutar gnugrep gnupatch patchelf attr acl + paxctl zlib pcre linuxHeaders ed gcc gcc.cc libsigsegv + ]; + */ + + overrides = self: super: { + gcc = cc; + + inherit (prevStage) + gzip bzip2 xz bash binutils coreutils diffutils findutils gawk + glibc gnumake gnused gnutar gnugrep gnupatch patchelf + attr acl paxctl zlib pcre; + }; }; + }) - /* outputs TODO - allowedRequisites = with stage4.pkgs; - [ gzip bzip2 xz bash binutils coreutils diffutils findutils gawk - glibc gnumake gnused gnutar gnugrep gnupatch patchelf attr acl - paxctl zlib pcre linuxHeaders ed gcc gcc.cc libsigsegv - ]; - */ - - overrides = pkgs: { - gcc = cc; - - inherit (stage4.pkgs) - gzip bzip2 xz bash binutils coreutils diffutils findutils gawk - glibc gnumake gnused gnutar gnugrep gnupatch patchelf - attr acl paxctl zlib pcre; - }; - }; - -} +] diff --git a/pkgs/stdenv/native/default.nix b/pkgs/stdenv/native/default.nix index 8396bd0cb01..57182fae523 100644 --- a/pkgs/stdenv/native/default.nix +++ b/pkgs/stdenv/native/default.nix @@ -1,10 +1,10 @@ -{ lib, allPackages +{ lib , system, platform, crossSystem, config }: assert crossSystem == null; -rec { +let shell = if system == "i686-freebsd" || system == "x86_64-freebsd" then "/usr/local/bin/bash" @@ -77,7 +77,7 @@ rec { # A function that builds a "native" stdenv (one that uses tools in # /usr etc.). makeStdenv = - { cc, fetchurl, extraPath ? [], overrides ? (pkgs: { }) }: + { cc, fetchurl, extraPath ? [], overrides ? (self: super: { }) }: import ../generic { preHook = @@ -101,50 +101,54 @@ rec { inherit system shell cc overrides config; }; +in - stdenvBoot0 = makeStdenv { - cc = null; - fetchurl = null; - }; +[ + ({}: rec { + __raw = true; - cc = import ../../build-support/cc-wrapper { - name = "cc-native"; - nativeTools = true; - nativeLibc = true; - nativePrefix = if system == "i686-solaris" then "/usr/gnu" else if system == "x86_64-solaris" then "/opt/local/gcc47" else "/usr"; - stdenv = stdenvBoot0; - }; + stdenv = makeStdenv { + cc = null; + fetchurl = null; + }; + cc = import ../../build-support/cc-wrapper { + name = "cc-native"; + nativeTools = true; + nativeLibc = true; + nativePrefix = { # switch + "i686-solaris" = "/usr/gnu"; + "x86_64-solaris" = "/opt/local/gcc47"; + }.${system} or "/usr"; + inherit stdenv; + }; - fetchurl = import ../../build-support/fetchurl { - stdenv = stdenvBoot0; - # Curl should be in /usr/bin or so. - curl = null; - }; + fetchurl = import ../../build-support/fetchurl { + inherit stdenv; + # Curl should be in /usr/bin or so. + curl = null; + }; + }) # First build a stdenv based only on tools outside the store. - stdenvBoot1 = makeStdenv { - inherit cc fetchurl; - } // {inherit fetchurl;}; + (prevStage: { + inherit system crossSystem platform config; + stdenv = makeStdenv { + inherit (prevStage) cc fetchurl; + } // { inherit (prevStage) fetchurl; }; + }) - stdenvBoot1Pkgs = allPackages { - inherit system platform crossSystem config; - allowCustomOverrides = false; - stdenv = stdenvBoot1; - }; + # Using that, build a stdenv that adds the ‘xz’ command (which most systems + # don't have, so we mustn't rely on the native environment providing it). + (prevStage: { + inherit system crossSystem platform config; + stdenv = makeStdenv { + inherit (prevStage.stdenv) cc fetchurl; + extraPath = [ prevStage.xz ]; + overrides = self: super: { inherit (prevStage) xz; }; + }; + }) - - # Using that, build a stdenv that adds the ‘xz’ command (which most - # systems don't have, so we mustn't rely on the native environment - # providing it). - stdenvBoot2 = makeStdenv { - inherit cc fetchurl; - extraPath = [ stdenvBoot1Pkgs.xz ]; - overrides = pkgs: { inherit (stdenvBoot1Pkgs) xz; }; - }; - - - stdenvNative = stdenvBoot2; -} +] diff --git a/pkgs/stdenv/nix/default.nix b/pkgs/stdenv/nix/default.nix index e58972e5c8a..a5f0a18464c 100644 --- a/pkgs/stdenv/nix/default.nix +++ b/pkgs/stdenv/nix/default.nix @@ -1,39 +1,53 @@ -{ stdenv, pkgs, config, lib }: +{ lib +, crossSystem, config +, bootStages +, ... +}: -import ../generic rec { - inherit config; +assert crossSystem == null; - preHook = - '' - export NIX_ENFORCE_PURITY="''${NIX_ENFORCE_PURITY-1}" - export NIX_ENFORCE_NO_NATIVE="''${NIX_ENFORCE_NO_NATIVE-1}" - export NIX_IGNORE_LD_THROUGH_GCC=1 - ''; +bootStages ++ [ + (prevStage: let + inherit (prevStage) stdenv; + inherit (stdenv) system platform; + in { + inherit system platform crossSystem config; - initialPath = (import ../common-path.nix) {pkgs = pkgs;}; + stdenv = import ../generic rec { + inherit config; - system = stdenv.system; + preHook = '' + export NIX_ENFORCE_PURITY="''${NIX_ENFORCE_PURITY-1}" + export NIX_ENFORCE_NO_NATIVE="''${NIX_ENFORCE_NO_NATIVE-1}" + export NIX_IGNORE_LD_THROUGH_GCC=1 + ''; - cc = import ../../build-support/cc-wrapper { - nativeTools = false; - nativePrefix = stdenv.lib.optionalString stdenv.isSunOS "/usr"; - nativeLibc = true; - inherit stdenv; - inherit (pkgs) binutils coreutils gnugrep; - cc = pkgs.gcc.cc; - isGNU = true; - shell = pkgs.bash + "/bin/sh"; - }; + initialPath = (import ../common-path.nix) { pkgs = prevStage; }; - shell = pkgs.bash + "/bin/sh"; + system = stdenv.system; - fetchurlBoot = stdenv.fetchurlBoot; + cc = import ../../build-support/cc-wrapper { + nativeTools = false; + nativePrefix = stdenv.lib.optionalString stdenv.isSunOS "/usr"; + nativeLibc = true; + inherit stdenv; + inherit (prevStage) binutils coreutils gnugrep; + cc = prevStage.gcc.cc; + isGNU = true; + shell = prevStage.bash + "/bin/sh"; + }; - overrides = pkgs_: { - inherit cc; - inherit (cc) binutils; - inherit (pkgs) - gzip bzip2 xz bash coreutils diffutils findutils gawk - gnumake gnused gnutar gnugrep gnupatch perl; - }; -} + shell = prevStage.bash + "/bin/sh"; + + fetchurlBoot = stdenv.fetchurlBoot; + + overrides = self: super: { + inherit cc; + inherit (cc) binutils; + inherit (prevStage) + gzip bzip2 xz bash coreutils diffutils findutils gawk + gnumake gnused gnutar gnugrep gnupatch perl; + }; + }; + }) +] diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index ef350dfef0c..21300ff90c0 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -5,9 +5,7 @@ * to merges. Please use the full-text search of your editor. ;) * Hint: ### starts category names. */ -{ system, noSysDirs, config, crossSystem, platform, lib -, nixpkgsFun -}: +{ lib, nixpkgsFun, noSysDirs, config}: self: pkgs: with pkgs; @@ -18,9 +16,6 @@ in { - # Make some arguments passed to all-packages.nix available - inherit system platform; - # Allow callPackage to fill in the pkgs argument inherit pkgs; @@ -4728,6 +4723,7 @@ in gambit = callPackage ../development/compilers/gambit { }; gcc = gcc5; + gcc-unwrapped = gcc.cc; wrapCCMulti = cc: if system == "x86_64-linux" then lowPrio ( diff --git a/pkgs/top-level/default.nix b/pkgs/top-level/default.nix index db3abb531f1..04daf9778ff 100644 --- a/pkgs/top-level/default.nix +++ b/pkgs/top-level/default.nix @@ -7,11 +7,11 @@ 3. Defaults to no non-standard config and no cross-compilation target - 4. Uses the above to infer the default standard environment (stdenv) if - none is provided + 4. Uses the above to infer the default standard environment's (stdenv's) + stages if no stdenv's are provided - 5. Builds the final stage --- a fully booted package set with the chosen - stdenv + 5. Folds the stages to yield the final fully booted package set for the + chosen stdenv Use `impure.nix` to also infer the `system` based on the one on which evaluation is taking place, and the configuration from environment variables @@ -23,9 +23,10 @@ , # Allow a configuration attribute set to be passed in as an argument. config ? {} -, # The standard environment for building packages, or rather a function - # providing it. See below for the arguments given to that function. - stdenvFunc ? import ../stdenv +, # A function booting the final package set for a specific standard + # environment. See below for the arguments given to that function, + # the type of list it returns. + stdenvStages ? import ../stdenv , crossSystem ? null , platform ? assert false; null @@ -76,10 +77,12 @@ in let inherit lib nixpkgsFun; } // newArgs); - stdenv = stdenvFunc { - inherit lib allPackages system platform crossSystem config; + boot = import ../stdenv/booter.nix { inherit lib allPackages; }; + + stages = stdenvStages { + inherit lib system platform crossSystem config; }; - pkgs = allPackages { inherit system stdenv config crossSystem platform; }; + pkgs = boot stages; in pkgs diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index 1d6305151ca..ebc6473e425 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -18,7 +18,7 @@ , # This is used because stdenv replacement and the stdenvCross do benefit from # the overridden configuration provided by the user, as opposed to the normal # bootstrapping stdenvs. - allowCustomOverrides ? true + allowCustomOverrides , # Non-GNU/Linux OSes are currently "impure" platforms, with their libc # outside of the store. Thus, GCC, GFortran, & co. must always look for @@ -47,27 +47,25 @@ let inherit lib; inherit (self) stdenv stdenvNoCC; inherit (self.xorg) lndir; }; - stdenvDefault = self: super: - { stdenv = stdenv // { inherit platform; }; }; + stdenvBootstappingAndPlatforms = self: super: { + stdenv = stdenv // { inherit platform; }; + inherit + system platform crossSystem; + }; allPackages = self: super: let res = import ./all-packages.nix - { inherit system noSysDirs config crossSystem platform lib nixpkgsFun; } + { inherit lib nixpkgsFun noSysDirs config; } res self; in res; aliases = self: super: import ./aliases.nix super; - # stdenvOverrides is used to avoid circular dependencies for building - # the standard build environment. This mechanism uses the override - # mechanism to implement some staged compilation of the stdenv. - # - # We don't want stdenv overrides in the case of cross-building, or - # otherwise the basic overridden packages will not be built with the - # crossStdenv adapter. + # stdenvOverrides is used to avoid having multiple of versions + # of certain dependencies that were used in bootstrapping the + # standard environment. stdenvOverrides = self: super: - lib.optionalAttrs (crossSystem == null && super.stdenv ? overrides) - (super.stdenv.overrides super); + (super.stdenv.overrides or (_: _: {})) self super; # Allow packages to be overridden globally via the `packageOverrides' # configuration option, which must be a function that takes `pkgs' @@ -82,7 +80,7 @@ let # The complete chain of package set builders, applied from top to bottom toFix = lib.foldl' (lib.flip lib.extends) (self: {}) [ - stdenvDefault + stdenvBootstappingAndPlatforms stdenvAdapters trivialBuilders allPackages