From ceb27efde0532bb663e3d8c4fe20f82ee4dc22e3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 24 Mar 2019 16:33:46 -0400 Subject: [PATCH 1/5] manual: Auto reformat --- doc/reviewing-contributions.xml | 3 ++- doc/stdenv.xml | 29 ++++++++++++++--------------- doc/submitting-changes.xml | 19 +++++++++---------- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/doc/reviewing-contributions.xml b/doc/reviewing-contributions.xml index 029299a50b1..6e3b6face3a 100644 --- a/doc/reviewing-contributions.xml +++ b/doc/reviewing-contributions.xml @@ -189,7 +189,8 @@ $ git rebase --onto nixos-unstable BASEBRANCH FETCH_HEAD - The nix-review + The + nix-review tool can be used to review a pull request content in a single command. PRNUMBER should be replaced by the number at the end of the pull request title. You can also provide the full github pull diff --git a/doc/stdenv.xml b/doc/stdenv.xml index a3990dec052..7e6c589d9fe 100644 --- a/doc/stdenv.xml +++ b/doc/stdenv.xml @@ -2633,21 +2633,20 @@ addEnvHooks "$hostOffset" myBashFunction happens. It prevents nix from cleaning up the build environment immediately and allows the user to attach to a build environment using the cntr command. Upon build error it will print - instructions on how to use cntr, which can be used - to enter the environment for debugging. Installing cntr and - running the command will provide shell access to the build sandbox of - failed build. At /var/lib/cntr the sandboxed - filesystem is mounted. All commands and files of the system are still - accessible within the shell. To execute commands from the sandbox use - the cntr exec subcommand. Note that cntr also needs - to be executed on the machine that is doing the build, which might not - be the case when remote builders are enabled. cntr is - only supported on Linux-based platforms. To use it first add - cntr to your - environment.systemPackages on NixOS or alternatively - to the root user on non-NixOS systems. Then in the package that is - supposed to be inspected, add breakpointHook to - nativeBuildInputs. + instructions on how to use cntr, which can be used to + enter the environment for debugging. Installing cntr and running the + command will provide shell access to the build sandbox of failed build. + At /var/lib/cntr the sandboxed filesystem is + mounted. All commands and files of the system are still accessible + within the shell. To execute commands from the sandbox use the cntr exec + subcommand. Note that cntr also needs to be executed + on the machine that is doing the build, which might not be the case when + remote builders are enabled. cntr is only supported + on Linux-based platforms. To use it first add cntr to + your environment.systemPackages on NixOS or + alternatively to the root user on non-NixOS systems. Then in the package + that is supposed to be inspected, add breakpointHook + to nativeBuildInputs. nativeBuildInputs = [ breakpointHook ]; diff --git a/doc/submitting-changes.xml b/doc/submitting-changes.xml index 33abfb634ea..bc090fd757c 100644 --- a/doc/submitting-changes.xml +++ b/doc/submitting-changes.xml @@ -354,23 +354,22 @@ Additional information. Tested compilation of all pkgs that depend on this change using <command>nix-review</command> - If you are updating a package's version, you can use nix-review to make sure all - packages that depend on the updated package still compile correctly. - The nix-review utility can look for and build all dependencies - either based on uncommited changes with the wip option or - specifying a github pull request number. + If you are updating a package's version, you can use nix-review to make + sure all packages that depend on the updated package still compile + correctly. The nix-review utility can look for and build + all dependencies either based on uncommited changes with the + wip option or specifying a github pull request number. - review changes from pull request number 12345: - nix-shell -p nix-review --run "nix-review pr 12345" + review changes from pull request number 12345: +nix-shell -p nix-review --run "nix-review pr 12345" - review uncommitted changes: - nix-shell -p nix-review --run "nix-review wip" + review uncommitted changes: +nix-shell -p nix-review --run "nix-review wip" -
From 70d71bbbe46eecfefc920bb1a919960d5feb9fc4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Mar 2019 13:34:38 -0400 Subject: [PATCH 2/5] top-level: Create `pkgs{Build,Host,Target}{Build,Host,Target}` This is needed to avoid confusing and repeated boilerplate for `fooForTarget`. The vast majority of use-cases can still use `buildPackages or `targetPackages`, which are now defined in terms of these. --- pkgs/stdenv/booter.nix | 26 +++++++++++---- pkgs/top-level/all-packages.nix | 3 -- pkgs/top-level/splice.nix | 23 +++++++------ pkgs/top-level/stage.nix | 59 ++++++++++++++++++++++++--------- 4 files changed, 74 insertions(+), 37 deletions(-) diff --git a/pkgs/stdenv/booter.nix b/pkgs/stdenv/booter.nix index 668dcabc049..1df05099fbf 100644 --- a/pkgs/stdenv/booter.nix +++ b/pkgs/stdenv/booter.nix @@ -95,13 +95,25 @@ stageFuns: let __hatPackages = nextStage; }; }; - in - if args.__raw or false - then args' - else allPackages ((builtins.removeAttrs args' ["selfBuild"]) // { - buildPackages = if args.selfBuild or true then null else prevStage; - targetPackages = if args.selfBuild or true then null else nextStage; - }); + thisStage = + if args.__raw or false + then args' + else allPackages ((builtins.removeAttrs args' ["selfBuild"]) // { + adjacentPackages = if args.selfBuild or true then null else rec { + pkgsBuildBuild = prevStage.buildPackages; + pkgsBuildHost = prevStage; + pkgsBuildTarget = + if args.stdenv.targetPlatform == args.stdenv.hostPlatform + then pkgsBuildHost + else assert args.stdenv.hostPlatform == args.stdenv.buildPlatform; thisStage; + pkgsHostHost = + if args.stdenv.hostPlatform == args.stdenv.targetPlatform + then thisStage + else assert args.stdenv.buildPlatform == args.stdenv.hostPlatform; pkgsBuildHost; + pkgsTargetTarget = nextStage; + }; + }); + in thisStage; # This is a hack for resolving cross-compiled compilers' run-time # deps. (That is, compilers that are themselves cross-compiled, as diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 6ace5f7c99c..ab5265ac885 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -25,9 +25,6 @@ let in { - # Allow callPackage to fill in the pkgs argument - inherit pkgs; - # A stdenv capable of building 32-bit binaries. On x86_64-linux, # it uses GCC compiled with multilib support; on i686-linux, it's # just the plain stdenv. diff --git a/pkgs/top-level/splice.nix b/pkgs/top-level/splice.nix index ec6ed357c68..a093442d369 100644 --- a/pkgs/top-level/splice.nix +++ b/pkgs/top-level/splice.nix @@ -96,19 +96,20 @@ let } @ args: if actuallySplice then spliceReal args else pkgsHostTarget; - splicedPackages = splicePackages rec { - pkgsBuildBuild = pkgs.buildPackages.buildPackages; - pkgsBuildHost = pkgs.buildPackages; - pkgsBuildTarget = - if pkgs.stdenv.targetPlatform == pkgs.stdenv.hostPlatform - then pkgsBuildHost - else assert pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform; pkgsHostTarget; - pkgsHostHost = {}; # unimplemented - pkgsHostTarget = pkgs; - pkgsTargetTarget = pkgs.targetPackages; + splicedPackages = splicePackages { + inherit (pkgs) + pkgsBuildBuild pkgsBuildHost pkgsBuildTarget + pkgsHostHost pkgsHostTarget + pkgsTargetTarget + ; } // { # These should never be spliced under any circumstances - inherit (pkgs) pkgs buildPackages targetPackages; + inherit (pkgs) + pkgsBuildBuild pkgsBuildHost pkgsBuildTarget + pkgsHostHost pkgsHostTarget + pkgsTargetTarget + buildPackages pkgs targetPackages + ; inherit (pkgs.stdenv) buildPlatform targetPlatform hostPlatform; }; diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index 0ee5c25b010..f04cdf338e8 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -21,18 +21,23 @@ ## Other parameters ## -, # The package set used at build-time. If null, `buildPackages` will - # be defined internally as the final produced package set itself. This allows - # us to avoid expensive splicing. - buildPackages - -, # The package set used in the next stage. If null, `targetPackages` will be - # defined internally as the final produced package set itself, just like with - # `buildPackages` and for the same reasons. +, # Either null or an object in the form: # - # THIS IS A HACK for compilers that don't think critically about cross- - # compilation. Please do *not* use unless you really know what you are doing. - targetPackages + # { + # pkgsBuildBuild = ...; + # pkgsBuildHost = ...; + # pkgsBuildTarget = ...; + # pkgsHostHost = ...; + # # pkgsHostTarget skipped on purpose. + # pkgsTargetTarget ...; + # } + # + # These are references to adjacent bootstrapping stages. The more familiar + # `buildPackages` and `targetPackages` are defined in terms of them. If null, + # they are instead defined internally as the current stage. This allows us to + # avoid expensive splicing. `pkgsHostTarget` is skipped because it is always + # defined as the current stage. + adjacentPackages , # The standard environment to use for building packages. stdenv @@ -70,11 +75,33 @@ let inherit (self) runtimeShell; }; - stdenvBootstappingAndPlatforms = self: super: { - buildPackages = (if buildPackages == null then self else buildPackages) - // { recurseForDerivations = false; }; - targetPackages = (if targetPackages == null then self else targetPackages) + stdenvBootstappingAndPlatforms = self: super: let + withFallback = thisPkgs: + (if adjacentPackages == null then self else thisPkgs) // { recurseForDerivations = false; }; + in { + # Here are package sets of from related stages. They are all in the form + # `pkgs{theirHost}{theirTarget}`. For example, `pkgsBuildHost` means their + # host platform is our build platform, and their target platform is our host + # platform. We only care about their host/target platforms, not their build + # platform, because the the former two alone affect the interface of the + # final package; the build platform is just an implementation detail that + # should not leak. + pkgsBuildBuild = withFallback adjacentPackages.pkgsBuildBuild; + pkgsBuildHost = withFallback adjacentPackages.pkgsBuildHost; + pkgsBuildTarget = withFallback adjacentPackages.pkgsBuildTarget; + pkgsHostHost = withFallback adjacentPackages.pkgsHostHost; + pkgsHostTarget = self // { recurseForDerivations = false; }; # always `self` + pkgsTargetTarget = withFallback adjacentPackages.pkgsTargetTarget; + + # Older names for package sets. Use these when only the host platform of the + # package set matter (i.e. use `buildPackages` where any of `pkgsBuild*` + # would do, and `targetPackages` when any of `pkgsTarget*` would do (if we + # had more than just `pkgsTargetTarget`).) + buildPackages = self.pkgsBuildHost; + pkgs = self.pkgsHostTarget; + targetPackages = self.pkgsTargetTarget; + inherit stdenv; }; @@ -87,7 +114,7 @@ let inherit (hostPlatform) system; }; - splice = self: super: import ./splice.nix lib self (buildPackages != null); + splice = self: super: import ./splice.nix lib self (adjacentPackages != null); allPackages = self: super: let res = import ./all-packages.nix From 655a29ff9ccf9b27e52893de24f9535bda7e3cd2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Mar 2019 17:03:53 -0400 Subject: [PATCH 3/5] ghc, go, guile: Use new `pkgs*` `pkgsBuildTarget` allows us to avoid repeated and confusing conditions. The others merely provide clarity for one the foreign package set's target platform matters. --- pkgs/development/compilers/ghc/8.2.2.nix | 10 ++++------ pkgs/development/compilers/ghc/8.4.4.nix | 10 ++++------ pkgs/development/compilers/ghc/8.6.4.nix | 10 ++++------ pkgs/development/compilers/ghc/head.nix | 10 ++++------ pkgs/development/compilers/go/1.11.nix | 15 ++++++--------- pkgs/development/compilers/go/1.12.nix | 15 ++++++--------- pkgs/development/interpreters/guile/1.8.nix | 4 ++-- pkgs/development/interpreters/guile/2.0.nix | 4 ++-- pkgs/development/interpreters/guile/default.nix | 4 ++-- 9 files changed, 34 insertions(+), 48 deletions(-) diff --git a/pkgs/development/compilers/ghc/8.2.2.nix b/pkgs/development/compilers/ghc/8.2.2.nix index 3e355dc302d..a88cf9c0116 100644 --- a/pkgs/development/compilers/ghc/8.2.2.nix +++ b/pkgs/development/compilers/ghc/8.2.2.nix @@ -1,4 +1,4 @@ -{ stdenv, targetPackages +{ stdenv, pkgsBuildTarget, targetPackages # build-tools , bootPkgs @@ -70,11 +70,9 @@ let ++ stdenv.lib.optional (!enableIntegerSimple) gmp ++ stdenv.lib.optional (platform.libc != "glibc") libiconv; - toolsForTarget = - if hostPlatform == buildPlatform then - [ targetPackages.stdenv.cc ] ++ stdenv.lib.optional useLLVM llvmPackages.llvm - else assert targetPlatform == hostPlatform; # build != host == target - [ stdenv.cc ] ++ stdenv.lib.optional useLLVM buildLlvmPackages.llvm; + toolsForTarget = [ + pkgsBuildTarget.targetPackages.stdenv.cc + ] ++ stdenv.lib.optional useLLVM buildLlvmPackages.llvm; targetCC = builtins.head toolsForTarget; diff --git a/pkgs/development/compilers/ghc/8.4.4.nix b/pkgs/development/compilers/ghc/8.4.4.nix index 874580c87aa..da72c351ec6 100644 --- a/pkgs/development/compilers/ghc/8.4.4.nix +++ b/pkgs/development/compilers/ghc/8.4.4.nix @@ -1,4 +1,4 @@ -{ stdenv, targetPackages +{ stdenv, pkgsBuildTarget, targetPackages # build-tools , bootPkgs @@ -72,11 +72,9 @@ let ++ stdenv.lib.optional (!enableIntegerSimple) gmp ++ stdenv.lib.optional (platform.libc != "glibc" && !targetPlatform.isWindows) libiconv; - toolsForTarget = - if hostPlatform == buildPlatform then - [ targetPackages.stdenv.cc ] ++ stdenv.lib.optional useLLVM llvmPackages.llvm - else assert targetPlatform == hostPlatform; # build != host == target - [ stdenv.cc ] ++ stdenv.lib.optional useLLVM buildLlvmPackages.llvm; + toolsForTarget = [ + pkgsBuildTarget.targetPackages.stdenv.cc + ] ++ stdenv.lib.optional useLLVM buildLlvmPackages.llvm; targetCC = builtins.head toolsForTarget; diff --git a/pkgs/development/compilers/ghc/8.6.4.nix b/pkgs/development/compilers/ghc/8.6.4.nix index 140cea22442..f970836fd69 100644 --- a/pkgs/development/compilers/ghc/8.6.4.nix +++ b/pkgs/development/compilers/ghc/8.6.4.nix @@ -1,4 +1,4 @@ -{ stdenv, targetPackages +{ stdenv, pkgsBuildTarget, targetPackages # build-tools , bootPkgs @@ -72,11 +72,9 @@ let ++ stdenv.lib.optional (!enableIntegerSimple) gmp ++ stdenv.lib.optional (platform.libc != "glibc" && !targetPlatform.isWindows) libiconv; - toolsForTarget = - if hostPlatform == buildPlatform then - [ targetPackages.stdenv.cc ] ++ stdenv.lib.optional useLLVM llvmPackages.llvm - else assert targetPlatform == hostPlatform; # build != host == target - [ stdenv.cc ] ++ stdenv.lib.optional useLLVM buildLlvmPackages.llvm; + toolsForTarget = [ + pkgsBuildTarget.targetPackages.stdenv.cc + ] ++ stdenv.lib.optional useLLVM buildLlvmPackages.llvm; targetCC = builtins.head toolsForTarget; diff --git a/pkgs/development/compilers/ghc/head.nix b/pkgs/development/compilers/ghc/head.nix index 7e670743f7f..087d5a2c678 100644 --- a/pkgs/development/compilers/ghc/head.nix +++ b/pkgs/development/compilers/ghc/head.nix @@ -1,4 +1,4 @@ -{ stdenv, targetPackages +{ stdenv, pkgsBuildTarget, targetPackages # build-tools , bootPkgs @@ -69,11 +69,9 @@ let ++ stdenv.lib.optional (!enableIntegerSimple) gmp ++ stdenv.lib.optional (platform.libc != "glibc" && !targetPlatform.isWindows) libiconv; - toolsForTarget = - if hostPlatform == buildPlatform then - [ targetPackages.stdenv.cc ] ++ stdenv.lib.optional useLLVM llvmPackages.llvm - else assert targetPlatform == hostPlatform; # build != host == target - [ stdenv.cc ] ++ stdenv.lib.optional useLLVM buildLlvmPackages.llvm; + toolsForTarget = [ + pkgsBuildTarget.targetPackages.stdenv.cc + ] ++ stdenv.lib.optional useLLVM buildLlvmPackages.llvm; targetCC = builtins.head toolsForTarget; diff --git a/pkgs/development/compilers/go/1.11.nix b/pkgs/development/compilers/go/1.11.nix index db3731c2fca..355b2559e98 100644 --- a/pkgs/development/compilers/go/1.11.nix +++ b/pkgs/development/compilers/go/1.11.nix @@ -1,7 +1,8 @@ { stdenv, fetchFromGitHub, tzdata, iana-etc, go_bootstrap, runCommand, writeScriptBin , perl, which, pkgconfig, patch, procps, pcre, cacert, llvm, Security, Foundation , mailcap, runtimeShell -, buildPackages, targetPackages }: +, buildPackages, pkgsTargetTarget +}: let @@ -152,16 +153,12 @@ stdenv.mkDerivation rec { # {CC,CXX}_FOR_TARGET must be only set for cross compilation case as go expect those # to be different from CC/CXX - CC_FOR_TARGET = if (stdenv.hostPlatform != stdenv.targetPlatform) then - "${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc" - else if (stdenv.buildPlatform != stdenv.targetPlatform) then - "${stdenv.cc.targetPrefix}cc" + CC_FOR_TARGET = if (stdenv.buildPlatform != stdenv.targetPlatform) then + "${pkgsTargetTarget.stdenv.cc}/bin/${pkgsTargetTarget.stdenv.cc.targetPrefix}cc" else null; - CXX_FOR_TARGET = if (stdenv.hostPlatform != stdenv.targetPlatform) then - "${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++" - else if (stdenv.buildPlatform != stdenv.targetPlatform) then - "${stdenv.cc.targetPrefix}c++" + CXX_FOR_TARGET = if (stdenv.buildPlatform != stdenv.targetPlatform) then + "${pkgsTargetTarget.stdenv.cc}/bin/${pkgsTargetTarget.stdenv.cc.targetPrefix}c++" else null; diff --git a/pkgs/development/compilers/go/1.12.nix b/pkgs/development/compilers/go/1.12.nix index d7bbd4eaf0f..bcd2eacc7ec 100644 --- a/pkgs/development/compilers/go/1.12.nix +++ b/pkgs/development/compilers/go/1.12.nix @@ -1,7 +1,8 @@ { stdenv, fetchurl, tzdata, iana-etc, go_bootstrap, runCommand, writeScriptBin , perl, which, pkgconfig, patch, procps, pcre, cacert, llvm, Security, Foundation , mailcap, runtimeShell -, buildPackages, targetPackages }: +, buildPackages, pkgsTargetTarget +}: let @@ -154,16 +155,12 @@ stdenv.mkDerivation rec { # {CC,CXX}_FOR_TARGET must be only set for cross compilation case as go expect those # to be different from CC/CXX - CC_FOR_TARGET = if (stdenv.hostPlatform != stdenv.targetPlatform) then - "${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc" - else if (stdenv.buildPlatform != stdenv.targetPlatform) then - "${stdenv.cc.targetPrefix}cc" + CC_FOR_TARGET = if (stdenv.buildPlatform != stdenv.targetPlatform) then + "${pkgsTargetTarget.stdenv.cc}/bin/${pkgsTargetTarget.stdenv.cc.targetPrefix}cc" else null; - CXX_FOR_TARGET = if (stdenv.hostPlatform != stdenv.targetPlatform) then - "${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++" - else if (stdenv.buildPlatform != stdenv.targetPlatform) then - "${stdenv.cc.targetPrefix}c++" + CXX_FOR_TARGET = if (stdenv.buildPlatform != stdenv.targetPlatform) then + "${pkgsTargetTarget.stdenv.cc}/bin/${pkgsTargetTarget.stdenv.cc.targetPrefix}c++" else null; diff --git a/pkgs/development/interpreters/guile/1.8.nix b/pkgs/development/interpreters/guile/1.8.nix index fd270dedefc..158987d4e0e 100644 --- a/pkgs/development/interpreters/guile/1.8.nix +++ b/pkgs/development/interpreters/guile/1.8.nix @@ -1,4 +1,4 @@ -{ stdenv, buildPackages +{ stdenv, pkgsBuildBuild, buildPackages , fetchurl, makeWrapper, gawk, pkgconfig , libtool, readline, gmp }: @@ -23,7 +23,7 @@ stdenv.mkDerivation rec { depsBuildBuild = [ buildPackages.stdenv.cc ] ++ stdenv.lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) - buildPackages.buildPackages.guile_1_8; + pkgsBuildBuild.guile_1_8; nativeBuildInputs = [ makeWrapper gawk pkgconfig ]; buildInputs = [ readline libtool ]; diff --git a/pkgs/development/interpreters/guile/2.0.nix b/pkgs/development/interpreters/guile/2.0.nix index 433271d9c85..17ca1d1dcd9 100644 --- a/pkgs/development/interpreters/guile/2.0.nix +++ b/pkgs/development/interpreters/guile/2.0.nix @@ -1,4 +1,4 @@ -{ stdenv, buildPackages +{ stdenv, pkgsBuildBuild, buildPackages , fetchpatch, fetchurl, makeWrapper, gawk, pkgconfig , libffi, libtool, readline, gmp, boehmgc, libunistring , coverageAnalysis ? null @@ -22,7 +22,7 @@ depsBuildBuild = [ buildPackages.stdenv.cc ] ++ stdenv.lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) - buildPackages.buildPackages.guile_2_0; + pkgsBuildBuild.guile_2_0; nativeBuildInputs = [ makeWrapper gawk pkgconfig ]; buildInputs = [ readline libtool libunistring libffi ]; diff --git a/pkgs/development/interpreters/guile/default.nix b/pkgs/development/interpreters/guile/default.nix index 1943b10bdca..5e458c6e2cc 100644 --- a/pkgs/development/interpreters/guile/default.nix +++ b/pkgs/development/interpreters/guile/default.nix @@ -1,4 +1,4 @@ -{ stdenv, buildPackages +{ stdenv, pkgsBuildBuild, buildPackages , fetchurl, makeWrapper, gawk, pkgconfig , libffi, libtool, readline, gmp, boehmgc, libunistring , coverageAnalysis ? null @@ -23,7 +23,7 @@ depsBuildBuild = [ buildPackages.stdenv.cc ] ++ stdenv.lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) - buildPackages.buildPackages.guile; + pkgsBuildBuild.guile; nativeBuildInputs = [ makeWrapper gawk pkgconfig ]; buildInputs = [ readline libtool libunistring libffi ]; From 5e5266f83fc2cce2b353601da0f29bd6805d4597 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 20 Mar 2019 18:21:00 -0400 Subject: [PATCH 4/5] manual: Document `pkgsFooBar` and more There was a bunch of stuff in the cross section that haddn't had any attention in a while. I might need to slim it down later, but this is good for now. --- doc/cross-compilation.xml | 374 +++++++++++++++++++++++++++++--------- doc/stdenv.xml | 12 +- 2 files changed, 299 insertions(+), 87 deletions(-) diff --git a/doc/cross-compilation.xml b/doc/cross-compilation.xml index dbaf6f104ec..d97f12f2566 100644 --- a/doc/cross-compilation.xml +++ b/doc/cross-compilation.xml @@ -12,11 +12,12 @@ computing power and memory to compile their own programs. One might think that cross-compilation is a fairly niche concern. However, there are significant advantages to rigorously distinguishing between build-time and - run-time environments! This applies even when one is developing and - deploying on the same machine. Nixpkgs is increasingly adopting the opinion - that packages should be written with cross-compilation in mind, and nixpkgs - should evaluate in a similar way (by minimizing cross-compilation-specific - special cases) whether or not one is cross-compiling. + run-time environments! Significant, because the benefits apply even when one + is developing and deploying on the same machine. Nixpkgs is increasingly + adopting the opinion that packages should be written with cross-compilation + in mind, and nixpkgs should evaluate in a similar way (by minimizing + cross-compilation-specific special cases) whether or not one is + cross-compiling. @@ -30,7 +31,7 @@
Packaging in a cross-friendly manner -
+
Platform parameters @@ -218,8 +219,20 @@
-
- Specifying Dependencies +
+ Theory of dependency categorization + + + + This is a rather philosophical description that isn't very + Nixpkgs-specific. For an overview of all the relevant attributes given to + mkDerivation, see + . For a description of how + everything is implemented, see + . + + In this section we explore the relationship between both runtime and @@ -227,84 +240,98 @@ - A runtime dependency between 2 packages implies that between them both the - host and target platforms match. This is directly implied by the meaning of - "host platform" and "runtime dependency": The package dependency exists - while both packages are running on a single host platform. + A run time dependency between two packages requires that their host + platforms match. This is directly implied by the meaning of "host platform" + and "runtime dependency": The package dependency exists while both packages + are running on a single host platform. - A build time dependency, however, implies a shift in platforms between the - depending package and the depended-on package. The meaning of a build time - dependency is that to build the depending package we need to be able to run - the depended-on's package. The depending package's build platform is - therefore equal to the depended-on package's host platform. Analogously, - the depending package's host platform is equal to the depended-on package's - target platform. + A build time dependency, however, has a shift in platforms between the + depending package and the depended-on package. "build time dependency" + means that to build the depending package we need to be able to run the + depended-on's package. The depending package's build platform is therefore + equal to the depended-on package's host platform. - In this manner, given the 3 platforms for one package, we can determine the - three platforms for all its transitive dependencies. This is the most - important guiding principle behind cross-compilation with Nixpkgs, and will - be called the sliding window principle. + If both the dependency and depending packages aren't compilers or other + machine-code-producing tools, we're done. And indeed + buildInputs and nativeBuildInputs + have covered these simpler build-time and run-time (respectively) changes + for many years. But if the depedency does produce machine code, we might + need to worry about it's target platform too. In principle, that target + platform might be any of the depending package's build, host, or target + platforms, but we prohibit dependencies from a "later" platform to an + earlier platform to limit confusion because we've never seen a legitimate + use for them. - Some examples will make this clearer. If a package is being built with a - (build, host, target) platform triple of (foo, - bar, bar), then its build-time dependencies would have a triple - of (foo, foo, bar), and those - packages' build-time dependencies would have a triple of - (foo, foo, foo). In other words, it should take two - "rounds" of following build-time dependency edges before one reaches a - fixed point where, by the sliding window principle, the platform triple no - longer changes. Indeed, this happens with cross-compilation, where only - rounds of native dependencies starting with the second necessarily coincide - with native packages. + Finally, if the depending package is a compiler or other + machine-code-producing tool, it might need dependencies that run at "emit + time". This is for compilers that (regrettably) insist on being in built + together with their source langauges' standard libraries. Assuming build != + host != target, a run-time dependency of the standard library cannot be run + at the compiler's build time or run time, but only at the run time of code + emitted by the compiler. - - - The depending package's target platform is unconstrained by the sliding - window principle, which makes sense in that one can in principle build - cross compilers targeting arbitrary platforms. - - - - How does this work in practice? Nixpkgs is now structured so that - build-time dependencies are taken from buildPackages, - whereas run-time dependencies are taken from the top level attribute set. - For example, buildPackages.gcc should be used at - build-time, while gcc should be used at run-time. Now, - for most of Nixpkgs's history, there was no - buildPackages, and most packages have not been - refactored to use it explicitly. Instead, one can use the six - (gasp) attributes used for specifying dependencies as - documented in . We "splice" - together the run-time and build-time package sets with - callPackage, and then mkDerivation - for each of four attributes pulls the right derivation out. This splicing - can be skipped when not cross-compiling as the package sets are the same, - but is a bit slow for cross-compiling. Because of this, a - best-of-both-worlds solution is in the works with no splicing or explicit - access of buildPackages needed. For now, feel free to - use either method. + Putting this all together, that means we have dependencies in the form + "host → target", in at most the following six combinations: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Possible dependency types
Dependency's host platformDependency's target platform
buildbuild
buildhost
buildtarget
hosthost
hosttarget
targettarget
 - - - There is also a "backlink" targetPackages, yielding a - package set whose buildPackages is the current package - set. This is a hack, though, to accommodate compilers with lousy build - systems. Please do not use this unless you are absolutely sure you are - packaging such a compiler and there is no other way. - - + + Some examples will make this table clearer. Suppose there's some package + that is being built with a (build, host, target) + platform triple of (foo, bar, baz). If it has a + build-time library dependency, that would be a "host → build" dependency + with a triple of (foo, foo, *) (the target platform is + irrelevant). If it needs a compiler to be built, that would be a "build → + host" dependency with a triple of (foo, foo, *) (the + target platform is irrelevant). That compiler, would be built with another + compiler, also "build → host" dependency, with a triple of (foo, + foo, foo). +
-
+
Cross packaging cookbook @@ -450,21 +477,202 @@ nix-build <nixpkgs> --arg crossSystem '{ config = "<arch>-<os>
Cross-compilation infrastructure - - To be written. - +
+ Implementation of dependencies - - If one explores Nixpkgs, they will see derivations with names like - gccCross. Such *Cross derivations is - a holdover from before we properly distinguished between the host and - target platforms—the derivation with "Cross" in the name covered the - build = host != target case, while the other covered the - host = target, with build platform the same or not based - on whether one was using its .nativeDrv or - .crossDrv. This ugliness will disappear soon. + The categorizes of dependencies developed in + are specified as + lists of derivations given to mkDerivation, as + documented in . In short, the + each list of dependencies for "host → target" of "foo → bar" is called + depsFooBar, with the exceptions for backwards + compatibility that depsBuildHost is instead called + nativeBuildInputs and depsHostTarget + is instead called buildInputs. Nixpkgs is now structured + so that each depsFooBar is automatically taken from + pkgsFooBar. (These pkgsFooBars are + quite new, so there is no special case for + nativeBuildInputs and buildInputs.) + For example, pkgsBuildHost.gcc should be used at + build-time, while pkgsHostTarget.gcc should be used at + run-time. - + + + Now, for most of Nixpkgs's history, there was no + pkgsFooBar attributes, and most packages have not been + refactored to use it explicitly. Prior to those, there were just + buildPackages, pkgs, and + targetPackages. Those are now redefined as aliases to + pkgsBuildHost, pkgsHostTarget, and + pkgsTargetTarget. It is fine, indeed if anything + recommended, to use them for libraries to show that the host platform is + irrelevant. + + + + But before that, there was just pkgs, even though both + buildInputs and nativeBuildInputs + existed. [Cross barely worked, and those were implemented with some hacks + on mkDerivation to override dependencies.] What this + means is the vast majority of packages do not use any explicit package set + to populate their dependencies, just using whatever + callPackage gives them even if they do correctly sort + their dependencies into the multiple lists described above. And indeed, + asking that users both sort their dependencies, and + take them from the right attribute set, is both too onerous and redundant, + so the recommend approach (for now) is to continue just categorizing by + list and not using an explicit package set. + + + + No make this work, we "splice" together the six + pkgsFooBar package sets and have + callPackage actually take its arguments from that. This + is currently implemented in pkgs/top-level/splice.nix. + mkDerivation then, for each dependency attribute, pulls + the right derivation out from the splice. This splicing can be skipped when + not cross-compiling as the package sets are the same, but still is a bit + slow for cross-compiling. We'd like to do something better, but haven't + come up with anything yet. + +
+ +
+ Bootstrapping + + + Each of the package sets described above come from a single bootstrapping + stage. While pkgs/top-level/default.nix, coordinates + the composition of stages at a high level, + pkgs/top-level/stage.nix "ties the knot" (creates the + fixed point) of each stage. The package sets are defined per-stage however, + so they can be thought of as edges between stages (the nodes) in a graph. + Compositions like pkgsBuildTarget.TargetPackages can be + thought of as paths to this graph. + + + + While there are many package sets, and thus many edges, the stages can also + be arranged in a linear chain. In other words, many of the edges are + redundant as far as connectivity is concerned. This hinges on the type of + bootstrapping we do. Currently for cross it is: + + + + (native, native, native) + + + + + (native, native, foreign) + + + + + (native, foreign, foreign) + + + + In each stage, pkgsBuildHost refers the the previous + stage, pkgsBuildBuild refers to the one before that, and + pkgsHostTarget refers to the current one, and + pkgsTargetTarget refers to the next one. When there is + no previous or next stage, they instead refer to the current stage. Note + how all the invariants about the mapping between dependency and depending + packages' build host and target platforms are preserved. + pkgsBuildTarget and pkgsHostHost are + more complex in that the stage fitting the requirements isn't always a + fixed chain of "prevs" and "nexts" away (modulo the "saturating" + self-references at the ends). We just special case instead. All the primary + edges are implemented is in pkgs/stdenv/booter.nix, + and secondarily aliases in pkgs/top-level/stage.nix. + + + + + Note the native stages are bootstrapped in legacy ways that predate the + current cross implementation. This is why the the bootstrapping stages + leading up to the final stages are ignored inthe previous paragraph. + + + + + If one looks at the 3 platform triples, one can see that they overlap such + that one could put them together into a chain like: + +(native, native, native, foreign, foreign) + + If one imagines the saturating self references at the end being replaced + with infinite stages, and then overlays those platform triples, one ends up + with the infinite tuple: + +(native..., native, native, native, foreign, foreign, foreign...) + + On can then imagine any sequence of platforms such that there are bootstrap + stages with their 3 platforms determined by "sliding a window" that is the + 3 tuple through the sequence. This was the original model for + bootstrapping. Without a target platform (assume a better world where all + compilers are multi-target and all standard libraries are built in their + own derivation), this is sufficient. Conversely if one wishes to cross + compile "faster", with a "Canadian Cross" bootstraping stage where + build != host != target, more bootstrapping stages are + needed since no sliding window providess the pesky + pkgsBuildTarget package set since it skips the Canadian + cross stage's "host". + + + + + It is much better to refer to buildPackages than + targetPackages, or more broadly package sets that do + not mention "target". There are three reasons for this. + + + First, it is because bootstrapping stages do not have a unique + targetPackages. For example a (x86-linux, + x86-linux, arm-linux) and (x86-linux, x86-linux, + x86-windows) package set both have a (x86-linux, + x86-linux, x86-linux) package set. Because there is no canonical + targetPackages for such a native (build == + host == target) package set, we set their + targetPackages + + + Second, it is because this is a frequent source of hard-to-follow + "infinite recursions" / cycles. When only packages sets that don't mention + target are used, the package set forms a directly acyclic graph. This + means that all cycles that exist are confirmed to one stage. This means + they are a lot smaller, so easier to follow in the code or a backtrace. It + also means they are present in native and cross builds alike, and so more + likely to be caught by CI and other users. + + + Thirdly, it is because everything target-mentioning only exists to + accommodate compilers with lousy build systems that insist on the compiler + itself and standard library being built together. Of course that is bad + because bigger derivation means longer rebuilds. It is also subpar because + it tends to make the standard libraries less like other libraries than + they could be, complicating code and build systems alike. Because of the + other problems, and because of these innate disadvantages, compilers ought + to be packaged another way where possible. + + + + + + If one explores Nixpkgs, they will see derivations with names like + gccCross. Such *Cross derivations is + a holdover from before we properly distinguished between the host and + target platforms—the derivation with "Cross" in the name covered the + build = host != target case, while the other covered + the host = target, with build platform the same or not + based on whether one was using its .nativeDrv or + .crossDrv. This ugliness will disappear soon. + + +
diff --git a/doc/stdenv.xml b/doc/stdenv.xml index 7e6c589d9fe..85efbc1dd9d 100644 --- a/doc/stdenv.xml +++ b/doc/stdenv.xml @@ -222,9 +222,10 @@ genericBuild But even if one is not cross compiling, the platforms imply whether or not the dependency is needed at run-time or build-time, a concept that makes - perfect sense outside of cross compilation. For now, the run-time/build-time - distinction is just a hint for mental clarity, but in the future it perhaps - could be enforced. + perfect sense outside of cross compilation. By default, the + run-time/build-time distinction is just a hint for mental clarity, but with + strictDeps set it is somewhat enforced even in the native + case. @@ -348,7 +349,10 @@ let f(h, h + 1, i) = i + h Overall, the unifying theme here is that propagation shouldn't be introducing transitive dependencies involving platforms the depending - package is unaware of. The offset bounds checking and definition of + package is unaware of. [One can imagine the dependending package asking for + dependencies with the platforms it knows about; other platforms it doesn't + know how to ask for. The platform description in that scenario is a kind of + unforagable capability.] The offset bounds checking and definition of mapOffset together ensure that this is the case. Discovering a new offset is discovering a new platform, and since those platforms weren't in the derivation "spec" of the needing package, they From 8ba4db0f11f7d1268b34c725f925c99be240b817 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 25 Mar 2019 17:50:13 -0400 Subject: [PATCH 5/5] manual: Fix typos Thanks @matthewbauer! Co-Authored-By: Ericson2314 --- doc/cross-compilation.xml | 34 +++++++++++++++++----------------- doc/stdenv.xml | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/doc/cross-compilation.xml b/doc/cross-compilation.xml index d97f12f2566..b7844da195d 100644 --- a/doc/cross-compilation.xml +++ b/doc/cross-compilation.xml @@ -259,8 +259,8 @@ machine-code-producing tools, we're done. And indeed buildInputs and nativeBuildInputs have covered these simpler build-time and run-time (respectively) changes - for many years. But if the depedency does produce machine code, we might - need to worry about it's target platform too. In principle, that target + for many years. But if the dependency does produce machine code, we might + need to worry about its target platform too. In principle, that target platform might be any of the depending package's build, host, or target platforms, but we prohibit dependencies from a "later" platform to an earlier platform to limit confusion because we've never seen a legitimate @@ -270,7 +270,7 @@ Finally, if the depending package is a compiler or other machine-code-producing tool, it might need dependencies that run at "emit - time". This is for compilers that (regrettably) insist on being in built + time". This is for compilers that (regrettably) insist on being built together with their source langauges' standard libraries. Assuming build != host != target, a run-time dependency of the standard library cannot be run at the compiler's build time or run time, but only at the run time of code @@ -485,9 +485,9 @@ nix-build <nixpkgs> --arg crossSystem '{ config = "<arch>-<os> are specified as lists of derivations given to mkDerivation, as - documented in . In short, the + documented in . In short, each list of dependencies for "host → target" of "foo → bar" is called - depsFooBar, with the exceptions for backwards + depsFooBar, with exceptions for backwards compatibility that depsBuildHost is instead called nativeBuildInputs and depsHostTarget is instead called buildInputs. Nixpkgs is now structured @@ -501,13 +501,13 @@ nix-build <nixpkgs> --arg crossSystem '{ config = "<arch>-<os> - Now, for most of Nixpkgs's history, there was no + Now, for most of Nixpkgs's history, there were no pkgsFooBar attributes, and most packages have not been refactored to use it explicitly. Prior to those, there were just buildPackages, pkgs, and targetPackages. Those are now redefined as aliases to pkgsBuildHost, pkgsHostTarget, and - pkgsTargetTarget. It is fine, indeed if anything + pkgsTargetTarget. It is acceptable, even recommended, to use them for libraries to show that the host platform is irrelevant. @@ -523,12 +523,12 @@ nix-build <nixpkgs> --arg crossSystem '{ config = "<arch>-<os> their dependencies into the multiple lists described above. And indeed, asking that users both sort their dependencies, and take them from the right attribute set, is both too onerous and redundant, - so the recommend approach (for now) is to continue just categorizing by + so the recommended approach (for now) is to continue just categorizing by list and not using an explicit package set. - No make this work, we "splice" together the six + To make this work, we "splice" together the six pkgsFooBar package sets and have callPackage actually take its arguments from that. This is currently implemented in pkgs/top-level/splice.nix. @@ -550,7 +550,7 @@ nix-build <nixpkgs> --arg crossSystem '{ config = "<arch>-<os> pkgs/top-level/stage.nix "ties the knot" (creates the fixed point) of each stage. The package sets are defined per-stage however, so they can be thought of as edges between stages (the nodes) in a graph. - Compositions like pkgsBuildTarget.TargetPackages can be + Compositions like pkgsBuildTarget.targetPackages can be thought of as paths to this graph. @@ -581,12 +581,12 @@ nix-build <nixpkgs> --arg crossSystem '{ config = "<arch>-<os> pkgsHostTarget refers to the current one, and pkgsTargetTarget refers to the next one. When there is no previous or next stage, they instead refer to the current stage. Note - how all the invariants about the mapping between dependency and depending + how all the invariants regarding the mapping between dependency and depending packages' build host and target platforms are preserved. pkgsBuildTarget and pkgsHostHost are more complex in that the stage fitting the requirements isn't always a fixed chain of "prevs" and "nexts" away (modulo the "saturating" - self-references at the ends). We just special case instead. All the primary + self-references at the ends). We just special case each instead. All the primary edges are implemented is in pkgs/stdenv/booter.nix, and secondarily aliases in pkgs/top-level/stage.nix. @@ -642,10 +642,10 @@ nix-build <nixpkgs> --arg crossSystem '{ config = "<arch>-<os> Second, it is because this is a frequent source of hard-to-follow - "infinite recursions" / cycles. When only packages sets that don't mention - target are used, the package set forms a directly acyclic graph. This - means that all cycles that exist are confirmed to one stage. This means - they are a lot smaller, so easier to follow in the code or a backtrace. It + "infinite recursions" / cycles. When only package sets that don't mention + target are used, the package set forms a directed acyclic graph. This + means that all cycles that exist are confined to one stage. This means + they are a lot smaller, and easier to follow in the code or a backtrace. It also means they are present in native and cross builds alike, and so more likely to be caught by CI and other users. @@ -653,7 +653,7 @@ nix-build <nixpkgs> --arg crossSystem '{ config = "<arch>-<os> Thirdly, it is because everything target-mentioning only exists to accommodate compilers with lousy build systems that insist on the compiler itself and standard library being built together. Of course that is bad - because bigger derivation means longer rebuilds. It is also subpar because + because bigger derivations means longer rebuilds. It is also problematic because it tends to make the standard libraries less like other libraries than they could be, complicating code and build systems alike. Because of the other problems, and because of these innate disadvantages, compilers ought diff --git a/doc/stdenv.xml b/doc/stdenv.xml index 85efbc1dd9d..74f815fc1d7 100644 --- a/doc/stdenv.xml +++ b/doc/stdenv.xml @@ -224,7 +224,7 @@ genericBuild the dependency is needed at run-time or build-time, a concept that makes perfect sense outside of cross compilation. By default, the run-time/build-time distinction is just a hint for mental clarity, but with - strictDeps set it is somewhat enforced even in the native + strictDeps set it is mostly enforced even in the native case.