From f5dfe78a1eb5ff8dfcc7ab37cfc132c5f31d3cef Mon Sep 17 00:00:00 2001 From: "Nicolas B. Pierron" Date: Sat, 17 Dec 2016 18:05:21 +0000 Subject: [PATCH] Add overlays mechanism to Nixpkgs. This patch add a new argument to Nixpkgs default expression named "overlays". By default, the value of the argument is either taken from the environment variable `NIXPKGS_OVERLAYS`, or from the directory `~/.nixpkgs/overlays/`. If the environment variable does not name a valid directory then this mechanism would fallback on the home directory. If the home directory does not exists it will fallback on an empty list of overlays. The overlays directory should contain the list of extra Nixpkgs stages which would be used to extend the content of Nixpkgs, with additional set of packages. The overlays, i-e directory, files, symbolic links are used in alphabetical order. The simplest overlay which extends Nixpkgs with nothing looks like: ```nix self: super: { } ``` More refined overlays can use `super` as the basis for building new packages, and `self` as a way to query the final result of the fix-point. An example of overlay which extends Nixpkgs with a small set of packages can be found at: https://github.com/nbp/nixpkgs-mozilla/blob/nixpkgs-overlay/moz-overlay.nix To use this file, checkout the repository and add a symbolic link to the `moz-overlay.nix` file in `~/.nixpkgs/overlays` directory. --- doc/functions.xml | 64 +------------- doc/languages-frameworks/python.md | 6 +- doc/manual.xml | 1 + doc/overlays.xml | 99 ++++++++++++++++++++++ nixos/doc/manual/release-notes/rl-1703.xml | 9 +- pkgs/stdenv/cross/default.nix | 8 +- pkgs/stdenv/custom/default.nix | 6 +- pkgs/stdenv/darwin/default.nix | 6 +- pkgs/stdenv/default.nix | 2 +- pkgs/stdenv/freebsd/default.nix | 4 +- pkgs/stdenv/linux/default.nix | 4 +- pkgs/stdenv/native/default.nix | 6 +- pkgs/top-level/default.nix | 5 +- pkgs/top-level/impure.nix | 19 ++++- pkgs/top-level/stage.nix | 24 ++---- 15 files changed, 160 insertions(+), 103 deletions(-) create mode 100644 doc/overlays.xml diff --git a/doc/functions.xml b/doc/functions.xml index f6a0a4352f6..6374c15ddf2 100644 --- a/doc/functions.xml +++ b/doc/functions.xml @@ -17,66 +17,6 @@ derivations or even the whole package set. -
- pkgs.overridePackages - - - This function inside the nixpkgs expression (pkgs) - can be used to override the set of packages itself. - - - Warning: this function is expensive and must not be used from within - the nixpkgs repository. - - - Example usage: - - let - pkgs = import <nixpkgs> {}; - newpkgs = pkgs.overridePackages (self: super: { - foo = super.foo.override { ... }; - }); - in ... - - - - The resulting newpkgs will have the new foo - expression, and all other expressions depending on foo will also - use the new foo expression. - - - - The behavior of this function is similar to config.packageOverrides. - - - - The self parameter refers to the final package set with the - applied overrides. Using this parameter may lead to infinite recursion if not - used consciously. - - - - The super parameter refers to the old package set. - It's equivalent to pkgs in the above example. - - - - Note that in previous versions of nixpkgs, this method replaced any changes from config.packageOverrides, - along with that from previous calls if this function was called repeatedly. - Now those previous changes will be preserved so this function can be "chained" meaningfully. - To recover the old behavior, make sure config.packageOverrides is unset, - and call this only once off a "freshly" imported nixpkgs: - - let - pkgs = import <nixpkgs> { config: {}; }; - newpkgs = pkgs.overridePackages ...; - in ... - - -
-
<pkg>.override @@ -91,9 +31,9 @@ Example usages: pkgs.foo.override { arg1 = val1; arg2 = val2; ... } - pkgs.overridePackages (self: super: { + import pkgs.path { overlays = [ (self: super: { foo = super.foo.override { barSupport = true ; }; - }) + })]}; mypkg = pkgs.callPackage ./mypkg.nix { mydep = pkgs.mydep.override { ... }; } diff --git a/doc/languages-frameworks/python.md b/doc/languages-frameworks/python.md index 82aeb112c93..48d9e0a0952 100644 --- a/doc/languages-frameworks/python.md +++ b/doc/languages-frameworks/python.md @@ -737,18 +737,18 @@ in (pkgs.python35.override {inherit packageOverrides;}).withPackages (ps: [ps.bl ``` The requested package `blaze` depends on `pandas` which itself depends on `scipy`. -If you want the whole of Nixpkgs to use your modifications, then you can use `pkgs.overridePackages` +If you want the whole of Nixpkgs to use your modifications, then you can use `overlays` as explained in this manual. In the following example we build a `inkscape` using a different version of `numpy`. ``` let pkgs = import {}; - newpkgs = pkgs.overridePackages ( pkgsself: pkgssuper: { + newpkgs = import pkgs.path { overlays = [ (pkgsself: pkgssuper: { python27 = let packageOverrides = self: super: { numpy = super.numpy_1_10; }; in pkgssuper.python27.override {inherit packageOverrides;}; - } ); + } ) ]; }; in newpkgs.inkscape ``` diff --git a/doc/manual.xml b/doc/manual.xml index 6ad66d48652..1c0dac6e4df 100644 --- a/doc/manual.xml +++ b/doc/manual.xml @@ -18,6 +18,7 @@ + diff --git a/doc/overlays.xml b/doc/overlays.xml new file mode 100644 index 00000000000..d194c6bfc89 --- /dev/null +++ b/doc/overlays.xml @@ -0,0 +1,99 @@ + + +Overlays + +This chapter describes how to extend and change Nixpkgs content using +overlays. Overlays are used to add phases in the fix-point used by Nixpkgs +to bind the dependencies of all packages. + + + +
+Installing Overlays + +Overlays are looked for in the following order, the first valid one is +considered, and all the rest are ignored: + + + + + + As argument of the imported attribute set. When importing Nixpkgs, + the overlays attribute argument can be set to a list of + functions, which would be describe in . + + + + + + As a directory pointed by the environment variable named +NIXPKGS_OVERLAYS. This directory can contain symbolic +links to Nix expressions. + + + + + + As the directory located at +~/.nixpkgs/overlays/. This directory can contain +symbolic links to Nix expressions. + + + + + + +For the second and third option, the directory contains either +directories providing a default.nix expression, or files, or symbolic links +to the entry Nix expression of the overlay. These Nix expressions are +following the syntax described in . + +To install an overlay, using the last option. Clone the repository of +the overlay, and add a symbolic link to it in the +~/.nixpkgs/overlays/ directory. + +
+ + + +
+Overlays Layout + +An overlay is a Nix expression, which is a function which accepts 2 +arguments. + + +self: super: + +{ + foo = super.foo.override { ... }; + bar = import ./pkgs/bar { + inherit (self) stdenv fetchurl; + inherit (self) ninja crawl dwarf-fortress; + }; +} + + +The first argument, usualy named self, corresponds +to the final package set. You should use this set to inherit all the +dependencies needed by your package expression. + +The second argument, usualy named super, +corresponds to the result of the evaluation of the previous stages of +Nixpkgs, it does not contain any of the packages added by the current +overlay nor any of the following overlays. This set is used in to override +existing packages, either by changing their dependencies or their +recipes. + +The value returned by this function should be a set similar to +pkgs/top-level/all-packages.nix, which contains either +extra packages defined by the overlay, or which overwrite Nixpkgs packages +with other custom defaults. This is similar to . + +
+ + diff --git a/nixos/doc/manual/release-notes/rl-1703.xml b/nixos/doc/manual/release-notes/rl-1703.xml index 6997155d94d..bbb9c53eaa3 100644 --- a/nixos/doc/manual/release-notes/rl-1703.xml +++ b/nixos/doc/manual/release-notes/rl-1703.xml @@ -11,7 +11,9 @@ has the following highlights: - + Nixpkgs is now extensible through overlays. + See the Nixpkgs manual for more information. @@ -96,6 +98,11 @@ following incompatible changes: networking.timeServers. + + + pkgs.overridePackages function no longer exists. Instead import Nixpkgs a second time using import pkgs.path { + overlays = [ ... ]; }. + diff --git a/pkgs/stdenv/cross/default.nix b/pkgs/stdenv/cross/default.nix index 359d45b78b9..16f41671b76 100644 --- a/pkgs/stdenv/cross/default.nix +++ b/pkgs/stdenv/cross/default.nix @@ -1,10 +1,10 @@ { lib -, system, platform, crossSystem, config +, system, platform, crossSystem, config, overlays }: let bootStages = import ../. { - inherit lib system platform; + inherit lib system platform overlays; crossSystem = null; # Ignore custom stdenvs when cross compiling for compatability config = builtins.removeAttrs config [ "replaceStdenv" ]; @@ -18,7 +18,7 @@ in bootStages ++ [ # should be used to build compilers and other such tools targeting the cross # platform. Then, `forceNativeDrv` can be removed. (vanillaPackages: { - inherit system platform crossSystem config; + inherit system platform crossSystem config overlays; # It's OK to change the built-time dependencies allowCustomOverrides = true; stdenv = vanillaPackages.stdenv // { @@ -30,7 +30,7 @@ in bootStages ++ [ # Run packages (buildPackages: { - inherit system platform crossSystem config; + inherit system platform crossSystem config overlays; stdenv = if crossSystem.useiOSCross or false then let inherit (buildPackages.darwin.ios-cross { diff --git a/pkgs/stdenv/custom/default.nix b/pkgs/stdenv/custom/default.nix index 59a6e871cfd..d7e9bf53bed 100644 --- a/pkgs/stdenv/custom/default.nix +++ b/pkgs/stdenv/custom/default.nix @@ -1,12 +1,12 @@ { lib -, system, platform, crossSystem, config +, system, platform, crossSystem, config, overlays }: assert crossSystem == null; let bootStages = import ../. { - inherit lib system platform crossSystem; + inherit lib system platform crossSystem overlays; # Remove config.replaceStdenv to ensure termination. config = builtins.removeAttrs config [ "replaceStdenv" ]; }; @@ -15,7 +15,7 @@ in bootStages ++ [ # Additional stage, built using custom stdenv (vanillaPackages: { - inherit system platform crossSystem config; + inherit system platform crossSystem config overlays; stdenv = config.replaceStdenv { pkgs = vanillaPackages; }; }) diff --git a/pkgs/stdenv/darwin/default.nix b/pkgs/stdenv/darwin/default.nix index 9c860bfb0f9..e3a87ea078f 100644 --- a/pkgs/stdenv/darwin/default.nix +++ b/pkgs/stdenv/darwin/default.nix @@ -1,5 +1,5 @@ { lib -, system, platform, crossSystem, config +, system, platform, crossSystem, config, overlays # Allow passing in bootstrap files directly so we can test the stdenv bootstrap process when changing the bootstrap tools , bootstrapFiles ? let @@ -98,7 +98,7 @@ in rec { }; in { - inherit system platform crossSystem config; + inherit system platform crossSystem config overlays; stdenv = thisStdenv; }; @@ -316,7 +316,7 @@ in rec { stage3 stage4 (prevStage: { - inherit system crossSystem platform config; + inherit system crossSystem platform config overlays; stdenv = stdenvDarwin prevStage; }) ]; diff --git a/pkgs/stdenv/default.nix b/pkgs/stdenv/default.nix index 0ca03ecdde1..f60ffec4b56 100644 --- a/pkgs/stdenv/default.nix +++ b/pkgs/stdenv/default.nix @@ -7,7 +7,7 @@ { # Args just for stdenvs' usage lib # Args to pass on to the pkgset builder, too -, system, platform, crossSystem, config +, system, platform, crossSystem, config, overlays } @ args: let diff --git a/pkgs/stdenv/freebsd/default.nix b/pkgs/stdenv/freebsd/default.nix index e0256e26f5f..2cb059deb34 100644 --- a/pkgs/stdenv/freebsd/default.nix +++ b/pkgs/stdenv/freebsd/default.nix @@ -1,5 +1,5 @@ { lib -, system, platform, crossSystem, config +, system, platform, crossSystem, config, overlays }: assert crossSystem == null; @@ -58,7 +58,7 @@ assert crossSystem == null; }) (prevStage: { - inherit system crossSystem platform config; + inherit system crossSystem platform config overlays; stdenv = import ../generic { name = "stdenv-freebsd-boot-3"; inherit system config; diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index 430122c36f7..bc8a03a485a 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -4,7 +4,7 @@ # compiler and linker that do not search in default locations, # ensuring purity of components produced by it. { lib -, system, platform, crossSystem, config +, system, platform, crossSystem, config, overlays , bootstrapFiles ? if system == "i686-linux" then import ./bootstrap-files/i686.nix @@ -91,7 +91,7 @@ let }; in { - inherit system platform crossSystem config; + inherit system platform crossSystem config overlays; stdenv = thisStdenv; }; diff --git a/pkgs/stdenv/native/default.nix b/pkgs/stdenv/native/default.nix index 57182fae523..4028638009e 100644 --- a/pkgs/stdenv/native/default.nix +++ b/pkgs/stdenv/native/default.nix @@ -1,5 +1,5 @@ { lib -, system, platform, crossSystem, config +, system, platform, crossSystem, config, overlays }: assert crossSystem == null; @@ -134,7 +134,7 @@ in # First build a stdenv based only on tools outside the store. (prevStage: { - inherit system crossSystem platform config; + inherit system crossSystem platform config overlays; stdenv = makeStdenv { inherit (prevStage) cc fetchurl; } // { inherit (prevStage) fetchurl; }; @@ -143,7 +143,7 @@ in # Using that, build a stdenv that adds the ‘xz’ command (which most systems # don't have, so we mustn't rely on the native environment providing it). (prevStage: { - inherit system crossSystem platform config; + inherit system crossSystem platform config overlays; stdenv = makeStdenv { inherit (prevStage.stdenv) cc fetchurl; extraPath = [ prevStage.xz ]; diff --git a/pkgs/top-level/default.nix b/pkgs/top-level/default.nix index 04daf9778ff..a146dad63bc 100644 --- a/pkgs/top-level/default.nix +++ b/pkgs/top-level/default.nix @@ -23,6 +23,9 @@ , # Allow a configuration attribute set to be passed in as an argument. config ? {} +, # List of overlays layers used to extend Nixpkgs. + overlays ? [] + , # A function booting the final package set for a specific standard # environment. See below for the arguments given to that function, # the type of list it returns. @@ -80,7 +83,7 @@ in let boot = import ../stdenv/booter.nix { inherit lib allPackages; }; stages = stdenvStages { - inherit lib system platform crossSystem config; + inherit lib system platform crossSystem config overlays; }; pkgs = boot stages; diff --git a/pkgs/top-level/impure.nix b/pkgs/top-level/impure.nix index e9066815927..1aa55f7293b 100644 --- a/pkgs/top-level/impure.nix +++ b/pkgs/top-level/impure.nix @@ -18,7 +18,24 @@ else if homeDir != "" && pathExists configFile2 then import configFile2 else {} +, # Overlays are used to extend Nixpkgs collection with additional + # collections of packages. These collection of packages are part of the + # fix-point made by Nixpkgs. + overlays ? let + inherit (builtins) getEnv pathExists readDir attrNames map sort + lessThan; + dirEnv = getEnv "NIXPKGS_OVERLAYS"; + dirHome = (getEnv "HOME") + "/.nixpkgs/overlays"; + dirCheck = dir: dir != "" && pathExists (dir + "/."); + overlays = dir: + let content = readDir dir; in + map (n: import "${dir}/${n}") (sort lessThan (attrNames content)); + in + if dirCheck dirEnv then overlays dirEnv + else if dirCheck dirHome then overlays dirHome + else [] + , ... } @ args: -import ./. (args // { inherit system config; }) +import ./. (args // { inherit system config overlays; }) diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index ebc6473e425..cbf65870eb7 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -30,6 +30,8 @@ , # The configuration attribute set config +, overlays # List of overlays to use in the fix-point. + , crossSystem , platform , lib @@ -79,7 +81,7 @@ let ((config.packageOverrides or (super: {})) super); # The complete chain of package set builders, applied from top to bottom - toFix = lib.foldl' (lib.flip lib.extends) (self: {}) [ + toFix = lib.foldl' (lib.flip lib.extends) (self: {}) ([ stdenvBootstappingAndPlatforms stdenvAdapters trivialBuilders @@ -87,20 +89,8 @@ let aliases stdenvOverrides configOverrides - ]; + ] ++ overlays); - # Use `overridePackages` to easily override this package set. - # Warning: this function is very expensive and must not be used - # from within the nixpkgs repository. - # - # Example: - # pkgs.overridePackages (self: super: { - # foo = super.foo.override { ... }; - # } - # - # The result is `pkgs' where all the derivations depending on `foo' - # will use the new version. - - # Return the complete set of packages. Warning: this function is very - # expensive! -in lib.makeExtensibleWithCustomName "overridePackages" toFix +in + # Return the complete set of packages. + lib.fix toFix