From 3049064aa58137a0385ec1a6f392e5a57587f463 Mon Sep 17 00:00:00 2001 From: Tristan Helmich Date: Sun, 22 Nov 2020 11:41:22 +0000 Subject: [PATCH 01/36] nixos/release-notes: Warn on wpa_supplicant changes --- nixos/doc/manual/release-notes/rl-2009.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-2009.xml b/nixos/doc/manual/release-notes/rl-2009.xml index afb09d7c5d2..3da8080958e 100644 --- a/nixos/doc/manual/release-notes/rl-2009.xml +++ b/nixos/doc/manual/release-notes/rl-2009.xml @@ -1344,6 +1344,12 @@ CREATE ROLE postgres LOGIN SUPERUSER; that makes it unsuitable to be a default app. + + + If you want to manage the configuration of wpa_supplicant outside of NixOS you must ensure that none of , or is being used or true. + Using any of those options will cause wpa_supplicant to be started with a NixOS generated configuration file instead of your own. + + From 0b61ed7af920e6248638d7b53d932c0470b9b054 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 2 Sep 2020 00:40:27 +0200 Subject: [PATCH 02/36] lib/options: Don't show internal suboption in the manual Initially https://github.com/NixOS/nixpkgs/pull/82897 prevented non-visible options from being rendered in the manual, but visible-but-internal options were still being recursed into. This fixes this, aligning the recurse condition here with the one in make-options-doc/default.nix --- lib/options.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/options.nix b/lib/options.nix index 87cd8b79796..5c042a6c6f2 100644 --- a/lib/options.nix +++ b/lib/options.nix @@ -192,7 +192,7 @@ rec { let ss = opt.type.getSubOptions opt.loc; in if ss != {} then optionAttrSetToDocList' opt.loc ss else []; in - [ docOption ] ++ optionals docOption.visible subOptions) (collect isOption options); + [ docOption ] ++ optionals (docOption.visible && ! docOption.internal) subOptions) (collect isOption options); /* This function recursively removes all derivation attributes from From df5ba82f74df75e96390995472f3e1e5179da21c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 1 Sep 2020 23:32:57 +0200 Subject: [PATCH 03/36] lib/modules: Implement module-builtin assertions This implements assertions/warnings supported by the module system directly, instead of just being a NixOS option (see nixos/modules/misc/assertions.nix). This has the following benefits: - It allows cleanly redoing the user interface. The new implementation specifically allows disabling assertions or converting them to warnings instead. - Assertions/warnings can now be thrown easily from within submodules, which previously wasn't possible and needed workarounds. --- lib/modules.nix | 153 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 1 deletion(-) diff --git a/lib/modules.nix b/lib/modules.nix index 3f2bfd478b0..0d761c632d0 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -46,6 +46,7 @@ let showFiles showOption unknownModule + literalExample ; in @@ -116,6 +117,98 @@ rec { turned off. ''; }; + + _module.assertions = mkOption { + description = '' + Assertions and warnings to trigger during module evaluation. The + attribute name will be displayed when it is triggered, allowing + users to disable/change these assertions again if necessary. See + the section on Warnings and Assertions in the manual for more + information. + ''; + example = literalExample '' + { + gpgSshAgent = { + enable = config.programs.gnupg.agent.enableSSHSupport && config.programs.ssh.startAgent; + message = "You can't use ssh-agent and GnuPG agent with SSH support enabled at the same time!"; + }; + + grafanaPassword = { + enable = config.services.grafana.database.password != ""; + message = "Grafana passwords will be stored as plaintext in the Nix store!"; + type = "warning"; + }; + } + ''; + default = {}; + internal = true; + type = types.attrsOf (types.submodule { + # TODO: Rename to assertion? Or allow also setting assertion? + options.enable = mkOption { + description = '' + Whether to enable this assertion. + + This is the inverse of asserting a condition: If a certain + condition should be true, then this + option should be set to false when that + case occurs + + ''; + type = types.bool; + }; + + options.type = mkOption { + description = '' + The type of the assertion. The default + "error" type will cause evaluation to fail, + while the "warning" type will only show a + warning. + ''; + type = types.enum [ "error" "warning" ]; + default = "error"; + example = "warning"; + }; + + options.message = mkOption { + description = '' + The assertion message to display if this assertion triggers. + To display option names in the message, add + options to the module function arguments + and use ''${options.path.to.option}. + ''; + type = types.str; + example = literalExample '' + Enabling both ''${options.services.foo.enable} and ''${options.services.bar.enable} is not possible. + ''; + }; + + options.triggerPath = mkOption { + description = '' + The config path which when evaluated should + trigger this assertion. By default this is + [], meaning evaluating + config at all will trigger the assertion. + On NixOS this default is changed to + [ "system" "build" "toplevel" such that + only a system evaluation triggers the assertions. + + Evaluating config from within the current + module evaluation doesn't cause a trigger. Only accessing it + from outside will do that. This means it's easy to miss + assertions if this option doesn't have an externally-accessed + value. + + ''; + # Mark as internal as it's easy to misuse it + internal = true; + type = types.uniq (types.listOf types.str); + # Default to [], causing assertions to be triggered when + # anything is evaluated. This is a safe and convenient default. + default = []; + example = [ "system" "build" "vm" ]; + }; + }); + }; }; config = { @@ -154,6 +247,64 @@ rec { # paths, meaning recursiveUpdate will never override any value else recursiveUpdate freeformConfig declaredConfig; + /* + Inject a list of assertions into a config value, corresponding to their + triggerPath (meaning when that path is accessed from the result of this + function, the assertion triggers). + */ + injectAssertions = assertions: config: let + # Partition into assertions that are triggered on this level and ones that aren't + parted = partition (a: length a.triggerPath == 0) assertions; + + # From the ones that are triggered, filter out ones that aren't enabled + # and group into warnings/errors + byType = groupBy (a: a.type) (filter (a: a.enable) parted.right); + + # Triggers semantically are just lib.id, but they print warning cause errors in addition + warningTrigger = value: lib.foldr (w: warn w.show) value (byType.warning or []); + errorTrigger = value: + if byType.error or [] == [] then value else + throw '' + Failed assertions: + ${concatMapStringsSep "\n" (a: "- ${a.show}") byType.error} + ''; + # Trigger for both warnings and errors + trigger = value: warningTrigger (errorTrigger value); + + # From the non-triggered assertions, split off the first element of triggerPath + # to get a mapping from nested attributes to a list of assertions for that attribute + nested = zipAttrs (map (a: { + ${head a.triggerPath} = a // { + triggerPath = tail a.triggerPath; + }; + }) parted.wrong); + + # Recursively inject assertions if config is an attribute set and we + # have assertions under its attributes + result = + if isAttrs config + then + mapAttrs (name: value: + if nested ? ${name} + then injectAssertions nested.${name} value + else value + ) config + else config; + in trigger result; + + # List of assertions for this module evaluation, where each assertion also + # has a `show` attribute for how to show it if triggered + assertions = mapAttrsToList (name: value: + let id = + if hasPrefix "_" name then "" + else "[${showOption prefix}${optionalString (prefix != []) "/"}${name}] "; + in value // { + show = "${id}${value.message}"; + } + ) config._module.assertions; + + finalConfig = injectAssertions assertions (removeAttrs config [ "_module" ]); + checkUnmatched = if config._module.check && config._module.freeformType == null && merged.unmatchedDefns != [] then let @@ -173,7 +324,7 @@ rec { result = builtins.seq checkUnmatched { inherit options; - config = removeAttrs config [ "_module" ]; + config = finalConfig; inherit (config) _module; }; in result; From 9523df7eb600e7fc2a88bc5227d9dfe12055a9bd Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 2 Sep 2020 00:17:26 +0200 Subject: [PATCH 04/36] nixos/assertions: Use module-builtin assertion implementation --- lib/modules.nix | 12 +++++------ nixos/modules/misc/assertions.nix | 21 ++++++++++++++++++- nixos/modules/system/activation/top-level.nix | 10 +-------- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/lib/modules.nix b/lib/modules.nix index 0d761c632d0..31200ae0b03 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -254,11 +254,11 @@ rec { */ injectAssertions = assertions: config: let # Partition into assertions that are triggered on this level and ones that aren't - parted = partition (a: length a.triggerPath == 0) assertions; + parted = lib.partition (a: length a.triggerPath == 0) assertions; # From the ones that are triggered, filter out ones that aren't enabled # and group into warnings/errors - byType = groupBy (a: a.type) (filter (a: a.enable) parted.right); + byType = lib.groupBy (a: a.type) (filter (a: a.enable) parted.right); # Triggers semantically are just lib.id, but they print warning cause errors in addition warningTrigger = value: lib.foldr (w: warn w.show) value (byType.warning or []); @@ -266,16 +266,16 @@ rec { if byType.error or [] == [] then value else throw '' Failed assertions: - ${concatMapStringsSep "\n" (a: "- ${a.show}") byType.error} + ${lib.concatMapStringsSep "\n" (a: "- ${a.show}") byType.error} ''; # Trigger for both warnings and errors trigger = value: warningTrigger (errorTrigger value); # From the non-triggered assertions, split off the first element of triggerPath # to get a mapping from nested attributes to a list of assertions for that attribute - nested = zipAttrs (map (a: { + nested = lib.zipAttrs (map (a: { ${head a.triggerPath} = a // { - triggerPath = tail a.triggerPath; + triggerPath = lib.tail a.triggerPath; }; }) parted.wrong); @@ -296,7 +296,7 @@ rec { # has a `show` attribute for how to show it if triggered assertions = mapAttrsToList (name: value: let id = - if hasPrefix "_" name then "" + if lib.hasPrefix "_" name then "" else "[${showOption prefix}${optionalString (prefix != []) "/"}${name}] "; in value // { show = "${id}${value.message}"; diff --git a/nixos/modules/misc/assertions.nix b/nixos/modules/misc/assertions.nix index 550b3ac97f6..e931611247f 100644 --- a/nixos/modules/misc/assertions.nix +++ b/nixos/modules/misc/assertions.nix @@ -1,4 +1,4 @@ -{ lib, ... }: +{ lib, config, ... }: with lib; @@ -29,6 +29,25 @@ with lib; ''; }; + _module.assertions = mkOption { + type = types.attrsOf (types.submodule { + triggerPath = mkDefault [ "system" "build" "toplevel" ]; + }); + }; + }; + + config._module.assertions = lib.listToAttrs (lib.imap1 (n: value: + let + name = "_${toString n}"; + isWarning = lib.isString value; + result = { + enable = if isWarning then true else ! value.assertion; + type = if isWarning then "warning" else "error"; + message = if isWarning then value else value.message; + }; + in nameValuePair name result + ) (config.assertions ++ config.warnings)); + # impl of assertions is in } diff --git a/nixos/modules/system/activation/top-level.nix b/nixos/modules/system/activation/top-level.nix index 03d7e749323..17b62ad9569 100644 --- a/nixos/modules/system/activation/top-level.nix +++ b/nixos/modules/system/activation/top-level.nix @@ -117,18 +117,10 @@ let perl = "${pkgs.perl}/bin/perl " + (concatMapStringsSep " " (lib: "-I${lib}/${pkgs.perl.libPrefix}") (with pkgs.perlPackages; [ FileSlurp NetDBus XMLParser XMLTwig ])); }; - # Handle assertions and warnings - - failedAssertions = map (x: x.message) (filter (x: !x.assertion) config.assertions); - - baseSystemAssertWarn = if failedAssertions != [] - then throw "\nFailed assertions:\n${concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}" - else showWarnings config.warnings baseSystem; - # Replace runtime dependencies system = fold ({ oldDependency, newDependency }: drv: pkgs.replaceDependency { inherit oldDependency newDependency drv; } - ) baseSystemAssertWarn config.system.replaceRuntimeDependencies; + ) baseSystem config.system.replaceRuntimeDependencies; in From 20131348db3b51f7fe41f2d4aa3cd8875a6b48f2 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 2 Sep 2020 15:43:16 +0200 Subject: [PATCH 05/36] lib/modules: Use module-builtin assertions for mkRemovedOptionModule and co. --- lib/modules.nix | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/lib/modules.nix b/lib/modules.nix index 31200ae0b03..1902db5c616 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -924,14 +924,16 @@ rec { visible = false; apply = x: throw "The option `${showOption optionName}' can no longer be used since it's been removed. ${replacementInstructions}"; }); - config.assertions = - let opt = getAttrFromPath optionName options; in [{ - assertion = !opt.isDefined; + config._module.assertions = + let opt = getAttrFromPath optionName options; in { + ${showOption optionName} = { + enable = mkDefault opt.isDefined; message = '' The option definition `${showOption optionName}' in ${showFiles opt.files} no longer has any effect; please remove it. ${replacementInstructions} ''; - }]; + }; + }; }; /* Return a module that causes a warning to be shown if the @@ -992,14 +994,19 @@ rec { })) from); config = { - warnings = filter (x: x != "") (map (f: - let val = getAttrFromPath f config; - opt = getAttrFromPath f options; - in - optionalString - (val != "_mkMergedOptionModule") - "The option `${showOption f}' defined in ${showFiles opt.files} has been changed to `${showOption to}' that has a different type. Please read `${showOption to}' documentation and update your configuration accordingly." - ) from); + _module.assertions = + let warningMessages = map (f: + let val = getAttrFromPath f config; + opt = getAttrFromPath f options; + in { + ${showOption f} = { + enable = mkDefault (val != "_mkMergedOptionModule"); + type = "warning"; + message = "The option `${showOption f}' defined in ${showFiles opt.files} has been changed to `${showOption to}' that has a different type. Please read `${showOption to}' documentation and update your configuration accordingly."; + }; + } + ) from; + in mkMerge warningMessages; } // setAttrByPath to (mkMerge (optional (any (f: (getAttrFromPath f config) != "_mkMergedOptionModule") from) @@ -1058,8 +1065,11 @@ rec { }); config = mkMerge [ { - warnings = optional (warn && fromOpt.isDefined) - "The option `${showOption from}' defined in ${showFiles fromOpt.files} has been renamed to `${showOption to}'."; + _module.assertions.${showOption from} = { + enable = mkDefault (warn && fromOpt.isDefined); + type = "warning"; + message = "The option `${showOption from}' defined in ${showFiles fromOpt.files} has been renamed to `${showOption to}'."; + }; } (if withPriority then mkAliasAndWrapDefsWithPriority (setAttrByPath to) fromOpt From 1e6a84b7af71f579f2467619f504d1cfe43bb3e9 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 2 Sep 2020 16:10:17 +0200 Subject: [PATCH 06/36] nixos/modules: Allow options to be coerced to a string for convenience --- lib/modules.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/modules.nix b/lib/modules.nix index 1902db5c616..9ff8f4701bb 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -665,6 +665,8 @@ rec { definitions = map (def: def.value) res.defsFinal; files = map (def: def.file) res.defsFinal; inherit (res) isDefined; + # This allows options to be correctly displayed using `${options.path.to.it}` + __toString = _: showOption loc; }; # Merge definitions of a value of a given type. From 3759a77fcda2e33f89023b8c6b1476e8fa413a8e Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 2 Sep 2020 18:01:10 +0200 Subject: [PATCH 07/36] nixos/modules: Expose the internal module in the top-level documentation --- lib/modules.nix | 15 ++++++++++----- lib/tests/misc.nix | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/modules.nix b/lib/modules.nix index 9ff8f4701bb..e3f7ca3581c 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -77,10 +77,15 @@ rec { # attribute. These options are fragile, as they are used by the # module system to change the interpretation of modules. internalModule = rec { - _file = ./modules.nix; + # FIXME: Using ./modules.nix directly breaks the doc for some reason + _file = "lib/modules.nix"; key = _file; + # These options are set to be internal only for prefix != [], aka it's + # a submodule evaluation. This way their docs are displayed only once + # as a top-level NixOS option, but will be hidden for all submodules, + # even though they are available there too options = { _module.args = mkOption { # Because things like `mkIf` are entirely useless for @@ -90,13 +95,13 @@ rec { # a `_module.args.pkgs = import (fetchTarball { ... }) {}` won't # start a download when `pkgs` wasn't evaluated. type = types.lazyAttrsOf types.unspecified; - internal = true; + internal = prefix != []; description = "Arguments passed to each module."; }; _module.check = mkOption { type = types.bool; - internal = true; + internal = prefix != []; default = check; description = "Whether to check whether all option definitions have matching declarations."; }; @@ -104,7 +109,7 @@ rec { _module.freeformType = mkOption { # Disallow merging for now, but could be implemented nicely with a `types.optionType` type = types.nullOr (types.uniq types.attrs); - internal = true; + internal = prefix != []; default = null; description = '' If set, merge all definitions that don't have an associated option @@ -141,7 +146,7 @@ rec { } ''; default = {}; - internal = true; + internal = prefix != []; type = types.attrsOf (types.submodule { # TODO: Rename to assertion? Or allow also setting assertion? options.enable = mkOption { diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index 35a5801c724..2d53ed81176 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -655,7 +655,7 @@ runTests { modules = [ module ]; }).options; - locs = filter (o: ! o.internal) (optionAttrSetToDocList options); + locs = filter (o: ! o.internal) (optionAttrSetToDocList (removeAttrs options [ "_module" ])); in map (o: o.loc) locs; expected = [ [ "foo" ] [ "foo" "" "bar" ] [ "foo" "bar" ] ]; }; From c4fb54e92a10f04bb70b31b397a50fdbc203bc66 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 2 Sep 2020 19:23:39 +0200 Subject: [PATCH 08/36] nixos/docs: Update assertion docs for new module-builtin ones --- nixos/doc/manual/development/assertions.xml | 131 ++++++++++++++------ 1 file changed, 95 insertions(+), 36 deletions(-) diff --git a/nixos/doc/manual/development/assertions.xml b/nixos/doc/manual/development/assertions.xml index 32f90cf2e7c..91506ba65a1 100644 --- a/nixos/doc/manual/development/assertions.xml +++ b/nixos/doc/manual/development/assertions.xml @@ -8,7 +8,7 @@ When configuration problems are detectable in a module, it is a good idea to write an assertion or warning. Doing so provides clear feedback to the user - and prevents errors after the build. + and can prevent errors before the build. @@ -20,55 +20,114 @@ NixOS module system. -
- Warnings +
+ Defining Warnings and Assertions - This is an example of using warnings. + Both warnings and assertions can be defined using the option. Each assertion needs an attribute name, under which you have to define an enable condition using and a message using . Note that the enable condition is inverse of what an assertion would be: To assert a value being true, the enable condition should be false in that case, so that it isn't triggered. For the assertion message, you can add options to the module arguments and use ${options.path.to.option} to print a context-aware string representation of the option path. Here is an example showing how this can be done. - +
-
- Assertions +
+ Ignoring Warnings and Assertions - This example, extracted from the - - syslogd module shows how to use - assertions. Since there can only be one active syslog - daemon at a time, an assertion is useful to prevent such a broken system - from being built. + Sometimes you can get warnings or assertions that don't apply to your specific case and you wish to ignore them, or at least make assertions non-fatal. You can do so for all assertions defined using by using the attribute name of the definition, which is conveniently printed using [...] when the assertion is triggered. For above example, the evaluation output when the assertions are triggered looks as follows: - +trace: warning: [grafanaPassword] The grafana password defined with + services.grafana.database.password will be stored as plaintext in the Nix store! +error: Failed assertions: +- [gpgSshAgent] You can't enable both programs.ssh.startAgent and + programs.gnupg.agent.enableSSHSupport! + + + The [grafanaPassword] and [gpgSshAgent] strings tell you that these were defined under the grafanaPassword and gpgSshAgent attributes of respectively. With this knowledge you can adjust them to your liking: + + + +{ lib, ... }: { + # Change the assertion into a non-fatal warning + _module.assertions.gpgSshAgent.type = "warning"; + + # We don't care about this warning, disable it + _module.assertions.grafanaPassword.enable = lib.mkForce false; +} + + + +
+
+ Warnings and Assertions in Submodules + + + Warnings and assertions can be defined within submodules in the same way. Here is an example: + + + +{ lib, ... }: { + + options.myServices = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule ({ config, options, ... }: { + options.port = lib.mkOption {}; + + config._module.assertions.portConflict = { + enable = config.port == 80; + message = "Port ${toString config.port} defined using" + + " ${options.port} is usually used for HTTP"; + type = "warning"; + }; + })); + }; + +} + + + + When this assertion is triggered, it shows both the submodule path along with the assertion attribute within that submodule, joined by a /. Note also how ${options.port} correctly shows the context of the option. + + + +trace: warning: [myServices.foo/portConflict] Port 80 defined using + myServices.foo.port is usually used for HTTP + + + + Therefore to disable such an assertion, you can do so by changing the option within the myServices.foo submodule: + + + +{ lib, ... }: { + myServices.foo._module.assertions.portConflict.enable = lib.mkForce false; +} + + + + + Assertions defined in submodules under types.listOf can't be ignored, since there's no way to change previously defined list items. + + +
From 900c4a5abd1061db2390224849cbc5eb8eb07059 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 2 Sep 2020 22:33:29 +0200 Subject: [PATCH 09/36] lib/tests: Implement generalized checkConfigCodeOutErr for module tests --- lib/tests/modules.sh | 55 +++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 309c5311361..012835e48c1 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -27,37 +27,50 @@ reportFailure() { fail=$((fail + 1)) } -checkConfigOutput() { +checkConfigCodeOutErr() { + local expectedExit=$1 + shift; local outputContains=$1 shift; - if evalConfig "$@" 2>/dev/null | grep --silent "$outputContains" ; then - pass=$((pass + 1)) - return 0; - else - echo 2>&1 "error: Expected result matching '$outputContains', while evaluating" + local errorContains=$1 + shift; + out=$(mktemp) + err=$(mktemp) + evalConfig "$@" 1>"$out" 2>"$err" + if [[ "$?" -ne "$expectedExit" ]]; then + echo 2>&1 "error: Expected exit code $expectedExit while evaluating" reportFailure "$@" return 1 fi + + if [[ -n "$outputContains" ]] && ! grep -zP --silent "$outputContains" "$out"; then + echo 2>&1 "error: Expected output matching '$outputContains', while evaluating" + reportFailure "$@" + return 1 + fi + + if [[ -n "$errorContains" ]] && ! grep -zP --silent "$errorContains" "$err"; then + echo 2>&1 "error: Expected error matching '$errorContains', while evaluating" + reportFailure "$@" + return 1 + fi + + pass=$((pass + 1)) + + # Clean up temp files + rm "$out" "$err" +} + +checkConfigOutput() { + local outputContains=$1 + shift; + checkConfigCodeOutErr 0 "$outputContains" "" "$@" } checkConfigError() { local errorContains=$1 - local err="" shift; - if err==$(evalConfig "$@" 2>&1 >/dev/null); then - echo 2>&1 "error: Expected error code, got exit code 0, while evaluating" - reportFailure "$@" - return 1 - else - if echo "$err" | grep -zP --silent "$errorContains" ; then - pass=$((pass + 1)) - return 0; - else - echo 2>&1 "error: Expected error matching '$errorContains', while evaluating" - reportFailure "$@" - return 1 - fi - fi + checkConfigCodeOutErr 1 "" "$errorContains" "$@" } # Check boolean option. From 3e39d6efdf65ce8fbf18471c0bb1062b28bfe984 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 2 Sep 2020 22:34:13 +0200 Subject: [PATCH 10/36] lib/tests: Add tests for module-builtin assertions --- lib/tests/modules.sh | 37 +++++++++++++++++++ lib/tests/modules/assertions/enable-false.nix | 8 ++++ lib/tests/modules/assertions/enable-lazy.nix | 17 +++++++++ lib/tests/modules/assertions/multi.nix | 23 ++++++++++++ .../modules/assertions/non-cascading.nix | 17 +++++++++ lib/tests/modules/assertions/simple.nix | 6 +++ .../assertions/submodule-attrsOf-attrsOf.nix | 13 +++++++ .../modules/assertions/submodule-attrsOf.nix | 13 +++++++ lib/tests/modules/assertions/submodule.nix | 13 +++++++ lib/tests/modules/assertions/trigger-lazy.nix | 15 ++++++++ .../modules/assertions/trigger-submodule.nix | 18 +++++++++ .../assertions/underscore-attributes.nix | 8 ++++ lib/tests/modules/assertions/warning.nix | 9 +++++ 13 files changed, 197 insertions(+) create mode 100644 lib/tests/modules/assertions/enable-false.nix create mode 100644 lib/tests/modules/assertions/enable-lazy.nix create mode 100644 lib/tests/modules/assertions/multi.nix create mode 100644 lib/tests/modules/assertions/non-cascading.nix create mode 100644 lib/tests/modules/assertions/simple.nix create mode 100644 lib/tests/modules/assertions/submodule-attrsOf-attrsOf.nix create mode 100644 lib/tests/modules/assertions/submodule-attrsOf.nix create mode 100644 lib/tests/modules/assertions/submodule.nix create mode 100644 lib/tests/modules/assertions/trigger-lazy.nix create mode 100644 lib/tests/modules/assertions/trigger-submodule.nix create mode 100644 lib/tests/modules/assertions/underscore-attributes.nix create mode 100644 lib/tests/modules/assertions/warning.nix diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 012835e48c1..65eb91c9927 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -275,6 +275,43 @@ checkConfigOutput true config.value.mkbefore ./types-anything/mk-mods.nix checkConfigOutput 1 config.value.nested.foo ./types-anything/mk-mods.nix checkConfigOutput baz config.value.nested.bar.baz ./types-anything/mk-mods.nix +## Module assertions +# Check that assertions are triggered by default for just evaluating config +checkConfigError 'Failed assertions:\n- \[test\] Assertion failed' config ./assertions/simple.nix +# Check that assertions are only triggered if they have a triggerPath that's evaluated +checkConfigError 'Failed assertions:\n- \[test\] Assertion failed' config.foo ./assertions/trigger-lazy.nix +checkConfigOutput true config.bar ./assertions/trigger-lazy.nix + +# The assertions enable condition should only be evaluated if the trigger is evaluated +checkConfigError 'enable evaluated' config.foo ./assertions/enable-lazy.nix +checkConfigOutput true config.bar ./assertions/enable-lazy.nix + +# Assertion is not triggered when enable is false +checkConfigOutput '{ }' config ./assertions/enable-false.nix + +# Warnings should be displayed on standard error +checkConfigCodeOutErr 0 '{ }' 'warning: \[test\] Warning message' config ./assertions/warning.nix + +# A triggerPath can be set to a submodule path +checkConfigOutput '{ baz = ; }' config.foo.bar ./assertions/trigger-submodule.nix +checkConfigError 'Failed assertions:\n- \[test\] Assertion failed' config.foo.bar.baz ./assertions/trigger-submodule.nix + +# Check that multiple assertions and warnings can be triggered at once +checkConfigError 'Failed assertions:\n- \[test1\] Assertion 1 failed\n- \[test2\] Assertion 2 failed' config ./assertions/multi.nix +checkConfigError 'trace: warning: \[test3\] Warning 3 failed\ntrace: warning: \[test4\] Warning 4 failed' config ./assertions/multi.nix + +# Submodules should be able to trigger assertions and display the submodule prefix in their error +checkConfigError 'Failed assertions:\n- \[foo/test\] Assertion failed' config.foo ./assertions/submodule.nix +checkConfigError 'Failed assertions:\n- \[foo.bar/test\] Assertion failed' config.foo.bar ./assertions/submodule-attrsOf.nix +checkConfigError 'Failed assertions:\n- \[foo.bar.baz/test\] Assertion failed' config.foo.bar.baz ./assertions/submodule-attrsOf-attrsOf.nix + +# Assertions aren't triggered when the trigger path is only evaluated from within the same module evaluation +# This behavior is necessary to allow assertions to depend on config values. This could potentially be changed in the future if all of NixOS' assertions are rewritten to not depend on any config values +checkConfigOutput true config.bar ./assertions/non-cascading.nix + +# Assertions with an attribute starting with _ shouldn't have their name displayed +checkConfigError 'Failed assertions:\n- Assertion failed' config ./assertions/underscore-attributes.nix + cat < Date: Mon, 30 Nov 2020 20:04:03 +0100 Subject: [PATCH 11/36] lib/modules: Rename _module.assertions to _module.checks --- lib/modules.nix | 66 +++++++++-------- lib/tests/modules.sh | 16 ++-- lib/tests/modules/assertions/enable-false.nix | 2 +- lib/tests/modules/assertions/enable-lazy.nix | 2 +- lib/tests/modules/assertions/multi.nix | 2 +- .../modules/assertions/non-cascading.nix | 2 +- lib/tests/modules/assertions/simple.nix | 2 +- .../assertions/submodule-attrsOf-attrsOf.nix | 2 +- .../modules/assertions/submodule-attrsOf.nix | 2 +- lib/tests/modules/assertions/submodule.nix | 2 +- lib/tests/modules/assertions/trigger-lazy.nix | 2 +- .../modules/assertions/trigger-submodule.nix | 2 +- .../assertions/underscore-attributes.nix | 2 +- lib/tests/modules/assertions/warning.nix | 2 +- nixos/doc/manual/development/assertions.xml | 74 +++++++++++++------ nixos/modules/misc/assertions.nix | 4 +- 16 files changed, 108 insertions(+), 76 deletions(-) diff --git a/lib/modules.nix b/lib/modules.nix index e3f7ca3581c..9aa638231bf 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -103,7 +103,13 @@ rec { type = types.bool; internal = prefix != []; default = check; - description = "Whether to check whether all option definitions have matching declarations."; + description = '' + Whether to check whether all option definitions have matching + declarations. + + Note that this has nothing to do with the similarly named + option + ''; }; _module.freeformType = mkOption { @@ -123,11 +129,11 @@ rec { ''; }; - _module.assertions = mkOption { + _module.checks = mkOption { description = '' - Assertions and warnings to trigger during module evaluation. The + Evaluation checks to trigger during module evaluation. The attribute name will be displayed when it is triggered, allowing - users to disable/change these assertions again if necessary. See + users to disable/change these checks if necessary. See the section on Warnings and Assertions in the manual for more information. ''; @@ -151,7 +157,7 @@ rec { # TODO: Rename to assertion? Or allow also setting assertion? options.enable = mkOption { description = '' - Whether to enable this assertion. + Whether to enable this check. This is the inverse of asserting a condition: If a certain condition should be true, then this @@ -164,7 +170,7 @@ rec { options.type = mkOption { description = '' - The type of the assertion. The default + The type of the check. The default "error" type will cause evaluation to fail, while the "warning" type will only show a warning. @@ -176,7 +182,7 @@ rec { options.message = mkOption { description = '' - The assertion message to display if this assertion triggers. + The message to display if this check triggers. To display option names in the message, add options to the module function arguments and use ''${options.path.to.option}. @@ -190,24 +196,24 @@ rec { options.triggerPath = mkOption { description = '' The config path which when evaluated should - trigger this assertion. By default this is + trigger this check. By default this is [], meaning evaluating - config at all will trigger the assertion. + config at all will trigger the check. On NixOS this default is changed to [ "system" "build" "toplevel" such that - only a system evaluation triggers the assertions. + only a system evaluation triggers the checks. Evaluating config from within the current module evaluation doesn't cause a trigger. Only accessing it from outside will do that. This means it's easy to miss - assertions if this option doesn't have an externally-accessed + failing checks if this option doesn't have an externally-accessed value. ''; # Mark as internal as it's easy to misuse it internal = true; type = types.uniq (types.listOf types.str); - # Default to [], causing assertions to be triggered when + # Default to [], causing checks to be triggered when # anything is evaluated. This is a safe and convenient default. default = []; example = [ "system" "build" "vm" ]; @@ -253,13 +259,13 @@ rec { else recursiveUpdate freeformConfig declaredConfig; /* - Inject a list of assertions into a config value, corresponding to their + Inject a list of checks into a config value, corresponding to their triggerPath (meaning when that path is accessed from the result of this - function, the assertion triggers). + function, the check triggers). */ - injectAssertions = assertions: config: let - # Partition into assertions that are triggered on this level and ones that aren't - parted = lib.partition (a: length a.triggerPath == 0) assertions; + injectChecks = checks: config: let + # Partition into checks that are triggered on this level and ones that aren't + parted = lib.partition (a: length a.triggerPath == 0) checks; # From the ones that are triggered, filter out ones that aren't enabled # and group into warnings/errors @@ -270,45 +276,45 @@ rec { errorTrigger = value: if byType.error or [] == [] then value else throw '' - Failed assertions: + Failed checks: ${lib.concatMapStringsSep "\n" (a: "- ${a.show}") byType.error} ''; # Trigger for both warnings and errors trigger = value: warningTrigger (errorTrigger value); - # From the non-triggered assertions, split off the first element of triggerPath - # to get a mapping from nested attributes to a list of assertions for that attribute + # From the non-triggered checks, split off the first element of triggerPath + # to get a mapping from nested attributes to a list of checks for that attribute nested = lib.zipAttrs (map (a: { ${head a.triggerPath} = a // { triggerPath = lib.tail a.triggerPath; }; }) parted.wrong); - # Recursively inject assertions if config is an attribute set and we - # have assertions under its attributes + # Recursively inject checks if config is an attribute set and we + # have checks under its attributes result = if isAttrs config then mapAttrs (name: value: if nested ? ${name} - then injectAssertions nested.${name} value + then injectChecks nested.${name} value else value ) config else config; in trigger result; - # List of assertions for this module evaluation, where each assertion also + # List of checks for this module evaluation, where each check also # has a `show` attribute for how to show it if triggered - assertions = mapAttrsToList (name: value: + checks = mapAttrsToList (name: value: let id = if lib.hasPrefix "_" name then "" else "[${showOption prefix}${optionalString (prefix != []) "/"}${name}] "; in value // { show = "${id}${value.message}"; } - ) config._module.assertions; + ) config._module.checks; - finalConfig = injectAssertions assertions (removeAttrs config [ "_module" ]); + finalConfig = injectChecks checks (removeAttrs config [ "_module" ]); checkUnmatched = if config._module.check && config._module.freeformType == null && merged.unmatchedDefns != [] then @@ -931,7 +937,7 @@ rec { visible = false; apply = x: throw "The option `${showOption optionName}' can no longer be used since it's been removed. ${replacementInstructions}"; }); - config._module.assertions = + config._module.checks = let opt = getAttrFromPath optionName options; in { ${showOption optionName} = { enable = mkDefault opt.isDefined; @@ -1001,7 +1007,7 @@ rec { })) from); config = { - _module.assertions = + _module.checks = let warningMessages = map (f: let val = getAttrFromPath f config; opt = getAttrFromPath f options; @@ -1072,7 +1078,7 @@ rec { }); config = mkMerge [ { - _module.assertions.${showOption from} = { + _module.checks.${showOption from} = { enable = mkDefault (warn && fromOpt.isDefined); type = "warning"; message = "The option `${showOption from}' defined in ${showFiles fromOpt.files} has been renamed to `${showOption to}'."; diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 65eb91c9927..43bcabdf816 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -277,9 +277,9 @@ checkConfigOutput baz config.value.nested.bar.baz ./types-anything/mk-mods.nix ## Module assertions # Check that assertions are triggered by default for just evaluating config -checkConfigError 'Failed assertions:\n- \[test\] Assertion failed' config ./assertions/simple.nix +checkConfigError 'Failed checks:\n- \[test\] Assertion failed' config ./assertions/simple.nix # Check that assertions are only triggered if they have a triggerPath that's evaluated -checkConfigError 'Failed assertions:\n- \[test\] Assertion failed' config.foo ./assertions/trigger-lazy.nix +checkConfigError 'Failed checks:\n- \[test\] Assertion failed' config.foo ./assertions/trigger-lazy.nix checkConfigOutput true config.bar ./assertions/trigger-lazy.nix # The assertions enable condition should only be evaluated if the trigger is evaluated @@ -294,23 +294,23 @@ checkConfigCodeOutErr 0 '{ }' 'warning: \[test\] Warning message' config ./asser # A triggerPath can be set to a submodule path checkConfigOutput '{ baz = ; }' config.foo.bar ./assertions/trigger-submodule.nix -checkConfigError 'Failed assertions:\n- \[test\] Assertion failed' config.foo.bar.baz ./assertions/trigger-submodule.nix +checkConfigError 'Failed checks:\n- \[test\] Assertion failed' config.foo.bar.baz ./assertions/trigger-submodule.nix # Check that multiple assertions and warnings can be triggered at once -checkConfigError 'Failed assertions:\n- \[test1\] Assertion 1 failed\n- \[test2\] Assertion 2 failed' config ./assertions/multi.nix +checkConfigError 'Failed checks:\n- \[test1\] Assertion 1 failed\n- \[test2\] Assertion 2 failed' config ./assertions/multi.nix checkConfigError 'trace: warning: \[test3\] Warning 3 failed\ntrace: warning: \[test4\] Warning 4 failed' config ./assertions/multi.nix # Submodules should be able to trigger assertions and display the submodule prefix in their error -checkConfigError 'Failed assertions:\n- \[foo/test\] Assertion failed' config.foo ./assertions/submodule.nix -checkConfigError 'Failed assertions:\n- \[foo.bar/test\] Assertion failed' config.foo.bar ./assertions/submodule-attrsOf.nix -checkConfigError 'Failed assertions:\n- \[foo.bar.baz/test\] Assertion failed' config.foo.bar.baz ./assertions/submodule-attrsOf-attrsOf.nix +checkConfigError 'Failed checks:\n- \[foo/test\] Assertion failed' config.foo ./assertions/submodule.nix +checkConfigError 'Failed checks:\n- \[foo.bar/test\] Assertion failed' config.foo.bar ./assertions/submodule-attrsOf.nix +checkConfigError 'Failed checks:\n- \[foo.bar.baz/test\] Assertion failed' config.foo.bar.baz ./assertions/submodule-attrsOf-attrsOf.nix # Assertions aren't triggered when the trigger path is only evaluated from within the same module evaluation # This behavior is necessary to allow assertions to depend on config values. This could potentially be changed in the future if all of NixOS' assertions are rewritten to not depend on any config values checkConfigOutput true config.bar ./assertions/non-cascading.nix # Assertions with an attribute starting with _ shouldn't have their name displayed -checkConfigError 'Failed assertions:\n- Assertion failed' config ./assertions/underscore-attributes.nix +checkConfigError 'Failed checks:\n- Assertion failed' config ./assertions/underscore-attributes.nix cat < - Warnings and Assertions + Evaluation Checks When configuration problems are detectable in a module, it is a good idea to - write an assertion or warning. Doing so provides clear feedback to the user - and can prevent errors before the build. + write a check for catching it early. Doing so can provide clear feedback to + the user and can prevent errors before the build. Although Nix has the abort and builtins.trace functions - to perform such tasks, they are not ideally suited for NixOS modules. Instead - of these functions, you can declare your warnings and assertions using the - NixOS module system. + to perform such tasks generally, they are not ideally suited for NixOS + modules. Instead of these functions, you can declare your evaluation checks + using the NixOS module system.
- Defining Warnings and Assertions + Defining Checks - Both warnings and assertions can be defined using the option. Each assertion needs an attribute name, under which you have to define an enable condition using and a message using . Note that the enable condition is inverse of what an assertion would be: To assert a value being true, the enable condition should be false in that case, so that it isn't triggered. For the assertion message, you can add options to the module arguments and use ${options.path.to.option} to print a context-aware string representation of the option path. Here is an example showing how this can be done. + Checks can be defined using the option. + Each check needs an attribute name, under which you have to define an enable + condition using and a + message using . Note that + the enable condition is inverse of what an assertion + would be: To assert a value being true, the enable condition should be false + in that case, so that it isn't triggered. For the check message, you can add + options to the module arguments and use + ${options.path.to.option} to print a context-aware string + representation of the option path. Here is an example showing how this can be + done. { config, options, ... }: { - _module.assertions.gpgSshAgent = { + _module.checks.gpgSshAgent = { enable = config.programs.gnupg.agent.enableSSHSupport && config.programs.ssh.startAgent; message = "You can't enable both ${options.programs.ssh.startAgent}" + " and ${options.programs.gnupg.agent.enableSSHSupport}!"; }; - _module.assertions.grafanaPassword = { + _module.checks.grafanaPassword = { enable = config.services.grafana.database.password != ""; message = "The grafana password defined with ${options.services.grafana.database.password}" + " will be stored as plaintext in the Nix store!"; @@ -48,41 +58,51 @@
- Ignoring Warnings and Assertions + Ignoring Checks - Sometimes you can get warnings or assertions that don't apply to your specific case and you wish to ignore them, or at least make assertions non-fatal. You can do so for all assertions defined using by using the attribute name of the definition, which is conveniently printed using [...] when the assertion is triggered. For above example, the evaluation output when the assertions are triggered looks as follows: + Sometimes you can get failing checks that don't apply to your specific case + and you wish to ignore them, or at least make errors non-fatal. You can do so + for all checks defined using by + using the attribute name of the definition, which is conveniently printed + using [...] when the check is triggered. For above + example, the evaluation output when the checks are triggered looks as + follows: trace: warning: [grafanaPassword] The grafana password defined with services.grafana.database.password will be stored as plaintext in the Nix store! -error: Failed assertions: +error: Failed checks: - [gpgSshAgent] You can't enable both programs.ssh.startAgent and programs.gnupg.agent.enableSSHSupport! - The [grafanaPassword] and [gpgSshAgent] strings tell you that these were defined under the grafanaPassword and gpgSshAgent attributes of respectively. With this knowledge you can adjust them to your liking: + The [grafanaPassword] and [gpgSshAgent] + strings tell you that these were defined under the grafanaPassword + and gpgSshAgent attributes of + respectively. With this knowledge + you can adjust them to your liking: { lib, ... }: { - # Change the assertion into a non-fatal warning - _module.assertions.gpgSshAgent.type = "warning"; + # Change the error into a non-fatal warning + _module.checks.gpgSshAgent.type = "warning"; # We don't care about this warning, disable it - _module.assertions.grafanaPassword.enable = lib.mkForce false; + _module.checks.grafanaPassword.enable = lib.mkForce false; }
- Warnings and Assertions in Submodules + Checks in Submodules - Warnings and assertions can be defined within submodules in the same way. Here is an example: + Evaluation checks can be defined within submodules in the same way. Here is an example: @@ -92,7 +112,7 @@ error: Failed assertions: type = lib.types.attrsOf (lib.types.submodule ({ config, options, ... }: { options.port = lib.mkOption {}; - config._module.assertions.portConflict = { + config._module.checks.portConflict = { enable = config.port == 80; message = "Port ${toString config.port} defined using" + " ${options.port} is usually used for HTTP"; @@ -105,7 +125,10 @@ error: Failed assertions: - When this assertion is triggered, it shows both the submodule path along with the assertion attribute within that submodule, joined by a /. Note also how ${options.port} correctly shows the context of the option. + When this check is triggered, it shows both the submodule path along with + the check attribute within that submodule, joined by a + /. Note also how ${options.port} + correctly shows the context of the option. @@ -114,18 +137,21 @@ trace: warning: [myServices.foo/portConflict] Port 80 defined using - Therefore to disable such an assertion, you can do so by changing the option within the myServices.foo submodule: + Therefore to disable such a check, you can do so by changing the + option within the + myServices.foo submodule: { lib, ... }: { - myServices.foo._module.assertions.portConflict.enable = lib.mkForce false; + myServices.foo._module.checks.portConflict.enable = lib.mkForce false; } - Assertions defined in submodules under types.listOf can't be ignored, since there's no way to change previously defined list items. + Checks defined in submodules under types.listOf can't be + ignored, since there's no way to change previously defined list items. diff --git a/nixos/modules/misc/assertions.nix b/nixos/modules/misc/assertions.nix index e931611247f..e8b1f5afca3 100644 --- a/nixos/modules/misc/assertions.nix +++ b/nixos/modules/misc/assertions.nix @@ -29,7 +29,7 @@ with lib; ''; }; - _module.assertions = mkOption { + _module.checks = mkOption { type = types.attrsOf (types.submodule { triggerPath = mkDefault [ "system" "build" "toplevel" ]; }); @@ -37,7 +37,7 @@ with lib; }; - config._module.assertions = lib.listToAttrs (lib.imap1 (n: value: + config._module.checks = lib.listToAttrs (lib.imap1 (n: value: let name = "_${toString n}"; isWarning = lib.isString value; From 8dea4df90323c43f9cc86a629f1581b91866e11d Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Mon, 30 Nov 2020 21:42:52 +0100 Subject: [PATCH 12/36] lib/modules: Remove _module.checks.*.triggerPath as it's not necessary Previously this option was thought to be necessary to avoid infinite recursion, but it actually isn't, since the check evaluation isn't fed back into the module fixed-point. --- lib/modules.nix | 99 +++++-------------- lib/tests/modules.sh | 15 --- lib/tests/modules/assertions/enable-lazy.nix | 17 ---- .../modules/assertions/non-cascading.nix | 17 ---- lib/tests/modules/assertions/trigger-lazy.nix | 15 --- .../modules/assertions/trigger-submodule.nix | 18 ---- nixos/modules/misc/assertions.nix | 6 -- 7 files changed, 22 insertions(+), 165 deletions(-) delete mode 100644 lib/tests/modules/assertions/enable-lazy.nix delete mode 100644 lib/tests/modules/assertions/non-cascading.nix delete mode 100644 lib/tests/modules/assertions/trigger-lazy.nix delete mode 100644 lib/tests/modules/assertions/trigger-submodule.nix diff --git a/lib/modules.nix b/lib/modules.nix index 9aa638231bf..2c827a01e01 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -192,32 +192,6 @@ rec { Enabling both ''${options.services.foo.enable} and ''${options.services.bar.enable} is not possible. ''; }; - - options.triggerPath = mkOption { - description = '' - The config path which when evaluated should - trigger this check. By default this is - [], meaning evaluating - config at all will trigger the check. - On NixOS this default is changed to - [ "system" "build" "toplevel" such that - only a system evaluation triggers the checks. - - Evaluating config from within the current - module evaluation doesn't cause a trigger. Only accessing it - from outside will do that. This means it's easy to miss - failing checks if this option doesn't have an externally-accessed - value. - - ''; - # Mark as internal as it's easy to misuse it - internal = true; - type = types.uniq (types.listOf types.str); - # Default to [], causing checks to be triggered when - # anything is evaluated. This is a safe and convenient default. - default = []; - example = [ "system" "build" "vm" ]; - }; }); }; }; @@ -258,63 +232,34 @@ rec { # paths, meaning recursiveUpdate will never override any value else recursiveUpdate freeformConfig declaredConfig; - /* - Inject a list of checks into a config value, corresponding to their - triggerPath (meaning when that path is accessed from the result of this - function, the check triggers). - */ - injectChecks = checks: config: let - # Partition into checks that are triggered on this level and ones that aren't - parted = lib.partition (a: length a.triggerPath == 0) checks; + # Triggers all checks defined by _module.checks before returning its argument + triggerChecks = let - # From the ones that are triggered, filter out ones that aren't enabled - # and group into warnings/errors - byType = lib.groupBy (a: a.type) (filter (a: a.enable) parted.right); + handleCheck = errors: name: + let + value = config._module.checks.${name}; + show = + # Assertions with a _ prefix aren't meant to be configurable + if lib.hasPrefix "_" name then value.message + else "[${showOption prefix}${optionalString (prefix != []) "/"}${name}] ${value.message}"; + in + if ! value.enable then errors + else if value.type == "warning" then lib.warn show errors + else if value.type == "error" then errors ++ [ show ] + else abort "Unknown check type ${value.type}"; - # Triggers semantically are just lib.id, but they print warning cause errors in addition - warningTrigger = value: lib.foldr (w: warn w.show) value (byType.warning or []); - errorTrigger = value: - if byType.error or [] == [] then value else - throw '' - Failed checks: - ${lib.concatMapStringsSep "\n" (a: "- ${a.show}") byType.error} - ''; - # Trigger for both warnings and errors - trigger = value: warningTrigger (errorTrigger value); + errors = lib.foldl' handleCheck [] (lib.attrNames config._module.checks); - # From the non-triggered checks, split off the first element of triggerPath - # to get a mapping from nested attributes to a list of checks for that attribute - nested = lib.zipAttrs (map (a: { - ${head a.triggerPath} = a // { - triggerPath = lib.tail a.triggerPath; - }; - }) parted.wrong); + errorMessage = '' + Failed checks: + ${lib.concatMapStringsSep "\n" (a: "- ${a}") errors} + ''; - # Recursively inject checks if config is an attribute set and we - # have checks under its attributes - result = - if isAttrs config - then - mapAttrs (name: value: - if nested ? ${name} - then injectChecks nested.${name} value - else value - ) config - else config; - in trigger result; + trigger = if errors == [] then null else throw errorMessage; - # List of checks for this module evaluation, where each check also - # has a `show` attribute for how to show it if triggered - checks = mapAttrsToList (name: value: - let id = - if lib.hasPrefix "_" name then "" - else "[${showOption prefix}${optionalString (prefix != []) "/"}${name}] "; - in value // { - show = "${id}${value.message}"; - } - ) config._module.checks; + in builtins.seq trigger; - finalConfig = injectChecks checks (removeAttrs config [ "_module" ]); + finalConfig = triggerChecks (removeAttrs config [ "_module" ]); checkUnmatched = if config._module.check && config._module.freeformType == null && merged.unmatchedDefns != [] then diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 43bcabdf816..9e85c90d15c 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -278,13 +278,6 @@ checkConfigOutput baz config.value.nested.bar.baz ./types-anything/mk-mods.nix ## Module assertions # Check that assertions are triggered by default for just evaluating config checkConfigError 'Failed checks:\n- \[test\] Assertion failed' config ./assertions/simple.nix -# Check that assertions are only triggered if they have a triggerPath that's evaluated -checkConfigError 'Failed checks:\n- \[test\] Assertion failed' config.foo ./assertions/trigger-lazy.nix -checkConfigOutput true config.bar ./assertions/trigger-lazy.nix - -# The assertions enable condition should only be evaluated if the trigger is evaluated -checkConfigError 'enable evaluated' config.foo ./assertions/enable-lazy.nix -checkConfigOutput true config.bar ./assertions/enable-lazy.nix # Assertion is not triggered when enable is false checkConfigOutput '{ }' config ./assertions/enable-false.nix @@ -292,10 +285,6 @@ checkConfigOutput '{ }' config ./assertions/enable-false.nix # Warnings should be displayed on standard error checkConfigCodeOutErr 0 '{ }' 'warning: \[test\] Warning message' config ./assertions/warning.nix -# A triggerPath can be set to a submodule path -checkConfigOutput '{ baz = ; }' config.foo.bar ./assertions/trigger-submodule.nix -checkConfigError 'Failed checks:\n- \[test\] Assertion failed' config.foo.bar.baz ./assertions/trigger-submodule.nix - # Check that multiple assertions and warnings can be triggered at once checkConfigError 'Failed checks:\n- \[test1\] Assertion 1 failed\n- \[test2\] Assertion 2 failed' config ./assertions/multi.nix checkConfigError 'trace: warning: \[test3\] Warning 3 failed\ntrace: warning: \[test4\] Warning 4 failed' config ./assertions/multi.nix @@ -305,10 +294,6 @@ checkConfigError 'Failed checks:\n- \[foo/test\] Assertion failed' config.foo ./ checkConfigError 'Failed checks:\n- \[foo.bar/test\] Assertion failed' config.foo.bar ./assertions/submodule-attrsOf.nix checkConfigError 'Failed checks:\n- \[foo.bar.baz/test\] Assertion failed' config.foo.bar.baz ./assertions/submodule-attrsOf-attrsOf.nix -# Assertions aren't triggered when the trigger path is only evaluated from within the same module evaluation -# This behavior is necessary to allow assertions to depend on config values. This could potentially be changed in the future if all of NixOS' assertions are rewritten to not depend on any config values -checkConfigOutput true config.bar ./assertions/non-cascading.nix - # Assertions with an attribute starting with _ shouldn't have their name displayed checkConfigError 'Failed checks:\n- Assertion failed' config ./assertions/underscore-attributes.nix diff --git a/lib/tests/modules/assertions/enable-lazy.nix b/lib/tests/modules/assertions/enable-lazy.nix deleted file mode 100644 index e2519022864..00000000000 --- a/lib/tests/modules/assertions/enable-lazy.nix +++ /dev/null @@ -1,17 +0,0 @@ -{ lib, ... }: { - - options.foo = lib.mkOption { - default = true; - }; - - options.bar = lib.mkOption { - default = true; - }; - - config._module.checks.test = { - enable = throw "enable evaluated"; - message = "Assertion failed"; - triggerPath = [ "foo" ]; - }; - -} diff --git a/lib/tests/modules/assertions/non-cascading.nix b/lib/tests/modules/assertions/non-cascading.nix deleted file mode 100644 index 7b9e333a11a..00000000000 --- a/lib/tests/modules/assertions/non-cascading.nix +++ /dev/null @@ -1,17 +0,0 @@ -{ lib, config, ... }: { - - options.foo = lib.mkOption { - default = true; - }; - - options.bar = lib.mkOption { - default = config.foo; - }; - - config._module.checks.foo = { - enable = true; - message = "Foo assertion"; - triggerPath = [ "foo" ]; - }; - -} diff --git a/lib/tests/modules/assertions/trigger-lazy.nix b/lib/tests/modules/assertions/trigger-lazy.nix deleted file mode 100644 index 9e9e3683b0c..00000000000 --- a/lib/tests/modules/assertions/trigger-lazy.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ lib, ... }: { - options.foo = lib.mkOption { - default = true; - }; - - options.bar = lib.mkOption { - default = true; - }; - - config._module.checks.test = { - enable = true; - message = "Assertion failed"; - triggerPath = [ "foo" ]; - }; -} diff --git a/lib/tests/modules/assertions/trigger-submodule.nix b/lib/tests/modules/assertions/trigger-submodule.nix deleted file mode 100644 index 27deb48e4a9..00000000000 --- a/lib/tests/modules/assertions/trigger-submodule.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ lib, ... }: { - - options.foo = lib.mkOption { - default = { bar = {}; }; - type = lib.types.attrsOf (lib.types.submodule { - options.baz = lib.mkOption { - default = true; - }; - }); - }; - - config._module.checks.test = { - enable = true; - message = "Assertion failed"; - triggerPath = [ "foo" "bar" "baz" ]; - }; - -} diff --git a/nixos/modules/misc/assertions.nix b/nixos/modules/misc/assertions.nix index e8b1f5afca3..6a26a2332f2 100644 --- a/nixos/modules/misc/assertions.nix +++ b/nixos/modules/misc/assertions.nix @@ -29,12 +29,6 @@ with lib; ''; }; - _module.checks = mkOption { - type = types.attrsOf (types.submodule { - triggerPath = mkDefault [ "system" "build" "toplevel" ]; - }); - }; - }; config._module.checks = lib.listToAttrs (lib.imap1 (n: value: From 991dfccbd1935aabb76a20245ca0108aadd38f3c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Mon, 30 Nov 2020 22:10:48 +0100 Subject: [PATCH 13/36] lib/modules: _module.check should always be internal Honestly this option should probably just be removed --- lib/modules.nix | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/modules.nix b/lib/modules.nix index 2c827a01e01..23dbe962491 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -73,19 +73,20 @@ rec { check ? true }: let - # This internal module declare internal options under the `_module' - # attribute. These options are fragile, as they are used by the - # module system to change the interpretation of modules. + # An internal module that's always added, defining special options which + # change the behavior of the module evaluation itself. This is under a + # `_`-prefixed namespace in order to prevent name clashes with + # user-defined options internalModule = rec { # FIXME: Using ./modules.nix directly breaks the doc for some reason _file = "lib/modules.nix"; key = _file; - # These options are set to be internal only for prefix != [], aka it's - # a submodule evaluation. This way their docs are displayed only once - # as a top-level NixOS option, but will be hidden for all submodules, - # even though they are available there too + # Most of these options are set to be internal only for prefix != [], + # aka it's a submodule evaluation. This way their docs are displayed + # only once as a top-level NixOS option, but will be hidden for all + # submodules, even though they are available there too options = { _module.args = mkOption { # Because things like `mkIf` are entirely useless for @@ -101,7 +102,7 @@ rec { _module.check = mkOption { type = types.bool; - internal = prefix != []; + internal = true; default = check; description = '' Whether to check whether all option definitions have matching From a343ff7e1433130ac293e96bb04ea5dbe363fff1 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 3 Sep 2020 14:11:41 +0200 Subject: [PATCH 14/36] makeInitrd: make uinitrd behaviour optional --- pkgs/build-support/kernel/make-initrd.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/kernel/make-initrd.nix b/pkgs/build-support/kernel/make-initrd.nix index ed5dbdaee17..3915335b8da 100644 --- a/pkgs/build-support/kernel/make-initrd.nix +++ b/pkgs/build-support/kernel/make-initrd.nix @@ -17,6 +17,7 @@ , compressor ? "gzip -9n" , prepend ? [] , lib +, makeUInitrd ? stdenvNoCC.hostPlatform.platform.kernelTarget == "uImage" }: let # !!! Move this into a public lib function, it is probably useful for others @@ -24,12 +25,10 @@ let lib.concatStringsSep "-" (filter (x: !(isList x)) (split "[^a-zA-Z0-9_=.?-]+" x)); in stdenvNoCC.mkDerivation rec { - inherit name; + inherit name makeUInitrd; builder = ./make-initrd.sh; - makeUInitrd = stdenvNoCC.hostPlatform.platform.kernelTarget == "uImage"; - nativeBuildInputs = [ perl cpio ] ++ stdenvNoCC.lib.optional makeUInitrd ubootTools; From 85e0ae78276bf7259d75888df89386f4a04214b3 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 3 Sep 2020 14:17:07 +0200 Subject: [PATCH 15/36] makeInitrd: don't assume uImage => arm mips for example might use uImages too --- pkgs/build-support/kernel/make-initrd.nix | 3 ++- pkgs/build-support/kernel/make-initrd.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/kernel/make-initrd.nix b/pkgs/build-support/kernel/make-initrd.nix index 3915335b8da..f0a61d3476e 100644 --- a/pkgs/build-support/kernel/make-initrd.nix +++ b/pkgs/build-support/kernel/make-initrd.nix @@ -18,6 +18,7 @@ , prepend ? [] , lib , makeUInitrd ? stdenvNoCC.hostPlatform.platform.kernelTarget == "uImage" +, uInitrdArch ? stdenvNoCC.hostPlatform.parsed.cpu.family }: let # !!! Move this into a public lib function, it is probably useful for others @@ -25,7 +26,7 @@ let lib.concatStringsSep "-" (filter (x: !(isList x)) (split "[^a-zA-Z0-9_=.?-]+" x)); in stdenvNoCC.mkDerivation rec { - inherit name makeUInitrd; + inherit name makeUInitrd uInitrdArch; builder = ./make-initrd.sh; diff --git a/pkgs/build-support/kernel/make-initrd.sh b/pkgs/build-support/kernel/make-initrd.sh index 0aeaedeb372..822883a681f 100644 --- a/pkgs/build-support/kernel/make-initrd.sh +++ b/pkgs/build-support/kernel/make-initrd.sh @@ -44,5 +44,5 @@ done if [ -n "$makeUInitrd" ]; then mv $out/initrd $out/initrd.gz - mkimage -A arm -O linux -T ramdisk -C gzip -d $out/initrd.gz $out/initrd + mkimage -A $uInitrdArch -O linux -T ramdisk -C gzip -d $out/initrd.gz $out/initrd fi From 3a3c9c95481c5382ab7b7ede24ffd2f38651d2df Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 3 Sep 2020 14:19:43 +0200 Subject: [PATCH 16/36] makeInitrd: include dotfiles at root --- pkgs/build-support/kernel/make-initrd.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/kernel/make-initrd.sh b/pkgs/build-support/kernel/make-initrd.sh index 822883a681f..8243736ba46 100644 --- a/pkgs/build-support/kernel/make-initrd.sh +++ b/pkgs/build-support/kernel/make-initrd.sh @@ -39,8 +39,8 @@ mkdir -p $out for PREP in $prepend; do cat $PREP >> $out/initrd done -(cd root && find * -print0 | xargs -0r touch -h -d '@1') -(cd root && find * -print0 | sort -z | cpio -o -H newc -R +0:+0 --reproducible --null | $compressor >> $out/initrd) +(cd root && find * .[^.*] -exec touch -h -d '@1' '{}' +) +(cd root && find * .[^.*] -print0 | sort -z | cpio -o -H newc -R +0:+0 --reproducible --null | $compressor >> $out/initrd) if [ -n "$makeUInitrd" ]; then mv $out/initrd $out/initrd.gz From 2ee35e1fcecdae598651fd9b1452f451ac221384 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 3 Sep 2020 15:58:10 +0200 Subject: [PATCH 17/36] lib/systems: fix kernelArch for x86_64 IA64 (Itanium) is something completely different and certainly not what we want! x86_64 code lives in arch/x86 just like "classic" x86. --- lib/systems/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/systems/default.nix b/lib/systems/default.nix index f6832945a23..e31a8e65f2a 100644 --- a/lib/systems/default.nix +++ b/lib/systems/default.nix @@ -83,7 +83,7 @@ rec { if final.isAarch32 then "arm" else if final.isAarch64 then "arm64" else if final.isx86_32 then "x86" - else if final.isx86_64 then "ia64" + else if final.isx86_64 then "x86" else if final.isMips then "mips" else final.parsed.cpu.name; From 14fbf575ecd47451dc1b2b3122f56ce8ac670030 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Fri, 4 Sep 2020 18:18:32 +0200 Subject: [PATCH 18/36] make-initrd: various improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Generate a link to the initramfs file with an appropriate file extension, guessed based on the compressor by default - Use correct metadata in u-boot images if generated, up to now this was hardcoded to gzip and would silently generate an erroneous image if another compressor was specified - Document all the parameters - Improve cross-building compatibility, by allowing passing either a string as before, or a function taking a package set and returning the path to a compressor in the "compressor" argument of the function. - Support more compression algorithms - Place compressor executable function and arguments in passthru, for reuse when appending initramfses Co-Authored-By: Dominik Xaver Hörl --- .../kernel/initrd-compressor-meta.nix | 53 +++++++++++ pkgs/build-support/kernel/make-initrd.nix | 91 ++++++++++++++++--- pkgs/build-support/kernel/make-initrd.sh | 9 +- 3 files changed, 135 insertions(+), 18 deletions(-) create mode 100644 pkgs/build-support/kernel/initrd-compressor-meta.nix diff --git a/pkgs/build-support/kernel/initrd-compressor-meta.nix b/pkgs/build-support/kernel/initrd-compressor-meta.nix new file mode 100644 index 00000000000..443e599a239 --- /dev/null +++ b/pkgs/build-support/kernel/initrd-compressor-meta.nix @@ -0,0 +1,53 @@ +rec { + cat = { + executable = pkgs: "cat"; + ubootName = "none"; + extension = ".cpio"; + }; + gzip = { + executable = pkgs: "${pkgs.gzip}/bin/gzip"; + defaultArgs = ["-9n"]; + ubootName = "gzip"; + extension = ".gz"; + }; + bzip2 = { + executable = pkgs: "${pkgs.bzip2}/bin/bzip2"; + ubootName = "bzip2"; + extension = ".bz2"; + }; + xz = { + executable = pkgs: "${pkgs.xz}/bin/xz"; + defaultArgs = ["--check=crc32" "--lzma2=dict=512KiB"]; + extension = ".xz"; + }; + lzma = { + executable = pkgs: "${pkgs.xz}/bin/lzma"; + defaultArgs = ["--check=crc32" "--lzma1=dict=512KiB"]; + ubootName = "lzma"; + extension = ".lzma"; + }; + lz4 = { + executable = pkgs: "${pkgs.lz4}/bin/lz4"; + defaultArgs = ["-l"]; + ubootName = "lz4"; + extension = ".lz4"; + }; + lzop = { + executable = pkgs: "${pkgs.lzop}/bin/lzop"; + ubootName = "lzo"; + extension = ".lzo"; + }; + zstd = { + executable = pkgs: "${pkgs.zstd}/bin/zstd"; + defaultArgs = ["-10"]; + ubootName = "zstd"; + extension = ".zst"; + }; + pigz = gzip // { + executable = pkgs: "${pkgs.pigz}/bin/pigz"; + }; + pixz = xz // { + executable = pkgs: "${pkgs.pixz}/bin/pixz"; + defaultArgs = []; + }; +} diff --git a/pkgs/build-support/kernel/make-initrd.nix b/pkgs/build-support/kernel/make-initrd.nix index f0a61d3476e..901eb311a88 100644 --- a/pkgs/build-support/kernel/make-initrd.nix +++ b/pkgs/build-support/kernel/make-initrd.nix @@ -1,24 +1,74 @@ -# Create an initial ramdisk containing the closure of the specified -# file system objects. An initial ramdisk is used during the initial +# Create an initramfs containing the closure of the specified +# file system objects. An initramfs is used during the initial # stages of booting a Linux system. It is loaded by the boot loader # along with the kernel image. It's supposed to contain everything # (such as kernel modules) necessary to allow us to mount the root # file system. Once the root file system is mounted, the `real' boot # script can be called. # -# An initrd is really just a gzipped cpio archive. -# -# Symlinks are created for each top-level file system object. E.g., -# `contents = {object = ...; symlink = /init;}' is a typical -# argument. - -{ stdenvNoCC, perl, cpio, contents, ubootTools +# An initramfs is a cpio archive, and may be compressed with a number +# of algorithms. +let + # Some metadata on various compression programs, relevant to naming + # the initramfs file and, if applicable, generating a u-boot image + # from it. + compressors = import ./initrd-compressor-meta.nix; + # Get the basename of the actual compression program from the whole + # compression command, for the purpose of guessing the u-boot + # compression type and filename extension. + compressorName = fullCommand: builtins.elemAt (builtins.match "([^ ]*/)?([^ ]+).*" fullCommand) 1; +in +{ stdenvNoCC, perl, cpio, ubootTools, lib, pkgsBuildHost +# Name of the derivation (not of the resulting file!) , name ? "initrd" -, compressor ? "gzip -9n" + +# Program used to compress the cpio archive; use "cat" for no compression. +# This can also be a function which takes a package set and returns the path to the compressor, +# such as `pkgs: "${pkgs.lzop}/bin/lzop"`. +, compressor ? "gzip" +, _compressorFunction ? + if lib.isFunction compressor then compressor + else if ! builtins.hasContext compressor && builtins.hasAttr compressor compressors then compressors.${compressor}.executable + else _: compressor +, _compressorExecutable ? _compressorFunction pkgsBuildHost +, _compressorName ? compressorName _compressorExecutable +, _compressorMeta ? compressors.${_compressorName} or {} + +# List of arguments to pass to the compressor program, or null to use its defaults +, compressorArgs ? null +, _compressorArgsReal ? if compressorArgs == null then _compressorMeta.defaultArgs or [] else compressorArgs + +# Filename extension to use for the compressed initramfs. This is +# included for clarity, but $out/initrd will always be a symlink to +# the final image. +# If this isn't guessed, you may want to complete the metadata above and send a PR :) +, extension ? _compressorMeta.extension or + (throw "Unrecognised compressor ${_compressorName}, please specify filename extension") + +# List of { object = path_or_derivation; symlink = "/path"; } +# The paths are copied into the initramfs in their nix store path +# form, then linked at the root according to `symlink`. +, contents + +# List of uncompressed cpio files to prepend to the initramfs. This +# can be used to add files in specified paths without them becoming +# symlinks to store paths. , prepend ? [] -, lib + +# Whether to wrap the initramfs in a u-boot image. , makeUInitrd ? stdenvNoCC.hostPlatform.platform.kernelTarget == "uImage" -, uInitrdArch ? stdenvNoCC.hostPlatform.parsed.cpu.family + +# If generating a u-boot image, the architecture to use. The default +# guess may not align with u-boot's nomenclature correctly, so it can +# be overridden. +# See https://gitlab.denx.de/u-boot/u-boot/-/blob/9bfb567e5f1bfe7de8eb41f8c6d00f49d2b9a426/common/image.c#L81-106 for a list. +, uInitrdArch ? stdenvNoCC.hostPlatform.kernelArch + +# The name of the compression, as recognised by u-boot. +# See https://gitlab.denx.de/u-boot/u-boot/-/blob/9bfb567e5f1bfe7de8eb41f8c6d00f49d2b9a426/common/image.c#L195-204 for a list. +# If this isn't guessed, you may want to complete the metadata above and send a PR :) +, uInitrdCompression ? _compressorMeta.ubootName or + (throw "Unrecognised compressor ${_compressorName}, please specify uInitrdCompression") }: let # !!! Move this into a public lib function, it is probably useful for others @@ -26,13 +76,26 @@ let lib.concatStringsSep "-" (filter (x: !(isList x)) (split "[^a-zA-Z0-9_=.?-]+" x)); in stdenvNoCC.mkDerivation rec { - inherit name makeUInitrd uInitrdArch; + inherit name makeUInitrd extension uInitrdArch prepend; + + ${if makeUInitrd then "uinitrdCompression" else null} = uInitrdCompression; builder = ./make-initrd.sh; nativeBuildInputs = [ perl cpio ] ++ stdenvNoCC.lib.optional makeUInitrd ubootTools; + compress = "${_compressorExecutable} ${lib.escapeShellArgs _compressorArgsReal}"; + + # Pass the function through, for reuse in append-initrd-secrets. The + # function is used instead of the string, in order to support + # cross-compilation (append-initrd-secrets running on a different + # architecture than what the main initramfs is built on). + passthru = { + compressorExecutableFunction = _compressorFunction; + compressorArgs = _compressorArgsReal; + }; + # !!! should use XML. objects = map (x: x.object) contents; symlinks = map (x: x.symlink) contents; @@ -47,6 +110,4 @@ in stdenvNoCC.mkDerivation rec { contents (lib.range 0 (lib.length contents - 1)); pathsFromGraph = ./paths-from-graph.pl; - - inherit compressor prepend; } diff --git a/pkgs/build-support/kernel/make-initrd.sh b/pkgs/build-support/kernel/make-initrd.sh index 8243736ba46..c0619ef14ae 100644 --- a/pkgs/build-support/kernel/make-initrd.sh +++ b/pkgs/build-support/kernel/make-initrd.sh @@ -40,9 +40,12 @@ for PREP in $prepend; do cat $PREP >> $out/initrd done (cd root && find * .[^.*] -exec touch -h -d '@1' '{}' +) -(cd root && find * .[^.*] -print0 | sort -z | cpio -o -H newc -R +0:+0 --reproducible --null | $compressor >> $out/initrd) +(cd root && find * .[^.*] -print0 | sort -z | cpio -o -H newc -R +0:+0 --reproducible --null | eval -- $compress >> "$out/initrd") if [ -n "$makeUInitrd" ]; then - mv $out/initrd $out/initrd.gz - mkimage -A $uInitrdArch -O linux -T ramdisk -C gzip -d $out/initrd.gz $out/initrd + mkimage -A $uInitrdArch -O linux -T ramdisk -C "$uInitrdCompression" -d $out/initrd"$extension" $out/initrd.img + # Compatibility symlink + ln -s "initrd.img" "$out/initrd" +else + ln -s "initrd" "$out/initrd$extension" fi From d4ef25db5d794474b2e1a0e5afc55d42d10ac49f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Xaver=20H=C3=B6rl?= Date: Sun, 23 Aug 2020 00:12:42 +0200 Subject: [PATCH 19/36] nixos/initrd: add compressorArgs, make compressor option public --- nixos/modules/system/boot/stage-1.nix | 29 ++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix index 0f5787a1921..3af80ed20cc 100644 --- a/nixos/modules/system/boot/stage-1.nix +++ b/nixos/modules/system/boot/stage-1.nix @@ -308,7 +308,7 @@ let # the initial RAM disk. initialRamdisk = pkgs.makeInitrd { name = "initrd-${kernel-name}"; - inherit (config.boot.initrd) compressor prepend; + inherit (config.boot.initrd) compressor compressorArgs prepend; contents = [ { object = bootStage1; @@ -334,7 +334,9 @@ let # Script to add secret files to the initrd at bootloader update time initialRamdiskSecretAppender = - pkgs.writeScriptBin "append-initrd-secrets" + let + compressorExe = initialRamdisk.compressorExecutableFunction pkgs; + in pkgs.writeScriptBin "append-initrd-secrets" '' #!${pkgs.bash}/bin/bash -e function usage { @@ -376,7 +378,7 @@ let } (cd "$tmp" && find . -print0 | sort -z | cpio -o -H newc -R +0:+0 --reproducible --null) | \ - ${config.boot.initrd.compressor} >> "$1" + ${compressorExe} ${lib.escapeShellArgs initialRamdisk.compressorArgs} >> "$1" ''; in @@ -511,13 +513,26 @@ in }; boot.initrd.compressor = mkOption { - internal = true; - default = "gzip -9n"; - type = types.str; - description = "The compressor to use on the initrd image."; + default = "gzip"; + type = types.unspecified; # We don't have a function type... + description = '' + The compressor to use on the initrd image. May be any of: + + - A string representing a command available in stdenv, e.g. "xz"; + - A function which, given the nixpkgs package set, returns the path to a compressor tool, e.g. pkgs: "''${pkgs.pigz}/bin/pigz" + - (not recommended, because it does not work when cross-compiling) the full path to a compressor tool, e.g. "''${pkgs.pigz}/bin/pigz" + + The given program should read data from stdin and write it to stdout compressed. + ''; example = "xz"; }; + boot.initrd.compressorArgs = mkOption { + default = null; + type = types.nullOr (types.listOf types.str); + description = "Arguments to pass to the compressor for the initrd image, or null to use the compressor's defaults."; + }; + boot.initrd.secrets = mkOption { default = {}; type = types.attrsOf (types.nullOr types.path); From 3136e49b8ef0c40f33f618ee56f1c8959e6cbf88 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Sat, 12 Dec 2020 14:39:58 +0100 Subject: [PATCH 20/36] nixos/tests: Add test for initrd secrets lz4 compression is excluded because it doesn't work for a reason which remains unclear to me. --- nixos/tests/all-tests.nix | 1 + nixos/tests/initrd-secrets.nix | 35 ++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 nixos/tests/initrd-secrets.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 0c06e3f4424..8cbac702308 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -168,6 +168,7 @@ in initrd-network-openvpn = handleTest ./initrd-network-openvpn {}; initrd-network-ssh = handleTest ./initrd-network-ssh {}; initrdNetwork = handleTest ./initrd-network.nix {}; + initrd-secrets = handleTest ./initrd-secrets.nix {}; installer = handleTest ./installer.nix {}; iodine = handleTest ./iodine.nix {}; ipfs = handleTest ./ipfs.nix {}; diff --git a/nixos/tests/initrd-secrets.nix b/nixos/tests/initrd-secrets.nix new file mode 100644 index 00000000000..10dd908502d --- /dev/null +++ b/nixos/tests/initrd-secrets.nix @@ -0,0 +1,35 @@ +{ system ? builtins.currentSystem +, config ? {} +, pkgs ? import ../.. { inherit system config; } +, lib ? pkgs.lib +, testing ? import ../lib/testing-python.nix { inherit system pkgs; } +}: +let + secretInStore = pkgs.writeText "topsecret" "iamasecret"; + testWithCompressor = compressor: testing.makeTest { + name = "initrd-secrets-${compressor}"; + + meta.maintainers = [ lib.maintainers.lheckemann ]; + + machine = { ... }: { + virtualisation.useBootLoader = true; + boot.initrd.secrets."/test" = secretInStore; + boot.initrd.postMountCommands = '' + cp /test /mnt-root/secret-from-initramfs + ''; + boot.initrd.compressor = compressor; + # zstd compression is only supported from 5.9 onwards. Remove when 5.10 becomes default. + boot.kernelPackages = pkgs.linuxPackages_latest; + }; + + testScript = '' + start_all() + machine.wait_for_unit("multi-user.target") + machine.succeed( + "cmp ${secretInStore} /secret-from-initramfs" + ) + ''; + }; +in lib.flip lib.genAttrs testWithCompressor [ + "cat" "gzip" "bzip2" "xz" "lzma" "lzop" "pigz" "pixz" "zstd" +] From a448295dcad4dcac5362d13c3a888eb749f92f22 Mon Sep 17 00:00:00 2001 From: Nikita Uvarov Date: Thu, 17 Dec 2020 13:26:08 +0100 Subject: [PATCH 21/36] pythonPackages.ciscomobilityexpress: init at 0.3.9 --- .../ciscomobilityexpress/default.nix | 20 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 ++ 2 files changed, 22 insertions(+) create mode 100644 pkgs/development/python-modules/ciscomobilityexpress/default.nix diff --git a/pkgs/development/python-modules/ciscomobilityexpress/default.nix b/pkgs/development/python-modules/ciscomobilityexpress/default.nix new file mode 100644 index 00000000000..420a57c07d0 --- /dev/null +++ b/pkgs/development/python-modules/ciscomobilityexpress/default.nix @@ -0,0 +1,20 @@ +{ buildPythonPackage, fetchPypi, lib, requests }: + +buildPythonPackage rec { + pname = "ciscomobilityexpress"; + version = "0.3.9"; + + src = fetchPypi { + inherit pname version; + sha256 = "0kj0i1963afxqw9apk0yxzj1f7kpi1949ggnkzkb8v90kxpgymma"; + }; + + propagatedBuildInputs = [ requests ]; + + meta = { + description = "Module to interact with Cisco Mobility Express APIs to fetch connected devices"; + homepage = "https://pypi.python.org/pypi/${pname}/"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ uvnikita ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 4c57648ae29..a93ebf9ba89 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -1175,6 +1175,8 @@ in { cirq = callPackage ../development/python-modules/cirq { }; + ciscomobilityexpress = callPackage ../development/python-modules/ciscomobilityexpress { }; + ciso8601 = callPackage ../development/python-modules/ciso8601 { }; citeproc-py = callPackage ../development/python-modules/citeproc-py { }; From fc3db0f443d7520c829ec10002ddacea97157a1f Mon Sep 17 00:00:00 2001 From: ajs124 Date: Thu, 17 Dec 2020 19:55:15 +0100 Subject: [PATCH 22/36] nginx: 1.19.5 -> 1.19.6 --- pkgs/servers/http/nginx/mainline.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/servers/http/nginx/mainline.nix b/pkgs/servers/http/nginx/mainline.nix index edb87258d6b..cadc1064ac0 100644 --- a/pkgs/servers/http/nginx/mainline.nix +++ b/pkgs/servers/http/nginx/mainline.nix @@ -1,6 +1,6 @@ { callPackage, ... }@args: callPackage ./generic.nix args { - version = "1.19.5"; - sha256 = "173rv8gacd9bakb0r9jmkr4pqgjw9mzpdh3f7x2d8ln4ssplc2jw"; + version = "1.19.6"; + sha256 = "1d9kzks8x1226prjbpdin4dz93fjnv304zlqybfqachx5fh9a4di"; } From e2a3d3f559f85b52b130f9f846e02f11e438821d Mon Sep 17 00:00:00 2001 From: ajs124 Date: Thu, 17 Dec 2020 19:58:07 +0100 Subject: [PATCH 23/36] nginx: add myself as maintainer --- pkgs/servers/http/nginx/generic.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/servers/http/nginx/generic.nix b/pkgs/servers/http/nginx/generic.nix index 6ec5b0a7851..2b2af8966e9 100644 --- a/pkgs/servers/http/nginx/generic.nix +++ b/pkgs/servers/http/nginx/generic.nix @@ -149,6 +149,6 @@ stdenv.mkDerivation { homepage = "http://nginx.org"; license = licenses.bsd2; platforms = platforms.all; - maintainers = with maintainers; [ thoughtpolice raskin fpletz globin ]; + maintainers = with maintainers; [ thoughtpolice raskin fpletz globin ajs124 ]; }; } From 767d80099cd8418b3cc7338eb24f9217fedb6449 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Mon, 30 Nov 2020 22:38:56 +0100 Subject: [PATCH 24/36] lib/modules: Introduce _module.checks.*.check Previously the .enable option was used to encode the condition as well, which lead to some oddness: - In order to encode an assertion, one had to invert it - To disable a check, one had to mkForce it By introducing a separate .check option this is solved because: - It can be used to encode assertions - Disabling is done separately with .enable option, whose default can be overridden without a mkForce --- lib/modules.nix | 36 +++++++++---------- lib/tests/modules.sh | 3 +- .../modules/assertions/condition-true.nix | 8 +++++ lib/tests/modules/assertions/enable-false.nix | 1 + lib/tests/modules/assertions/multi.nix | 8 ++--- lib/tests/modules/assertions/simple.nix | 2 +- .../assertions/submodule-attrsOf-attrsOf.nix | 2 +- .../modules/assertions/submodule-attrsOf.nix | 2 +- lib/tests/modules/assertions/submodule.nix | 2 +- .../assertions/underscore-attributes.nix | 2 +- lib/tests/modules/assertions/warning.nix | 2 +- nixos/doc/manual/development/assertions.xml | 34 +++++++++--------- nixos/modules/misc/assertions.nix | 2 +- 13 files changed, 56 insertions(+), 48 deletions(-) create mode 100644 lib/tests/modules/assertions/condition-true.nix diff --git a/lib/modules.nix b/lib/modules.nix index 23dbe962491..468c373d6aa 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -155,17 +155,22 @@ rec { default = {}; internal = prefix != []; type = types.attrsOf (types.submodule { - # TODO: Rename to assertion? Or allow also setting assertion? options.enable = mkOption { description = '' - Whether to enable this check. - - This is the inverse of asserting a condition: If a certain - condition should be true, then this - option should be set to false when that - case occurs - + Whether to enable this check. Set this to false to not trigger + any errors or warning messages. This is useful for ignoring a + check in case it doesn't make sense in certain scenarios. ''; + default = true; + type = types.bool; + }; + + options.check = mkOption { + description = '' + The condition that must succeed in order for this check to be + successful and not trigger a warning or error. + ''; + readOnly = true; type = types.bool; }; @@ -189,9 +194,7 @@ rec { and use ''${options.path.to.option}. ''; type = types.str; - example = literalExample '' - Enabling both ''${options.services.foo.enable} and ''${options.services.bar.enable} is not possible. - ''; + example = "Enabling both \${options.services.foo.enable} and \${options.services.bar.enable} is not possible."; }; }); }; @@ -244,7 +247,7 @@ rec { if lib.hasPrefix "_" name then value.message else "[${showOption prefix}${optionalString (prefix != []) "/"}${name}] ${value.message}"; in - if ! value.enable then errors + if value.enable -> value.check then errors else if value.type == "warning" then lib.warn show errors else if value.type == "error" then errors ++ [ show ] else abort "Unknown check type ${value.type}"; @@ -885,8 +888,7 @@ rec { }); config._module.checks = let opt = getAttrFromPath optionName options; in { - ${showOption optionName} = { - enable = mkDefault opt.isDefined; + ${showOption optionName} = lib.mkIf opt.isDefined { message = '' The option definition `${showOption optionName}' in ${showFiles opt.files} no longer has any effect; please remove it. ${replacementInstructions} @@ -958,8 +960,7 @@ rec { let val = getAttrFromPath f config; opt = getAttrFromPath f options; in { - ${showOption f} = { - enable = mkDefault (val != "_mkMergedOptionModule"); + ${showOption f} = lib.mkIf (val != "_mkMergedOptionModule") { type = "warning"; message = "The option `${showOption f}' defined in ${showFiles opt.files} has been changed to `${showOption to}' that has a different type. Please read `${showOption to}' documentation and update your configuration accordingly."; }; @@ -1024,8 +1025,7 @@ rec { }); config = mkMerge [ { - _module.checks.${showOption from} = { - enable = mkDefault (warn && fromOpt.isDefined); + _module.checks.${showOption from} = mkIf (warn && fromOpt.isDefined) { type = "warning"; message = "The option `${showOption from}' defined in ${showFiles fromOpt.files} has been renamed to `${showOption to}'."; }; diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 9e85c90d15c..775be9f7209 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -279,7 +279,8 @@ checkConfigOutput baz config.value.nested.bar.baz ./types-anything/mk-mods.nix # Check that assertions are triggered by default for just evaluating config checkConfigError 'Failed checks:\n- \[test\] Assertion failed' config ./assertions/simple.nix -# Assertion is not triggered when enable is false +# Assertion is not triggered when enable is false or condition is true +checkConfigOutput '{ }' config ./assertions/condition-true.nix checkConfigOutput '{ }' config ./assertions/enable-false.nix # Warnings should be displayed on standard error diff --git a/lib/tests/modules/assertions/condition-true.nix b/lib/tests/modules/assertions/condition-true.nix new file mode 100644 index 00000000000..7ca0817a239 --- /dev/null +++ b/lib/tests/modules/assertions/condition-true.nix @@ -0,0 +1,8 @@ +{ + + _module.checks.test = { + check = true; + message = "Assertion failed"; + }; + +} diff --git a/lib/tests/modules/assertions/enable-false.nix b/lib/tests/modules/assertions/enable-false.nix index c326c086f03..11f753bb32e 100644 --- a/lib/tests/modules/assertions/enable-false.nix +++ b/lib/tests/modules/assertions/enable-false.nix @@ -2,6 +2,7 @@ _module.checks.test = { enable = false; + check = false; message = "Assertion failed"; }; diff --git a/lib/tests/modules/assertions/multi.nix b/lib/tests/modules/assertions/multi.nix index ebbe17f3a55..1e2e14b8643 100644 --- a/lib/tests/modules/assertions/multi.nix +++ b/lib/tests/modules/assertions/multi.nix @@ -2,20 +2,20 @@ _module.checks = { test1 = { - enable = true; + check = false; message = "Assertion 1 failed"; }; test2 = { - enable = true; + check = false; message = "Assertion 2 failed"; }; test3 = { - enable = true; + check = false; message = "Warning 3 failed"; type = "warning"; }; test4 = { - enable = true; + check = false; message = "Warning 4 failed"; type = "warning"; }; diff --git a/lib/tests/modules/assertions/simple.nix b/lib/tests/modules/assertions/simple.nix index a63b8090f91..115d89a3036 100644 --- a/lib/tests/modules/assertions/simple.nix +++ b/lib/tests/modules/assertions/simple.nix @@ -1,6 +1,6 @@ { _module.checks.test = { - enable = true; + check = false; message = "Assertion failed"; }; } diff --git a/lib/tests/modules/assertions/submodule-attrsOf-attrsOf.nix b/lib/tests/modules/assertions/submodule-attrsOf-attrsOf.nix index a5f92aa93c7..27a63d1e432 100644 --- a/lib/tests/modules/assertions/submodule-attrsOf-attrsOf.nix +++ b/lib/tests/modules/assertions/submodule-attrsOf-attrsOf.nix @@ -4,7 +4,7 @@ default = { bar.baz = {}; }; type = lib.types.attrsOf (lib.types.attrsOf (lib.types.submodule { _module.checks.test = { - enable = true; + check = false; message = "Assertion failed"; }; })); diff --git a/lib/tests/modules/assertions/submodule-attrsOf.nix b/lib/tests/modules/assertions/submodule-attrsOf.nix index 450cad0804d..aac5937cf7e 100644 --- a/lib/tests/modules/assertions/submodule-attrsOf.nix +++ b/lib/tests/modules/assertions/submodule-attrsOf.nix @@ -4,7 +4,7 @@ default = { bar = {}; }; type = lib.types.attrsOf (lib.types.submodule { _module.checks.test = { - enable = true; + check = false; message = "Assertion failed"; }; }); diff --git a/lib/tests/modules/assertions/submodule.nix b/lib/tests/modules/assertions/submodule.nix index a46734a326b..4e7e0b1bd61 100644 --- a/lib/tests/modules/assertions/submodule.nix +++ b/lib/tests/modules/assertions/submodule.nix @@ -4,7 +4,7 @@ default = {}; type = lib.types.submodule { _module.checks.test = { - enable = true; + check = false; message = "Assertion failed"; }; }; diff --git a/lib/tests/modules/assertions/underscore-attributes.nix b/lib/tests/modules/assertions/underscore-attributes.nix index c28c9dcd918..f9ee5c5787b 100644 --- a/lib/tests/modules/assertions/underscore-attributes.nix +++ b/lib/tests/modules/assertions/underscore-attributes.nix @@ -1,7 +1,7 @@ { _module.checks._test = { - enable = true; + check = false; message = "Assertion failed"; }; diff --git a/lib/tests/modules/assertions/warning.nix b/lib/tests/modules/assertions/warning.nix index 8fed9871aa2..72598ba3fdd 100644 --- a/lib/tests/modules/assertions/warning.nix +++ b/lib/tests/modules/assertions/warning.nix @@ -1,7 +1,7 @@ { _module.checks.test = { - enable = true; + check = false; type = "warning"; message = "Warning message"; }; diff --git a/nixos/doc/manual/development/assertions.xml b/nixos/doc/manual/development/assertions.xml index a873345ef43..31d09f958af 100644 --- a/nixos/doc/manual/development/assertions.xml +++ b/nixos/doc/manual/development/assertions.xml @@ -25,28 +25,26 @@ Checks can be defined using the option. - Each check needs an attribute name, under which you have to define an enable - condition using and a - message using . Note that - the enable condition is inverse of what an assertion - would be: To assert a value being true, the enable condition should be false - in that case, so that it isn't triggered. For the check message, you can add + Each check needs an attribute name, under which you can define a trigger + assertion using and a + message using . + For the message, you can add options to the module arguments and use ${options.path.to.option} to print a context-aware string - representation of the option path. Here is an example showing how this can be + representation of an option path. Here is an example showing how this can be done. { config, options, ... }: { _module.checks.gpgSshAgent = { - enable = config.programs.gnupg.agent.enableSSHSupport && config.programs.ssh.startAgent; - message = "You can't enable both ${options.programs.ssh.startAgent}" - + " and ${options.programs.gnupg.agent.enableSSHSupport}!"; + check = config.programs.gnupg.agent.enableSSHSupport -> !config.programs.ssh.startAgent; + message = "If you have ${options.programs.gnupg.agent.enableSSHSupport} enabled," + + " you can't enable ${options.programs.ssh.startAgent} as well!"; }; _module.checks.grafanaPassword = { - enable = config.services.grafana.database.password != ""; + check = config.services.grafana.database.password == ""; message = "The grafana password defined with ${options.services.grafana.database.password}" + " will be stored as plaintext in the Nix store!"; # This is a non-fatal warning @@ -74,8 +72,8 @@ trace: warning: [grafanaPassword] The grafana password defined with services.grafana.database.password will be stored as plaintext in the Nix store! error: Failed checks: -- [gpgSshAgent] You can't enable both programs.ssh.startAgent and - programs.gnupg.agent.enableSSHSupport! +- [gpgSshAgent] If you have programs.gnupg.agent.enableSSHSupport + enabled, you can't enable programs.ssh.startAgent as well! @@ -87,12 +85,12 @@ error: Failed checks: -{ lib, ... }: { +{ # Change the error into a non-fatal warning _module.checks.gpgSshAgent.type = "warning"; # We don't care about this warning, disable it - _module.checks.grafanaPassword.enable = lib.mkForce false; + _module.checks.grafanaPassword.enable = false; } @@ -113,7 +111,7 @@ error: Failed checks: options.port = lib.mkOption {}; config._module.checks.portConflict = { - enable = config.port == 80; + check = config.port != 80; message = "Port ${toString config.port} defined using" + " ${options.port} is usually used for HTTP"; type = "warning"; @@ -143,8 +141,8 @@ trace: warning: [myServices.foo/portConflict] Port 80 defined using -{ lib, ... }: { - myServices.foo._module.checks.portConflict.enable = lib.mkForce false; +{ + myServices.foo._module.checks.portConflict.enable = false; } diff --git a/nixos/modules/misc/assertions.nix b/nixos/modules/misc/assertions.nix index 6a26a2332f2..d7cdb32491d 100644 --- a/nixos/modules/misc/assertions.nix +++ b/nixos/modules/misc/assertions.nix @@ -36,7 +36,7 @@ with lib; name = "_${toString n}"; isWarning = lib.isString value; result = { - enable = if isWarning then true else ! value.assertion; + check = if isWarning then false else value.assertion; type = if isWarning then "warning" else "error"; message = if isWarning then value else value.message; }; From 834cc5d5fa42cc6fd2825696370b9cb5572a134c Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 17 Dec 2020 22:59:49 +0100 Subject: [PATCH 25/36] nixos/initrd: docbookise "compressor" description --- nixos/modules/system/boot/stage-1.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix index 3af80ed20cc..86bfde6349c 100644 --- a/nixos/modules/system/boot/stage-1.nix +++ b/nixos/modules/system/boot/stage-1.nix @@ -518,9 +518,11 @@ in description = '' The compressor to use on the initrd image. May be any of: - - A string representing a command available in stdenv, e.g. "xz"; - - A function which, given the nixpkgs package set, returns the path to a compressor tool, e.g. pkgs: "''${pkgs.pigz}/bin/pigz" - - (not recommended, because it does not work when cross-compiling) the full path to a compressor tool, e.g. "''${pkgs.pigz}/bin/pigz" + + The name of one of the predefined compressors, see pkgs/build-support/kernel/initrd-compressor-meta.nix for the definitions. + A function which, given the nixpkgs package set, returns the path to a compressor tool, e.g. pkgs: "''${pkgs.pigz}/bin/pigz" + (not recommended, because it does not work when cross-compiling) the full path to a compressor tool, e.g. "''${pkgs.pigz}/bin/pigz" + The given program should read data from stdin and write it to stdout compressed. ''; From 1013dac80408343c0ac0d7a75e2a6bdbd85a3aac Mon Sep 17 00:00:00 2001 From: Stefan Frijters Date: Thu, 17 Dec 2020 23:55:55 +0100 Subject: [PATCH 26/36] hipchat: Fix source url --- .../networking/instant-messengers/hipchat/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/applications/networking/instant-messengers/hipchat/default.nix b/pkgs/applications/networking/instant-messengers/hipchat/default.nix index 7213332a549..63cbb69b187 100644 --- a/pkgs/applications/networking/instant-messengers/hipchat/default.nix +++ b/pkgs/applications/networking/instant-messengers/hipchat/default.nix @@ -43,7 +43,7 @@ in stdenv.mkDerivation { inherit version; src = fetchurl { - url = "https://atlassian.artifactoryonline.com/atlassian/hipchat-apt-client/pool/HipChat4-${version}-Linux.deb"; + url = "https://atlassian.artifactoryonline.com/artifactory/hipchat-apt-client/pool/HipChat4-${version}-Linux.deb"; sha256 = "03pz8wskafn848yvciq29kwdvqcgjrk6sjnm8nk9acl89xf0sn96"; }; From a6a70d14a9f7b885e65a51c5e6bd02145884ee50 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 17 Dec 2020 21:50:51 +0100 Subject: [PATCH 27/36] lib/modules: Prefix mkRemovedOptionModule & co. check names To avoid name clashes Co-authored-by: Robert Hensing --- lib/modules.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/modules.nix b/lib/modules.nix index 468c373d6aa..5548c5f7049 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -888,7 +888,7 @@ rec { }); config._module.checks = let opt = getAttrFromPath optionName options; in { - ${showOption optionName} = lib.mkIf opt.isDefined { + ${"removed-" + showOption optionName} = lib.mkIf opt.isDefined { message = '' The option definition `${showOption optionName}' in ${showFiles opt.files} no longer has any effect; please remove it. ${replacementInstructions} @@ -960,7 +960,7 @@ rec { let val = getAttrFromPath f config; opt = getAttrFromPath f options; in { - ${showOption f} = lib.mkIf (val != "_mkMergedOptionModule") { + ${"merged" + showOption f} = lib.mkIf (val != "_mkMergedOptionModule") { type = "warning"; message = "The option `${showOption f}' defined in ${showFiles opt.files} has been changed to `${showOption to}' that has a different type. Please read `${showOption to}' documentation and update your configuration accordingly."; }; @@ -1025,7 +1025,7 @@ rec { }); config = mkMerge [ { - _module.checks.${showOption from} = mkIf (warn && fromOpt.isDefined) { + _module.checks.${"renamed-" + showOption from} = mkIf (warn && fromOpt.isDefined) { type = "warning"; message = "The option `${showOption from}' defined in ${showFiles fromOpt.files} has been renamed to `${showOption to}'."; }; From be94a4cf235ca11e05a549a1128bc2ae19ab4293 Mon Sep 17 00:00:00 2001 From: Michael Weiss Date: Fri, 18 Dec 2020 16:21:37 +0100 Subject: [PATCH 28/36] ungoogled-chromium: Try to fix an evaluation error on Hydra This should fix a regression from #106475 (hopefully this is the only issue, my current implementation with channel+ungoogled isn't ideal): https://github.com/NixOS/nixpkgs/pull/106475#issuecomment-748131224 --- .../networking/browsers/chromium/default.nix | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pkgs/applications/networking/browsers/chromium/default.nix b/pkgs/applications/networking/browsers/chromium/default.nix index 0cbfbc33270..6e7c2307d64 100644 --- a/pkgs/applications/networking/browsers/chromium/default.nix +++ b/pkgs/applications/networking/browsers/chromium/default.nix @@ -54,15 +54,17 @@ let pkgSuffix = if channel == "dev" then "unstable" else channel; pkgName = "google-chrome-${pkgSuffix}"; - chromeSrc = fetchurl { - urls = map (repo: "${repo}/${pkgName}/${pkgName}_${version}-1_amd64.deb") [ - "https://dl.google.com/linux/chrome/deb/pool/main/g" - "http://95.31.35.30/chrome/pool/main/g" - "http://mirror.pcbeta.com/google/chrome/deb/pool/main/g" - "http://repo.fdzh.org/chrome/deb/pool/main/g" - ]; - sha256 = chromium.upstream-info.sha256bin64; - }; + chromeSrc = if channel == "ungoogled-chromium" + then throw "Google Chrome is not supported for the ungoogled-chromium channel." + else fetchurl { + urls = map (repo: "${repo}/${pkgName}/${pkgName}_${version}-1_amd64.deb") [ + "https://dl.google.com/linux/chrome/deb/pool/main/g" + "http://95.31.35.30/chrome/pool/main/g" + "http://mirror.pcbeta.com/google/chrome/deb/pool/main/g" + "http://repo.fdzh.org/chrome/deb/pool/main/g" + ]; + sha256 = chromium.upstream-info.sha256bin64; + }; mkrpath = p: "${lib.makeSearchPathOutput "lib" "lib64" p}:${lib.makeLibraryPath p}"; widevineCdm = stdenv.mkDerivation { From 88f3fcd763dc712ed62474df21ca3465b4671f87 Mon Sep 17 00:00:00 2001 From: Tim Steinbach Date: Fri, 18 Dec 2020 10:10:26 -0500 Subject: [PATCH 29/36] python3Packages.botocore: 1.19.38 -> 1.19.39 --- pkgs/development/python-modules/botocore/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/botocore/default.nix b/pkgs/development/python-modules/botocore/default.nix index 1b953ab6fc4..8c295050ce9 100644 --- a/pkgs/development/python-modules/botocore/default.nix +++ b/pkgs/development/python-modules/botocore/default.nix @@ -12,11 +12,11 @@ buildPythonPackage rec { pname = "botocore"; - version = "1.19.38"; # N.B: if you change this, change boto3 and awscli to a matching version + version = "1.19.39"; # N.B: if you change this, change boto3 and awscli to a matching version src = fetchPypi { inherit pname version; - sha256 = "12ipyrm5180lf00q6v669mrfkpw6x4rhzd7fsp6qzz3g1hdwn7hz"; + sha256 = "1h7skfzglnrz3ghfn8x8d74pfwwrklafd1y0nvbsnwm0k1h3il70"; }; propagatedBuildInputs = [ From 081cbe2ed9f010c1720b766cad178b54da7b7199 Mon Sep 17 00:00:00 2001 From: Tim Steinbach Date: Fri, 18 Dec 2020 10:10:40 -0500 Subject: [PATCH 30/36] python3Packages.boto3: 1.16.38 -> 1.16.39 --- pkgs/development/python-modules/boto3/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/boto3/default.nix b/pkgs/development/python-modules/boto3/default.nix index 2814e6ffb64..bc147ef8ec2 100644 --- a/pkgs/development/python-modules/boto3/default.nix +++ b/pkgs/development/python-modules/boto3/default.nix @@ -13,11 +13,11 @@ buildPythonPackage rec { pname = "boto3"; - version = "1.16.38"; # N.B: if you change this, change botocore too + version = "1.16.39"; # N.B: if you change this, change botocore too src = fetchPypi { inherit pname version; - sha256 = "1xxvpf0q8xiz1cr5q1m4pdpzbhjriw3j6afi5dwvrrq9sh3x7pqx"; + sha256 = "0j1qhfz2fi8hnfm5lhl6b3k0lh5r0vhr5bjm5aawf16l1wq18mm0"; }; propagatedBuildInputs = [ botocore jmespath s3transfer ] ++ lib.optionals (!isPy3k) [ futures ]; From fd1cc29974de2255d407a706f23aacdfb76f543b Mon Sep 17 00:00:00 2001 From: Tim Steinbach Date: Fri, 18 Dec 2020 10:11:58 -0500 Subject: [PATCH 31/36] awscli: 1.18.198 -> 1.18.199 --- pkgs/tools/admin/awscli/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/tools/admin/awscli/default.nix b/pkgs/tools/admin/awscli/default.nix index 9e0a165b051..edac64308c1 100644 --- a/pkgs/tools/admin/awscli/default.nix +++ b/pkgs/tools/admin/awscli/default.nix @@ -28,11 +28,11 @@ let in with py.pkgs; buildPythonApplication rec { pname = "awscli"; - version = "1.18.198"; # N.B: if you change this, change botocore to a matching version too + version = "1.18.199"; # N.B: if you change this, change botocore to a matching version too src = fetchPypi { inherit pname version; - sha256 = "0zcjx2gh9s1mak9cc9bmydg0f68id4rwhhpcaqqkcd3p37swyr2b"; + sha256 = "09ncnglxy3ph0i4zh93cxgwsxy3hgsy6pvnln1845p2nwvjsw434"; }; postPatch = '' From 9e6737710c4fb2613850e699178b23d54f1a3261 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 18 Dec 2020 16:42:42 +0100 Subject: [PATCH 32/36] Revert "Module-builtin assertions, disabling assertions and submodule assertions" --- lib/modules.nix | 164 +++--------------- lib/options.nix | 2 +- lib/tests/misc.nix | 2 +- lib/tests/modules.sh | 82 +++------ .../modules/assertions/condition-true.nix | 8 - lib/tests/modules/assertions/enable-false.nix | 9 - lib/tests/modules/assertions/multi.nix | 23 --- lib/tests/modules/assertions/simple.nix | 6 - .../assertions/submodule-attrsOf-attrsOf.nix | 13 -- .../modules/assertions/submodule-attrsOf.nix | 13 -- lib/tests/modules/assertions/submodule.nix | 13 -- .../assertions/underscore-attributes.nix | 8 - lib/tests/modules/assertions/warning.nix | 9 - nixos/doc/manual/development/assertions.xml | 161 +++++------------ nixos/modules/misc/assertions.nix | 15 +- nixos/modules/system/activation/top-level.nix | 10 +- 16 files changed, 96 insertions(+), 442 deletions(-) delete mode 100644 lib/tests/modules/assertions/condition-true.nix delete mode 100644 lib/tests/modules/assertions/enable-false.nix delete mode 100644 lib/tests/modules/assertions/multi.nix delete mode 100644 lib/tests/modules/assertions/simple.nix delete mode 100644 lib/tests/modules/assertions/submodule-attrsOf-attrsOf.nix delete mode 100644 lib/tests/modules/assertions/submodule-attrsOf.nix delete mode 100644 lib/tests/modules/assertions/submodule.nix delete mode 100644 lib/tests/modules/assertions/underscore-attributes.nix delete mode 100644 lib/tests/modules/assertions/warning.nix diff --git a/lib/modules.nix b/lib/modules.nix index 5548c5f7049..3f2bfd478b0 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -46,7 +46,6 @@ let showFiles showOption unknownModule - literalExample ; in @@ -73,20 +72,14 @@ rec { check ? true }: let - # An internal module that's always added, defining special options which - # change the behavior of the module evaluation itself. This is under a - # `_`-prefixed namespace in order to prevent name clashes with - # user-defined options + # This internal module declare internal options under the `_module' + # attribute. These options are fragile, as they are used by the + # module system to change the interpretation of modules. internalModule = rec { - # FIXME: Using ./modules.nix directly breaks the doc for some reason - _file = "lib/modules.nix"; + _file = ./modules.nix; key = _file; - # Most of these options are set to be internal only for prefix != [], - # aka it's a submodule evaluation. This way their docs are displayed - # only once as a top-level NixOS option, but will be hidden for all - # submodules, even though they are available there too options = { _module.args = mkOption { # Because things like `mkIf` are entirely useless for @@ -96,7 +89,7 @@ rec { # a `_module.args.pkgs = import (fetchTarball { ... }) {}` won't # start a download when `pkgs` wasn't evaluated. type = types.lazyAttrsOf types.unspecified; - internal = prefix != []; + internal = true; description = "Arguments passed to each module."; }; @@ -104,19 +97,13 @@ rec { type = types.bool; internal = true; default = check; - description = '' - Whether to check whether all option definitions have matching - declarations. - - Note that this has nothing to do with the similarly named - option - ''; + description = "Whether to check whether all option definitions have matching declarations."; }; _module.freeformType = mkOption { # Disallow merging for now, but could be implemented nicely with a `types.optionType` type = types.nullOr (types.uniq types.attrs); - internal = prefix != []; + internal = true; default = null; description = '' If set, merge all definitions that don't have an associated option @@ -129,75 +116,6 @@ rec { turned off. ''; }; - - _module.checks = mkOption { - description = '' - Evaluation checks to trigger during module evaluation. The - attribute name will be displayed when it is triggered, allowing - users to disable/change these checks if necessary. See - the section on Warnings and Assertions in the manual for more - information. - ''; - example = literalExample '' - { - gpgSshAgent = { - enable = config.programs.gnupg.agent.enableSSHSupport && config.programs.ssh.startAgent; - message = "You can't use ssh-agent and GnuPG agent with SSH support enabled at the same time!"; - }; - - grafanaPassword = { - enable = config.services.grafana.database.password != ""; - message = "Grafana passwords will be stored as plaintext in the Nix store!"; - type = "warning"; - }; - } - ''; - default = {}; - internal = prefix != []; - type = types.attrsOf (types.submodule { - options.enable = mkOption { - description = '' - Whether to enable this check. Set this to false to not trigger - any errors or warning messages. This is useful for ignoring a - check in case it doesn't make sense in certain scenarios. - ''; - default = true; - type = types.bool; - }; - - options.check = mkOption { - description = '' - The condition that must succeed in order for this check to be - successful and not trigger a warning or error. - ''; - readOnly = true; - type = types.bool; - }; - - options.type = mkOption { - description = '' - The type of the check. The default - "error" type will cause evaluation to fail, - while the "warning" type will only show a - warning. - ''; - type = types.enum [ "error" "warning" ]; - default = "error"; - example = "warning"; - }; - - options.message = mkOption { - description = '' - The message to display if this check triggers. - To display option names in the message, add - options to the module function arguments - and use ''${options.path.to.option}. - ''; - type = types.str; - example = "Enabling both \${options.services.foo.enable} and \${options.services.bar.enable} is not possible."; - }; - }); - }; }; config = { @@ -236,35 +154,6 @@ rec { # paths, meaning recursiveUpdate will never override any value else recursiveUpdate freeformConfig declaredConfig; - # Triggers all checks defined by _module.checks before returning its argument - triggerChecks = let - - handleCheck = errors: name: - let - value = config._module.checks.${name}; - show = - # Assertions with a _ prefix aren't meant to be configurable - if lib.hasPrefix "_" name then value.message - else "[${showOption prefix}${optionalString (prefix != []) "/"}${name}] ${value.message}"; - in - if value.enable -> value.check then errors - else if value.type == "warning" then lib.warn show errors - else if value.type == "error" then errors ++ [ show ] - else abort "Unknown check type ${value.type}"; - - errors = lib.foldl' handleCheck [] (lib.attrNames config._module.checks); - - errorMessage = '' - Failed checks: - ${lib.concatMapStringsSep "\n" (a: "- ${a}") errors} - ''; - - trigger = if errors == [] then null else throw errorMessage; - - in builtins.seq trigger; - - finalConfig = triggerChecks (removeAttrs config [ "_module" ]); - checkUnmatched = if config._module.check && config._module.freeformType == null && merged.unmatchedDefns != [] then let @@ -284,7 +173,7 @@ rec { result = builtins.seq checkUnmatched { inherit options; - config = finalConfig; + config = removeAttrs config [ "_module" ]; inherit (config) _module; }; in result; @@ -625,8 +514,6 @@ rec { definitions = map (def: def.value) res.defsFinal; files = map (def: def.file) res.defsFinal; inherit (res) isDefined; - # This allows options to be correctly displayed using `${options.path.to.it}` - __toString = _: showOption loc; }; # Merge definitions of a value of a given type. @@ -886,15 +773,14 @@ rec { visible = false; apply = x: throw "The option `${showOption optionName}' can no longer be used since it's been removed. ${replacementInstructions}"; }); - config._module.checks = - let opt = getAttrFromPath optionName options; in { - ${"removed-" + showOption optionName} = lib.mkIf opt.isDefined { + config.assertions = + let opt = getAttrFromPath optionName options; in [{ + assertion = !opt.isDefined; message = '' The option definition `${showOption optionName}' in ${showFiles opt.files} no longer has any effect; please remove it. ${replacementInstructions} ''; - }; - }; + }]; }; /* Return a module that causes a warning to be shown if the @@ -955,18 +841,14 @@ rec { })) from); config = { - _module.checks = - let warningMessages = map (f: - let val = getAttrFromPath f config; - opt = getAttrFromPath f options; - in { - ${"merged" + showOption f} = lib.mkIf (val != "_mkMergedOptionModule") { - type = "warning"; - message = "The option `${showOption f}' defined in ${showFiles opt.files} has been changed to `${showOption to}' that has a different type. Please read `${showOption to}' documentation and update your configuration accordingly."; - }; - } - ) from; - in mkMerge warningMessages; + warnings = filter (x: x != "") (map (f: + let val = getAttrFromPath f config; + opt = getAttrFromPath f options; + in + optionalString + (val != "_mkMergedOptionModule") + "The option `${showOption f}' defined in ${showFiles opt.files} has been changed to `${showOption to}' that has a different type. Please read `${showOption to}' documentation and update your configuration accordingly." + ) from); } // setAttrByPath to (mkMerge (optional (any (f: (getAttrFromPath f config) != "_mkMergedOptionModule") from) @@ -1025,10 +907,8 @@ rec { }); config = mkMerge [ { - _module.checks.${"renamed-" + showOption from} = mkIf (warn && fromOpt.isDefined) { - type = "warning"; - message = "The option `${showOption from}' defined in ${showFiles fromOpt.files} has been renamed to `${showOption to}'."; - }; + warnings = optional (warn && fromOpt.isDefined) + "The option `${showOption from}' defined in ${showFiles fromOpt.files} has been renamed to `${showOption to}'."; } (if withPriority then mkAliasAndWrapDefsWithPriority (setAttrByPath to) fromOpt diff --git a/lib/options.nix b/lib/options.nix index 5c042a6c6f2..87cd8b79796 100644 --- a/lib/options.nix +++ b/lib/options.nix @@ -192,7 +192,7 @@ rec { let ss = opt.type.getSubOptions opt.loc; in if ss != {} then optionAttrSetToDocList' opt.loc ss else []; in - [ docOption ] ++ optionals (docOption.visible && ! docOption.internal) subOptions) (collect isOption options); + [ docOption ] ++ optionals docOption.visible subOptions) (collect isOption options); /* This function recursively removes all derivation attributes from diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index 2d53ed81176..35a5801c724 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -655,7 +655,7 @@ runTests { modules = [ module ]; }).options; - locs = filter (o: ! o.internal) (optionAttrSetToDocList (removeAttrs options [ "_module" ])); + locs = filter (o: ! o.internal) (optionAttrSetToDocList options); in map (o: o.loc) locs; expected = [ [ "foo" ] [ "foo" "" "bar" ] [ "foo" "bar" ] ]; }; diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 775be9f7209..309c5311361 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -27,50 +27,37 @@ reportFailure() { fail=$((fail + 1)) } -checkConfigCodeOutErr() { - local expectedExit=$1 - shift; - local outputContains=$1 - shift; - local errorContains=$1 - shift; - out=$(mktemp) - err=$(mktemp) - evalConfig "$@" 1>"$out" 2>"$err" - if [[ "$?" -ne "$expectedExit" ]]; then - echo 2>&1 "error: Expected exit code $expectedExit while evaluating" - reportFailure "$@" - return 1 - fi - - if [[ -n "$outputContains" ]] && ! grep -zP --silent "$outputContains" "$out"; then - echo 2>&1 "error: Expected output matching '$outputContains', while evaluating" - reportFailure "$@" - return 1 - fi - - if [[ -n "$errorContains" ]] && ! grep -zP --silent "$errorContains" "$err"; then - echo 2>&1 "error: Expected error matching '$errorContains', while evaluating" - reportFailure "$@" - return 1 - fi - - pass=$((pass + 1)) - - # Clean up temp files - rm "$out" "$err" -} - checkConfigOutput() { local outputContains=$1 shift; - checkConfigCodeOutErr 0 "$outputContains" "" "$@" + if evalConfig "$@" 2>/dev/null | grep --silent "$outputContains" ; then + pass=$((pass + 1)) + return 0; + else + echo 2>&1 "error: Expected result matching '$outputContains', while evaluating" + reportFailure "$@" + return 1 + fi } checkConfigError() { local errorContains=$1 + local err="" shift; - checkConfigCodeOutErr 1 "" "$errorContains" "$@" + if err==$(evalConfig "$@" 2>&1 >/dev/null); then + echo 2>&1 "error: Expected error code, got exit code 0, while evaluating" + reportFailure "$@" + return 1 + else + if echo "$err" | grep -zP --silent "$errorContains" ; then + pass=$((pass + 1)) + return 0; + else + echo 2>&1 "error: Expected error matching '$errorContains', while evaluating" + reportFailure "$@" + return 1 + fi + fi } # Check boolean option. @@ -275,29 +262,6 @@ checkConfigOutput true config.value.mkbefore ./types-anything/mk-mods.nix checkConfigOutput 1 config.value.nested.foo ./types-anything/mk-mods.nix checkConfigOutput baz config.value.nested.bar.baz ./types-anything/mk-mods.nix -## Module assertions -# Check that assertions are triggered by default for just evaluating config -checkConfigError 'Failed checks:\n- \[test\] Assertion failed' config ./assertions/simple.nix - -# Assertion is not triggered when enable is false or condition is true -checkConfigOutput '{ }' config ./assertions/condition-true.nix -checkConfigOutput '{ }' config ./assertions/enable-false.nix - -# Warnings should be displayed on standard error -checkConfigCodeOutErr 0 '{ }' 'warning: \[test\] Warning message' config ./assertions/warning.nix - -# Check that multiple assertions and warnings can be triggered at once -checkConfigError 'Failed checks:\n- \[test1\] Assertion 1 failed\n- \[test2\] Assertion 2 failed' config ./assertions/multi.nix -checkConfigError 'trace: warning: \[test3\] Warning 3 failed\ntrace: warning: \[test4\] Warning 4 failed' config ./assertions/multi.nix - -# Submodules should be able to trigger assertions and display the submodule prefix in their error -checkConfigError 'Failed checks:\n- \[foo/test\] Assertion failed' config.foo ./assertions/submodule.nix -checkConfigError 'Failed checks:\n- \[foo.bar/test\] Assertion failed' config.foo.bar ./assertions/submodule-attrsOf.nix -checkConfigError 'Failed checks:\n- \[foo.bar.baz/test\] Assertion failed' config.foo.bar.baz ./assertions/submodule-attrsOf-attrsOf.nix - -# Assertions with an attribute starting with _ shouldn't have their name displayed -checkConfigError 'Failed checks:\n- Assertion failed' config ./assertions/underscore-attributes.nix - cat < - Evaluation Checks + Warnings and Assertions When configuration problems are detectable in a module, it is a good idea to - write a check for catching it early. Doing so can provide clear feedback to - the user and can prevent errors before the build. + write an assertion or warning. Doing so provides clear feedback to the user + and prevents errors after the build. Although Nix has the abort and builtins.trace functions - to perform such tasks generally, they are not ideally suited for NixOS - modules. Instead of these functions, you can declare your evaluation checks - using the NixOS module system. + to perform such tasks, they are not ideally suited for NixOS modules. Instead + of these functions, you can declare your warnings and assertions using the + NixOS module system. -
- Defining Checks +
+ Warnings - Checks can be defined using the option. - Each check needs an attribute name, under which you can define a trigger - assertion using and a - message using . - For the message, you can add - options to the module arguments and use - ${options.path.to.option} to print a context-aware string - representation of an option path. Here is an example showing how this can be - done. + This is an example of using warnings. -{ config, options, ... }: { - _module.checks.gpgSshAgent = { - check = config.programs.gnupg.agent.enableSSHSupport -> !config.programs.ssh.startAgent; - message = "If you have ${options.programs.gnupg.agent.enableSSHSupport} enabled," - + " you can't enable ${options.programs.ssh.startAgent} as well!"; - }; - - _module.checks.grafanaPassword = { - check = config.services.grafana.database.password == ""; - message = "The grafana password defined with ${options.services.grafana.database.password}" - + " will be stored as plaintext in the Nix store!"; - # This is a non-fatal warning - type = "warning"; - }; + -
-
- Ignoring Checks +
+ Assertions - Sometimes you can get failing checks that don't apply to your specific case - and you wish to ignore them, or at least make errors non-fatal. You can do so - for all checks defined using by - using the attribute name of the definition, which is conveniently printed - using [...] when the check is triggered. For above - example, the evaluation output when the checks are triggered looks as - follows: - - - -trace: warning: [grafanaPassword] The grafana password defined with - services.grafana.database.password will be stored as plaintext in the Nix store! -error: Failed checks: -- [gpgSshAgent] If you have programs.gnupg.agent.enableSSHSupport - enabled, you can't enable programs.ssh.startAgent as well! - - - - The [grafanaPassword] and [gpgSshAgent] - strings tell you that these were defined under the grafanaPassword - and gpgSshAgent attributes of - respectively. With this knowledge - you can adjust them to your liking: + This example, extracted from the + + syslogd module shows how to use + assertions. Since there can only be one active syslog + daemon at a time, an assertion is useful to prevent such a broken system + from being built. + - - -
-
- Checks in Submodules - - - Evaluation checks can be defined within submodules in the same way. Here is an example: - - - -{ lib, ... }: { - - options.myServices = lib.mkOption { - type = lib.types.attrsOf (lib.types.submodule ({ config, options, ... }: { - options.port = lib.mkOption {}; - - config._module.checks.portConflict = { - check = config.port != 80; - message = "Port ${toString config.port} defined using" - + " ${options.port} is usually used for HTTP"; - type = "warning"; - }; - })); - }; - -} - - - - When this check is triggered, it shows both the submodule path along with - the check attribute within that submodule, joined by a - /. Note also how ${options.port} - correctly shows the context of the option. - - - -trace: warning: [myServices.foo/portConflict] Port 80 defined using - myServices.foo.port is usually used for HTTP - - - - Therefore to disable such a check, you can do so by changing the - option within the - myServices.foo submodule: - - - -{ - myServices.foo._module.checks.portConflict.enable = false; -} - - - - - Checks defined in submodules under types.listOf can't be - ignored, since there's no way to change previously defined list items. - - -
diff --git a/nixos/modules/misc/assertions.nix b/nixos/modules/misc/assertions.nix index d7cdb32491d..550b3ac97f6 100644 --- a/nixos/modules/misc/assertions.nix +++ b/nixos/modules/misc/assertions.nix @@ -1,4 +1,4 @@ -{ lib, config, ... }: +{ lib, ... }: with lib; @@ -30,18 +30,5 @@ with lib; }; }; - - config._module.checks = lib.listToAttrs (lib.imap1 (n: value: - let - name = "_${toString n}"; - isWarning = lib.isString value; - result = { - check = if isWarning then false else value.assertion; - type = if isWarning then "warning" else "error"; - message = if isWarning then value else value.message; - }; - in nameValuePair name result - ) (config.assertions ++ config.warnings)); - # impl of assertions is in } diff --git a/nixos/modules/system/activation/top-level.nix b/nixos/modules/system/activation/top-level.nix index 17b62ad9569..03d7e749323 100644 --- a/nixos/modules/system/activation/top-level.nix +++ b/nixos/modules/system/activation/top-level.nix @@ -117,10 +117,18 @@ let perl = "${pkgs.perl}/bin/perl " + (concatMapStringsSep " " (lib: "-I${lib}/${pkgs.perl.libPrefix}") (with pkgs.perlPackages; [ FileSlurp NetDBus XMLParser XMLTwig ])); }; + # Handle assertions and warnings + + failedAssertions = map (x: x.message) (filter (x: !x.assertion) config.assertions); + + baseSystemAssertWarn = if failedAssertions != [] + then throw "\nFailed assertions:\n${concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}" + else showWarnings config.warnings baseSystem; + # Replace runtime dependencies system = fold ({ oldDependency, newDependency }: drv: pkgs.replaceDependency { inherit oldDependency newDependency drv; } - ) baseSystem config.system.replaceRuntimeDependencies; + ) baseSystemAssertWarn config.system.replaceRuntimeDependencies; in From 1cbaca3bf230228ba7c9580bd89ce78053df3fca Mon Sep 17 00:00:00 2001 From: Nikita Uvarov Date: Fri, 18 Dec 2020 17:33:00 +0100 Subject: [PATCH 33/36] home-assistant: update component-packages --- pkgs/servers/home-assistant/component-packages.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/servers/home-assistant/component-packages.nix b/pkgs/servers/home-assistant/component-packages.nix index 2ac4d3beccd..446ecf172be 100644 --- a/pkgs/servers/home-assistant/component-packages.nix +++ b/pkgs/servers/home-assistant/component-packages.nix @@ -113,7 +113,7 @@ "channels" = ps: with ps; [ ]; # missing inputs: pychannels "circuit" = ps: with ps; [ ]; # missing inputs: circuit-webhook "cisco_ios" = ps: with ps; [ pexpect ]; - "cisco_mobility_express" = ps: with ps; [ ]; # missing inputs: ciscomobilityexpress + "cisco_mobility_express" = ps: with ps; [ ciscomobilityexpress ]; "cisco_webex_teams" = ps: with ps; [ ]; # missing inputs: webexteamssdk "citybikes" = ps: with ps; [ ]; "clementine" = ps: with ps; [ ]; # missing inputs: python-clementine-remote From 648eece4bfb5a3360bed16670a7cf36e2169f07e Mon Sep 17 00:00:00 2001 From: Tim Steinbach Date: Fri, 18 Dec 2020 12:07:30 -0500 Subject: [PATCH 34/36] yq: Add test --- nixos/tests/all-tests.nix | 1 + nixos/tests/yq.nix | 12 ++++++++++++ pkgs/development/python-modules/yq/default.nix | 3 +++ 3 files changed, 16 insertions(+) create mode 100644 nixos/tests/yq.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index e770c8763d8..7f1a681186f 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -411,6 +411,7 @@ in xterm = handleTest ./xterm.nix {}; yabar = handleTest ./yabar.nix {}; yggdrasil = handleTest ./yggdrasil.nix {}; + yq = handleTest ./yq.nix {}; zfs = handleTest ./zfs.nix {}; zigbee2mqtt = handleTest ./zigbee2mqtt.nix {}; zoneminder = handleTest ./zoneminder.nix {}; diff --git a/nixos/tests/yq.nix b/nixos/tests/yq.nix new file mode 100644 index 00000000000..7c0e8e3d055 --- /dev/null +++ b/nixos/tests/yq.nix @@ -0,0 +1,12 @@ +import ./make-test-python.nix ({ pkgs, ... }: { + name = "yq"; + meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ nequissimus ]; }; + + nodes.yq = { pkgs, ... }: { environment.systemPackages = with pkgs; [ jq yq ]; }; + + testScript = '' + assert "hello:\n foo: bar\n" in yq.succeed( + 'echo \'{"hello":{"foo":"bar"}}\' | yq -y .' + ) + ''; +}) diff --git a/pkgs/development/python-modules/yq/default.nix b/pkgs/development/python-modules/yq/default.nix index b3e651fd2b0..9f54dba8b90 100644 --- a/pkgs/development/python-modules/yq/default.nix +++ b/pkgs/development/python-modules/yq/default.nix @@ -1,4 +1,5 @@ { lib +, nixosTests , buildPythonPackage , fetchPypi , pkgs @@ -46,6 +47,8 @@ buildPythonPackage rec { pythonImportsCheck = [ "yq" ]; + passthru.tests = { inherit (nixosTests) yq; }; + meta = with lib; { description = "Command-line YAML processor - jq wrapper for YAML documents"; homepage = "https://github.com/kislyuk/yq"; From 9b846b960029296bc81cfe2301ed452698ecf216 Mon Sep 17 00:00:00 2001 From: Michael Weiss Date: Fri, 18 Dec 2020 17:52:21 +0100 Subject: [PATCH 35/36] chromium: Improve update.py (documentation + linting fixes) --- .../networking/browsers/chromium/update.py | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/pkgs/applications/networking/browsers/chromium/update.py b/pkgs/applications/networking/browsers/chromium/update.py index 57fe268e72f..74459866bf7 100755 --- a/pkgs/applications/networking/browsers/chromium/update.py +++ b/pkgs/applications/networking/browsers/chromium/update.py @@ -1,6 +1,9 @@ #! /usr/bin/env nix-shell #! nix-shell -i python -p python3 nix nix-prefetch-git +"""This script automatically updates chromium, google-chrome, chromedriver, and ungoogled-chromium +via upstream-info.json.""" + import csv import json import re @@ -19,40 +22,53 @@ BUCKET_URL = 'https://commondatastorage.googleapis.com/chromium-browser-official JSON_PATH = dirname(abspath(__file__)) + '/upstream-info.json' + def load_json(path): + """Loads the given JSON file.""" with open(path, 'r') as f: return json.load(f) + def nix_prefetch_url(url, algo='sha256'): + """Prefetches the content of the given URL.""" print(f'nix-prefetch-url {url}') out = subprocess.check_output(['nix-prefetch-url', '--type', algo, url]) return out.decode('utf-8').rstrip() + def nix_prefetch_git(url, rev): + """Prefetches the requested Git revision of the given repository URL.""" print(f'nix-prefetch-git {url} {rev}') out = subprocess.check_output(['nix-prefetch-git', '--quiet', '--url', url, '--rev', rev]) return json.loads(out) + def get_file_revision(revision, file_path): + """Fetches the requested Git revision of the given Chromium file.""" url = f'https://raw.githubusercontent.com/chromium/chromium/{revision}/{file_path}' with urlopen(url) as http_response: return http_response.read() + def get_matching_chromedriver(version): + """Gets the matching chromedriver version for the given Chromium version.""" # See https://chromedriver.chromium.org/downloads/version-selection build = re.sub('.[0-9]+$', '', version) chromedriver_version_url = f'https://chromedriver.storage.googleapis.com/LATEST_RELEASE_{build}' with urlopen(chromedriver_version_url) as http_response: chromedriver_version = http_response.read().decode() def get_chromedriver_url(system): - return f'https://chromedriver.storage.googleapis.com/{chromedriver_version}/chromedriver_{system}.zip' + return ('https://chromedriver.storage.googleapis.com/' + + f'{chromedriver_version}/chromedriver_{system}.zip') return { 'version': chromedriver_version, 'sha256_linux': nix_prefetch_url(get_chromedriver_url('linux64')), 'sha256_darwin': nix_prefetch_url(get_chromedriver_url('mac64')) } + def get_channel_dependencies(channel): + """Gets all dependencies for the given Chromium version.""" deps = get_file_revision(channel['version'], 'DEPS') gn_pattern = b"'gn_version': 'git_revision:([0-9a-f]{40})'" gn_commit = re.search(gn_pattern, deps).group(1).decode() @@ -69,6 +85,7 @@ def get_channel_dependencies(channel): channels = {} last_channels = load_json(JSON_PATH) + print(f'GET {HISTORY_URL}', file=sys.stderr) with urlopen(HISTORY_URL) as resp: builds = csv.DictReader(iterdecode(resp, 'utf-8')) @@ -92,7 +109,9 @@ with urlopen(HISTORY_URL) as resp: try: channel['sha256'] = nix_prefetch_url(f'{BUCKET_URL}/chromium-{build["version"]}.tar.xz') - channel['sha256bin64'] = nix_prefetch_url(f'{DEB_URL}/google-chrome-{suffix}/google-chrome-{suffix}_{build["version"]}-1_amd64.deb') + channel['sha256bin64'] = nix_prefetch_url( + f'{DEB_URL}/google-chrome-{suffix}/' + + f'google-chrome-{suffix}_{build["version"]}-1_amd64.deb') except subprocess.CalledProcessError: # This build isn't actually available yet. Continue to # the next one. @@ -104,21 +123,23 @@ with urlopen(HISTORY_URL) as resp: channels[channel_name] = channel + with open(JSON_PATH, 'w') as out: def get_channel_key(item): + """Orders Chromium channels by their name.""" channel_name = item[0] if channel_name == 'stable': return 0 - elif channel_name == 'beta': + if channel_name == 'beta': return 1 - elif channel_name == 'dev': + if channel_name == 'dev': return 2 - elif channel_name == 'ungoogled-chromium': + if channel_name == 'ungoogled-chromium': return 3 - else: - print(f'Error: Unexpected channel: {channel_name}', file=sys.stderr) - sys.exit(1) - channels['ungoogled-chromium'] = last_channels['ungoogled-chromium'] # Keep ungoogled-chromium unchanged + print(f'Error: Unexpected channel: {channel_name}', file=sys.stderr) + sys.exit(1) + # Keep ungoogled-chromium unchanged: + channels['ungoogled-chromium'] = last_channels['ungoogled-chromium'] sorted_channels = OrderedDict(sorted(channels.items(), key=get_channel_key)) json.dump(sorted_channels, out, indent=2) out.write('\n') From 94bee1090426c1aaee8e20cc5bbb1e30da86c792 Mon Sep 17 00:00:00 2001 From: Michael Weiss Date: Fri, 18 Dec 2020 19:08:37 +0100 Subject: [PATCH 36/36] ungoogled-chromium: Support automatic updates via update.py --- .../networking/browsers/chromium/update.py | 39 ++++++++++++++++--- .../browsers/chromium/upstream-info.json | 8 ++-- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/pkgs/applications/networking/browsers/chromium/update.py b/pkgs/applications/networking/browsers/chromium/update.py index 74459866bf7..b4d16fa149f 100755 --- a/pkgs/applications/networking/browsers/chromium/update.py +++ b/pkgs/applications/networking/browsers/chromium/update.py @@ -67,9 +67,9 @@ def get_matching_chromedriver(version): } -def get_channel_dependencies(channel): +def get_channel_dependencies(version): """Gets all dependencies for the given Chromium version.""" - deps = get_file_revision(channel['version'], 'DEPS') + deps = get_file_revision(version, 'DEPS') gn_pattern = b"'gn_version': 'git_revision:([0-9a-f]{40})'" gn_commit = re.search(gn_pattern, deps).group(1).decode() gn = nix_prefetch_git('https://gn.googlesource.com/gn', gn_commit) @@ -82,6 +82,35 @@ def get_channel_dependencies(channel): } } + +def get_latest_ungoogled_chromium_tag(): + """Returns the latest ungoogled-chromium tag using the GitHub API.""" + api_tag_url = 'https://api.github.com/repos/Eloston/ungoogled-chromium/tags?per_page=1' + with urlopen(api_tag_url) as http_response: + tag_data = json.load(http_response) + return tag_data[0]['name'] + + +def get_ungoogled_chromium_channel(): + """Returns a dictionary for the ungoogled-chromium channel.""" + latest_tag = get_latest_ungoogled_chromium_tag() + version = latest_tag.split('-')[0] + if version == last_channels['ungoogled-chromium']['version']: + # No update available -> keep the cached information (no refetching required): + return last_channels['ungoogled-chromium'] + channel = { + 'version': version, + 'sha256': nix_prefetch_url(f'{BUCKET_URL}/chromium-{version}.tar.xz'), + 'deps': get_channel_dependencies(version) + } + repo_url = 'https://github.com/Eloston/ungoogled-chromium.git' + channel['deps']['ungoogled-patches'] = { + 'rev': latest_tag, + 'sha256': nix_prefetch_git(repo_url, latest_tag)['sha256'] + } + return channel + + channels = {} last_channels = load_json(JSON_PATH) @@ -117,7 +146,7 @@ with urlopen(HISTORY_URL) as resp: # the next one. continue - channel['deps'] = get_channel_dependencies(channel) + channel['deps'] = get_channel_dependencies(channel['version']) if channel_name == 'stable': channel['chromedriver'] = get_matching_chromedriver(channel['version']) @@ -138,8 +167,8 @@ with open(JSON_PATH, 'w') as out: return 3 print(f'Error: Unexpected channel: {channel_name}', file=sys.stderr) sys.exit(1) - # Keep ungoogled-chromium unchanged: - channels['ungoogled-chromium'] = last_channels['ungoogled-chromium'] + # Get the special ungoogled-chromium channel: + channels['ungoogled-chromium'] = get_ungoogled_chromium_channel() sorted_channels = OrderedDict(sorted(channels.items(), key=get_channel_key)) json.dump(sorted_channels, out, indent=2) out.write('\n') diff --git a/pkgs/applications/networking/browsers/chromium/upstream-info.json b/pkgs/applications/networking/browsers/chromium/upstream-info.json index 21121428cc1..2cd673e9c55 100644 --- a/pkgs/applications/networking/browsers/chromium/upstream-info.json +++ b/pkgs/applications/networking/browsers/chromium/upstream-info.json @@ -47,15 +47,15 @@ "version": "87.0.4280.88", "sha256": "1h09g9b2zxad85vd146ymvg3w2kpngpi78yig3dn1vrmhwr4aiiy", "deps": { - "ungoogled-patches": { - "rev": "87.0.4280.88-1", - "sha256": "0w2137w8hfcgl6f938hqnb4ffp33v5r8vdzxrvs814w7dszkiqgg" - }, "gn": { "version": "2020-09-09", "url": "https://gn.googlesource.com/gn", "rev": "e002e68a48d1c82648eadde2f6aafa20d08c36f2", "sha256": "0x4c7amxwzxs39grqs3dnnz0531mpf1p75niq7zhinyfqm86i4dk" + }, + "ungoogled-patches": { + "rev": "87.0.4280.88-1", + "sha256": "0w2137w8hfcgl6f938hqnb4ffp33v5r8vdzxrvs814w7dszkiqgg" } } }