From 5ae3fb2c3804a74251c359d603e00bd947aaea15 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Sat, 7 Mar 2020 02:44:56 +0100 Subject: [PATCH 1/6] lib/strings: Add floatToString --- lib/strings.nix | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/strings.nix b/lib/strings.nix index 74e3eaa0722..0baa942355c 100644 --- a/lib/strings.nix +++ b/lib/strings.nix @@ -612,6 +612,22 @@ rec { */ fixedWidthNumber = width: n: fixedWidthString width "0" (toString n); + /* Convert a float to a string, but emit a warning when precision is lost + during the conversion + + Example: + floatToString 0.000001 + => "0.000001" + floatToString 0.0000001 + => trace: warning: Imprecise conversion from float to string 0.000000 + "0.000000" + */ + floatToString = float: let + result = toString float; + precise = float == builtins.fromJSON result; + in if precise then result + else lib.warn "Imprecise conversion from float to string ${result}" result; + /* Check whether a value can be coerced to a string */ isCoercibleToString = x: builtins.elem (builtins.typeOf x) [ "path" "string" "null" "int" "float" "bool" ] || From 9df69cba054c3b5d3a3b67778e0d2947eba3f297 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 13 Dec 2019 00:24:30 +0100 Subject: [PATCH 2/6] lib/generators: Extend mkValueStringDefault with float support --- lib/generators.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/generators.nix b/lib/generators.nix index efe6ea6031d..abd237eb7d3 100644 --- a/lib/generators.nix +++ b/lib/generators.nix @@ -48,8 +48,10 @@ rec { else if isAttrs v then err "attrsets" v # functions can’t be printed of course else if isFunction v then err "functions" v - # let’s not talk about floats. There is no sensible `toString` for them. - else if isFloat v then err "floats" v + # Floats currently can't be converted to precise strings, + # condition warning on nix version once this isn't a problem anymore + # See https://github.com/NixOS/nix/pull/3480 + else if isFloat v then libStr.floatToString v else err "this value is" (toString v); From 888c923880ed08352a5cadc4e2476c6d1032e1e2 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 26 Mar 2020 02:54:23 +0100 Subject: [PATCH 3/6] pkgs: Add pkgs-lib structure --- pkgs/pkgs-lib/default.nix | 7 +++++++ pkgs/pkgs-lib/tests/default.nix | 5 +++++ pkgs/top-level/all-packages.nix | 3 +++ pkgs/top-level/release.nix | 2 ++ 4 files changed, 17 insertions(+) create mode 100644 pkgs/pkgs-lib/default.nix create mode 100644 pkgs/pkgs-lib/tests/default.nix diff --git a/pkgs/pkgs-lib/default.nix b/pkgs/pkgs-lib/default.nix new file mode 100644 index 00000000000..1f74747d311 --- /dev/null +++ b/pkgs/pkgs-lib/default.nix @@ -0,0 +1,7 @@ +# pkgs-lib is for functions and values that can't be in lib because +# they depend on some packages. This notably is *not* for supporting package +# building, instead pkgs/build-support is the place for that. +{ lib, pkgs }: { + +} + diff --git a/pkgs/pkgs-lib/tests/default.nix b/pkgs/pkgs-lib/tests/default.nix new file mode 100644 index 00000000000..ee71596a145 --- /dev/null +++ b/pkgs/pkgs-lib/tests/default.nix @@ -0,0 +1,5 @@ +# Call nix-build on this file to run all tests in this directory +{ pkgs ? import ../../.. {} }: +let +in pkgs.linkFarm "nixpkgs-pkgs-lib-tests" [ +] diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 6427bb4bbc2..4fca1254bcd 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -523,6 +523,9 @@ in #package writers writers = callPackage ../build-support/writers {}; + # lib functions depending on pkgs + inherit (import ../pkgs-lib { inherit lib pkgs; }); + ### TOOLS _0x0 = callPackage ../tools/misc/0x0 { }; diff --git a/pkgs/top-level/release.nix b/pkgs/top-level/release.nix index c11858f09c8..5fc6e91b311 100644 --- a/pkgs/top-level/release.nix +++ b/pkgs/top-level/release.nix @@ -34,6 +34,7 @@ let manual = import ../../doc { inherit pkgs nixpkgs; }; lib-tests = import ../../lib/tests/release.nix { inherit pkgs; }; + pkgs-lib-tests = import ../pkgs-lib/tests { inherit pkgs; }; darwin-tested = if supportDarwin then pkgs.releaseTools.aggregate { name = "nixpkgs-darwin-${jobs.tarball.version}"; @@ -91,6 +92,7 @@ let [ jobs.tarball jobs.manual jobs.lib-tests + jobs.pkgs-lib-tests jobs.stdenv.x86_64-linux jobs.linux.x86_64-linux jobs.pandoc.x86_64-linux From b6c540a87cf4cdf5c37cc7975f1afb0c07f8106c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 26 Mar 2020 02:55:16 +0100 Subject: [PATCH 4/6] pkgs-lib: Implement settings formats for JSON, INI, YAML and TOML --- pkgs/pkgs-lib/default.nix | 6 +- pkgs/pkgs-lib/formats.nix | 109 ++++++++++++++++++++++++++++++++ pkgs/top-level/all-packages.nix | 2 +- 3 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 pkgs/pkgs-lib/formats.nix diff --git a/pkgs/pkgs-lib/default.nix b/pkgs/pkgs-lib/default.nix index 1f74747d311..113dcebf8c6 100644 --- a/pkgs/pkgs-lib/default.nix +++ b/pkgs/pkgs-lib/default.nix @@ -2,6 +2,10 @@ # they depend on some packages. This notably is *not* for supporting package # building, instead pkgs/build-support is the place for that. { lib, pkgs }: { - + # setting format types and generators. These do not fit in lib/types.nix, + # because they depend on pkgs for rendering some formats + formats = import ./formats.nix { + inherit lib pkgs; + }; } diff --git a/pkgs/pkgs-lib/formats.nix b/pkgs/pkgs-lib/formats.nix new file mode 100644 index 00000000000..14589f8ecdc --- /dev/null +++ b/pkgs/pkgs-lib/formats.nix @@ -0,0 +1,109 @@ +{ lib, pkgs }: +rec { + + /* + + Every following entry represents a format for program configuration files + used for `settings`-style options (see https://github.com/NixOS/rfcs/pull/42). + Each entry should look as follows: + + = : { + # ^^ Parameters for controlling the format + + # The module system type most suitable for representing such a format + # The description needs to be overwritten for recursive types + type = ...; + + # generate :: Name -> Value -> Path + # A function for generating a file with a value of such a type + generate = ...; + + }); + */ + + + json = {}: { + + type = with lib.types; let + valueType = nullOr (oneOf [ + bool + int + float + str + (attrsOf valueType) + (listOf valueType) + ]) // { + description = "JSON value"; + }; + in valueType; + + generate = name: value: pkgs.runCommandNoCC name { + nativeBuildInputs = [ pkgs.jq ]; + value = builtins.toJSON value; + passAsFile = [ "value" ]; + } '' + jq . "$valuePath"> $out + ''; + + }; + + # YAML has been a strict superset of JSON since 1.2 + yaml = {}: + let jsonSet = json {}; + in jsonSet // { + type = jsonSet.type // { + description = "YAML value"; + }; + }; + + ini = { listsAsDuplicateKeys ? false, ... }@args: { + + type = with lib.types; let + + singleIniAtom = nullOr (oneOf [ + bool + int + float + str + ]) // { + description = "INI atom (null, bool, int, float or string)"; + }; + + iniAtom = + if listsAsDuplicateKeys then + coercedTo singleIniAtom lib.singleton (listOf singleIniAtom) // { + description = singleIniAtom.description + " or a list of them for duplicate keys"; + } + else + singleIniAtom; + + in attrsOf (attrsOf iniAtom); + + generate = name: value: pkgs.writeText name (lib.generators.toINI args value); + + }; + + toml = {}: json {} // { + type = with lib.types; let + valueType = oneOf [ + bool + int + float + str + (attrsOf valueType) + (listOf valueType) + ] // { + description = "TOML value"; + }; + in valueType; + + generate = name: value: pkgs.runCommandNoCC name { + nativeBuildInputs = [ pkgs.remarshal ]; + value = builtins.toJSON value; + passAsFile = [ "value" ]; + } '' + json2toml "$valuePath" "$out" + ''; + + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 4fca1254bcd..07e5fca1685 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -524,7 +524,7 @@ in writers = callPackage ../build-support/writers {}; # lib functions depending on pkgs - inherit (import ../pkgs-lib { inherit lib pkgs; }); + inherit (import ../pkgs-lib { inherit lib pkgs; }) formats; ### TOOLS From 9c1565a042ecd2f03b800f14f3cdc5ba0d464b5d Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 26 Mar 2020 02:56:07 +0100 Subject: [PATCH 5/6] pkgs-lib: Add tests for formats --- pkgs/pkgs-lib/tests/default.nix | 2 + pkgs/pkgs-lib/tests/formats.nix | 157 ++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 pkgs/pkgs-lib/tests/formats.nix diff --git a/pkgs/pkgs-lib/tests/default.nix b/pkgs/pkgs-lib/tests/default.nix index ee71596a145..f3549ea9b0f 100644 --- a/pkgs/pkgs-lib/tests/default.nix +++ b/pkgs/pkgs-lib/tests/default.nix @@ -1,5 +1,7 @@ # Call nix-build on this file to run all tests in this directory { pkgs ? import ../../.. {} }: let + formats = import ./formats.nix { inherit pkgs; }; in pkgs.linkFarm "nixpkgs-pkgs-lib-tests" [ + { name = "formats"; path = import ./formats.nix { inherit pkgs; }; } ] diff --git a/pkgs/pkgs-lib/tests/formats.nix b/pkgs/pkgs-lib/tests/formats.nix new file mode 100644 index 00000000000..bf6be8595e1 --- /dev/null +++ b/pkgs/pkgs-lib/tests/formats.nix @@ -0,0 +1,157 @@ +{ pkgs }: +let + inherit (pkgs) lib formats; +in +with lib; +let + + evalFormat = format: args: def: + let + formatSet = format args; + config = formatSet.type.merge [] (imap1 (n: def: { + value = def; + file = "def${toString n}"; + }) [ def ]); + in formatSet.generate "test-format-file" config; + + runBuildTest = name: { drv, expected }: pkgs.runCommandNoCC name {} '' + if diff ${drv} ${builtins.toFile "expected" expected}; then + touch $out + else + echo "Got: $(cat ${drv})" + echo "Should be: ${expected}" + exit 1 + fi + ''; + + runBuildTests = tests: pkgs.linkFarm "nixpkgs-pkgs-lib-format-tests" (mapAttrsToList (name: value: { inherit name; path = runBuildTest name value; }) (filterAttrs (name: value: value != null) tests)); + +in runBuildTests { + + testJsonAtoms = { + drv = evalFormat formats.json {} { + null = null; + false = false; + true = true; + int = 10; + float = 3.141; + str = "foo"; + attrs.foo = null; + list = [ null null ]; + }; + expected = '' + { + "attrs": { + "foo": null + }, + "false": false, + "float": 3.141, + "int": 10, + "list": [ + null, + null + ], + "null": null, + "str": "foo", + "true": true + } + ''; + }; + + testYamlAtoms = { + drv = evalFormat formats.yaml {} { + null = null; + false = false; + true = true; + float = 3.141; + str = "foo"; + attrs.foo = null; + list = [ null null ]; + }; + expected = '' + { + "attrs": { + "foo": null + }, + "false": false, + "float": 3.141, + "list": [ + null, + null + ], + "null": null, + "str": "foo", + "true": true + } + ''; + }; + + testIniAtoms = { + drv = evalFormat formats.ini {} { + foo = { + bool = true; + int = 10; + float = 3.141; + str = "string"; + }; + }; + expected = '' + [foo] + bool=true + float=3.141000 + int=10 + str=string + ''; + }; + + testIniDuplicateKeys = { + drv = evalFormat formats.ini { listsAsDuplicateKeys = true; } { + foo = { + bar = [ null true "test" 1.2 10 ]; + baz = false; + qux = "qux"; + }; + }; + expected = '' + [foo] + bar=null + bar=true + bar=test + bar=1.200000 + bar=10 + baz=false + qux=qux + ''; + }; + + testTomlAtoms = { + drv = evalFormat formats.toml {} { + false = false; + true = true; + int = 10; + float = 3.141; + str = "foo"; + attrs.foo = "foo"; + list = [ 1 2 ]; + level1.level2.level3.level4 = "deep"; + }; + expected = '' + false = false + float = 3.141 + int = 10 + list = [1, 2] + str = "foo" + true = true + + [attrs] + foo = "foo" + + [level1] + + [level1.level2] + + [level1.level2.level3] + level4 = "deep" + ''; + }; +} From 83b16885f526d6ab7b39e98159e2b48024f3238c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 6 Mar 2020 15:02:07 +0100 Subject: [PATCH 6/6] nixos/docs: Add documentation for settings options --- .../manual/development/settings-options.xml | 179 ++++++++++++++++++ .../manual/development/writing-modules.xml | 1 + 2 files changed, 180 insertions(+) create mode 100644 nixos/doc/manual/development/settings-options.xml diff --git a/nixos/doc/manual/development/settings-options.xml b/nixos/doc/manual/development/settings-options.xml new file mode 100644 index 00000000000..84895adb444 --- /dev/null +++ b/nixos/doc/manual/development/settings-options.xml @@ -0,0 +1,179 @@ +
+ Options for Program Settings + + + Many programs have configuration files where program-specific settings can be declared. File formats can be separated into two categories: + + + + Nix-representable ones: These can trivially be mapped to a subset of Nix syntax. E.g. JSON is an example, since its values like {"foo":{"bar":10}} can be mapped directly to Nix: { foo = { bar = 10; }; }. Other examples are INI, YAML and TOML. The following section explains the convention for these settings. + + + + + Non-nix-representable ones: These can't be trivially mapped to a subset of Nix syntax. Most generic programming languages are in this group, e.g. bash, since the statement if true; then echo hi; fi doesn't have a trivial representation in Nix. + + + Currently there are no fixed conventions for these, but it is common to have a configFile option for setting the configuration file path directly. The default value of configFile can be an auto-generated file, with convenient options for controlling the contents. For example an option of type attrsOf str can be used for representing environment variables which generates a section like export FOO="foo". Often it can also be useful to also include an extraConfig option of type lines to allow arbitrary text after the autogenerated part of the file. + + + + +
+ Nix-representable Formats (JSON, YAML, TOML, INI, ...) + + By convention, formats like this are handled with a generic settings option, representing the full program configuration as a Nix value. The type of this option should represent the format. The most common formats have a predefined type and string generator already declared under pkgs.formats: + + + + pkgs.formats.json { } + + + + A function taking an empty attribute set (for future extensibility) and returning a set with JSON-specific attributes type and generate as specified below. + + + + + + pkgs.formats.yaml { } + + + + A function taking an empty attribute set (for future extensibility) and returning a set with YAML-specific attributes type and generate as specified below. + + + + + + pkgs.formats.ini { listsAsDuplicateKeys ? false, ... } + + + + A function taking an attribute set with values + + + + listsAsDuplicateKeys + + + + A boolean for controlling whether list values can be used to represent duplicate INI keys + + + + + It returns a set with INI-specific attributes type and generate as specified below. + + + + + + pkgs.formats.toml { } + + + + A function taking an empty attribute set (for future extensibility) and returning a set with TOML-specific attributes type and generate as specified below. + + + + + + + + These functions all return an attribute set with these values: + + + + type + + + + A module system type representing a value of the format + + + + + + generate filename jsonValue + + + + A function that can render a value of the format to a file. Returns a file path. + + + This function puts the value contents in the Nix store. So this should be avoided for secrets. + + + + + + + + + Module with conventional <literal>settings</literal> option + + The following shows a module for an example program that uses a JSON configuration file. It demonstrates how above values can be used, along with some other related best practices. See the comments for explanations. + + +{ options, config, lib, pkgs, ... }: +let + cfg = config.services.foo; + # Define the settings format used for this program + settingsFormat = pkgs.formats.json {}; +in { + + options.services.foo = { + enable = lib.mkEnableOption "foo service"; + + settings = lib.mkOption { + # Setting this type allows for correct merging behavior + type = settingsFormat.type; + default = {}; + description = '' + Configuration for foo, see + <link xlink:href="https://example.com/docs/foo"/> + for supported values. + ''; + }; + }; + + config = lib.mkIf cfg.enable { + # We can assign some default settings here to make the service work by just + # enabling it. We use `mkDefault` for values that can be changed without + # problems + services.foo.settings = { + # Fails at runtime without any value set + log_level = lib.mkDefault "WARN"; + + # We assume systemd's `StateDirectory` is used, so we require this value, + # therefore no mkDefault + data_path = "/var/lib/foo"; + + # Since we use this to create a user we need to know the default value at + # eval time + user = lib.mkDefault "foo"; + }; + + environment.etc."foo.json".source = + # The formats generator function takes a filename and the Nix value + # representing the format value and produces a filepath with that value + # rendered in the format + settingsFormat.generate "foo-config.json" cfg.settings; + + # We know that the `user` attribute exists because we set a default value + # for it above, allowing us to use it without worries here + users.users.${cfg.settings.user} = {} + + # ... + }; +} + + +
+ +
diff --git a/nixos/doc/manual/development/writing-modules.xml b/nixos/doc/manual/development/writing-modules.xml index bbf793bb0be..602f134f9cb 100644 --- a/nixos/doc/manual/development/writing-modules.xml +++ b/nixos/doc/manual/development/writing-modules.xml @@ -183,4 +183,5 @@ in { +