Merge pull request #75584 from Infinisil/settings-formats

Configuration file formats for JSON, INI, YAML and TOML

lib/generators.nix

@@ -48,8 +48,10 @@ rec {
     else if isAttrs    v then err "attrsets" v
     # functions can’t be printed of course
     else if isFunction v then err "functions" v
-    # let’s not talk about floats. There is no sensible `toString` for them.
+    # Floats currently can't be converted to precise strings,
-    else if isFloat    v then err "floats" v
+    # condition warning on nix version once this isn't a problem anymore
+    # See https://github.com/NixOS/nix/pull/3480
+    else if isFloat    v then libStr.floatToString v
     else err "this value is" (toString v);
 
 

lib/strings.nix

@@ -612,6 +612,22 @@ rec {
   */
   fixedWidthNumber = width: n: fixedWidthString width "0" (toString n);
 
+  /* Convert a float to a string, but emit a warning when precision is lost
+     during the conversion
+
+     Example:
+       floatToString 0.000001
+       => "0.000001"
+       floatToString 0.0000001
+       => trace: warning: Imprecise conversion from float to string 0.000000
+          "0.000000"
+  */
+  floatToString = float: let
+    result = toString float;
+    precise = float == builtins.fromJSON result;
+  in if precise then result
+    else lib.warn "Imprecise conversion from float to string ${result}" result;
+
   /* Check whether a value can be coerced to a string */
   isCoercibleToString = x:
     builtins.elem (builtins.typeOf x) [ "path" "string" "null" "int" "float" "bool" ] ||

nixos/doc/manual/development/settings-options.xml

@@ -0,0 +1,179 @@
+<section xmlns="http://docbook.org/ns/docbook"
+        xmlns:xlink="http://www.w3.org/1999/xlink"
+        xmlns:xi="http://www.w3.org/2001/XInclude"
+        version="5.0"
+        xml:id="sec-settings-options">
+ <title>Options for Program Settings</title>
+
+ <para>
+   Many programs have configuration files where program-specific settings can be declared. File formats can be separated into two categories:
+   <itemizedlist>
+     <listitem>
+       <para>
+         Nix-representable ones: These can trivially be mapped to a subset of Nix syntax. E.g. JSON is an example, since its values like <literal>{"foo":{"bar":10}}</literal> can be mapped directly to Nix: <literal>{ foo = { bar = 10; }; }</literal>. Other examples are INI, YAML and TOML. The following section explains the convention for these settings.
+       </para>
+     </listitem>
+     <listitem>
+       <para>
+         Non-nix-representable ones: These can't be trivially mapped to a subset of Nix syntax. Most generic programming languages are in this group, e.g. bash, since the statement <literal>if true; then echo hi; fi</literal> doesn't have a trivial representation in Nix.
+       </para>
+       <para>
+         Currently there are no fixed conventions for these, but it is common to have a <literal>configFile</literal> option for setting the configuration file path directly. The default value of <literal>configFile</literal> can be an auto-generated file, with convenient options for controlling the contents. For example an option of type <literal>attrsOf str</literal> can be used for representing environment variables which generates a section like <literal>export FOO="foo"</literal>. Often it can also be useful to also include an <literal>extraConfig</literal> option of type <literal>lines</literal> to allow arbitrary text after the autogenerated part of the file.
+       </para>
+     </listitem>
+   </itemizedlist>
+ </para>
+ <section xml:id="sec-settings-nix-representable">
+   <title>Nix-representable Formats (JSON, YAML, TOML, INI, ...)</title>
+   <para>
+     By convention, formats like this are handled with a generic <literal>settings</literal> option, representing the full program configuration as a Nix value. The type of this option should represent the format. The most common formats have a predefined type and string generator already declared under <literal>pkgs.formats</literal>:
+     <variablelist>
+       <varlistentry>
+         <term>
+           <varname>pkgs.formats.json</varname> { }
+         </term>
+         <listitem>
+           <para>
+             A function taking an empty attribute set (for future extensibility) and returning a set with JSON-specific attributes <varname>type</varname> and <varname>generate</varname> as specified <link linkend='pkgs-formats-result'>below</link>.
+           </para>
+         </listitem>
+       </varlistentry>
+       <varlistentry>
+         <term>
+           <varname>pkgs.formats.yaml</varname> { }
+         </term>
+         <listitem>
+           <para>
+             A function taking an empty attribute set (for future extensibility) and returning a set with YAML-specific attributes <varname>type</varname> and <varname>generate</varname> as specified <link linkend='pkgs-formats-result'>below</link>.
+           </para>
+         </listitem>
+       </varlistentry>
+       <varlistentry>
+         <term>
+           <varname>pkgs.formats.ini</varname> { <replaceable>listsAsDuplicateKeys</replaceable> ? false, ... }
+         </term>
+         <listitem>
+           <para>
+             A function taking an attribute set with values
+             <variablelist>
+               <varlistentry>
+                 <term>
+                   <varname>listsAsDuplicateKeys</varname>
+                 </term>
+                 <listitem>
+                   <para>
+                     A boolean for controlling whether list values can be used to represent duplicate INI keys
+                   </para>
+                 </listitem>
+               </varlistentry>
+             </variablelist>
+            It returns a set with INI-specific attributes <varname>type</varname> and <varname>generate</varname> as specified <link linkend='pkgs-formats-result'>below</link>.
+           </para>
+         </listitem>
+       </varlistentry>
+       <varlistentry>
+         <term>
+           <varname>pkgs.formats.toml</varname> { }
+         </term>
+         <listitem>
+           <para>
+             A function taking an empty attribute set (for future extensibility) and returning a set with TOML-specific attributes <varname>type</varname> and <varname>generate</varname> as specified <link linkend='pkgs-formats-result'>below</link>.
+           </para>
+         </listitem>
+       </varlistentry>
+     </variablelist>
+
+   </para>
+   <para xml:id="pkgs-formats-result">
+     These functions all return an attribute set with these values:
+    <variablelist>
+      <varlistentry>
+        <term>
+          <varname>type</varname>
+        </term>
+        <listitem>
+          <para>
+            A module system type representing a value of the format
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term>
+          <varname>generate</varname> <replaceable>filename</replaceable> <replaceable>jsonValue</replaceable>
+        </term>
+        <listitem>
+          <para>
+           A function that can render a value of the format to a file. Returns a file path.
+           <note>
+            <para>
+             This function puts the value contents in the Nix store. So this should be avoided for secrets.
+            </para>
+           </note>
+          </para>
+        </listitem>
+      </varlistentry>
+    </variablelist>
+   </para>
+   <example xml:id="ex-settings-nix-representable">
+     <title>Module with conventional <literal>settings</literal> option</title>
+     <para>
+       The following shows a module for an example program that uses a JSON configuration file. It demonstrates how above values can be used, along with some other related best practices. See the comments for explanations.
+     </para>
+<programlisting>
+{ options, config, lib, pkgs, ... }:
+let
+  cfg = config.services.foo;
+  # Define the settings format used for this program
+  settingsFormat = pkgs.formats.json {};
+in {
+
+  options.services.foo = {
+    enable = lib.mkEnableOption "foo service";
+
+    settings = lib.mkOption {
+      # Setting this type allows for correct merging behavior
+      type = settingsFormat.type;
+      default = {};
+      description = ''
+        Configuration for foo, see
+        &lt;link xlink:href="https://example.com/docs/foo"/&gt;
+        for supported values.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    # We can assign some default settings here to make the service work by just
+    # enabling it. We use `mkDefault` for values that can be changed without
+    # problems
+    services.foo.settings = {
+      # Fails at runtime without any value set
+      log_level = lib.mkDefault "WARN";
+
+      # We assume systemd's `StateDirectory` is used, so we require this value,
+      # therefore no mkDefault
+      data_path = "/var/lib/foo";
+
+      # Since we use this to create a user we need to know the default value at
+      # eval time
+      user = lib.mkDefault "foo";
+    };
+
+    environment.etc."foo.json".source =
+      # The formats generator function takes a filename and the Nix value
+      # representing the format value and produces a filepath with that value
+      # rendered in the format
+      settingsFormat.generate "foo-config.json" cfg.settings;
+
+    # We know that the `user` attribute exists because we set a default value
+    # for it above, allowing us to use it without worries here
+    users.users.${cfg.settings.user} = {}
+
+    # ...
+  };
+}
+</programlisting>
+   </example>
+ </section>
+
+</section>

nixos/doc/manual/development/writing-modules.xml

@@ -183,4 +183,5 @@ in {
  <xi:include href="meta-attributes.xml" />
  <xi:include href="importing-modules.xml" />
  <xi:include href="replace-modules.xml" />
+ <xi:include href="settings-options.xml" />
 </chapter>

pkgs/pkgs-lib/default.nix

@@ -0,0 +1,11 @@
+# pkgs-lib is for functions and values that can't be in lib because
+# they depend on some packages. This notably is *not* for supporting package
+# building, instead pkgs/build-support is the place for that.
+{ lib, pkgs }: {
+  # setting format types and generators. These do not fit in lib/types.nix,
+  # because they depend on pkgs for rendering some formats
+  formats = import ./formats.nix {
+    inherit lib pkgs;
+  };
+}
+

pkgs/pkgs-lib/formats.nix

@@ -0,0 +1,109 @@
+{ lib, pkgs }:
+rec {
+
+  /*
+
+  Every following entry represents a format for program configuration files
+  used for `settings`-style options (see https://github.com/NixOS/rfcs/pull/42).
+  Each entry should look as follows:
+
+    <format> = <parameters>: {
+      #        ^^ Parameters for controlling the format
+
+      # The module system type most suitable for representing such a format
+      # The description needs to be overwritten for recursive types
+      type = ...;
+
+      # generate :: Name -> Value -> Path
+      # A function for generating a file with a value of such a type
+      generate = ...;
+
+    });
+  */
+
+
+  json = {}: {
+
+    type = with lib.types; let
+      valueType = nullOr (oneOf [
+        bool
+        int
+        float
+        str
+        (attrsOf valueType)
+        (listOf valueType)
+      ]) // {
+        description = "JSON value";
+      };
+    in valueType;
+
+    generate = name: value: pkgs.runCommandNoCC name {
+      nativeBuildInputs = [ pkgs.jq ];
+      value = builtins.toJSON value;
+      passAsFile = [ "value" ];
+    } ''
+      jq . "$valuePath"> $out
+    '';
+
+  };
+
+  # YAML has been a strict superset of JSON since 1.2
+  yaml = {}:
+    let jsonSet = json {};
+    in jsonSet // {
+      type = jsonSet.type // {
+        description = "YAML value";
+      };
+    };
+
+  ini = { listsAsDuplicateKeys ? false, ... }@args: {
+
+    type = with lib.types; let
+
+      singleIniAtom = nullOr (oneOf [
+        bool
+        int
+        float
+        str
+      ]) // {
+        description = "INI atom (null, bool, int, float or string)";
+      };
+
+      iniAtom =
+        if listsAsDuplicateKeys then
+          coercedTo singleIniAtom lib.singleton (listOf singleIniAtom) // {
+            description = singleIniAtom.description + " or a list of them for duplicate keys";
+          }
+        else
+          singleIniAtom;
+
+    in attrsOf (attrsOf iniAtom);
+
+    generate = name: value: pkgs.writeText name (lib.generators.toINI args value);
+
+  };
+
+  toml = {}: json {} // {
+    type = with lib.types; let
+      valueType = oneOf [
+        bool
+        int
+        float
+        str
+        (attrsOf valueType)
+        (listOf valueType)
+      ] // {
+        description = "TOML value";
+      };
+    in valueType;
+
+    generate = name: value: pkgs.runCommandNoCC name {
+      nativeBuildInputs = [ pkgs.remarshal ];
+      value = builtins.toJSON value;
+      passAsFile = [ "value" ];
+    } ''
+      json2toml "$valuePath" "$out"
+    '';
+
+  };
+}

pkgs/pkgs-lib/tests/default.nix

@@ -0,0 +1,7 @@
+# Call nix-build on this file to run all tests in this directory
+{ pkgs ? import ../../.. {} }:
+let
+  formats = import ./formats.nix { inherit pkgs; };
+in pkgs.linkFarm "nixpkgs-pkgs-lib-tests" [
+  { name = "formats"; path = import ./formats.nix { inherit pkgs; }; }
+]

pkgs/pkgs-lib/tests/formats.nix

@@ -0,0 +1,157 @@
+{ pkgs }:
+let
+  inherit (pkgs) lib formats;
+in
+with lib;
+let
+
+  evalFormat = format: args: def:
+    let
+      formatSet = format args;
+      config = formatSet.type.merge [] (imap1 (n: def: {
+        value = def;
+        file = "def${toString n}";
+      }) [ def ]);
+    in formatSet.generate "test-format-file" config;
+
+  runBuildTest = name: { drv, expected }: pkgs.runCommandNoCC name {} ''
+    if diff ${drv} ${builtins.toFile "expected" expected}; then
+      touch $out
+    else
+      echo "Got: $(cat ${drv})"
+      echo "Should be: ${expected}"
+      exit 1
+    fi
+  '';
+
+  runBuildTests = tests: pkgs.linkFarm "nixpkgs-pkgs-lib-format-tests" (mapAttrsToList (name: value: { inherit name; path = runBuildTest name value; }) (filterAttrs (name: value: value != null) tests));
+
+in runBuildTests {
+
+  testJsonAtoms = {
+    drv = evalFormat formats.json {} {
+      null = null;
+      false = false;
+      true = true;
+      int = 10;
+      float = 3.141;
+      str = "foo";
+      attrs.foo = null;
+      list = [ null null ];
+    };
+    expected = ''
+      {
+        "attrs": {
+          "foo": null
+        },
+        "false": false,
+        "float": 3.141,
+        "int": 10,
+        "list": [
+          null,
+          null
+        ],
+        "null": null,
+        "str": "foo",
+        "true": true
+      }
+    '';
+  };
+
+  testYamlAtoms = {
+    drv = evalFormat formats.yaml {} {
+      null = null;
+      false = false;
+      true = true;
+      float = 3.141;
+      str = "foo";
+      attrs.foo = null;
+      list = [ null null ];
+    };
+    expected = ''
+      {
+        "attrs": {
+          "foo": null
+        },
+        "false": false,
+        "float": 3.141,
+        "list": [
+          null,
+          null
+        ],
+        "null": null,
+        "str": "foo",
+        "true": true
+      }
+    '';
+  };
+
+  testIniAtoms = {
+    drv = evalFormat formats.ini {} {
+      foo = {
+        bool = true;
+        int = 10;
+        float = 3.141;
+        str = "string";
+      };
+    };
+    expected = ''
+      [foo]
+      bool=true
+      float=3.141000
+      int=10
+      str=string
+    '';
+  };
+
+  testIniDuplicateKeys = {
+    drv = evalFormat formats.ini { listsAsDuplicateKeys = true; } {
+      foo = {
+        bar = [ null true "test" 1.2 10 ];
+        baz = false;
+        qux = "qux";
+      };
+    };
+    expected = ''
+      [foo]
+      bar=null
+      bar=true
+      bar=test
+      bar=1.200000
+      bar=10
+      baz=false
+      qux=qux
+    '';
+  };
+
+  testTomlAtoms = {
+    drv = evalFormat formats.toml {} {
+      false = false;
+      true = true;
+      int = 10;
+      float = 3.141;
+      str = "foo";
+      attrs.foo = "foo";
+      list = [ 1 2 ];
+      level1.level2.level3.level4 = "deep";
+    };
+    expected = ''
+      false = false
+      float = 3.141
+      int = 10
+      list = [1, 2]
+      str = "foo"
+      true = true
+
+      [attrs]
+      foo = "foo"
+
+      [level1]
+
+      [level1.level2]
+
+      [level1.level2.level3]
+      level4 = "deep"
+    '';
+  };
+}

pkgs/top-level/all-packages.nix

@@ -525,6 +525,9 @@ in
   #package writers
   writers = callPackage ../build-support/writers {};
 
+  # lib functions depending on pkgs
+  inherit (import ../pkgs-lib { inherit lib pkgs; }) formats;
+
   ### TOOLS
 
   _0x0 = callPackage ../tools/misc/0x0 { };

pkgs/top-level/release.nix

@@ -34,6 +34,7 @@ let
 
       manual = import ../../doc { inherit pkgs nixpkgs; };
       lib-tests = import ../../lib/tests/release.nix { inherit pkgs; };
+      pkgs-lib-tests = import ../pkgs-lib/tests { inherit pkgs; };
 
       darwin-tested = if supportDarwin then pkgs.releaseTools.aggregate
         { name = "nixpkgs-darwin-${jobs.tarball.version}";
@@ -91,6 +92,7 @@ let
             [ jobs.tarball
               jobs.manual
               jobs.lib-tests
+              jobs.pkgs-lib-tests
               jobs.stdenv.x86_64-linux
               jobs.linux.x86_64-linux
               jobs.pandoc.x86_64-linux