Merge pull request #20507 from Profpatsch/lib-ini-generator

lib: File generators from Nix Expressions

doc/functions.xml

@@ -8,7 +8,16 @@
   The nixpkgs repository has several utility functions to manipulate Nix expressions.
 </para>
 
-<section xml:id="sec-pkgs-overridePackages">
+<section xml:id="sec-overrides">
+  <title>Overriding</title>
+
+  <para>
+    Sometimes one wants to override parts of
+    <literal>nixpkgs</literal>, e.g. derivation attributes, the results of
+    derivations or even the whole package set.
+  </para>
+
+  <section xml:id="sec-pkgs-overridePackages">
     <title>pkgs.overridePackages</title>
 
     <para>
@@ -27,7 +36,7 @@
     newpkgs = pkgs.overridePackages (self: super: {
       foo = super.foo.override { ... };
     };
-in ...</programlisting>
+  in ...</programlisting>
     </para>
 
     <para>
@@ -63,12 +72,12 @@ in ...</programlisting>
       <programlisting>let
     pkgs = import &lt;nixpkgs&gt; { config: {}; };
     newpkgs = pkgs.overridePackages ...;
-in ...</programlisting>
+  in ...</programlisting>
     </para>
 
-</section>
+  </section>
 
-<section xml:id="sec-pkg-override">
+  <section xml:id="sec-pkg-override">
     <title>&lt;pkg&gt;.override</title>
 
     <para>
@@ -84,10 +93,10 @@ in ...</programlisting>
       <programlisting>pkgs.foo.override { arg1 = val1; arg2 = val2; ... }</programlisting>
       <programlisting>pkgs.overridePackages (self: super: {
     foo = super.foo.override { barSupport = true ; };
-})</programlisting>
+  })</programlisting>
       <programlisting>mypkg = pkgs.callPackage ./mypkg.nix {
     mydep = pkgs.mydep.override { ... };
-})</programlisting>
+  })</programlisting>
     </para>
 
     <para>
@@ -97,9 +106,9 @@ in ...</programlisting>
       the given new arguments.
     </para>
 
-</section>
+  </section>
 
-<section xml:id="sec-pkg-overrideAttrs">
+  <section xml:id="sec-pkg-overrideAttrs">
     <title>&lt;pkg&gt;.overrideAttrs</title>
 
     <para>
@@ -116,7 +125,7 @@ in ...</programlisting>
 
       <programlisting>helloWithDebug = pkgs.hello.overrideAttrs (oldAttrs: rec {
     separateDebugInfo = true;
-});</programlisting>
+  });</programlisting>
     </para>
 
     <para>
@@ -148,10 +157,10 @@ in ...</programlisting>
       </para>
     </note>
 
-</section>
+  </section>
 
 
-<section xml:id="sec-pkg-overrideDerivation">
+  <section xml:id="sec-pkg-overrideDerivation">
     <title>&lt;pkg&gt;.overrideDerivation</title>
 
     <warning>
@@ -196,7 +205,7 @@ in ...</programlisting>
       sha256 = "11nq06d131y4wmf3drm0yk502d2xc6n5qy82cg88rb9nqd2lj41k";
     };
     patches = [];
-});</programlisting>
+  });</programlisting>
     </para>
 
     <para>
@@ -224,9 +233,9 @@ in ...</programlisting>
       </para>
     </note>
 
-</section>
+  </section>
 
-<section xml:id="sec-lib-makeOverridable">
+  <section xml:id="sec-lib-makeOverridable">
     <title>lib.makeOverridable</title>
 
     <para>
@@ -239,7 +248,7 @@ in ...</programlisting>
       Example usage:
 
       <programlisting>f = { a, b }: { result = a+b; }
-c = lib.makeOverridable f { a = 1; b = 2; }</programlisting>
+  c = lib.makeOverridable f { a = 1; b = 2; }</programlisting>
 
     </para>
 
@@ -256,6 +265,40 @@ c = lib.makeOverridable f { a = 1; b = 2; }</programlisting>
       <varname>(c.override { a = 4; }).result</varname> is 6.
     </para>
 
+  </section>
+
+</section>
+
+<section xml:id="sec-generators">
+  <title>Generators</title>
+
+  <para>
+    Generators are functions that create file formats from nix
+    data structures, e. g. for configuration files.
+    There are generators available for: <literal>INI</literal>,
+    <literal>JSON</literal> and <literal>YAML</literal>
+  </para>
+
+  <para>
+    All generators follow a similar call interface: <code>generatorName
+    configFunctions data</code>, where <literal>configFunctions</literal> is a
+    set of user-defined functions that format variable parts of the content.
+    They each have common defaults, so often they do not need to be set
+    manually. An example is <code>mkSectionName ? (name: libStr.escape [ "[" "]"
+    ] name)</code> from the <literal>INI</literal> generator. It gets the name
+    of a section and returns a sanitized name. The default
+    <literal>mkSectionName</literal> escapes <literal>[</literal> and
+    <literal>]</literal> with a backslash. 
+  </para>
+
+  <note><para>Nix store paths can be converted to strings by enclosing a
+  derivation attribute like so: <code>"${drv}"</code>.</para></note>
+
+  <para>
+    Detailed documentation for each generator can be found in
+    <literal>lib/generators.nix</literal>.
+  </para>
+
 </section>
 
 
@@ -370,25 +413,25 @@ c = lib.makeOverridable f { a = 1; b = 2; }</programlisting>
 </section>
 
 <section xml:id="sec-pkgs-dockerTools">
- <title>pkgs.dockerTools</title>
+<title>pkgs.dockerTools</title>
 
- <para>
+<para>
   <varname>pkgs.dockerTools</varname> is a set of functions for creating and
   manipulating Docker images according to the
   <link xlink:href="https://github.com/docker/docker/blob/master/image/spec/v1.md#docker-image-specification-v100">
   Docker Image Specification v1.0.0
   </link>. Docker itself is not used to perform any of the operations done by these
   functions.
- </para>
+</para>
 
- <warning>
+<warning>
   <para>
   The <varname>dockerTools</varname> API is unstable and may be subject to
   backwards-incompatible changes in the future.
   </para>
- </warning>
+</warning>
 
- <section xml:id="ssec-pkgs-dockerTools-buildImage">
+<section xml:id="ssec-pkgs-dockerTools-buildImage">
   <title>buildImage</title>
 
   <para>
@@ -541,9 +584,9 @@ c = lib.makeOverridable f { a = 1; b = 2; }</programlisting>
   using its <varname>buildArgs</varname> attribute.
   </para>
 
- </section>
+</section>
 
- <section xml:id="ssec-pkgs-dockerTools-fetchFromRegistry">
+<section xml:id="ssec-pkgs-dockerTools-fetchFromRegistry">
   <title>pullImage</title>
 
   <para>
@@ -618,9 +661,9 @@ c = lib.makeOverridable f { a = 1; b = 2; }</programlisting>
   </callout>
   </calloutlist>
 
- </section>
+</section>
 
- <section xml:id="ssec-pkgs-dockerTools-exportImage">
+<section xml:id="ssec-pkgs-dockerTools-exportImage">
   <title>exportImage</title>
 
   <para>
@@ -664,9 +707,9 @@ c = lib.makeOverridable f { a = 1; b = 2; }</programlisting>
   The <varname>name</varname> argument is the name of the derivation output,
   which defaults to <varname>fromImage.name</varname>.
   </para>
- </section>
+</section>
 
- <section xml:id="ssec-pkgs-dockerTools-shadowSetup">
+<section xml:id="ssec-pkgs-dockerTools-shadowSetup">
   <title>shadowSetup</title>
 
   <para>
@@ -700,7 +743,7 @@ c = lib.makeOverridable f { a = 1; b = 2; }</programlisting>
   manipulate users and groups.
   </para>
 
- </section>
+</section>
 
 </section>
 

lib/default.nix

@@ -27,6 +27,7 @@ let
 
   # misc
   debug = import ./debug.nix;
+  generators = import ./generators.nix;
   misc = import ./deprecated.nix;
 
   # domain-specific
@@ -39,7 +40,7 @@ in
             customisation maintainers meta sources
             modules options types
             licenses platforms systems
-            debug misc
+            debug generators misc
             sandbox fetchers;
   }
   # !!! don't include everything at top-level; perhaps only the most

lib/generators.nix

@@ -0,0 +1,72 @@
+/* Functions that generate widespread file
+ * formats from nix data structures.
+ *
+ * They all follow a similar interface:
+ * generator { config-attrs } data
+ *
+ * Tests can be found in ./tests.nix
+ * Documentation in the manual, #sec-generators
+ */
+with import ./trivial.nix;
+let
+  libStr = import ./strings.nix;
+  libAttr = import ./attrsets.nix;
+
+  flipMapAttrs = flip libAttr.mapAttrs;
+in
+
+rec {
+
+  /* Generates an INI-style config file from an
+   * attrset of sections to an attrset of key-value pairs.
+   *
+   * generators.toINI {} {
+   *   foo = { hi = "${pkgs.hello}"; ciao = "bar"; };
+   *   baz = { "also, integers" = 42; };
+   * }
+   *
+   *> [baz]
+   *> also, integers=42
+   *>
+   *> [foo]
+   *> ciao=bar
+   *> hi=/nix/store/y93qql1p5ggfnaqjjqhxcw0vqw95rlz0-hello-2.10
+   *
+   * The mk* configuration attributes can generically change
+   * the way sections and key-value strings are generated.
+   *
+   * For more examples see the test cases in ./tests.nix.
+   */
+  toINI = {
+    # apply transformations (e.g. escapes) to section names
+    mkSectionName ? (name: libStr.escape [ "[" "]" ] name),
+    # format a setting line from key and value
+    mkKeyValue    ? (k: v: "${libStr.escape ["="] k}=${toString v}")
+  }: attrsOfAttrs:
+    let
+        # map function to string for each key val
+        mapAttrsToStringsSep = sep: mapFn: attrs:
+          libStr.concatStringsSep sep
+            (libAttr.mapAttrsToList mapFn attrs);
+        mkLine = k: v: mkKeyValue k v + "\n";
+        mkSection = sectName: sectValues: ''
+          [${mkSectionName sectName}]
+        '' + libStr.concatStrings (libAttr.mapAttrsToList mkLine sectValues);
+    in
+      # map input to ini sections
+      mapAttrsToStringsSep "\n" mkSection attrsOfAttrs;
+
+
+  /* Generates JSON from an arbitrary (non-function) value.
+    * For more information see the documentation of the builtin.
+    */
+  toJSON = {}: builtins.toJSON;
+
+
+  /* YAML has been a strict superset of JSON since 1.2, so we
+    * use toJSON. Before it only had a few differences referring
+    * to implicit typing rules, so it should work with older
+    * parsers as well.
+    */
+  toYAML = {}@args: toJSON args;
+}

lib/tests.nix

@@ -130,4 +130,78 @@ runTests {
     expected = false;
   };
 
+
+  /* Generator tests */
+  # these tests assume attributes are converted to lists
+  # in alphabetical order
+
+  testToINIEmpty = {
+    expr = generators.toINI {} {};
+    expected = "";
+  };
+
+  testToINIEmptySection = {
+    expr = generators.toINI {} { foo = {}; bar = {}; };
+    expected = ''
+      [bar]
+
+      [foo]
+    '';
+  };
+
+  testToINIDefaultEscapes = {
+    expr = generators.toINI {} {
+      "no [ and ] allowed unescaped" = {
+        "and also no = in keys" = 42;
+      };
+    };
+    expected = ''
+      [no \[ and \] allowed unescaped]
+      and also no \= in keys=42
+    '';
+  };
+
+  testToINIDefaultFull = {
+    expr = generators.toINI {} {
+      "section 1" = {
+        attribute1 = 5;
+        x = "Me-se JarJar Binx";
+      };
+      "foo[]" = {
+        "he\\h=he" = "this is okay";
+      };
+    };
+    expected = ''
+      [foo\[\]]
+      he\h\=he=this is okay
+
+      [section 1]
+      attribute1=5
+      x=Me-se JarJar Binx
+    '';
+  };
+
+  /* right now only invocation check */
+  testToJSONSimple =
+    let val = {
+      foobar = [ "baz" 1 2 3 ];
+    };
+    in {
+      expr = generators.toJSON {} val;
+      # trival implementation
+      expected = builtins.toJSON val;
+  };
+
+  /* right now only invocation check */
+  testToYAMLSimple =
+    let val = {
+      list = [ { one = 1; } { two = 2; } ];
+      all = 42;
+    };
+    in {
+      expr = generators.toYAML {} val;
+      # trival implementation
+      expected = builtins.toJSON val;
+  };
+
 }