Module system improvements for NixOS as a submodule (#75031)

Module system improvements for NixOS as a submodule

lib/default.nix

@@ -102,7 +102,7 @@ let
       commitIdFromGitRepo cleanSourceWith pathHasContext
       canCleanSource;
     inherit (modules) evalModules closeModules unifyModuleSyntax
-      applyIfFunction unpackSubmodule packSubmodule mergeModules
+      applyIfFunction mergeModules
       mergeModules' mergeOptionDecls evalOptionValue mergeDefinitions
       pushDownProperties dischargeProperties filterOverrides
       sortProperties fixupOptionType mkIf mkAssert mkMerge mkOverride

lib/modules.nix

@@ -103,42 +103,42 @@ rec {
       toClosureList = file: parentKey: imap1 (n: x:
         if isAttrs x || isFunction x then
           let key = "${parentKey}:anon-${toString n}"; in
-          unifyModuleSyntax file key (unpackSubmodule (applyIfFunction key) x args)
+          unifyModuleSyntax file key (applyIfFunction key x args)
         else
           let file = toString x; key = toString x; in
           unifyModuleSyntax file key (applyIfFunction key (import x) args));
     in
       builtins.genericClosure {
         startSet = toClosureList unknownModule "" modules;
-        operator = m: toClosureList m.file m.key m.imports;
+        operator = m: toClosureList m._file m.key m.imports;
       };
 
   /* Massage a module into canonical form, that is, a set consisting
      of ‘options’, ‘config’ and ‘imports’ attributes. */
   unifyModuleSyntax = file: key: m:
-    let metaSet = if m ? meta
+    let addMeta = config: if m ? meta
-      then { meta = m.meta; }
+      then mkMerge [ config { meta = m.meta; } ]
-      else {};
+      else config;
     in
     if m ? config || m ? options then
       let badAttrs = removeAttrs m ["_file" "key" "disabledModules" "imports" "options" "config" "meta"]; in
       if badAttrs != {} then
         throw "Module `${key}' has an unsupported attribute `${head (attrNames badAttrs)}'. This is caused by assignments to the top-level attributes `config' or `options'."
       else
-        { file = m._file or file;
+        { _file = m._file or file;
           key = toString m.key or key;
           disabledModules = m.disabledModules or [];
           imports = m.imports or [];
           options = m.options or {};
-          config = mkMerge [ (m.config or {}) metaSet ];
+          config = addMeta (m.config or {});
         }
     else
-      { file = m._file or file;
+      { _file = m._file or file;
         key = toString m.key or key;
         disabledModules = m.disabledModules or [];
         imports = m.require or [] ++ m.imports or [];
         options = {};
-        config = mkMerge [ (removeAttrs m ["_file" "key" "disabledModules" "require" "imports"]) metaSet ];
+        config = addMeta (removeAttrs m ["_file" "key" "disabledModules" "require" "imports"]);
       };
 
   applyIfFunction = key: f: args@{ config, options, lib, ... }: if isFunction f then
@@ -171,17 +171,6 @@ rec {
   else
     f;
 
-  /* We have to pack and unpack submodules. We cannot wrap the expected
-     result of the function as we would no longer be able to list the arguments
-     of the submodule. (see applyIfFunction) */
-  unpackSubmodule = unpack: m: args:
-    if isType "submodule" m then
-      { _file = m.file; } // (unpack m.submodule args)
-    else unpack m args;
-
-  packSubmodule = file: m:
-    { _type = "submodule"; file = file; submodule = m; };
-
   /* Merge a list of modules.  This will recurse over the option
      declarations in all modules, combining them into a single set.
      At the same time, for each option declaration, it will merge the
@@ -189,7 +178,7 @@ rec {
      in the ‘value’ attribute of each option. */
   mergeModules = prefix: modules:
     mergeModules' prefix modules
-      (concatMap (m: map (config: { inherit (m) file; inherit config; }) (pushDownProperties m.config)) modules);
+      (concatMap (m: map (config: { file = m._file; inherit config; }) (pushDownProperties m.config)) modules);
 
   mergeModules' = prefix: options: configs:
     let
@@ -223,7 +212,7 @@ rec {
                ) {} modules;
       # an attrset 'name' => list of submodules that declare ‘name’.
       declsByName = byName "options" (module: option:
-          [{ inherit (module) file; options = option; }]
+          [{ inherit (module) _file; options = option; }]
         ) options;
       # an attrset 'name' => list of submodules that define ‘name’.
       defnsByName = byName "config" (module: value:
@@ -250,7 +239,7 @@ rec {
               firstOption = findFirst (m: isOption m.options) "" decls;
               firstNonOption = findFirst (m: !isOption m.options) "" decls;
             in
-              throw "The option `${showOption loc}' in `${firstOption.file}' is a prefix of options in `${firstNonOption.file}'."
+              throw "The option `${showOption loc}' in `${firstOption._file}' is a prefix of options in `${firstNonOption._file}'."
           else
             mergeModules' loc decls defns
       ))
@@ -267,7 +256,14 @@ rec {
 
      'opts' is a list of modules.  Each module has an options attribute which
      correspond to the definition of 'loc' in 'opt.file'. */
-  mergeOptionDecls = loc: opts:
+  mergeOptionDecls =
+   let
+    packSubmodule = file: m:
+      { _file = file; imports = [ m ]; };
+    coerceOption = file: opt:
+      if isFunction opt then packSubmodule file opt
+      else packSubmodule file { options = opt; };
+   in loc: opts:
     foldl' (res: opt:
       let t  = res.type;
           t' = opt.options.type;
@@ -284,7 +280,7 @@ rec {
          bothHave "apply" ||
          (bothHave "type" && (! typesMergeable))
       then
-        throw "The option `${showOption loc}' in `${opt.file}' is already declared in ${showFiles res.declarations}."
+        throw "The option `${showOption loc}' in `${opt._file}' is already declared in ${showFiles res.declarations}."
       else
         let
           /* Add the modules of the current option to the list of modules
@@ -293,16 +289,14 @@ rec {
              current option declaration as the file use for the submodule.  If the
              submodule defines any filename, then we ignore the enclosing option file. */
           options' = toList opt.options.options;
-          coerceOption = file: opt:
+
-            if isFunction opt then packSubmodule file opt
-            else packSubmodule file { options = opt; };
           getSubModules = opt.options.type.getSubModules or null;
           submodules =
-            if getSubModules != null then map (packSubmodule opt.file) getSubModules ++ res.options
+            if getSubModules != null then map (packSubmodule opt._file) getSubModules ++ res.options
-            else if opt.options ? options then map (coerceOption opt.file) options' ++ res.options
+            else if opt.options ? options then map (coerceOption opt._file) options' ++ res.options
             else res.options;
         in opt.options // res //
-          { declarations = res.declarations ++ [opt.file];
+          { declarations = res.declarations ++ [opt._file];
             options = submodules;
           } // typeSet
     ) { inherit loc; declarations = []; options = []; } opts;

lib/tests/modules.sh

@@ -164,6 +164,24 @@ checkConfigOutput "true" config.enableAlias ./alias-with-priority.nix
 checkConfigOutput "false" config.enable ./alias-with-priority-can-override.nix
 checkConfigOutput "false" config.enableAlias ./alias-with-priority-can-override.nix
 
+# submoduleWith
+
+## specialArgs should work
+checkConfigOutput "foo" config.submodule.foo ./declare-submoduleWith-special.nix
+
+## shorthandOnlyDefines config behaves as expected
+checkConfigOutput "true" config.submodule.config ./declare-submoduleWith-shorthand.nix ./define-submoduleWith-shorthand.nix
+checkConfigError 'is not of type `boolean' config.submodule.config ./declare-submoduleWith-shorthand.nix ./define-submoduleWith-noshorthand.nix
+checkConfigError 'value is a boolean while a set was expected' config.submodule.config ./declare-submoduleWith-noshorthand.nix ./define-submoduleWith-shorthand.nix
+checkConfigOutput "true" config.submodule.config ./declare-submoduleWith-noshorthand.nix ./define-submoduleWith-noshorthand.nix
+
+## submoduleWith should merge all modules in one swoop
+checkConfigOutput "true" config.submodule.inner ./declare-submoduleWith-modules.nix
+checkConfigOutput "true" config.submodule.outer ./declare-submoduleWith-modules.nix
+
+## Paths should be allowed as values and work as expected
+checkConfigOutput "true" config.submodule.enable ./declare-submoduleWith-path.nix
+
 cat <<EOF
 ====== module tests ======
 $pass Pass

lib/tests/modules/declare-submoduleWith-modules.nix

@@ -0,0 +1,30 @@
+{ lib, ... }: {
+  options.submodule = lib.mkOption {
+    type = lib.types.submoduleWith {
+      modules = [
+        {
+          options.inner = lib.mkOption {
+            type = lib.types.bool;
+            default = false;
+          };
+        }
+        {
+          outer = true;
+        }
+      ];
+    };
+    default = {};
+  };
+
+  config.submodule = lib.mkMerge [
+    ({ lib, ... }: {
+      options.outer = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+      };
+    })
+    {
+      inner = true;
+    }
+  ];
+}

lib/tests/modules/declare-submoduleWith-noshorthand.nix

@@ -0,0 +1,13 @@
+{ lib, ... }: let
+  sub.options.config = lib.mkOption {
+    type = lib.types.bool;
+    default = false;
+  };
+in {
+  options.submodule = lib.mkOption {
+    type = lib.types.submoduleWith {
+      modules = [ sub ];
+    };
+    default = {};
+  };
+}

lib/tests/modules/declare-submoduleWith-path.nix

@@ -0,0 +1,12 @@
+{ lib, ... }: {
+  options.submodule = lib.mkOption {
+    type = lib.types.submoduleWith {
+      modules = [
+        ./declare-enable.nix
+      ];
+    };
+    default = {};
+  };
+
+  config.submodule = ./define-enable.nix;
+}

lib/tests/modules/declare-submoduleWith-shorthand.nix

@@ -0,0 +1,14 @@
+{ lib, ... }: let
+  sub.options.config = lib.mkOption {
+    type = lib.types.bool;
+    default = false;
+  };
+in {
+  options.submodule = lib.mkOption {
+    type = lib.types.submoduleWith {
+      modules = [ sub ];
+      shorthandOnlyDefinesConfig = true;
+    };
+    default = {};
+  };
+}

lib/tests/modules/declare-submoduleWith-special.nix

@@ -0,0 +1,17 @@
+{ lib, ... }: {
+  options.submodule = lib.mkOption {
+    type = lib.types.submoduleWith {
+      modules = [
+        ({ lib, ... }: {
+          options.foo = lib.mkOption {
+            default = lib.foo;
+          };
+        })
+      ];
+      specialArgs.lib = lib // {
+        foo = "foo";
+      };
+    };
+    default = {};
+  };
+}

lib/tests/modules/define-submoduleWith-noshorthand.nix

@@ -0,0 +1,3 @@
+{
+  submodule.config.config = true;
+}

lib/tests/modules/define-submoduleWith-shorthand.nix

@@ -0,0 +1,3 @@
+{
+  submodule.config = true;
+}

lib/types.nix

@@ -358,25 +358,43 @@ rec {
     };
 
     # A submodule (like typed attribute set). See NixOS manual.
-    submodule = opts:
+    submodule = modules: submoduleWith {
+      shorthandOnlyDefinesConfig = true;
+      modules = toList modules;
+    };
+
+    submoduleWith =
+      { modules
+      , specialArgs ? {}
+      , shorthandOnlyDefinesConfig ? false
+      }@attrs:
       let
-        opts' = toList opts;
         inherit (lib.modules) evalModules;
+
+        coerce = unify: value: if isFunction value
+          then setFunctionArgs (args: unify (value args)) (functionArgs value)
+          else unify (if shorthandOnlyDefinesConfig then { config = value; } else value);
+
+        allModules = defs: modules ++ imap1 (n: { value, file }:
+          if isAttrs value || isFunction value then
+            # Annotate the value with the location of its definition for better error messages
+            coerce (lib.modules.unifyModuleSyntax file "${toString file}-${toString n}") value
+          else value
+        ) defs;
+
       in
       mkOptionType rec {
         name = "submodule";
-        check = x: isAttrs x || isFunction x;
+        check = x: isAttrs x || isFunction x || path.check x;
         merge = loc: defs:
-          let
+          (evalModules {
-            coerce = def: if isFunction def then def else { config = def; };
+            modules = allModules defs;
-            modules = opts' ++ map (def: { _file = def.file; imports = [(coerce def.value)]; }) defs;
+            inherit specialArgs;
-          in (evalModules {
-            inherit modules;
             args.name = last loc;
             prefix = loc;
           }).config;
         getSubOptions = prefix: (evalModules
-          { modules = opts'; inherit prefix;
+          { inherit modules prefix specialArgs;
             # This is a work-around due to the fact that some sub-modules,
             # such as the one included in an attribute set, expects a "args"
             # attribute to be given to the sub-module. As the option
@@ -394,13 +412,29 @@ rec {
             # It shouldn't cause an issue since this is cosmetic for the manual.
             args.name = "‹name›";
           }).options;
-        getSubModules = opts';
+        getSubModules = modules;
-        substSubModules = m: submodule m;
+        substSubModules = m: submoduleWith (attrs // {
-        functor = (defaultFunctor name) // {
+          modules = m;
-          # Merging of submodules is done as part of mergeOptionDecls, as we have to annotate
+        });
-          # each submodule with its location.
+        functor = defaultFunctor name // {
-          payload = [];
+          type = types.submoduleWith;
-          binOp = lhs: rhs: [];
+          payload = {
+            modules = modules;
+            specialArgs = specialArgs;
+            shorthandOnlyDefinesConfig = shorthandOnlyDefinesConfig;
+          };
+          binOp = lhs: rhs: {
+            modules = lhs.modules ++ rhs.modules;
+            specialArgs =
+              let intersecting = builtins.intersectAttrs lhs.specialArgs rhs.specialArgs;
+              in if intersecting == {}
+              then lhs.specialArgs // rhs.specialArgs
+              else throw "A submoduleWith option is declared multiple times with the same specialArgs \"${toString (attrNames intersecting)}\"";
+            shorthandOnlyDefinesConfig =
+              if lhs.shorthandOnlyDefinesConfig == rhs.shorthandOnlyDefinesConfig
+              then lhs.shorthandOnlyDefinesConfig
+              else throw "A submoduleWith option is declared multiple times with conflicting shorthandOnlyDefinesConfig values";
+          };
         };
       };
 

nixos/doc/manual/development/option-types.xml

@@ -257,14 +257,68 @@
     <listitem>
      <para>
       A set of sub options <replaceable>o</replaceable>.
-      <replaceable>o</replaceable> can be an attribute set or a function
+      <replaceable>o</replaceable> can be an attribute set, a function
-      returning an attribute set. Submodules are used in composed types to
+      returning an attribute set, or a path to a file containing such a value. Submodules are used in
-      create modular options. Submodule are detailed in
+      composed types to create modular options. This is equivalent to
+      <literal>types.submoduleWith { modules = toList o; shorthandOnlyDefinesConfig = true; }</literal>.
+      Submodules are detailed in
       <xref
           linkend='section-option-types-submodule' />.
      </para>
     </listitem>
    </varlistentry>
+   <varlistentry>
+     <term>
+       <varname>types.submoduleWith</varname> {
+        <replaceable>modules</replaceable>,
+        <replaceable>specialArgs</replaceable> ? {},
+        <replaceable>shorthandOnlyDefinesConfig</replaceable> ? false }
+     </term>
+     <listitem>
+       <para>
+         Like <varname>types.submodule</varname>, but more flexible and with better defaults.
+         It has parameters
+         <itemizedlist>
+           <listitem><para>
+             <replaceable>modules</replaceable>
+             A list of modules to use by default for this submodule type. This gets combined
+             with all option definitions to build the final list of modules that will be included.
+             <note><para>
+               Only options defined with this argument are included in rendered documentation.
+             </para></note>
+           </para></listitem>
+           <listitem><para>
+             <replaceable>specialArgs</replaceable>
+             An attribute set of extra arguments to be passed to the module functions.
+             The option <literal>_module.args</literal> should be used instead
+             for most arguments since it allows overriding. <replaceable>specialArgs</replaceable> should only be
+             used for arguments that can&apos;t go through the module fixed-point, because of
+             infinite recursion or other problems. An example is overriding the
+             <varname>lib</varname> argument, because <varname>lib</varname> itself is used
+             to define <literal>_module.args</literal>, which makes using
+             <literal>_module.args</literal> to define it impossible.
+           </para></listitem>
+           <listitem><para>
+             <replaceable>shorthandOnlyDefinesConfig</replaceable>
+             Whether definitions of this type should default to the <literal>config</literal>
+             section of a module (see <xref linkend='ex-module-syntax'/>) if it is an attribute
+             set. Enabling this only has a benefit when the submodule defines an option named
+             <literal>config</literal> or <literal>options</literal>. In such a case it would
+             allow the option to be set with <literal>the-submodule.config = "value"</literal>
+             instead of requiring <literal>the-submodule.config.config = "value"</literal>.
+             This is because only when modules <emphasis>don&apos;t</emphasis> set the
+             <literal>config</literal> or <literal>options</literal> keys, all keys are interpreted
+             as option definitions in the <literal>config</literal> section. Enabling this option
+             implicitly puts all attributes in the <literal>config</literal> section.
+           </para>
+           <para>
+             With this option enabled, defining a non-<literal>config</literal> section requires
+             using a function: <literal>the-submodule = { ... }: { options = { ... }; }</literal>.
+           </para></listitem>
+         </itemizedlist>
+       </para>
+     </listitem>
+   </varlistentry>
   </variablelist>
  </section>
 

nixos/modules/services/networking/syncthing.nix

@@ -112,12 +112,12 @@ in {
               addresses = [ "tcp://192.168.0.10:51820" ];
             };
           };
-          type = types.attrsOf (types.submodule ({ config, ... }: {
+          type = types.attrsOf (types.submodule ({ name, ... }: {
             options = {
 
               name = mkOption {
                 type = types.str;
-                default = config._module.args.name;
+                default = name;
                 description = ''
                   Name of the device
                 '';
@@ -175,7 +175,7 @@ in {
               devices = [ "bigbox" ];
             };
           };
-          type = types.attrsOf (types.submodule ({ config, ... }: {
+          type = types.attrsOf (types.submodule ({ name, ... }: {
             options = {
 
               enable = mkOption {
@@ -190,7 +190,7 @@ in {
 
               path = mkOption {
                 type = types.str;
-                default = config._module.args.name;
+                default = name;
                 description = ''
                   The path to the folder which should be shared.
                 '';
@@ -198,7 +198,7 @@ in {
 
               id = mkOption {
                 type = types.str;
-                default = config._module.args.name;
+                default = name;
                 description = ''
                   The id of the folder. Must be the same on all devices.
                 '';
@@ -206,7 +206,7 @@ in {
 
               label = mkOption {
                 type = types.str;
-                default = config._module.args.name;
+                default = name;
                 description = ''
                   The label of the folder.
                 '';