Merge pull request 'feat(inventory/instances): add option for extraModules to roles' (#3830) from flake-models into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3830

lib/inventory/build-inventory/default.nix

@@ -30,16 +30,13 @@ let
           # config.distributedServices.allMachines.${name} or [ ];
           { config, ... }:
           {
-
             distributedServices = clanLib.inventory.mapInstances {
               inherit (config) inventory;
               inherit localModuleSet;
               inherit flakeInputs;
               prefix = prefix ++ [ "distributedServices" ];
             };
-            machines = lib.mapAttrs (_machineName: v: {
+            machines = config.distributedServices.allMachines;
-              machineImports = v;
-            }) config.distributedServices.allMachines;
 
           }
         )

lib/inventory/build-inventory/interface.nix

@@ -410,6 +410,49 @@ in
                       default = { };
                       type = clanLib.types.uniqueDeferredSerializableModule;
                     };
+                    extraModules = lib.mkOption {
+                      description = ''
+                        List of additionally imported `.nix` expressions.
+
+                        Supported types:
+
+                        - **Strings**: Interpreted relative to the 'directory' passed to buildClan.
+                        - **Paths**: should be relative to the current file.
+                        - **Any**: Nix expression must be serializable to JSON.
+
+                        !!! Note
+                            **The import only happens if the machine is part of the service or role.**
+
+                        Other types are passed through to the nixos configuration.
+
+                        ???+ Example
+                            To import the `special.nix` file
+
+                            ```
+                            . Clan Directory
+                            ├── flake.nix
+                            ...
+                            └── modules
+                                ├── special.nix
+                                └── ...
+                            ```
+
+                            ```nix
+                            {
+                              extraModules = [ "modules/special.nix" ];
+                            }
+                            ```
+                      '';
+                      apply = value: if lib.isString value then value else builtins.seq (builtins.toJSON value) value;
+                      default = [ ];
+                      type = types.listOf (
+                        types.oneOf [
+                          types.str
+                          types.path
+                          (types.attrsOf types.anything)
+                        ]
+                      );
+                    };
                   };
                 }
               );

lib/inventory/distributed-service/inventory-adapter.nix

@@ -114,7 +114,9 @@ in
               };
             in
             # instances.<instanceName>.roles.<roleName> =
-            {
+            # Remove "tags", they are resolved into "machines"
+            (removeAttrs role [ "tags" ])
+            // {
               machines = lib.genAttrs resolvedMachines.machines (
                 machineName:
                 let
@@ -136,10 +138,6 @@ in
                   };
                 }
               );
-              # Maps to settings for the role.
-              # In other words this sets the following path of a clan.service module:
-              # instances.<instanceName>.roles.<roleName>.settings
-              settings = role.settings;
             }
           ) instance.roles;
         in
@@ -157,6 +155,7 @@ in
           modules =
             [
               # Import the resolved module.
+              # i.e. clan.modules.admin
               (builtins.head instances).instance.resolvedModule
             ] # Include all the instances that correlate to the resolved module
             ++ (builtins.map (v: {
@@ -185,20 +184,18 @@ in
         }
       ) { } importedModuleWithInstances;
 
-      # TODO: Return an attribute set of resources instead of a plain list of nixosModules
+      allMachines = lib.mapAttrs (machineName: _: {
-      allMachines = lib.foldlAttrs (
+        # This is the list of nixosModules for each machine
-        acc: _module_ident: eval:
+        machineImports = lib.foldlAttrs (
-        acc
+          acc: _module_ident: eval:
-        // lib.mapAttrs (
+          acc ++ [ eval.config.result.final.${machineName}.nixosModule or { } ]
-          machineName: result: acc.${machineName} or [ ] ++ [ result.nixosModule ]
+        ) [ ] importedModulesEvaluated;
-        ) eval.config.result.final
+      }) inventory.machines or { };
-      ) { } importedModulesEvaluated;
     in
     {
       inherit
         importedModuleWithInstances
         grouped
-
         allMachines
         importedModulesEvaluated
         ;

lib/inventory/distributed-service/service-module.nix

@@ -214,6 +214,11 @@ in
                               description = "Settings of 'role': ${name}";
                               default = { };
                             };
+
+                            options.extraModules = lib.mkOption {
+                              default = [ ];
+                              type = types.listOf (types.deferredModule);
+                            };
                           }
                         )
                       ];
@@ -405,12 +410,43 @@ in
     # Intermediate result by mapping over the 'roles', 'instances', and 'machines'.
     # During this step the 'perMachine' and 'perInstance' are applied.
     # The result-set for a single machine can then be found by collecting all 'nixosModules' recursively.
+
+    /**
+      allRoles :: {
+        <roleName> :: {
+          allInstances :: {
+            <instanceName> :: {
+              allMachines :: {
+                <machineName> :: {
+                  nixosModule :: NixOSModule;
+                  services :: { }; # TODO: nested services
+                };
+              };
+            };
+          };
+        };
+      }
+    */
     result.allRoles = mkOption {
       readOnly = true;
       default = lib.mapAttrs (roleName: roleCfg: {
         allInstances = lib.mapAttrs (instanceName: instanceCfg: {
           allMachines = lib.mapAttrs (
-            machineName: _machineCfg: roleCfg.perInstance instanceName machineName
+            machineName: _machineCfg:
+            let
+              instanceRes = roleCfg.perInstance instanceName machineName;
+            in
+            {
+              nixosModule = {
+                imports = [
+                  # Result of the applied 'perInstance = {...}: {  nixosModule = { ... }; }'
+                  instanceRes.nixosModule
+                ] ++ instanceCfg.roles.${roleName}.extraModules;
+              };
+              # TODO: nested services
+              services = { };
+            }
+
           ) instanceCfg.roles.${roleName}.machines or { };
         }) config.instances;
       }) config.roles;
@@ -434,6 +470,9 @@ in
               ) [ ] instance.roles
             );
           # The service machines are defined by collecting all instance machines
+          # returns "allMachines" that are part of the service in the form:
+          # serviceMachines :: { ${machineName} :: MachineOrigin; }
+          # MachineOrigin :: { instances :: [ string ]; roles :: [ string ]; }
           serviceMachines = lib.foldlAttrs (
             acc: instanceName: instance:
             acc
@@ -470,8 +509,6 @@ in
       default = lib.mapAttrs (
         machineName: machineResult:
         let
-          # config.result.allRoles.client.allInstances.bar.allMachines.test
-          # instanceResults = config.result.allRoles.client.allInstances.bar.allMachines.${machineName};
           instanceResults = lib.foldlAttrs (
             acc: roleName: role:
             acc

lib/inventory/distributed-service/tests/per_instance_args.nix

@@ -88,26 +88,38 @@ let
       roles.peer.tags.all = { };
     };
   };
+
+  /*
+    1 { imports = [ { instanceName = "instance_foo"; machine = { name = "jon"; roles = [ "controller" "pe 1 null
+    . er" ]; }; roles = { controller = { machines = { jon = { settings = { }; }; }; settings = { }; }; pe .
+    . er = { machines = { jon = { settings = { timeout = "foo-peer-jon"; }; }; }; settings = { timeout =  .
+    . "foo-peer"; }; }; }; settings = { timeout = "foo-peer-jon"; }; vendoredSettings = { timeout = "conf .
+    . ig.thing"; }; } ]; }                                                                                .
+  */
+  unwrapModule = m: (builtins.head m.imports);
 in
 {
   # settings should evaluate
   test_per_instance_arguments = {
-    expr = {
+    expr =
-      instanceName =
+      let
-        res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances."instance_foo".allMachines.jon.nixosModule.instanceName;
+        m = (
+          unwrapModule
+            res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances."instance_foo".allMachines.jon.nixosModule
+        );
+      in
+      {
+        instanceName = m.instanceName;
 
-      # settings are specific.
+        # settings are specific.
-      # Below we access:
+        # Below we access:
-      # instance = instance_foo
+        # instance = instance_foo
-      # roles = peer
+        # roles = peer
-      # machines = jon
+        # machines = jon
-      settings =
+        settings = m.settings;
-        res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances.instance_foo.allMachines.jon.nixosModule.settings;
+        machine = m.machine;
-      machine =
+        roles = m.roles;
-        res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances.instance_foo.allMachines.jon.nixosModule.machine;
+      };
-      roles =
-        res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances.instance_foo.allMachines.jon.nixosModule.roles;
-    };
     expected = {
       instanceName = "instance_foo";
       settings = {
@@ -147,10 +159,12 @@ in
     };
   };
 
+  # TODO: Cannot be tested like this anymore
   test_per_instance_settings_vendoring = {
     expr =
-
+      (unwrapModule
-      res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances."instance_foo".allMachines.jon.nixosModule.vendoredSettings;
+        res.importedModulesEvaluated.self-A.config.result.allRoles.peer.allInstances."instance_foo".allMachines.jon.nixosModule
+      ).vendoredSettings;
     expected = {
       timeout = "config.thing";
     };

pkgs/clan-cli/clan_lib/nix_models/clan.py

@@ -38,11 +38,13 @@ class InventoryInstanceRoleTag(TypedDict):
 
 
 
+InventoryInstanceRoleExtramodulesType = list[dict[str, Any] | str]
 InventoryInstanceRoleMachinesType = dict[str, InventoryInstanceRoleMachine]
 InventoryInstanceRoleSettingsType = Unknown
 InventoryInstanceRoleTagsType = dict[str, InventoryInstanceRoleTag]
 
 class InventoryInstanceRole(TypedDict):
+    extraModules: NotRequired[InventoryInstanceRoleExtramodulesType]
     machines: NotRequired[InventoryInstanceRoleMachinesType]
     settings: NotRequired[InventoryInstanceRoleSettingsType]
     tags: NotRequired[InventoryInstanceRoleTagsType]

pkgs/classgen/main.py

@@ -2,7 +2,7 @@
 import argparse
 import json
 import logging
-import sys
+import traceback
 from collections.abc import Callable, Iterable
 from functools import partial
 from pathlib import Path
@@ -38,6 +38,7 @@ def map_json_type(
         for t in json_type:
             res.extend(map_json_type(t))
         return sort_types(set(res))
+
     if isinstance(json_type, dict):
         items = json_type.get("items")
         if items:
@@ -46,6 +47,13 @@ def map_json_type(
         if not json_type.get("type") and json_type.get("tsType") == "unknown":
             return ["Unknown"]
 
+        union = json_type.get("oneOf")
+        if union:
+            res: list[str] = []
+            for t in union:
+                res.extend(map_json_type(t, nested_types, parent))
+            return sort_types(set(res))
+
         return sort_types(map_json_type(json_type.get("type"), nested_types))
     if json_type == "string":
         return ["str"]
@@ -67,6 +75,7 @@ def map_json_type(
         return [f"""dict[str, {" | ".join(sort_types(nested_types))}]"""]
     if json_type == "null":
         return ["None"]
+
     msg = f"Python type not found for {json_type}"
     raise Error(msg)
 
@@ -432,15 +441,13 @@ def main() -> None:
         default=None,
     )
     parser.set_defaults(func=run_gen)
-
     args = parser.parse_args()
-
+    args.func(args)
-    try:
-        args.func(args)
-    except Error as e:
-        print(e)
-        sys.exit(1)
 
 
 if __name__ == "__main__":
-    main()
+    try:
+        main()
+    except Exception:
+        print("An error occurred:")
+        traceback.print_exc()