Merge pull request 'init inventory.instances and clan.service modules' (#3102) from hsjobeki/clan-core:clan-services into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3102

lib/build-clan/interface.nix

@@ -142,6 +142,8 @@ in
           inventoryFile = lib.mkOption { type = lib.types.raw; };
           # The machine 'imports' generated by the inventory per machine
           inventoryClass = lib.mkOption { type = lib.types.raw; };
+          # new attribute
+          distributedServices = lib.mkOption { type = lib.types.raw; };
           # clan-core's modules
           clanModules = lib.mkOption { type = lib.types.raw; };
           source = lib.mkOption { type = lib.types.raw; };

lib/build-clan/module.nix

@@ -165,7 +165,18 @@ let
 in
 {
   imports = [
-    ./auto-imports.nix
+    # Temporarily disable auto-imports since the type of the modules is not a plain path anymore we cant "merge" multiple definitions
+    # That this feature worked previously seems like a coincidence.
+    # TODO(@Qubasa): make sure modules are not imported twice.
+    # Example error:
+    # The option `inventory.modules.admin' is defined multiple times while it's expected to be unique.
+    # - In `/nix/store/a0iqxl7r1spqsf2b886kn3i5sj8p37nc-source/lib/build-clan/auto-imports.nix': /nix/store/a0iqxl7r1spqsf2b886kn3i5sj8p37nc-source/clanModules/admin
+    # - In `/nix/store/a0iqxl7r1spqsf2b886kn3i5sj8p37nc-source/lib/build-clan/module.nix': /nix/store/a0iqxl7r1spqsf2b886kn3i5sj8p37nc-source/clanModules/admin
+    #
+    # After the inventory refactoring we might not need this anymore
+    # People can just import the module they want to use: `module = { input = "inputName"; name = "moduleName"; };`
+    # ./auto-imports.nix
+
     # Merge the inventory file
     {
       inventory = _: {
@@ -199,6 +210,10 @@ in
   clanInternals = {
     moduleSchemas = clan-core.lib.modules.getModulesSchema config.inventory.modules;
     inherit inventoryClass;
+    distributedServices = import ../distributed-service/inventory-adapter.nix {
+      inherit lib inventory;
+      flake = config.self;
+    };
     inherit (clan-core) clanModules;
     inherit inventoryFile;
     inventoryValuesPrios =

lib/distributed-service/flake-module.nix

@@ -0,0 +1,34 @@
+{ self, inputs, ... }:
+let
+  inputOverrides = builtins.concatStringsSep " " (
+    builtins.map (input: " --override-input ${input} ${inputs.${input}}") (builtins.attrNames inputs)
+  );
+in
+{
+  perSystem =
+    {
+      pkgs,
+      lib,
+      system,
+      ...
+    }:
+    {
+      # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.<attrName>
+      legacyPackages.evalTest-distributedServices = import ./tests {
+        inherit lib self;
+      };
+
+      checks = {
+        lib-distributedServices-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
+          export HOME="$(realpath .)"
+          export NIX_ABORT_ON_WARN=1
+          nix-unit --eval-store "$HOME" \
+            --extra-experimental-features flakes \
+            ${inputOverrides} \
+            --flake ${self}#legacyPackages.${system}.evalTest-distributedServices
+
+          touch $out
+        '';
+      };
+    };
+}

lib/distributed-service/inventory-adapter.nix

@@ -0,0 +1,190 @@
+# Adapter function between the inventory.instances and the clan.service module
+#
+# Data flow:
+# - inventory.instances -> Adapter -> clan.service module -> Service Resources (i.e. NixosModules per Machine, Vars per Service, etc.)
+#
+# What this file does:
+#
+# - Resolves the [Module] to an actual module-path and imports it.
+# - Groups together all the same modules into a single import and creates all instances for it.
+# - Resolves the inventory tags into machines. Tags don't exist at the service level.
+#   Also combines the settings for 'machines' and 'tags'.
+{
+  lib,
+  # This is used to resolve the module imports from 'flake.inputs'
+  flake,
+  # The clan inventory
+  inventory,
+}:
+let
+  # Returns the list of machine names
+  # { ... } -> [ string ]
+  resolveTags =
+    {
+      # Available InventoryMachines :: { {name} :: { tags = [ string ]; }; }
+      machines,
+      # Requested members :: { machines, tags }
+      # Those will be resolved against the available machines
+      members,
+      # Not needed for resolution - only for error reporting
+      roleName,
+      instanceName,
+    }:
+    {
+      machines =
+        members.machines or [ ]
+        ++ (builtins.foldl' (
+          acc: tag:
+          let
+            # For error printing
+            availableTags = lib.foldlAttrs (
+              acc: _: v:
+              v.tags or [ ] ++ acc
+            ) [ ] (machines);
+
+            tagMembers = builtins.attrNames (lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) machines);
+          in
+          if tagMembers == [ ] then
+            lib.warn ''
+              Service instance '${instanceName}': - ${roleName} tags: no machine with tag '${tag}' found.
+              Available tags: ${builtins.toJSON (lib.unique availableTags)}
+            '' acc
+          else
+            acc ++ tagMembers
+        ) [ ] members.tags or [ ]);
+    };
+
+  machineHasTag = machineName: tagName: lib.elem tagName inventory.machines.${machineName}.tags;
+
+  # map the instances into the module
+  importedModuleWithInstances = lib.mapAttrs (
+    instanceName: instance:
+    let
+      # TODO:
+      resolvedModuleSet =
+        # If the module.name is self then take the modules defined in the flake
+        # Otherwise its an external input which provides the modules via 'clan.modules' attribute
+        if instance.module.input == null then
+          inventory.modules
+        else
+          let
+            input =
+              flake.inputs.${instance.module.input} or (throw ''
+                Flake doesn't provide input with name '${instance.module.input}'
+
+                Choose one of the following inputs:
+                - ${
+                  builtins.concatStringsSep "\n- " (
+                    lib.attrNames (lib.filterAttrs (_name: input: input ? clan) flake.inputs)
+                  )
+                }
+
+                To import a local module from 'inventory.modules' remove the 'input' attribute from the module definition
+                Remove the following line from the module definition:
+
+                ...
+                - module.input = "${instance.module.input}"
+
+
+              '');
+            clanAttrs =
+              input.clan
+                or (throw "It seems the flake input ${instance.module.input} doesn't export any clan resources");
+          in
+          clanAttrs.modules;
+
+      resolvedModule =
+        resolvedModuleSet.${instance.module.name}
+          or (throw "flake doesn't provide clan-module with name ${instance.module.name}");
+
+      # Every instance includes machines via roles
+      # :: { client :: ... }
+      instanceRoles = lib.mapAttrs (
+        roleName: role:
+        let
+          resolvedMachines = resolveTags {
+            members = {
+              # Explicit members
+              machines = lib.attrNames role.machines;
+              # Resolved Members
+              tags = lib.attrNames role.tags;
+            };
+            inherit (inventory) machines;
+            inherit instanceName roleName;
+          };
+        in
+        # instances.<instanceName>.roles.<roleName> =
+        {
+          machines = lib.genAttrs resolvedMachines.machines (
+            machineName:
+            let
+              machineSettings = instance.roles.${roleName}.machines.${machineName}.settings or { };
+              settingsViaTags = lib.filterAttrs (
+                tagName: _: machineHasTag machineName tagName
+              ) instance.roles.${roleName}.tags;
+            in
+            {
+              # TODO: Do we want to wrap settings with
+              # setDefaultModuleLocation "inventory.instances.${instanceName}.roles.${roleName}.tags.${tagName}";
+              settings = {
+                imports = [
+                  machineSettings
+                ] ++ lib.attrValues (lib.mapAttrs (_tagName: v: v.settings) settingsViaTags);
+              };
+            }
+          );
+          # Maps to settings for the role.
+          # In other words this sets the following path of a clan.service module:
+          # instances.<instanceName>.roles.<roleName>.settings
+          settings = role.settings;
+        }
+      ) instance.roles;
+    in
+    {
+      inherit (instance) module;
+      inherit resolvedModule instanceRoles;
+    }
+  ) inventory.instances;
+
+  # TODO: Eagerly check the _class of the resolved module
+  evals = lib.mapAttrs (
+    _module_ident: instances:
+    (lib.evalModules {
+      class = "clan.service";
+      modules =
+        [
+          ./service-module.nix
+          # Import the resolved module
+          (builtins.head instances).instance.resolvedModule
+        ]
+        # Include all the instances that correlate to the resolved module
+        ++ (builtins.map (v: {
+          instances.${v.instanceName}.roles = v.instance.instanceRoles;
+        }) instances);
+    })
+  ) grouped;
+
+  # Group the instances by the module they resolve to
+  # This is necessary to evaluate the module in a single pass
+  # :: { <module.input>_<module.name> :: [ { name, value } ] }
+  # Since 'perMachine' needs access to all the instances we should include them as a whole
+  grouped = lib.foldlAttrs (
+    acc: instanceName: instance:
+    let
+      inputName = if instance.module.input == null then "self" else instance.module.input;
+      id = inputName + "-" + instance.module.name;
+    in
+    acc
+    // {
+      ${id} = acc.${id} or [ ] ++ [
+        {
+          inherit instanceName instance;
+        }
+      ];
+    }
+  ) { } importedModuleWithInstances;
+in
+{
+  inherit importedModuleWithInstances grouped;
+  inherit evals;
+}

lib/distributed-service/service-module.nix

@@ -0,0 +1,513 @@
+{ lib, config, ... }:
+let
+  inherit (lib) mkOption types;
+  inherit (types) attrsWith submoduleWith;
+
+  # TODO:
+  # Remove once this gets merged upstream; performs in O(n*log(n) instead of O(n^2))
+  # https://github.com/NixOS/nixpkgs/pull/355616/files
+  uniqueStrings = list: builtins.attrNames (builtins.groupBy lib.id list);
+
+  checkInstanceRoles =
+    instanceName: instanceRoles:
+    let
+      unmatchedRoles = lib.filter (roleName: !lib.elem roleName (lib.attrNames config.roles)) (
+        lib.attrNames instanceRoles
+      );
+    in
+    if unmatchedRoles == [ ] then
+      true
+    else
+      throw ''
+        inventory instance: 'instances.${instanceName}' defines the following roles:
+        ${builtins.toJSON unmatchedRoles}
+
+        But the clan-service module '${config.manifest.name}' defines roles:
+        ${builtins.toJSON (lib.attrNames config.roles)}
+      '';
+
+  # checkInstanceSettings =
+  #   instanceName: instanceSettings:
+  #   let
+  #     unmatchedRoles = 1;
+  #   in
+  #   unmatchedRoles;
+
+  /**
+    Merges the role- and machine-settings using the role interface
+
+    Arguments:
+
+    - roleName: The name of the role
+    - instanceName: The name of the instance
+    - settings: The settings of the machine. Leave empty to get the role settings
+
+    Returns: evalModules result
+
+    The caller is responsible to use .config or .extendModules
+  */
+  # TODO: evaluate against the role.settings statically and use extendModules to get the machineSettings
+  # Doing this might improve performance
+  evalMachineSettings =
+    {
+      roleName,
+      instanceName,
+      machineName ? null,
+      settings,
+    }:
+    lib.evalModules {
+      # Prefix for better error reporting
+      # This prints the path where the option should be defined rather than the plain path within settings
+      # "The option `instances.foo.roles.server.machines.test.settings.<>' was accessed but has no value defined. Try setting the option."
+      prefix =
+        [
+          "instances"
+          instanceName
+          "roles"
+          roleName
+        ]
+        ++ (lib.optionals (machineName != null) [
+          "machines"
+          machineName
+        ])
+        ++ [ "settings" ];
+
+      # This may lead to better error reporting
+      # And catch errors if anyone tried to import i.e. a nixosConfiguration
+      # Set some class: i.e "network.server.settings"
+      class = lib.concatStringsSep "." [
+        config.manifest.name
+        roleName
+        "settings"
+      ];
+
+      modules = [
+        (lib.setDefaultModuleLocation "Via clan.service module: roles.${roleName}.interface"
+          config.roles.${roleName}.interface
+        )
+        (lib.setDefaultModuleLocation "inventory.instances.${instanceName}.roles.${roleName}.settings"
+          config.instances.${instanceName}.roles.${roleName}.settings
+        )
+        settings
+        # Dont set the module location here
+        # This should already be set by the tags resolver
+        # config.instances.${instanceName}.roles.${roleName}.machines.${machineName}.settings
+      ];
+    };
+
+  /**
+    Makes a module extensible
+    returning its config
+    and making it extensible via '__functor' polymorphism
+
+    Example:
+
+    ```nix-repl
+    res = makeExtensibleConfig (evalModules { options.foo = mkOption { default = 42; };)
+    res
+    =>
+    {
+      foo = 42;
+      _functor = <function>;
+    }
+
+    # This allows to override using mkDefault, mkForce, etc.
+    res { foo = 100; }
+    =>
+    {
+      foo = 100;
+      _functor = <function>;
+    }
+    ```
+  */
+  makeExtensibleConfig =
+    f: args:
+    let
+      makeModuleExtensible =
+        eval:
+        eval.config
+        // {
+          __functor = _self: m: makeModuleExtensible (eval.extendModules { modules = lib.toList m; });
+        };
+    in
+    makeModuleExtensible (f args);
+
+  /**
+    Apply the settings to the instance
+
+    Takes a [ServiceInstance] :: { roles :: { roleName :: { machines :: { machineName :: { settings :: { ... } } } } } }
+    Returns the same object but evaluates the settings against the interface.
+
+    We need this because 'perMachine' shouldn't gain access the raw deferred module.
+  */
+  applySettings =
+    instanceName: instance:
+    lib.mapAttrs (roleName: role: {
+      machines = lib.mapAttrs (machineName: v: {
+        # TODO: evaluate the settings against the interface
+        # settings = (evalMachineSettings { inherit roleName instanceName; inherit (v) settings; }).config;
+        settings = (
+          makeExtensibleConfig evalMachineSettings {
+            inherit roleName instanceName machineName;
+            inherit (v) settings;
+          }
+        );
+      }) role.machines;
+      # TODO: evaluate the settings against the interface
+      settings = (
+        makeExtensibleConfig evalMachineSettings {
+          inherit roleName instanceName;
+          inherit (role) settings;
+        }
+      );
+    }) instance.roles;
+in
+{
+  options = {
+    instances = mkOption {
+      default = throw ''
+        The clan service module ${config.manifest.name} doesn't define any instances.
+
+        Did you forget to create instances via 'inventory.instances' ?
+      '';
+
+      type = attrsWith {
+        placeholder = "instanceName";
+        elemType = submoduleWith {
+          modules = [
+            (
+              { name, ... }:
+              {
+                # options.settings = mkOption {
+                #   description = "settings of 'instance': ${name}";
+                #   default = {};
+                #   apply = v: lib.seq (checkInstanceSettings name v) v;
+                # };
+                options.roles = mkOption {
+                  default = throw ''
+                    Instance '${name}' of service '${config.manifest.name}' mut define members via 'roles'.
+
+                    To include a machine:
+                    'instances.${name}.roles.<role-name>.machines.<your-machine-name>' must be set.
+                  '';
+                  type = attrsWith {
+                    placeholder = "roleName";
+                    elemType = submoduleWith {
+                      modules = [
+                        (
+                          { ... }:
+                          {
+                            # instances.{instanceName}.roles.{roleName}.machines
+                            options.machines = mkOption {
+                              type = attrsWith {
+                                placeholder = "machineName";
+                                elemType = submoduleWith {
+                                  modules = [
+                                    (m: {
+                                      options.settings = mkOption {
+                                        type = types.raw;
+                                        description = "Settings of '${name}-machine': ${m.name}.";
+                                        default = { };
+                                      };
+                                    })
+                                  ];
+                                };
+                              };
+                            };
+
+                            # instances.{instanceName}.roles.{roleName}.settings
+                            # options._settings = mkOption { };
+                            # options._settingsViaTags = mkOption { };
+                            # A deferred module that combines _settingsViaTags with _settings
+                            options.settings = mkOption {
+                              type = types.raw;
+                              description = "Settings of 'role': ${name}";
+                              default = { };
+                            };
+                          }
+                        )
+                      ];
+                    };
+                  };
+                  apply = v: lib.seq (checkInstanceRoles name v) v;
+                };
+              }
+            )
+          ];
+        };
+      };
+    };
+
+    manifest = mkOption {
+      description = "Meta information about this module itself";
+      type = submoduleWith {
+        modules = [
+          {
+            options = {
+              name = mkOption {
+                description = ''
+                  The name of the module
+
+                  Mainly used to create an error context while evaluating.
+                  This helps backtracking which module was included; And where an error came from originally.
+                '';
+                type = types.str;
+              };
+            };
+          }
+        ];
+      };
+    };
+    roles = mkOption {
+      default = throw ''
+        Role behavior of service '${config.manifest.name}' must be defined.
+        A 'clan.service' module should always define its behavior via 'roles'
+        ---
+        To add the role:
+        `roles.client = {}`
+
+        To define multiple instance behavior:
+        `roles.client.perInstance = { ... }: {}`
+      '';
+      type = attrsWith {
+        placeholder = "roleName";
+        elemType = submoduleWith {
+          modules = [
+            (
+              { name, ... }:
+              let
+                roleName = name;
+              in
+              {
+                options.interface = mkOption {
+                  type = types.deferredModule;
+                  # TODO: Default to an empty module
+                  # need to test that an the empty module can be evaluated to empty settings
+                  default = { };
+                };
+                options.perInstance = mkOption {
+                  type = types.deferredModuleWith {
+                    staticModules = [
+                      # Common output format
+                      # As described by adr
+                      # { nixosModule, services, ... }
+                      (
+                        { ... }:
+                        {
+                          options.nixosModule = mkOption { default = { }; };
+                          options.services = mkOption {
+                            type = attrsWith {
+                              placeholder = "serviceName";
+                              elemType = submoduleWith {
+                                modules = [ ./service-module.nix ];
+                              };
+                            };
+                            default = { };
+                          };
+                        }
+                      )
+                    ];
+                  };
+                  default = { };
+                  apply =
+                    /**
+                      This apply transforms the module into a function that takes arguments and returns an evaluated module
+                      The arguments of the function are determined by its scope:
+                      -> 'perInstance' maps over all instances and over all machines hence it takes 'instanceName' and 'machineName' as iterator arguments
+                    */
+                    v: instanceName: machineName:
+                    (lib.evalModules {
+                      specialArgs = {
+                        inherit instanceName;
+                        machine = {
+                          name = machineName;
+                          roles = applySettings instanceName config.instances.${instanceName};
+                        };
+                        settings = (
+                          makeExtensibleConfig evalMachineSettings {
+                            inherit roleName instanceName machineName;
+                            settings =
+                              config.instances.${instanceName}.roles.${roleName}.machines.${machineName}.settings or { };
+                          }
+                        );
+                      };
+                      modules = [ v ];
+                    }).config;
+                };
+              }
+            )
+          ];
+        };
+      };
+    };
+
+    perMachine = mkOption {
+      type = types.deferredModuleWith {
+        staticModules = [
+          # Common output format
+          # As described by adr
+          # { nixosModule, services, ... }
+          (
+            { ... }:
+            {
+              options.nixosModule = mkOption { default = { }; };
+              options.services = mkOption {
+                type = attrsWith {
+                  placeholder = "serviceName";
+                  elemType = submoduleWith {
+                    modules = [ ./service-module.nix ];
+                  };
+                };
+                default = { };
+              };
+            }
+          )
+        ];
+      };
+      default = { };
+      apply =
+        v: machineName: machineScope:
+        (lib.evalModules {
+          specialArgs = {
+            /**
+              This apply transforms the module into a function that takes arguments and returns an evaluated module
+              The arguments of the function are determined by its scope:
+              -> 'perMachine' maps over all machines of a service 'machineName' and a helper 'scope' (some aggregated attributes) as iterator arguments
+              The 'scope' attribute is used to collect the 'roles' of all 'instances' where the machine is part of and inject both into the specialArgs
+            */
+            machine = {
+              name = machineName;
+              roles =
+                let
+                  collectRoles =
+                    instances:
+                    lib.foldlAttrs (
+                      r: _instanceName: instance:
+                      r
+                      ++ lib.foldlAttrs (
+                        r2: roleName: _role:
+                        r2 ++ [ roleName ]
+                      ) [ ] instance.roles
+                    ) [ ] instances;
+                in
+                uniqueStrings (collectRoles machineScope.instances);
+            };
+            inherit (machineScope) instances;
+
+            # There are no machine settings.
+            # Settings are always role specific, having settings that apply to a machine globally would mean to merge all role and all instance settings into a single module.
+            # But that will likely cause conflicts because it is inherently wrong.
+            settings = throw ''
+              'perMachine' doesn't have a 'settings' argument.
+
+              Alternatives:
+              - 'instances.<instanceName>.roles.<roleName>.settings' should be used instead.
+              - 'instances.<instanceName>.roles.<roleName>.machines.<machineName>.settings' should be used instead.
+
+              If that is insufficient, you might also consider using 'roles.<roleName>.perInstance' instead of 'perMachine'.
+            '';
+          };
+
+          modules = [ v ];
+        }).config;
+    };
+    # ---
+    # Place the result in _module.result to mark them as "internal" and discourage usage/overrides
+    #
+    # ---
+    # Intermediate result by mapping over the 'roles', 'instances', and 'machines'.
+    # During this step the 'perMachine' and 'perInstance' are applied.
+    # The result-set for a single machine can then be found by collecting all 'nixosModules' recursively.
+    result.allRoles = mkOption {
+      readOnly = true;
+      default = lib.mapAttrs (roleName: roleCfg: {
+        allInstances = lib.mapAttrs (instanceName: instanceCfg: {
+          allMachines = lib.mapAttrs (
+            machineName: _machineCfg: roleCfg.perInstance instanceName machineName
+          ) instanceCfg.roles.${roleName}.machines or { };
+        }) config.instances;
+      }) config.roles;
+    };
+
+    result.allMachines = mkOption {
+      readOnly = true;
+      default =
+        let
+          collectMachinesFromInstance =
+            instance:
+            uniqueStrings (
+              lib.foldlAttrs (
+                acc: _roleName: role:
+                acc ++ (lib.attrNames role.machines)
+              ) [ ] instance.roles
+            );
+          # The service machines are defined by collecting all instance machines
+          serviceMachines = lib.foldlAttrs (
+            acc: instanceName: instance:
+            acc
+            // lib.genAttrs (collectMachinesFromInstance instance) (machineName:
+            # Store information why this machine is part of the service
+            # MachineOrigin :: { instances :: [ string ]; }
+            {
+              # Helper attribute to
+              instances = [ instanceName ] ++ acc.${machineName}.instances or [ ];
+              # All roles of the machine ?
+              roles = lib.foldlAttrs (
+                acc2: roleName: role:
+                if builtins.elem machineName (lib.attrNames role.machines) then acc2 ++ [ roleName ] else acc2
+              ) [ ] instance.roles;
+            })
+          ) { } config.instances;
+
+          allMachines = lib.mapAttrs (_machineName: MachineOrigin: {
+            # Filter out instances of which the machine is not part of
+            instances = lib.mapAttrs (_n: v: { roles = v; }) (
+              lib.filterAttrs (instanceName: _: builtins.elem instanceName MachineOrigin.instances) (
+                # Instances with evaluated settings
+                lib.mapAttrs applySettings config.instances
+              )
+            );
+          }) serviceMachines;
+        in
+        # allMachines;
+        lib.mapAttrs config.perMachine allMachines;
+    };
+
+    result.final = mkOption {
+      readOnly = true;
+      default = lib.mapAttrs (
+        machineName: machineResult:
+        let
+          # config.result.allRoles.client.allInstances.bar.allMachines.test
+          # instanceResults = config.result.allRoles.client.allInstances.bar.allMachines.${machineName};
+          instanceResults = lib.foldlAttrs (
+            acc: roleName: role:
+            acc
+            ++ lib.foldlAttrs (
+              acc: instanceName: instance:
+              if instance.allMachines.${machineName}.nixosModule or { } != { } then
+                acc
+                ++ [
+                  (lib.setDefaultModuleLocation
+                    "Via instances.${instanceName}.roles.${roleName}.machines.${machineName}"
+                    instance.allMachines.${machineName}.nixosModule
+                  )
+                ]
+              else
+                acc
+            ) [ ] role.allInstances
+          ) [ ] config.result.allRoles;
+        in
+        {
+          inherit instanceResults;
+          nixosModule = {
+            imports = [
+              # For error backtracing. This module was produced by the 'perMachine' function
+              (lib.setDefaultModuleLocation "via perMachine" machineResult.nixosModule)
+            ] ++ instanceResults;
+          };
+        }
+      ) config.result.allMachines;
+    };
+  };
+}

lib/distributed-service/tests/default.nix

@@ -0,0 +1,327 @@
+{
+  lib,
+  ...
+}:
+let
+  inherit (lib)
+    evalModules
+    ;
+
+  evalInventory =
+    m:
+    (evalModules {
+      # Static modules
+      modules = [
+        ../../inventory/build-inventory/interface.nix
+        {
+          modules.test = { };
+        }
+        m
+      ];
+    }).config;
+
+  flakeFixture = {
+    inputs = { };
+  };
+
+  callInventoryAdapter =
+    inventoryModule:
+    import ../inventory-adapter.nix {
+      inherit lib;
+      flake = flakeFixture;
+      inventory = evalInventory inventoryModule;
+    };
+in
+{
+  test_simple =
+    let
+      res = callInventoryAdapter {
+        # Authored module
+        # A minimal module looks like this
+        # It isn't exactly doing anything but it's a valid module that produces an output
+        modules."simple-module" = {
+          _class = "clan.service";
+          manifest = {
+            name = "netwitness";
+          };
+        };
+        # User config
+        instances."instance_foo" = {
+          module = {
+            name = "simple-module";
+          };
+        };
+      };
+    in
+    {
+      # Test that the module is mapped into the output
+      # We might change the attribute name in the future
+      expr = res.evals ? "self-simple-module";
+      expected = true;
+    };
+
+  # A module can be imported multiple times
+  # A module can also have multiple instances within the same module
+  # This mean modules must be grouped together, imported once
+  # All instances should be included within one evaluation to make all of them available
+  test_module_grouping =
+    let
+      res = callInventoryAdapter {
+        # Authored module
+        # A minimal module looks like this
+        # It isn't exactly doing anything but it's a valid module that produces an output
+        modules."A" = {
+          _class = "clan.service";
+          manifest = {
+            name = "A-name";
+          };
+
+          perMachine = { }: { };
+        };
+        modules."B" = {
+          _class = "clan.service";
+          manifest = {
+            name = "B-name";
+          };
+
+          perMachine = { }: { };
+        };
+        # User config
+        instances."instance_foo" = {
+          module = {
+            name = "A";
+          };
+        };
+        instances."instance_bar" = {
+          module = {
+            name = "B";
+          };
+        };
+        instances."instance_baz" = {
+          module = {
+            name = "A";
+          };
+        };
+      };
+    in
+    {
+      # Test that the module is mapped into the output
+      # We might change the attribute name in the future
+      expr = lib.mapAttrs (_n: v: builtins.length v) res.grouped;
+      expected = {
+        self-A = 2;
+        self-B = 1;
+      };
+    };
+
+  test_creates_all_instances =
+    let
+      res = callInventoryAdapter {
+        # Authored module
+        # A minimal module looks like this
+        # It isn't exactly doing anything but it's a valid module that produces an output
+        modules."A" = {
+          _class = "clan.service";
+          manifest = {
+            name = "network";
+          };
+
+          perMachine = { }: { };
+        };
+        instances."instance_foo" = {
+          module = {
+            name = "A";
+          };
+        };
+        instances."instance_bar" = {
+          module = {
+            name = "A";
+          };
+        };
+        instances."instance_zaza" = {
+          module = {
+            name = "B";
+          };
+        };
+      };
+    in
+    {
+      # Test that the module is mapped into the output
+      # We might change the attribute name in the future
+      expr = lib.attrNames res.evals.self-A.config.instances;
+      expected = [
+        "instance_bar"
+        "instance_foo"
+      ];
+    };
+
+  # Membership via roles
+  test_add_machines_directly =
+    let
+      res = callInventoryAdapter {
+        # Authored module
+        # A minimal module looks like this
+        # It isn't exactly doing anything but it's a valid module that produces an output
+        modules."A" = {
+          _class = "clan.service";
+          manifest = {
+            name = "network";
+          };
+          # Define a role without special behavior
+          roles.peer = { };
+
+          # perMachine = {}: {};
+        };
+        machines = {
+          jon = { };
+          sara = { };
+          hxi = { };
+        };
+        instances."instance_foo" = {
+          module = {
+            name = "A";
+          };
+          roles.peer.machines.jon = { };
+        };
+        instances."instance_bar" = {
+          module = {
+            name = "A";
+          };
+          roles.peer.machines.sara = { };
+        };
+        instances."instance_zaza" = {
+          module = {
+            name = "B";
+          };
+          roles.peer.tags.all = { };
+        };
+      };
+    in
+    {
+      # Test that the module is mapped into the output
+      # We might change the attribute name in the future
+      expr = lib.attrNames res.evals.self-A.config.result.allMachines;
+      expected = [
+        "jon"
+        "sara"
+      ];
+    };
+
+  # Membership via tags
+  test_add_machines_via_tags =
+    let
+      res = callInventoryAdapter {
+        # Authored module
+        # A minimal module looks like this
+        # It isn't exactly doing anything but it's a valid module that produces an output
+        modules."A" = {
+          _class = "clan.service";
+          manifest = {
+            name = "network";
+          };
+          # Define a role without special behavior
+          roles.peer = { };
+
+          # perMachine = {}: {};
+        };
+        machines = {
+          jon = {
+            tags = [ "foo" ];
+          };
+          sara = {
+            tags = [ "foo" ];
+          };
+          hxi = { };
+        };
+        instances."instance_foo" = {
+          module = {
+            name = "A";
+          };
+          roles.peer.tags.foo = { };
+        };
+        instances."instance_zaza" = {
+          module = {
+            name = "B";
+          };
+          roles.peer.tags.all = { };
+        };
+      };
+    in
+    {
+      # Test that the module is mapped into the output
+      # We might change the attribute name in the future
+      expr = lib.attrNames res.evals.self-A.config.result.allMachines;
+      expected = [
+        "jon"
+        "sara"
+      ];
+    };
+
+  per_machine_args = import ./per_machine_args.nix { inherit lib callInventoryAdapter; };
+  # test_per_machine_receives_instances =
+  #   let
+  #     res = callInventoryAdapter {
+  #       # Authored module
+  #       # A minimal module looks like this
+  #       # It isn't exactly doing anything but it's a valid module that produces an output
+  #       modules."A" = {
+  #         _class = "clan.service";
+  #         manifest = {
+  #           name = "network";
+  #         };
+  #         # Define a role without special behavior
+  #         roles.peer = { };
+
+  #         perMachine =
+  #           { instances, ... }:
+  #           {
+  #             nixosModule = instances;
+  #           };
+  #       };
+  #       machines = {
+  #         jon = { };
+  #         sara = { };
+  #       };
+  #       instances."instance_foo" = {
+  #         module = {
+  #           name = "A";
+  #         };
+  #         roles.peer.machines.jon = { };
+  #       };
+  #       instances."instance_bar" = {
+  #         module = {
+  #           name = "A";
+  #         };
+  #         roles.peer.machines.sara = { };
+  #       };
+  #       instances."instance_zaza" = {
+  #         module = {
+  #           name = "B";
+  #         };
+  #         roles.peer.tags.all = { };
+  #       };
+  #     };
+  #   in
+  #   {
+  #     expr = {
+  #       hasMachineSettings =
+  #         res.evals.self-A.config.result.allMachines.jon.nixosModule. # { {instanceName} :: { roles :: { {roleName} :: { machines :: { {machineName} :: { settings :: {} } } } } } }
+  #         instance_foo.roles.peer.machines.jon ? settings;
+  #       machineSettingsEmpty =
+  #         lib.filterAttrs (n: _v: n != "__functor" ) res.evals.self-A.config.result.allMachines.jon.nixosModule. # { {instanceName} :: { roles :: { {roleName} :: { machines :: { {machineName} :: { settings :: {} } } } } } }
+  #         instance_foo.roles.peer.machines.jon.settings;
+  #       hasRoleSettings =
+  #         res.evals.self-A.config.result.allMachines.jon.nixosModule. # { {instanceName} :: { roles :: { {roleName} :: { machines :: { {machineName} :: { settings :: {} } } } } } }
+  #         instance_foo.roles.peer ? settings;
+  #       roleSettingsEmpty =
+  #         lib.filterAttrs (n: _v: n != "__functor" ) res.evals.self-A.config.result.allMachines.jon.nixosModule. # { {instanceName} :: { roles :: { {roleName} :: { machines :: { {machineName} :: { settings :: {} } } } } } }
+  #         instance_foo.roles.peer.settings;
+  #     };
+  #     expected = {
+  #       hasMachineSettings = true;
+  #       machineSettingsEmpty = {};
+  #       hasRoleSettings = true;
+  #       roleSettingsEmpty = {};
+  #     };
+  #   };
+}

lib/distributed-service/tests/per_machine_args.nix

@@ -0,0 +1,107 @@
+{ lib, callInventoryAdapter }:
+
+let # Authored module
+  # A minimal module looks like this
+  # It isn't exactly doing anything but it's a valid module that produces an output
+  modules."A" = {
+    _class = "clan.service";
+    manifest = {
+      name = "network";
+    };
+    # Define two roles with unmergeable interfaces
+    # Both define some 'timeout' but with completely different types.
+    roles.peer.interface =
+      { lib, ... }:
+      {
+        options.timeout = lib.mkOption {
+          type = lib.types.str;
+        };
+      };
+    roles.server.interface =
+      { lib, ... }:
+      {
+        options.timeout = lib.mkOption {
+          type = lib.types.submodule;
+        };
+      };
+
+    perMachine =
+      { instances, ... }:
+      {
+        nixosModule = instances;
+      };
+  };
+  machines = {
+    jon = { };
+    sara = { };
+  };
+  res = callInventoryAdapter {
+    inherit modules machines;
+    instances."instance_foo" = {
+      module = {
+        name = "A";
+      };
+      roles.peer.machines.jon = {
+        settings.timeout = lib.mkForce "foo-peer-jon";
+      };
+      roles.peer = {
+        settings.timeout = "foo-peer";
+      };
+    };
+    instances."instance_bar" = {
+      module = {
+        name = "A";
+      };
+      roles.peer.machines.jon = {
+        settings.timeout = "bar-peer-jon";
+      };
+    };
+    instances."instance_zaza" = {
+      module = {
+        name = "B";
+      };
+      roles.peer.tags.all = { };
+    };
+  };
+
+  filterInternals = lib.filterAttrs (n: _v: !lib.hasPrefix "_" n);
+in
+
+{
+
+  # settings should evaluate
+  test_per_machine_receives_instance_settings = {
+    expr = {
+      hasMachineSettings =
+        res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.machines.jon
+        ? settings;
+
+      # settings are specific.
+      # Below we access:
+      # instance = instance_foo
+      # roles = peer
+      # machines = jon
+      specificMachineSettings = filterInternals res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.machines.jon.settings;
+
+      hasRoleSettings =
+        res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer ? settings;
+
+      # settings are specific.
+      # Below we access:
+      # instance = instance_foo
+      # roles = peer
+      # machines = *
+      specificRoleSettings = filterInternals res.evals.self-A.config.result.allMachines.jon.nixosModule.instance_foo.roles.peer.settings;
+    };
+    expected = {
+      hasMachineSettings = true;
+      specificMachineSettings = {
+        timeout = "foo-peer-jon";
+      };
+      hasRoleSettings = true;
+      specificRoleSettings = {
+        timeout = "foo-peer";
+      };
+    };
+  };
+}

lib/flake-module.nix

@@ -16,6 +16,7 @@ in
     ./inventory/flake-module.nix
     ./build-clan/flake-module.nix
     ./values/flake-module.nix
+    ./distributed-service/flake-module.nix
   ];
   flake.lib = import ./default.nix {
     inherit lib inputs;

lib/inventory/build-inventory/interface.nix

@@ -103,7 +103,9 @@ in
       default = options;
     };
     modules = lib.mkOption {
-      type = types.attrsOf types.path;
+      # Don't define the type yet
+      # We manually transform the value with types.deferredModule.merge later to keep them serializable
+      type = types.attrsOf types.raw;
       default = { };
       defaultText = "clanModules of clan-core";
       description = ''
@@ -275,7 +277,73 @@ in
         )
       );
     };
-
+    instances = lib.mkOption {
+      # Keep as internal until all de-/serialization issues are resolved
+      visible = false;
+      internal = true;
+      description = "Multi host service module instances";
+      type = types.attrsOf (
+        types.submodule {
+          options = {
+            # ModuleSpec
+            module = lib.mkOption {
+              type = types.submodule {
+                options.input = lib.mkOption {
+                  type = types.nullOr types.str;
+                  default = null;
+                  defaultText = "Name of the input. Default to 'null' which means the module is local";
+                  description = ''
+                    Name of the input. Default to 'null' which means the module is local
+                  '';
+                };
+                options.name = lib.mkOption {
+                  type = types.str;
+                };
+              };
+            };
+            roles = lib.mkOption {
+              default = { };
+              type = types.attrsOf (
+                types.submodule {
+                  options = {
+                    # TODO: deduplicate
+                    machines = lib.mkOption {
+                      type = types.attrsOf (
+                        types.submodule {
+                          options.settings = lib.mkOption {
+                            default = { };
+                            # Dont transform the value with `types.deferredModule` here. We need to keep it json serializable
+                            # TODO: We need a custom serializer for deferredModule
+                            type = types.deferredModule;
+                          };
+                        }
+                      );
+                      default = { };
+                    };
+                    tags = lib.mkOption {
+                      type = types.attrsOf (
+                        types.submodule {
+                          options.settings = lib.mkOption {
+                            default = { };
+                            type = types.deferredModule;
+                          };
+                        }
+                      );
+                      default = { };
+                    };
+                    settings = lib.mkOption {
+                      default = { };
+                      type = types.deferredModule;
+                    };
+                  };
+                }
+              );
+            };
+          };
+        }
+      );
+      default = { };
+    };
     services = lib.mkOption {
       description = ''
         Services of the inventory.

lib/inventory/tests/clanModule/README.md

@@ -1,4 +0,0 @@
----
-features = [ "inventory" ]
----
-Description

lib/inventory/tests/clanModule/roles/default.nix

@@ -1,6 +0,0 @@
-{ ... }:
-{
-  _class = "clan";
-  perInstance = { };
-  perService = { };
-}

lib/inventory/tests/default.nix

@@ -2,7 +2,6 @@
 let
   inventory = (
     import ../build-inventory {
-
       inherit lib clan-core;
     }
   );
@@ -17,11 +16,9 @@ in
             A = { };
           };
           services = {
-            clanModule = { };
             legacyModule = { };
           };
           modules = {
-            clanModule = ./clanModule;
             legacyModule = ./legacyModule;
           };
         };
@@ -30,17 +27,11 @@ in
     in
     {
       expr = {
-        clanModule = lib.filterAttrs (
-          name: _: name == "isClanModule"
-        ) compiled.machines.A.compiledServices.clanModule;
         legacyModule = lib.filterAttrs (
           name: _: name == "isClanModule"
         ) compiled.machines.A.compiledServices.legacyModule;
       };
       expected = {
-        clanModule = {
-          isClanModule = true;
-        };
         legacyModule = {
           isClanModule = false;
         };

pkgs/clan-cli/clan_cli/inventory/classes.py

@@ -31,6 +31,6 @@ Service = dict[str, Any]
 class Inventory(TypedDict):
     machines: NotRequired[dict[str, Machine]]
     meta: NotRequired[Meta]
-    modules: NotRequired[dict[str, str]]
+    modules: NotRequired[dict[str, Any]]
     services: NotRequired[dict[str, Service]]
-    tags: NotRequired[dict[str, list[str]]]
+    tags: NotRequired[dict[str, Any]]

pkgs/clan-cli/clan_cli/inventory/update.sh

@@ -5,4 +5,4 @@ set -euo pipefail
 jsonSchema=$(nix build .#schemas.inventory-schema-abstract --print-out-paths)/schema.json
 SCRIPT_DIR=$(dirname "$0")
 cd "$SCRIPT_DIR"
-nix run .#classgen -- "$jsonSchema" "../../../clan-cli/clan_cli/inventory/classes.py" --stop-at "Service"
+nix run .#classgen -- "$jsonSchema" "../../../clan-cli/clan_cli/inventory/classes.py"

pkgs/clan-cli/default.nix

@@ -60,7 +60,7 @@ let
     ln -sf ${nixpkgs'} $out/clan_cli/nixpkgs
     cp -r ${../../templates} $out/clan_cli/templates
 
-    ${classgen}/bin/classgen ${inventory-schema-abstract}/schema.json $out/clan_cli/inventory/classes.py --stop-at "Service"
+    ${classgen}/bin/classgen ${inventory-schema-abstract}/schema.json $out/clan_cli/inventory/classes.py
   '';
 
   # Create a custom nixpkgs for use within the project

pkgs/clan-cli/flake-module.nix

@@ -169,7 +169,7 @@
           ];
 
           installPhase = ''
-            ${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json ./clan_cli/inventory/classes.py --stop-at "Service"
+            ${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json ./clan_cli/inventory/classes.py
 
             python docs.py reference
             mkdir -p $out
@@ -188,7 +188,7 @@
           ];
 
           installPhase = ''
-            ${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json ./clan_cli/inventory/classes.py --stop-at "Service"
+            ${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json ./clan_cli/inventory/classes.py
             mkdir -p $out
             # Retrieve python API Typescript types
             python api.py > $out/API.json
@@ -214,7 +214,7 @@
             classFile = "classes.py";
           };
           installPhase = ''
-            ${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json b_classes.py --stop-at "Service"
+            ${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json b_classes.py
             file1=$classFile
             file2=b_classes.py
 

pkgs/clan-cli/shell.nix

@@ -46,6 +46,6 @@ mkShell {
 
     # Generate classes.py from inventory schema
     # This file is in .gitignore
-    ${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json $PKG_ROOT/clan_cli/inventory/classes.py --stop-at "Service"
+    ${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json $PKG_ROOT/clan_cli/inventory/classes.py
   '';
 }

pkgs/classgen/main.py

@@ -32,6 +32,8 @@ def map_json_type(
         return {"str"}
     if json_type == "integer":
         return {"int"}
+    if json_type == "number":
+        return {"float"}
     if json_type == "boolean":
         return {"bool"}
     # In Python, "number" is analogous to the float type.
@@ -52,7 +54,11 @@ def map_json_type(
 
 known_classes = set()
 root_class = "Inventory"
-stop_at = None
+# TODO: make this configurable
+# For now this only includes static top-level attributes of the inventory.
+attrs = ["machines", "meta", "services"]
+
+static: dict[str, str] = {"Service": "dict[str, Any]"}
 
 
 def field_def_from_default_type(
@@ -191,19 +197,32 @@ def get_field_def(
 
 
 # Recursive function to generate dataclasses from JSON schema
-def generate_dataclass(schema: dict[str, Any], class_name: str = root_class) -> str:
+def generate_dataclass(
+    schema: dict[str, Any],
+    attr_path: list[str],
+    class_name: str = root_class,
+) -> str:
     properties = schema.get("properties", {})
 
     required_fields = []
     fields_with_default = []
     nested_classes: list[str] = []
-    if stop_at and class_name == stop_at:
+
-        # Skip generating classes below the stop_at property
+    # if We are at the top level, and the attribute name is in shallow
-        return f"{class_name} = dict[str, Any]"
+    # return f"{class_name} = dict[str, Any]"
+    if class_name in static:
+        return f"{class_name} = {static[class_name]}"
 
     for prop, prop_info in properties.items():
+        # If we are at the top level, and the attribute name is not explicitly included we only do shallow
         field_name = prop.replace("-", "_")
 
+        if len(attr_path) == 0 and prop not in attrs:
+            field_def = f"{field_name}: NotRequired[dict[str, Any]]"
+            fields_with_default.append(field_def)
+            # breakpoint()
+            continue
+
         prop_type = prop_info.get("type", None)
         union_variants = prop_info.get("oneOf", [])
         enum_variants = prop_info.get("enum", [])
@@ -241,7 +260,9 @@ def generate_dataclass(schema: dict[str, Any], class_name: str = root_class) ->
 
                 if nested_class_name not in known_classes:
                     nested_classes.append(
-                        generate_dataclass(inner_type, nested_class_name)
+                        generate_dataclass(
+                            inner_type, [*attr_path, prop], nested_class_name
+                        )
                     )
                     known_classes.add(nested_class_name)
 
@@ -257,7 +278,9 @@ def generate_dataclass(schema: dict[str, Any], class_name: str = root_class) ->
                 field_types = {nested_class_name}
                 if nested_class_name not in known_classes:
                     nested_classes.append(
-                        generate_dataclass(prop_info, nested_class_name)
+                        generate_dataclass(
+                            prop_info, [*attr_path, prop], nested_class_name
+                        )
                     )
                     known_classes.add(nested_class_name)
         else:
@@ -322,6 +345,8 @@ def generate_dataclass(schema: dict[str, Any], class_name: str = root_class) ->
             )
             required_fields.append(field_def)
 
+    # breakpoint()
+
     fields_str = "\n    ".join(required_fields + fields_with_default)
     nested_classes_str = "\n\n".join(nested_classes)
 
@@ -336,14 +361,11 @@ def generate_dataclass(schema: dict[str, Any], class_name: str = root_class) ->
 
 def run_gen(args: argparse.Namespace) -> None:
     print(f"Converting {args.input} to {args.output}")
-    if args.stop_at:
-        global stop_at
-        stop_at = args.stop_at
 
     dataclass_code = ""
     with args.input.open() as f:
         schema = json.load(f)
-        dataclass_code = generate_dataclass(schema)
+        dataclass_code = generate_dataclass(schema, [])
 
     with args.output.open("w") as f:
         f.write(

pkgs/webview-ui/app/src/api/wifi.ts

@@ -11,6 +11,8 @@ export async function get_iwd_service(base_path: string, machine_name: string) {
   if (r.status == "error") {
     return null;
   }
+  // @FIXME: Clean this up once we implement the feature
+  // @ts-expect-error: This doesn't check currently
   const inventory: Inventory = r.data;
 
   const instance_key = instance_name(machine_name);