From d6714355b5cbb1bd96676f0ffc8958dfe31fdedc Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 29 Apr 2025 15:12:44 +0200 Subject: [PATCH 1/2] refactor(clan.service): make evalClanService a standalone function to interact with standalone modules --- lib/inventory/default.nix | 5 +- .../distributed-service/inventory-adapter.nix | 323 +++++++++--------- 2 files changed, 172 insertions(+), 156 deletions(-) diff --git a/lib/inventory/default.nix b/lib/inventory/default.nix index 9dec489c8..d53a20e06 100644 --- a/lib/inventory/default.nix +++ b/lib/inventory/default.nix @@ -1,8 +1,11 @@ { lib, clanLib }: +let + services = clanLib.callLib ./distributed-service/inventory-adapter.nix { }; +in { + inherit (services) evalClanService mapInstances; inherit (import ./build-inventory { inherit lib clanLib; }) buildInventory; interface = ./build-inventory/interface.nix; - mapInstances = clanLib.callLib ./distributed-service/inventory-adapter.nix { }; # Returns the list of machine names # { ... } -> [ string ] resolveTags = diff --git a/lib/inventory/distributed-service/inventory-adapter.nix b/lib/inventory/distributed-service/inventory-adapter.nix index 03447ca65..92a134ac2 100644 --- a/lib/inventory/distributed-service/inventory-adapter.nix +++ b/lib/inventory/distributed-service/inventory-adapter.nix @@ -14,167 +14,180 @@ clanLib, ... }: -{ - # This is used to resolve the module imports from 'flake.inputs' - flakeInputs, - # The clan inventory - inventory, -}: let - # machineHasTag = machineName: tagName: lib.elem tagName inventory.machines.${machineName}.tags; - - # map the instances into the module - importedModuleWithInstances = lib.mapAttrs ( - instanceName: instance: - let - # TODO: - resolvedModuleSet = - # If the module.name is self then take the modules defined in the flake - # Otherwise its an external input which provides the modules via 'clan.modules' attribute - if instance.module.input == null then - inventory.modules - else - let - input = - flakeInputs.${instance.module.input} or (throw '' - Flake doesn't provide input with name '${instance.module.input}' - - Choose one of the following inputs: - - ${ - builtins.concatStringsSep "\n- " ( - lib.attrNames (lib.filterAttrs (_name: input: input ? clan) flakeInputs) - ) - } - - To import a local module from 'inventory.modules' remove the 'input' attribute from the module definition - Remove the following line from the module definition: - - ... - - module.input = "${instance.module.input}" - - - ''); - clanAttrs = - input.clan - or (throw "It seems the flake input ${instance.module.input} doesn't export any clan resources"); - in - clanAttrs.modules; - - resolvedModule = - resolvedModuleSet.${instance.module.name} - or (throw "flake doesn't provide clan-module with name ${instance.module.name}"); - - # Every instance includes machines via roles - # :: { client :: ... } - instanceRoles = lib.mapAttrs ( - roleName: role: - let - resolvedMachines = clanLib.inventory.resolveTags { - members = { - # Explicit members - machines = lib.attrNames role.machines; - # Resolved Members - tags = lib.attrNames role.tags; - }; - inherit (inventory) machines; - inherit instanceName roleName; - }; - in - # instances..roles. = - { - machines = lib.genAttrs resolvedMachines.machines ( - machineName: - let - machineSettings = instance.roles.${roleName}.machines.${machineName}.settings or { }; - in - # TODO: tag settings - # Wait for this feature until option introspection for 'settings' is done. - # This might get too complex to handle otherwise. - # settingsViaTags = lib.filterAttrs ( - # tagName: _: machineHasTag machineName tagName - # ) instance.roles.${roleName}.tags; - { - # TODO: Do we want to wrap settings with - # setDefaultModuleLocation "inventory.instances.${instanceName}.roles.${roleName}.tags.${tagName}"; - settings = { - imports = [ - machineSettings - ]; # ++ lib.attrValues (lib.mapAttrs (_tagName: v: v.settings) settingsViaTags); - }; - } - ); - # Maps to settings for the role. - # In other words this sets the following path of a clan.service module: - # instances..roles..settings - settings = role.settings; - } - ) instance.roles; - in - { - inherit (instance) module; - inherit resolvedModule instanceRoles; - } - ) inventory.instances or { }; - - # TODO: Eagerly check the _class of the resolved module - importedModulesEvaluated = lib.mapAttrs ( - module_ident: instances: + evalClanService = + { modules, id }: (lib.evalModules { class = "clan.service"; - modules = - [ - ./service-module.nix - # Import the resolved module. - (builtins.head instances).instance.resolvedModule + modules = [ + ./service-module.nix - # feature modules - (lib.modules.importApply ./api-feature.nix { - inherit clanLib; - attrName = module_ident; - }) - ] - # Include all the instances that correlate to the resolved module - ++ (builtins.map (v: { - instances.${v.instanceName}.roles = v.instance.instanceRoles; - }) instances); - }) - ) grouped; - - # Group the instances by the module they resolve to - # This is necessary to evaluate the module in a single pass - # :: { _ :: [ { name, value } ] } - # Since 'perMachine' needs access to all the instances we should include them as a whole - grouped = lib.foldlAttrs ( - acc: instanceName: instance: - let - inputName = if instance.module.input == null then "self" else instance.module.input; - id = inputName + "-" + instance.module.name; - in - acc - // { - ${id} = acc.${id} or [ ] ++ [ - { - inherit instanceName instance; - } + # feature modules + (lib.modules.importApply ./api-feature.nix { + inherit clanLib; + attrName = id; + }) ]; - } - ) { } importedModuleWithInstances; - - # TODO: Return an attribute set of resources instead of a plain list of nixosModules - allMachines = lib.foldlAttrs ( - acc: _module_ident: eval: - acc - // lib.mapAttrs ( - machineName: result: acc.${machineName} or [ ] ++ [ result.nixosModule ] - ) eval.config.result.final - ) { } importedModulesEvaluated; + }); in { - inherit - importedModuleWithInstances - grouped + inherit evalClanService; + mapInstances = + { + # This is used to resolve the module imports from 'flake.inputs' + flakeInputs, + # The clan inventory + inventory, + }: + let + # machineHasTag = machineName: tagName: lib.elem tagName inventory.machines.${machineName}.tags; + + # map the instances into the module + importedModuleWithInstances = lib.mapAttrs ( + instanceName: instance: + let + # TODO: + resolvedModuleSet = + # If the module.name is self then take the modules defined in the flake + # Otherwise its an external input which provides the modules via 'clan.modules' attribute + if instance.module.input == null then + inventory.modules + else + let + input = + flakeInputs.${instance.module.input} or (throw '' + Flake doesn't provide input with name '${instance.module.input}' + + Choose one of the following inputs: + - ${ + builtins.concatStringsSep "\n- " ( + lib.attrNames (lib.filterAttrs (_name: input: input ? clan) flakeInputs) + ) + } + + To import a local module from 'inventory.modules' remove the 'input' attribute from the module definition + Remove the following line from the module definition: + + ... + - module.input = "${instance.module.input}" + + + ''); + clanAttrs = + input.clan + or (throw "It seems the flake input ${instance.module.input} doesn't export any clan resources"); + in + clanAttrs.modules; + + resolvedModule = + resolvedModuleSet.${instance.module.name} + or (throw "flake doesn't provide clan-module with name ${instance.module.name}"); + + # Every instance includes machines via roles + # :: { client :: ... } + instanceRoles = lib.mapAttrs ( + roleName: role: + let + resolvedMachines = clanLib.inventory.resolveTags { + members = { + # Explicit members + machines = lib.attrNames role.machines; + # Resolved Members + tags = lib.attrNames role.tags; + }; + inherit (inventory) machines; + inherit instanceName roleName; + }; + in + # instances..roles. = + { + machines = lib.genAttrs resolvedMachines.machines ( + machineName: + let + machineSettings = instance.roles.${roleName}.machines.${machineName}.settings or { }; + in + # TODO: tag settings + # Wait for this feature until option introspection for 'settings' is done. + # This might get too complex to handle otherwise. + # settingsViaTags = lib.filterAttrs ( + # tagName: _: machineHasTag machineName tagName + # ) instance.roles.${roleName}.tags; + { + # TODO: Do we want to wrap settings with + # setDefaultModuleLocation "inventory.instances.${instanceName}.roles.${roleName}.tags.${tagName}"; + settings = { + imports = [ + machineSettings + ]; # ++ lib.attrValues (lib.mapAttrs (_tagName: v: v.settings) settingsViaTags); + }; + } + ); + # Maps to settings for the role. + # In other words this sets the following path of a clan.service module: + # instances..roles..settings + settings = role.settings; + } + ) instance.roles; + in + { + inherit (instance) module; + inherit resolvedModule instanceRoles; + } + ) inventory.instances or { }; + + # TODO: Eagerly check the _class of the resolved module + importedModulesEvaluated = lib.mapAttrs ( + module_ident: instances: + evalClanService { + id = module_ident; + modules = + [ + # Import the resolved module. + (builtins.head instances).instance.resolvedModule + ] # Include all the instances that correlate to the resolved module + ++ (builtins.map (v: { + instances.${v.instanceName}.roles = v.instance.instanceRoles; + }) instances); + } + ) grouped; + + # Group the instances by the module they resolve to + # This is necessary to evaluate the module in a single pass + # :: { _ :: [ { name, value } ] } + # Since 'perMachine' needs access to all the instances we should include them as a whole + grouped = lib.foldlAttrs ( + acc: instanceName: instance: + let + inputName = if instance.module.input == null then "self" else instance.module.input; + id = inputName + "-" + instance.module.name; + in + acc + // { + ${id} = acc.${id} or [ ] ++ [ + { + inherit instanceName instance; + } + ]; + } + ) { } importedModuleWithInstances; + + # TODO: Return an attribute set of resources instead of a plain list of nixosModules + allMachines = lib.foldlAttrs ( + acc: _module_ident: eval: + acc + // lib.mapAttrs ( + machineName: result: acc.${machineName} or [ ] ++ [ result.nixosModule ] + ) eval.config.result.final + ) { } importedModulesEvaluated; + in + { + inherit + importedModuleWithInstances + grouped + + allMachines + importedModulesEvaluated + ; + }; - allMachines - importedModulesEvaluated - ; } From 662787f96e7219b21c10d17744fe2cabaf4e8871 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 29 Apr 2025 15:31:02 +0200 Subject: [PATCH 2/2] Checks: add json-compat check wrapper to ensure all clan.modules stay json-compatible --- checks/flake-module.nix | 28 ++++++++++++++++++- clanServices/hello-world/flake-module.nix | 3 ++ .../distributed-service/inventory-adapter.nix | 9 +++--- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/checks/flake-module.nix b/checks/flake-module.nix index 33c3d41f4..04480c7e1 100644 --- a/checks/flake-module.nix +++ b/checks/flake-module.nix @@ -64,7 +64,33 @@ in self'.legacyPackages.homeConfigurations or { } ); in - nixosTests // flakeOutputs; + nixosTests + // flakeOutputs + // { + # TODO: Automatically provide this check to downstream users to check their modules + clan-modules-json-compatible = + let + allSchemas = lib.mapAttrs ( + _n: m: + let + schema = + (self.clanLib.inventory.evalClanService { + modules = [ m ]; + key = "checks"; + }).config.result.api.schema; + in + schema + ) self.clan.modules; + in + pkgs.runCommand "combined-result" + { + schemaFile = builtins.toFile "schemas.json" (builtins.toJSON allSchemas); + } + '' + mkdir -p $out + cat $schemaFile > $out/allSchemas.json + ''; + }; legacyPackages = { nixosTests = let diff --git a/clanServices/hello-world/flake-module.nix b/clanServices/hello-world/flake-module.nix index c76eca5a8..9a1c14a27 100644 --- a/clanServices/hello-world/flake-module.nix +++ b/clanServices/hello-world/flake-module.nix @@ -13,6 +13,9 @@ in clan.inventory.modules = { hello-world = module; }; + clan.modules = { + hello-world = module; + }; perSystem = { pkgs, ... }: let diff --git a/lib/inventory/distributed-service/inventory-adapter.nix b/lib/inventory/distributed-service/inventory-adapter.nix index 92a134ac2..e899564c8 100644 --- a/lib/inventory/distributed-service/inventory-adapter.nix +++ b/lib/inventory/distributed-service/inventory-adapter.nix @@ -16,18 +16,17 @@ }: let evalClanService = - { modules, id }: + { modules, key }: (lib.evalModules { class = "clan.service"; modules = [ ./service-module.nix - # feature modules (lib.modules.importApply ./api-feature.nix { inherit clanLib; - attrName = id; + attrName = key; }) - ]; + ] ++ modules; }); in { @@ -139,7 +138,7 @@ in importedModulesEvaluated = lib.mapAttrs ( module_ident: instances: evalClanService { - id = module_ident; + key = module_ident; modules = [ # Import the resolved module.