From 58cfcf3d25b1183dd9c91ec0c49534131bc9f3e9 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 31 Aug 2025 12:48:58 +0200 Subject: [PATCH 1/7] api/modules: delete instances.py duplicate --- pkgs/clan-cli/clan_lib/services/instances.py | 112 ------------------- 1 file changed, 112 deletions(-) delete mode 100644 pkgs/clan-cli/clan_lib/services/instances.py diff --git a/pkgs/clan-cli/clan_lib/services/instances.py b/pkgs/clan-cli/clan_lib/services/instances.py deleted file mode 100644 index 70a43f3ff..000000000 --- a/pkgs/clan-cli/clan_lib/services/instances.py +++ /dev/null @@ -1,112 +0,0 @@ -from typing import TypedDict - -from clan_lib.api import API -from clan_lib.errors import ClanError -from clan_lib.flake.flake import Flake -from clan_lib.nix_models.clan import ( - InventoryInstanceModule, - InventoryInstanceRolesType, - InventoryInstancesType, - InventoryMachinesType, -) -from clan_lib.persist.inventory_store import InventoryStore -from clan_lib.persist.util import set_value_by_path -from clan_lib.services.modules import ( - get_service_module, -) - -# TODO: move imports out of cli/__init__.py causing import cycles -# from clan_lib.machines.actions import list_machines - - -@API.register -def list_service_instances(flake: Flake) -> InventoryInstancesType: - """Returns all currently present service instances including their full configuration""" - inventory_store = InventoryStore(flake) - inventory = inventory_store.read() - return inventory.get("instances", {}) - - -def collect_tags(machines: InventoryMachinesType) -> set[str]: - res = set() - for machine in machines.values(): - res |= set(machine.get("tags", [])) - - return res - - -# Removed 'module' ref - Needs to be passed seperately -class InstanceConfig(TypedDict): - roles: InventoryInstanceRolesType - - -@API.register -def create_service_instance( - flake: Flake, - module_ref: InventoryInstanceModule, - instance_name: str, - instance_config: InstanceConfig, -) -> None: - module = get_service_module(flake, module_ref) - - inventory_store = InventoryStore(flake) - inventory = inventory_store.read() - - instances = inventory.get("instances", {}) - if instance_name in instances: - msg = f"service instance '{instance_name}' already exists." - raise ClanError(msg) - - target_roles = instance_config.get("roles") - if not target_roles: - msg = "Creating a service instance requires adding roles" - raise ClanError(msg) - - available_roles = set(module.get("roles", {}).keys()) - - unavailable_roles = list(filter(lambda r: r not in available_roles, target_roles)) - if unavailable_roles: - msg = f"Unknown roles: {unavailable_roles}. Use one of {available_roles}" - raise ClanError(msg) - - role_configs = instance_config.get("roles") - if not role_configs: - return - - ## Validate machine references - all_machines = inventory.get("machines", {}) - available_machine_refs = set(all_machines.keys()) - available_tag_refs = collect_tags(all_machines) - - for role_name, role_members in role_configs.items(): - machine_refs = role_members.get("machines") - msg = f"Role: '{role_name}' - " - if machine_refs: - unavailable_machines = list( - filter(lambda m: m not in available_machine_refs, machine_refs), - ) - if unavailable_machines: - msg += f"Unknown machine reference: {unavailable_machines}. Use one of {available_machine_refs}" - raise ClanError(msg) - - tag_refs = role_members.get("tags") - if tag_refs: - unavailable_tags = list( - filter(lambda m: m not in available_tag_refs, tag_refs), - ) - - if unavailable_tags: - msg += ( - f"Unknown tags: {unavailable_tags}. Use one of {available_tag_refs}" - ) - raise ClanError(msg) - - # TODO: - # Validate instance_config roles settings against role schema - - set_value_by_path(inventory, f"instances.{instance_name}", instance_config) - set_value_by_path(inventory, f"instances.{instance_name}.module", module_ref) - inventory_store.write( - inventory, - message=f"services: instance '{instance_name}' init", - ) From 453f2649d3ba4dfdc6386dfa952caeea7040e81d Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 31 Aug 2025 12:49:26 +0200 Subject: [PATCH 2/7] clanInternals: expose builtin modules --- lib/modules/clan/module.nix | 2 ++ lib/modules/inventoryClass/service-list-from-inputs.nix | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/lib/modules/clan/module.nix b/lib/modules/clan/module.nix index 1c643af28..ef28ce9cf 100644 --- a/lib/modules/clan/module.nix +++ b/lib/modules/clan/module.nix @@ -245,6 +245,8 @@ in in { config, ... }: { + staticModules = clan-core.clan.modules; + distributedServices = clanLib.inventory.mapInstances { inherit (clanConfig) inventory exportsModule; inherit flakeInputs directory; diff --git a/lib/modules/inventoryClass/service-list-from-inputs.nix b/lib/modules/inventoryClass/service-list-from-inputs.nix index c0368437c..1ba0c4fab 100644 --- a/lib/modules/inventoryClass/service-list-from-inputs.nix +++ b/lib/modules/inventoryClass/service-list-from-inputs.nix @@ -23,6 +23,12 @@ let }; in { + options.staticModules = lib.mkOption { + readOnly = true; + type = lib.types.raw; + + apply = moduleSet: lib.mapAttrs (inspectModule "") moduleSet; + }; options.modulesPerSource = lib.mkOption { # { sourceName :: { moduleName :: {} }} readOnly = true; From 5137d19b0f1dc2276beaf344add3f3ba291ce32f Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 31 Aug 2025 12:49:58 +0200 Subject: [PATCH 3/7] nix_modules: fix and update None types --- pkgs/clan-cli/clan_lib/nix_models/clan.py | 4 ++-- pkgs/classgen/main.py | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/nix_models/clan.py b/pkgs/clan-cli/clan_lib/nix_models/clan.py index ea055bf16..234fbf235 100644 --- a/pkgs/clan-cli/clan_lib/nix_models/clan.py +++ b/pkgs/clan-cli/clan_lib/nix_models/clan.py @@ -13,7 +13,7 @@ class Unknown: InventoryInstanceModuleNameType = str -InventoryInstanceModuleInputType = str +InventoryInstanceModuleInputType = str | None class InventoryInstanceModule(TypedDict): name: str @@ -163,7 +163,7 @@ class Template(TypedDict): -ClanDirectoryType = dict[str, Any] | list[Any] | bool | float | int | str +ClanDirectoryType = dict[str, Any] | list[Any] | bool | float | int | str | None ClanInventoryType = Inventory ClanMachinesType = dict[str, Unknown] ClanMetaType = Unknown diff --git a/pkgs/classgen/main.py b/pkgs/classgen/main.py index 66a82baf0..f7ea2c758 100644 --- a/pkgs/classgen/main.py +++ b/pkgs/classgen/main.py @@ -218,13 +218,12 @@ def get_field_def( default_factory: str | None = None, type_appendix: str = "", ) -> tuple[str, str]: - if "None" in field_types or default or default_factory: - if "None" in field_types: - field_types.remove("None") - serialised_types = " | ".join(sort_types(field_types)) + type_appendix + _field_types = set(field_types) + if "None" in _field_types or default or default_factory: + serialised_types = " | ".join(sort_types(_field_types)) + type_appendix serialised_types = f"{serialised_types}" else: - serialised_types = " | ".join(sort_types(field_types)) + type_appendix + serialised_types = " | ".join(sort_types(_field_types)) + type_appendix return (field_name, serialised_types) From c447aec9d325fb969c7b288abce4057cd2fa606e Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 31 Aug 2025 12:50:36 +0200 Subject: [PATCH 4/7] api/modules: improve logic for builtin modules --- .../fixtures/flakes/lib_clan/flake.nix | 8 +- pkgs/clan-cli/clan_lib/services/modules.py | 186 ++++++++++-------- .../clan_lib/services/modules_test.py | 50 +++-- pkgs/clan-cli/clan_lib/tests/test_create.py | 8 +- 4 files changed, 146 insertions(+), 106 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/fixtures/flakes/lib_clan/flake.nix b/pkgs/clan-cli/clan_lib/fixtures/flakes/lib_clan/flake.nix index daf1caea9..9c907b8ce 100644 --- a/pkgs/clan-cli/clan_lib/fixtures/flakes/lib_clan/flake.nix +++ b/pkgs/clan-cli/clan_lib/fixtures/flakes/lib_clan/flake.nix @@ -1,11 +1,11 @@ { - inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz"; - inputs.nixpkgs.follows = "clan-core/nixpkgs"; + inputs.Y2xhbi1jaW9yZS1uZXZlci1kZXBlbmQtb24tbWU.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz"; + inputs.nixpkgs.follows = "Y2xhbi1jaW9yZS1uZXZlci1kZXBlbmQtb24tbWU/nixpkgs"; outputs = - { self, clan-core, ... }: + { self, Y2xhbi1jaW9yZS1uZXZlci1kZXBlbmQtb24tbWU, ... }: let - clan = clan-core.lib.clan ({ + clan = Y2xhbi1jaW9yZS1uZXZlci1kZXBlbmQtb24tbWU.lib.clan ({ inherit self; imports = [ ./clan.nix diff --git a/pkgs/clan-cli/clan_lib/services/modules.py b/pkgs/clan-cli/clan_lib/services/modules.py index 33a7e8902..22a831ad4 100644 --- a/pkgs/clan-cli/clan_lib/services/modules.py +++ b/pkgs/clan-cli/clan_lib/services/modules.py @@ -60,7 +60,7 @@ class ModuleManifest: raise ValueError(msg) @classmethod - def from_dict(cls, data: dict) -> "ModuleManifest": + def from_dict(cls, data: dict[str, Any]) -> "ModuleManifest": """Create an instance of this class from a dictionary. Drops any keys that are not defined in the dataclass. """ @@ -147,23 +147,51 @@ def extract_frontmatter[T]( @dataclass -class ModuleInfo(TypedDict): +class ModuleInfo: manifest: ModuleManifest roles: dict[str, None] -class Module(TypedDict): - module: InventoryInstanceModule +@dataclass +class Module: + # To use this module specify: InventoryInstanceModule :: { input, name } (foreign key) + usage_ref: InventoryInstanceModule info: ModuleInfo + native: bool # Equivalent to input == None + + +@dataclass +class ClanModules: + modules: list[Module] + core_input_name: str @API.register -def list_service_modules(flake: Flake) -> list[Module]: +def list_service_modules(flake: Flake) -> ClanModules: """Show information about a module""" - modules = flake.select("clanInternals.inventoryClass.modulesPerSource") + # inputName.moduleName -> ModuleInfo + modules: dict[str, dict[str, Any]] = flake.select( + "clanInternals.inventoryClass.modulesPerSource" + ) - if "clan-core" not in modules: - msg = "Cannot find 'clan-core' input in the flake. Make sure your clan-core input is named 'clan-core'" + # moduleName -> ModuleInfo + builtin_modules: dict[str, Any] = flake.select( + "clanInternals.inventoryClass.staticModules" + ) + + first_name, first_module = next(iter(builtin_modules.items())) + clan_input_name = None + for input_name, module_set in modules.items(): + if first_name in module_set: + # Compare the manifest name + module_set[first_name]["manifest"]["name"] = first_module["manifest"][ + "name" + ] + clan_input_name = input_name + break + + if clan_input_name is None: + msg = "Could not determine the clan-core input name" raise ClanError(msg) res: list[Module] = [] @@ -171,86 +199,63 @@ def list_service_modules(flake: Flake) -> list[Module]: for module_name, module_info in module_set.items(): res.append( Module( - module={"name": module_name, "input": input_name}, + usage_ref={ + "name": module_name, + "input": None if input_name == clan_input_name else input_name, + }, info=ModuleInfo( - manifest=ModuleManifest.from_dict( - module_info.get("manifest"), - ), roles=module_info.get("roles", {}), + manifest=ModuleManifest.from_dict(module_info["manifest"]), ), + native=(input_name == clan_input_name), ) ) - return res + return ClanModules(res, clan_input_name) -@API.register -def get_service_module( +def resolve_service_module_ref( flake: Flake, module_ref: InventoryInstanceModuleType, -) -> ModuleInfo: - """Returns the module information for a given module reference - - :param module_ref: The module reference to get the information for - :return: Dict of module information - :raises ClanError: If the module_ref is invalid or missing required fields - """ - input_name, module_name = check_service_module_ref(flake, module_ref) - - avilable_modules = list_service_modules(flake) - module_set: list[Module] = [ - m for m in avilable_modules if m["module"].get("input", None) == input_name - ] - - if not module_set: - msg = f"Module set for input '{input_name}' not found" - raise ClanError(msg) - - module = next((m for m in module_set if m["module"]["name"] == module_name), None) - - if module is None: - msg = f"Module '{module_name}' not found in input '{input_name}'" - raise ClanError(msg) - - return module["info"] - - -def check_service_module_ref( - flake: Flake, - module_ref: InventoryInstanceModuleType, -) -> tuple[str, str]: +) -> Module: """Checks if the module reference is valid :param module_ref: The module reference to check :raises ClanError: If the module_ref is invalid or missing required fields """ - avilable_modules = list_service_modules(flake) + service_modules = list_service_modules(flake) + avilable_modules = service_modules.modules input_ref = module_ref.get("input", None) - if input_ref is None: - msg = "Setting module_ref.input is currently required" - raise ClanError(msg) - module_set = [ - m for m in avilable_modules if m["module"].get("input", None) == input_ref - ] + if input_ref is None or input_ref == service_modules.core_input_name: + # Take only the native modules + module_set = [m for m in avilable_modules if m.native] + else: + # Match the input ref + module_set = [ + m for m in avilable_modules if m.usage_ref.get("input", None) == input_ref + ] - if module_set is None: - inputs = {m["module"].get("input") for m in avilable_modules} + if not module_set: + inputs = {m.usage_ref.get("input") for m in avilable_modules} msg = f"module set for input '{input_ref}' not found" msg += f"\nAvilable input_refs: {inputs}" + msg += "\nOmit the input field to use the built-in modules\n" + msg += "\n".join([m.usage_ref["name"] for m in avilable_modules if m.native]) raise ClanError(msg) module_name = module_ref.get("name") if not module_name: msg = "Module name is required in module_ref" raise ClanError(msg) - module = next((m for m in module_set if m["module"]["name"] == module_name), None) + + module = next((m for m in module_set if m.usage_ref["name"] == module_name), None) if module is None: msg = f"module with name '{module_name}' not found" raise ClanError(msg) - return (input_ref, module_name) + return module @API.register @@ -264,7 +269,16 @@ def get_service_module_schema( :return: Dict of schemas for the service module roles :raises ClanError: If the module_ref is invalid or missing required fields """ - input_name, module_name = check_service_module_ref(flake, module_ref) + input_name, module_name = module_ref.get("input"), module_ref["name"] + module = resolve_service_module_ref(flake, module_ref) + + if module is None: + msg = f"Module '{module_name}' not found in input '{input_name}'" + raise ClanError(msg) + + if input_name is None: + msg = "Not implemented for: input_name is None" + raise ClanError(msg) return flake.select( f"clanInternals.inventoryClass.moduleSchemas.{input_name}.{module_name}", @@ -278,7 +292,8 @@ def create_service_instance( roles: InventoryInstanceRolesType, ) -> None: """Show information about a module""" - input_name, module_name = check_service_module_ref(flake, module_ref) + input_name, module_name = module_ref.get("input"), module_ref["name"] + module = resolve_service_module_ref(flake, module_ref) inventory_store = InventoryStore(flake) @@ -299,10 +314,10 @@ def create_service_instance( all_machines = inventory.get("machines", {}) available_machine_refs = set(all_machines.keys()) - schema = get_service_module_schema(flake, module_ref) + allowed_roles = module.info.roles for role_name, role_members in roles.items(): - if role_name not in schema: - msg = f"Role '{role_name}' is not defined in the module schema" + if role_name not in allowed_roles: + msg = f"Role '{role_name}' is not defined in the module" raise ClanError(msg) machine_refs = role_members.get("machines") @@ -319,13 +334,21 @@ def create_service_instance( # settings = role_members.get("settings", {}) # Create a new instance with the given roles - new_instance: InventoryInstance = { - "module": { - "name": module_name, - "input": input_name, - }, - "roles": roles, - } + if not input_name: + new_instance: InventoryInstance = { + "module": { + "name": module_name, + }, + "roles": roles, + } + else: + new_instance = { + "module": { + "name": module_name, + "input": input_name, + }, + "roles": roles, + } set_value_by_path(inventory, f"instances.{instance_name}", new_instance) inventory_store.write( @@ -335,8 +358,10 @@ def create_service_instance( ) -class InventoryInstanceInfo(TypedDict): - module: Module +@dataclass +class InventoryInstanceInfo: + resolved: Module + module: InventoryInstanceModule roles: InventoryInstanceRolesType @@ -345,24 +370,19 @@ def list_service_instances(flake: Flake) -> dict[str, InventoryInstanceInfo]: """Returns all currently present service instances including their full configuration""" inventory_store = InventoryStore(flake) inventory = inventory_store.read() - service_modules = { - (mod["module"]["name"], mod["module"].get("input", "clan-core")): mod - for mod in list_service_modules(flake) - } + instances = inventory.get("instances", {}) res: dict[str, InventoryInstanceInfo] = {} for instance_name, instance in instances.items(): - module_key = ( - instance.get("module", {})["name"], - instance.get("module", {}).get("input") - or "clan-core", # Replace None (or falsey) with "clan-core" - ) - module = service_modules.get(module_key) + persisted_ref = instance.get("module", {"name": instance_name}) + module = resolve_service_module_ref(flake, persisted_ref) + if module is None: - msg = f"Module '{module_key}' for instance '{instance_name}' not found" + msg = f"Module for instance '{instance_name}' not found" raise ClanError(msg) res[instance_name] = InventoryInstanceInfo( - module=module, + resolved=module, + module=persisted_ref, roles=instance.get("roles", {}), ) return res diff --git a/pkgs/clan-cli/clan_lib/services/modules_test.py b/pkgs/clan-cli/clan_lib/services/modules_test.py index bf157a42a..0e01a1f7f 100644 --- a/pkgs/clan-cli/clan_lib/services/modules_test.py +++ b/pkgs/clan-cli/clan_lib/services/modules_test.py @@ -3,7 +3,7 @@ from collections.abc import Callable import pytest from clan_cli.tests.fixtures_flakes import nested_dict from clan_lib.flake.flake import Flake -from clan_lib.services.modules import list_service_instances +from clan_lib.services.modules import list_service_instances, list_service_modules @pytest.mark.with_core @@ -11,25 +11,43 @@ def test_list_service_instances( clan_flake: Callable[..., Flake], ) -> None: config = nested_dict() - config["inventory"]["machines"]["alice"] = {} - config["inventory"]["machines"]["bob"] = {} - # implicit module selection (defaults to clan-core/admin) - config["inventory"]["instances"]["admin"]["roles"]["default"]["tags"]["all"] = {} # explicit module selection - config["inventory"]["instances"]["my-sshd"]["module"]["input"] = "clan-core" - config["inventory"]["instances"]["my-sshd"]["module"]["name"] = "sshd" + # We use this random string in test to avoid code dependencies on the input name + config["inventory"]["instances"]["foo"]["module"]["input"] = ( + "Y2xhbi1jaW9yZS1uZXZlci1kZXBlbmQtb24tbWU" + ) + config["inventory"]["instances"]["foo"]["module"]["name"] = "sshd" # input = null - config["inventory"]["instances"]["my-sshd-2"]["module"]["input"] = None - config["inventory"]["instances"]["my-sshd-2"]["module"]["name"] = "sshd" + config["inventory"]["instances"]["bar"]["module"]["input"] = None + config["inventory"]["instances"]["bar"]["module"]["name"] = "sshd" + + # Omit input + config["inventory"]["instances"]["baz"]["module"]["name"] = "sshd" # external input flake = clan_flake(config) + service_modules = list_service_modules(flake) + + assert len(service_modules.modules) + assert any(m.usage_ref["name"] == "sshd" for m in service_modules.modules) + instances = list_service_instances(flake) - assert list(instances.keys()) == ["admin", "my-sshd", "my-sshd-2"] - assert instances["admin"]["module"]["module"].get("input") == "clan-core" - assert instances["admin"]["module"]["module"].get("name") == "admin" - assert instances["my-sshd"]["module"]["module"].get("input") == "clan-core" - assert instances["my-sshd"]["module"]["module"].get("name") == "sshd" - assert instances["my-sshd-2"]["module"]["module"].get("input") == "clan-core" - assert instances["my-sshd-2"]["module"]["module"].get("name") == "sshd" + assert set(instances.keys()) == {"foo", "bar", "baz"} + + # Reference to a built-in module + assert instances["foo"].resolved.usage_ref.get("input") is None + assert instances["foo"].resolved.usage_ref.get("name") == "sshd" + assert instances["foo"].resolved.info.manifest.name == "clan-core/sshd" + # Actual module + assert ( + instances["foo"].module.get("input") + == "Y2xhbi1jaW9yZS1uZXZlci1kZXBlbmQtb24tbWU" + ) + + # Module exposes the input name? + assert instances["bar"].resolved.usage_ref.get("input") is None + assert instances["bar"].resolved.usage_ref.get("name") == "sshd" + + assert instances["baz"].resolved.usage_ref.get("input") is None + assert instances["baz"].resolved.usage_ref.get("name") == "sshd" diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index 036af0c10..5276cb07b 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -214,10 +214,12 @@ def test_clan_create_api( store = InventoryStore(clan_dir_flake) inventory = store.read() - modules = list_service_modules(clan_dir_flake) + service_modules = list_service_modules(clan_dir_flake) - admin_module = next(m for m in modules if m["module"]["name"] == "admin") - assert admin_module["info"]["manifest"].name == "clan-core/admin" + admin_module = next( + m for m in service_modules.modules if m.usage_ref.get("name") == "admin" + ) + assert admin_module.info.manifest.name == "clan-core/admin" set_value_by_path(inventory, "instances", inventory_conf.instances) store.write( From 3862ad2a063cbdf4610d465b0b0873d4498c9f30 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 31 Aug 2025 15:46:36 +0200 Subject: [PATCH 5/7] api/modules: add foreign key to instances --- .../ui/src/workflows/Service/Service.tsx | 23 ++++---- pkgs/clan-cli/clan_lib/services/modules.py | 55 +++++++++++++++++-- .../clan_lib/services/modules_test.py | 52 ++++++++++++++++++ 3 files changed, 114 insertions(+), 16 deletions(-) diff --git a/pkgs/clan-app/ui/src/workflows/Service/Service.tsx b/pkgs/clan-app/ui/src/workflows/Service/Service.tsx index e5e0ca3e7..fc1311360 100644 --- a/pkgs/clan-app/ui/src/workflows/Service/Service.tsx +++ b/pkgs/clan-app/ui/src/workflows/Service/Service.tsx @@ -49,7 +49,7 @@ type ModuleItem = ServiceModules[number]; interface Module { value: string; - input?: string; + input?: string | null; label: string; description: string; raw: ModuleItem; @@ -68,19 +68,20 @@ const SelectService = () => { createEffect(() => { if (serviceModulesQuery.data && serviceInstancesQuery.data) { setModuleOptions( - serviceModulesQuery.data.map((m) => ({ - value: `${m.module.name}:${m.module.input}`, - label: m.module.name, - description: m.info.manifest.description, - input: m.module.input, - raw: m, + serviceModulesQuery.data.modules.map((currService) => ({ + value: `${currService.usage_ref.name}:${currService.usage_ref.input}`, + label: currService.info.manifest.name, + description: currService.info.manifest.description, + input: currService.usage_ref.input, + raw: currService, // TODO: include the instances that use this module instances: Object.entries(serviceInstancesQuery.data) .filter( - ([name, i]) => - i.module.module.name === m.module.name && - (!i.module.module.input || - i.module.module.input === m.module.input), + ([name, instance]) => + instance.resolved.usage_ref.name === + currService.usage_ref.name && + instance.resolved.usage_ref.input === + currService.usage_ref.input, ) .map(([name, _]) => name), })), diff --git a/pkgs/clan-cli/clan_lib/services/modules.py b/pkgs/clan-cli/clan_lib/services/modules.py index 22a831ad4..ba87ad31d 100644 --- a/pkgs/clan-cli/clan_lib/services/modules.py +++ b/pkgs/clan-cli/clan_lib/services/modules.py @@ -11,6 +11,7 @@ from clan_lib.nix_models.clan import ( InventoryInstanceModule, InventoryInstanceModuleType, InventoryInstanceRolesType, + InventoryInstancesType, ) from clan_lib.persist.inventory_store import InventoryStore from clan_lib.persist.util import set_value_by_path @@ -157,7 +158,8 @@ class Module: # To use this module specify: InventoryInstanceModule :: { input, name } (foreign key) usage_ref: InventoryInstanceModule info: ModuleInfo - native: bool # Equivalent to input == None + native: bool + instance_refs: list[str] @dataclass @@ -166,6 +168,41 @@ class ClanModules: core_input_name: str +def find_instance_refs_for_module( + instances: InventoryInstancesType, + module_ref: InventoryInstanceModule, + core_input_name: str, +) -> list[str]: + """Find all usages of a given module by its module_ref + + If the module is native: + module_ref.input := None + .module.name := None + + If module is from explicit input + .module.name != None + module_ref.input could be None, if explicit input refers to a native module + + """ + res: list[str] = [] + for instance_name, instance in instances.items(): + local_ref = instance.get("module") + if not local_ref: + continue + + local_name: str = local_ref.get("name", instance_name) + local_input: str | None = local_ref.get("input") + + # Normal match + if ( + local_name == module_ref.get("name") + and local_input == module_ref.get("input") + ) or (local_input == core_input_name and local_name == module_ref.get("name")): + res.append(instance_name) + + return res + + @API.register def list_service_modules(flake: Flake) -> ClanModules: """Show information about a module""" @@ -178,6 +215,8 @@ def list_service_modules(flake: Flake) -> ClanModules: builtin_modules: dict[str, Any] = flake.select( "clanInternals.inventoryClass.staticModules" ) + inventory_store = InventoryStore(flake) + instances = inventory_store.read().get("instances", {}) first_name, first_module = next(iter(builtin_modules.items())) clan_input_name = None @@ -197,12 +236,18 @@ def list_service_modules(flake: Flake) -> ClanModules: res: list[Module] = [] for input_name, module_set in modules.items(): for module_name, module_info in module_set.items(): + module_ref = InventoryInstanceModule( + { + "name": module_name, + "input": None if input_name == clan_input_name else input_name, + } + ) res.append( Module( - usage_ref={ - "name": module_name, - "input": None if input_name == clan_input_name else input_name, - }, + instance_refs=find_instance_refs_for_module( + instances, module_ref, clan_input_name + ), + usage_ref=module_ref, info=ModuleInfo( roles=module_info.get("roles", {}), manifest=ModuleManifest.from_dict(module_info["manifest"]), diff --git a/pkgs/clan-cli/clan_lib/services/modules_test.py b/pkgs/clan-cli/clan_lib/services/modules_test.py index 0e01a1f7f..49088d7f7 100644 --- a/pkgs/clan-cli/clan_lib/services/modules_test.py +++ b/pkgs/clan-cli/clan_lib/services/modules_test.py @@ -1,15 +1,20 @@ from collections.abc import Callable +from typing import TYPE_CHECKING import pytest from clan_cli.tests.fixtures_flakes import nested_dict from clan_lib.flake.flake import Flake from clan_lib.services.modules import list_service_instances, list_service_modules +if TYPE_CHECKING: + from clan_lib.nix_models.clan import Clan + @pytest.mark.with_core def test_list_service_instances( clan_flake: Callable[..., Flake], ) -> None: + # ATTENTION! This method lacks Typechecking config = nested_dict() # explicit module selection # We use this random string in test to avoid code dependencies on the input name @@ -51,3 +56,50 @@ def test_list_service_instances( assert instances["baz"].resolved.usage_ref.get("input") is None assert instances["baz"].resolved.usage_ref.get("name") == "sshd" + + +@pytest.mark.with_core +def test_list_service_modules( + clan_flake: Callable[..., Flake], +) -> None: + # Nice! This is typechecked :) + clan_config: Clan = { + "inventory": { + "instances": { + # No module spec -> resolves to clan-core/admin + "admin": {}, + # Partial module spec + "admin2": {"module": {"name": "admin"}}, + # Full explicit module spec + "admin3": { + "module": { + "name": "admin", + "input": "Y2xhbi1jaW9yZS1uZXZlci1kZXBlbmQtb24tbWU", + } + }, + } + } + } + flake = clan_flake(clan_config) + + service_modules = list_service_modules(flake) + + # Detects the input name right + assert service_modules.core_input_name == "Y2xhbi1jaW9yZS1uZXZlci1kZXBlbmQtb24tbWU" + assert len(service_modules.modules) + + admin_service = next( + m for m in service_modules.modules if m.usage_ref.get("name") == "admin" + ) + assert admin_service + + assert admin_service.usage_ref == {"name": "admin", "input": None} + assert set(admin_service.instance_refs) == {"admin", "admin2", "admin3"} + + # Negative test: Assert not used + sshd_service = next( + m for m in service_modules.modules if m.usage_ref.get("name") == "sshd" + ) + assert sshd_service + assert sshd_service.usage_ref == {"name": "sshd", "input": None} + assert set(sshd_service.instance_refs) == set({}) From b9573636d8e9c319f63be4b557c3286b07b12947 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 31 Aug 2025 15:58:26 +0200 Subject: [PATCH 6/7] ui/modules: simplify ui logic --- .../src/workflows/Service/Service.stories.tsx | 81 ++++++++----------- .../ui/src/workflows/Service/Service.tsx | 29 +++---- 2 files changed, 43 insertions(+), 67 deletions(-) diff --git a/pkgs/clan-app/ui/src/workflows/Service/Service.stories.tsx b/pkgs/clan-app/ui/src/workflows/Service/Service.stories.tsx index ef0b5c04e..92359c054 100644 --- a/pkgs/clan-app/ui/src/workflows/Service/Service.stories.tsx +++ b/pkgs/clan-app/ui/src/workflows/Service/Service.stories.tsx @@ -24,59 +24,42 @@ const mockFetcher: Fetcher = ( ): ApiCall => { // TODO: Make this configurable for every story const resultData: Partial = { - list_service_modules: [ - { - module: { name: "Borgbackup", input: "clan-core" }, - info: { - manifest: { - name: "Borgbackup", - description: "This is module A", - }, - roles: { - client: null, - server: null, + list_service_modules: { + core_input_name: "clan-core", + modules: [ + { + usage_ref: { name: "Borgbackup", input: null }, + instance_refs: [], + native: true, + info: { + manifest: { + name: "Borgbackup", + description: "This is module A", + }, + roles: { + client: null, + server: null, + }, }, }, - }, - { - module: { name: "Zerotier", input: "clan-core" }, - info: { - manifest: { - name: "Zerotier", - description: "This is module B", - }, - roles: { - peer: null, - moon: null, - controller: null, + { + usage_ref: { name: "Zerotier", input: "fublub" }, + instance_refs: [], + native: false, + info: { + manifest: { + name: "Zerotier", + description: "This is module B", + }, + roles: { + peer: null, + moon: null, + controller: null, + }, }, }, - }, - { - module: { name: "Admin", input: "clan-core" }, - info: { - manifest: { - name: "Admin", - description: "This is module B", - }, - roles: { - default: null, - }, - }, - }, - { - module: { name: "Garage", input: "lo-l" }, - info: { - manifest: { - name: "Garage", - description: "This is module B", - }, - roles: { - default: null, - }, - }, - }, - ], + ], + }, list_machines: { jon: { name: "jon", diff --git a/pkgs/clan-app/ui/src/workflows/Service/Service.tsx b/pkgs/clan-app/ui/src/workflows/Service/Service.tsx index fc1311360..e2b74c714 100644 --- a/pkgs/clan-app/ui/src/workflows/Service/Service.tsx +++ b/pkgs/clan-app/ui/src/workflows/Service/Service.tsx @@ -45,7 +45,7 @@ import { } from "@/src/scene/highlightStore"; import { useClickOutside } from "@/src/hooks/useClickOutside"; -type ModuleItem = ServiceModules[number]; +type ModuleItem = ServiceModules["modules"][number]; interface Module { value: string; @@ -74,16 +74,7 @@ const SelectService = () => { description: currService.info.manifest.description, input: currService.usage_ref.input, raw: currService, - // TODO: include the instances that use this module - instances: Object.entries(serviceInstancesQuery.data) - .filter( - ([name, instance]) => - instance.resolved.usage_ref.name === - currService.usage_ref.name && - instance.resolved.usage_ref.input === - currService.usage_ref.input, - ) - .map(([name, _]) => name), + instances: currService.instance_refs, })), ); } @@ -98,8 +89,8 @@ const SelectService = () => { if (!module) return; set("module", { - name: module.raw.module.name, - input: module.raw.module.input, + name: module.raw.usage_ref.name, + input: module.raw.usage_ref.input, raw: module.raw, }); // TODO: Ideally we need to ask @@ -185,11 +176,13 @@ const SelectService = () => { inverted class="flex justify-between" > - + {item.description} - - by {item.input} + + + by {item.input} + @@ -540,7 +533,7 @@ export interface InventoryInstance { name: string; module: { name: string; - input?: string; + input?: string | null; }; roles: Record; } @@ -553,7 +546,7 @@ interface RoleType { export interface ServiceStoreType { module: { name: string; - input: string; + input?: string | null; raw?: ModuleItem; }; roles: Record; From 565391bd8cdc9977da8aecede2b60419f7e870fb Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 31 Aug 2025 16:14:32 +0200 Subject: [PATCH 7/7] ui/modules: deduplicate information --- pkgs/clan-app/ui/src/hooks/queries.ts | 1 + .../ui/src/workflows/Service/Service.tsx | 22 +++++++------------ 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/pkgs/clan-app/ui/src/hooks/queries.ts b/pkgs/clan-app/ui/src/hooks/queries.ts index 7450e3cc8..da81a5785 100644 --- a/pkgs/clan-app/ui/src/hooks/queries.ts +++ b/pkgs/clan-app/ui/src/hooks/queries.ts @@ -143,6 +143,7 @@ export const useMachineStateQuery = (clanURI: string, machineName: string) => { const client = useApiClient(); return useQuery(() => ({ queryKey: ["clans", encodeBase64(clanURI), "machine", machineName, "state"], + staleTime: 60_000, // 1 minute stale time queryFn: async () => { const apiCall = client.fetch("get_machine_state", { machine: { diff --git a/pkgs/clan-app/ui/src/workflows/Service/Service.tsx b/pkgs/clan-app/ui/src/workflows/Service/Service.tsx index e2b74c714..59f913b9e 100644 --- a/pkgs/clan-app/ui/src/workflows/Service/Service.tsx +++ b/pkgs/clan-app/ui/src/workflows/Service/Service.tsx @@ -49,11 +49,8 @@ type ModuleItem = ServiceModules["modules"][number]; interface Module { value: string; - input?: string | null; label: string; - description: string; raw: ModuleItem; - instances: string[]; } const SelectService = () => { @@ -70,11 +67,8 @@ const SelectService = () => { setModuleOptions( serviceModulesQuery.data.modules.map((currService) => ({ value: `${currService.usage_ref.name}:${currService.usage_ref.input}`, - label: currService.info.manifest.name, - description: currService.info.manifest.description, - input: currService.usage_ref.input, + label: currService.usage_ref.name, raw: currService, - instances: currService.instance_refs, })), ); } @@ -100,14 +94,14 @@ const SelectService = () => { // For now: // Create a new instance, if there are no instances yet // Update the first instance, if there is one - if (module.instances.length === 0) { + if (module.raw.instance_refs.length === 0) { set("action", "create"); } else { if (!serviceInstancesQuery.data) return; if (!machinesQuery.data) return; set("action", "update"); - const instanceName = module.instances[0]; + const instanceName = module.raw.instance_refs[0]; const instance = serviceInstancesQuery.data[instanceName]; console.log("Editing existing instance", module); @@ -157,7 +151,7 @@ const SelectService = () => {
- 0}> + 0}>
Added @@ -176,12 +170,12 @@ const SelectService = () => { inverted class="flex justify-between" > - - {item.description} + + {item.raw.info.manifest.description} - + - by {item.input} + by {item.raw.usage_ref.input}