From 5f19e76cd0f085ce29a1d428da520a37f876b37f Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 13 Jul 2025 11:52:13 +0200 Subject: [PATCH 1/6] api/modules: remove redundant localModules --- .../inventoryClass/service-list-from-inputs.nix | 5 ----- pkgs/clan-cli/clan_cli/tests/test_modules.py | 3 +-- pkgs/clan-cli/clan_lib/api/modules.py | 13 +++++-------- pkgs/clan-cli/clan_lib/tests/test_create.py | 2 +- 4 files changed, 7 insertions(+), 16 deletions(-) diff --git a/lib/modules/inventoryClass/service-list-from-inputs.nix b/lib/modules/inventoryClass/service-list-from-inputs.nix index 7e913bed1..7e5e91585 100644 --- a/lib/modules/inventoryClass/service-list-from-inputs.nix +++ b/lib/modules/inventoryClass/service-list-from-inputs.nix @@ -35,11 +35,6 @@ in inputName: v: lib.mapAttrs (inspectModule inputName) v.clan.modules ) inputsWithModules; }; - options.localModules = lib.mkOption { - readOnly = true; - type = lib.types.raw; - default = config.modulesPerSource.self; - }; options.templatesPerSource = lib.mkOption { # { sourceName :: { moduleName :: {} }} readOnly = true; diff --git a/pkgs/clan-cli/clan_cli/tests/test_modules.py b/pkgs/clan-cli/clan_cli/tests/test_modules.py index 94de8a815..d8374ba9a 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_modules.py +++ b/pkgs/clan-cli/clan_cli/tests/test_modules.py @@ -29,8 +29,7 @@ def test_list_modules(test_flake_with_core: FlakeForTest) -> None: base_path = test_flake_with_core.path modules_info = list_modules(str(base_path)) - assert "localModules" in modules_info - assert "modulesPerSource" in modules_info + assert "modules" in modules_info @pytest.mark.impure diff --git a/pkgs/clan-cli/clan_lib/api/modules.py b/pkgs/clan-cli/clan_lib/api/modules.py index 7e306a2c0..7ed03d95f 100644 --- a/pkgs/clan-cli/clan_lib/api/modules.py +++ b/pkgs/clan-cli/clan_lib/api/modules.py @@ -154,22 +154,19 @@ class ModuleInfo(TypedDict): roles: dict[str, None] -class ModuleLists(TypedDict): - modulesPerSource: dict[str, dict[str, ModuleInfo]] - localModules: dict[str, ModuleInfo] +class ModuleList(TypedDict): + modules: dict[str, dict[str, ModuleInfo]] @API.register -def list_modules(base_path: str) -> ModuleLists: +def list_modules(base_path: str) -> ModuleList: """ Show information about a module """ flake = Flake(base_path) - modules = flake.select( - "clanInternals.inventoryClass.{?modulesPerSource,?localModules}" - ) + modules = flake.select("clanInternals.inventoryClass.modulesPerSource") - return modules + return ModuleList({"modules": modules}) @dataclass diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index 4e54ef904..210b3bb21 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -208,7 +208,7 @@ def test_clan_create_api( modules = list_modules(str(clan_dir_flake.path)) assert ( - modules["modulesPerSource"]["clan-core"]["admin"]["manifest"]["name"] + modules["modules"]["clan-core"]["admin"]["manifest"]["name"] == "clan-core/admin" ) From 943dde4bbf00b04a48f50abac4ad40f5db80b159 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 13 Jul 2025 11:57:27 +0200 Subject: [PATCH 2/6] lib/modules: move from api to services module --- docs/nix/render_options/__init__.py | 4 ++-- pkgs/clan-cli/clan_cli/tests/test_modules.py | 2 +- pkgs/clan-cli/clan_lib/machines/list.py | 2 +- pkgs/clan-cli/clan_lib/{api => services}/modules.py | 3 +-- pkgs/clan-cli/clan_lib/tests/test_create.py | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) rename pkgs/clan-cli/clan_lib/{api => services}/modules.py (99%) diff --git a/docs/nix/render_options/__init__.py b/docs/nix/render_options/__init__.py index 15109a7ee..aa5da870d 100644 --- a/docs/nix/render_options/__init__.py +++ b/docs/nix/render_options/__init__.py @@ -29,13 +29,13 @@ from dataclasses import dataclass, field from pathlib import Path from typing import Any -from clan_lib.api.modules import ( +from clan_lib.errors import ClanError +from clan_lib.services.modules import ( CategoryInfo, Frontmatter, extract_frontmatter, get_roles, ) -from clan_lib.errors import ClanError # Get environment variables CLAN_CORE_PATH = Path(os.environ["CLAN_CORE_PATH"]) diff --git a/pkgs/clan-cli/clan_cli/tests/test_modules.py b/pkgs/clan-cli/clan_cli/tests/test_modules.py index d8374ba9a..c192d929d 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_modules.py +++ b/pkgs/clan-cli/clan_cli/tests/test_modules.py @@ -5,7 +5,6 @@ from typing import TYPE_CHECKING import pytest from clan_cli.machines.create import CreateOptions, create_machine from clan_cli.tests.fixtures_flakes import FlakeForTest -from clan_lib.api.modules import list_modules from clan_lib.flake import Flake from clan_lib.nix import nix_eval, run from clan_lib.nix_models.clan import ( @@ -16,6 +15,7 @@ from clan_lib.nix_models.clan import ( ) from clan_lib.persist.inventory_store import InventoryStore from clan_lib.persist.util import set_value_by_path +from clan_lib.services.modules import list_modules if TYPE_CHECKING: from .age_keys import KeyPair diff --git a/pkgs/clan-cli/clan_lib/machines/list.py b/pkgs/clan-cli/clan_lib/machines/list.py index 7fbe4b3ac..dd2b002b8 100644 --- a/pkgs/clan-cli/clan_lib/machines/list.py +++ b/pkgs/clan-cli/clan_lib/machines/list.py @@ -6,12 +6,12 @@ from clan_cli.machines.hardware import HardwareConfig from clan_lib.api import API from clan_lib.api.disk import MachineDiskMatter -from clan_lib.api.modules import parse_frontmatter from clan_lib.dirs import specific_machine_dir from clan_lib.flake import Flake from clan_lib.machines.actions import get_machine, list_machines from clan_lib.machines.machines import Machine from clan_lib.nix_models.clan import InventoryMachine +from clan_lib.services.modules import parse_frontmatter log = logging.getLogger(__name__) diff --git a/pkgs/clan-cli/clan_lib/api/modules.py b/pkgs/clan-cli/clan_lib/services/modules.py similarity index 99% rename from pkgs/clan-cli/clan_lib/api/modules.py rename to pkgs/clan-cli/clan_lib/services/modules.py index 7ed03d95f..934d2d43a 100644 --- a/pkgs/clan-cli/clan_lib/api/modules.py +++ b/pkgs/clan-cli/clan_lib/services/modules.py @@ -4,11 +4,10 @@ from dataclasses import dataclass, field from pathlib import Path from typing import Any, TypedDict +from clan_lib.api import API from clan_lib.errors import ClanError from clan_lib.flake import Flake -from . import API - class CategoryInfo(TypedDict): color: str diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index 210b3bb21..7f0efcd1d 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -17,7 +17,6 @@ from clan_cli.secrets.users import add_user from clan_cli.vars.generate import get_generators, run_generators from clan_lib.api.disk import hw_main_disk_options, set_machine_disk_schema -from clan_lib.api.modules import list_modules from clan_lib.cmd import RunOpts, run from clan_lib.dirs import specific_machine_dir from clan_lib.errors import ClanError @@ -33,6 +32,7 @@ from clan_lib.nix_models.clan import ( from clan_lib.nix_models.clan import InventoryMachineDeploy as MachineDeploy from clan_lib.persist.inventory_store import InventoryStore from clan_lib.persist.util import set_value_by_path +from clan_lib.services.modules import list_modules from clan_lib.ssh.remote import Remote, check_machine_ssh_login log = logging.getLogger(__name__) From ceb0221daf55739ed61d89d671fb2dcdcb21b4d8 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 13 Jul 2025 12:01:58 +0200 Subject: [PATCH 3/6] lib/disks: move from api to templates --- pkgs/clan-cli/clan_cli/templates/apply_disk.py | 2 +- pkgs/clan-cli/clan_lib/machines/list.py | 2 +- pkgs/clan-cli/clan_lib/{api => templates}/disk.py | 2 +- pkgs/clan-cli/clan_lib/tests/test_create.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename pkgs/clan-cli/clan_lib/{api => templates}/disk.py (99%) diff --git a/pkgs/clan-cli/clan_cli/templates/apply_disk.py b/pkgs/clan-cli/clan_cli/templates/apply_disk.py index 1836a63a2..1f85c199f 100644 --- a/pkgs/clan-cli/clan_cli/templates/apply_disk.py +++ b/pkgs/clan-cli/clan_cli/templates/apply_disk.py @@ -3,8 +3,8 @@ import logging from collections.abc import Sequence from typing import Any -from clan_lib.api.disk import set_machine_disk_schema from clan_lib.machines.machines import Machine +from clan_lib.templates.disk import set_machine_disk_schema from clan_cli.completions import ( add_dynamic_completer, diff --git a/pkgs/clan-cli/clan_lib/machines/list.py b/pkgs/clan-cli/clan_lib/machines/list.py index dd2b002b8..333b71141 100644 --- a/pkgs/clan-cli/clan_lib/machines/list.py +++ b/pkgs/clan-cli/clan_lib/machines/list.py @@ -5,13 +5,13 @@ from dataclasses import dataclass from clan_cli.machines.hardware import HardwareConfig from clan_lib.api import API -from clan_lib.api.disk import MachineDiskMatter from clan_lib.dirs import specific_machine_dir from clan_lib.flake import Flake from clan_lib.machines.actions import get_machine, list_machines from clan_lib.machines.machines import Machine from clan_lib.nix_models.clan import InventoryMachine from clan_lib.services.modules import parse_frontmatter +from clan_lib.templates.disk import MachineDiskMatter log = logging.getLogger(__name__) diff --git a/pkgs/clan-cli/clan_lib/api/disk.py b/pkgs/clan-cli/clan_lib/templates/disk.py similarity index 99% rename from pkgs/clan-cli/clan_lib/api/disk.py rename to pkgs/clan-cli/clan_lib/templates/disk.py index 207fccefc..49a897bc0 100644 --- a/pkgs/clan-cli/clan_lib/api/disk.py +++ b/pkgs/clan-cli/clan_lib/templates/disk.py @@ -6,12 +6,12 @@ from typing import Any, TypedDict from uuid import uuid4 from clan_lib.api import API -from clan_lib.api.modules import Frontmatter, extract_frontmatter from clan_lib.dirs import TemplateType, clan_templates from clan_lib.errors import ClanError from clan_lib.git import commit_file from clan_lib.machines.hardware import HardwareConfig, get_machine_hardware_config from clan_lib.machines.machines import Machine +from clan_lib.services.modules import Frontmatter, extract_frontmatter log = logging.getLogger(__name__) diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index 7f0efcd1d..23c38a1af 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -16,7 +16,6 @@ from clan_cli.secrets.sops import maybe_get_admin_public_keys from clan_cli.secrets.users import add_user from clan_cli.vars.generate import get_generators, run_generators -from clan_lib.api.disk import hw_main_disk_options, set_machine_disk_schema from clan_lib.cmd import RunOpts, run from clan_lib.dirs import specific_machine_dir from clan_lib.errors import ClanError @@ -34,6 +33,7 @@ from clan_lib.persist.inventory_store import InventoryStore from clan_lib.persist.util import set_value_by_path from clan_lib.services.modules import list_modules from clan_lib.ssh.remote import Remote, check_machine_ssh_login +from clan_lib.templates.disk import hw_main_disk_options, set_machine_disk_schema log = logging.getLogger(__name__) From 079f5d10339902f05fd1032ae9101031e9261d03 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 13 Jul 2025 12:04:23 +0200 Subject: [PATCH 4/6] lib/modules: rename 'list_modules' to 'list_service_modules' --- pkgs/clan-cli/clan_cli/tests/test_modules.py | 4 ++-- pkgs/clan-cli/clan_lib/services/modules.py | 2 +- pkgs/clan-cli/clan_lib/tests/test_create.py | 4 ++-- pkgs/clan-cli/openapi.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/tests/test_modules.py b/pkgs/clan-cli/clan_cli/tests/test_modules.py index c192d929d..339887883 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_modules.py +++ b/pkgs/clan-cli/clan_cli/tests/test_modules.py @@ -15,7 +15,7 @@ from clan_lib.nix_models.clan import ( ) from clan_lib.persist.inventory_store import InventoryStore from clan_lib.persist.util import set_value_by_path -from clan_lib.services.modules import list_modules +from clan_lib.services.modules import list_service_modules if TYPE_CHECKING: from .age_keys import KeyPair @@ -27,7 +27,7 @@ from clan_lib.machines.machines import Machine as MachineMachine @pytest.mark.with_core def test_list_modules(test_flake_with_core: FlakeForTest) -> None: base_path = test_flake_with_core.path - modules_info = list_modules(str(base_path)) + modules_info = list_service_modules(str(base_path)) assert "modules" in modules_info diff --git a/pkgs/clan-cli/clan_lib/services/modules.py b/pkgs/clan-cli/clan_lib/services/modules.py index 934d2d43a..ee058905b 100644 --- a/pkgs/clan-cli/clan_lib/services/modules.py +++ b/pkgs/clan-cli/clan_lib/services/modules.py @@ -158,7 +158,7 @@ class ModuleList(TypedDict): @API.register -def list_modules(base_path: str) -> ModuleList: +def list_service_modules(base_path: str) -> ModuleList: """ Show information about a module """ diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index 23c38a1af..26ecaa9c8 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -31,7 +31,7 @@ from clan_lib.nix_models.clan import ( from clan_lib.nix_models.clan import InventoryMachineDeploy as MachineDeploy from clan_lib.persist.inventory_store import InventoryStore from clan_lib.persist.util import set_value_by_path -from clan_lib.services.modules import list_modules +from clan_lib.services.modules import list_service_modules from clan_lib.ssh.remote import Remote, check_machine_ssh_login from clan_lib.templates.disk import hw_main_disk_options, set_machine_disk_schema @@ -206,7 +206,7 @@ def test_clan_create_api( store = InventoryStore(clan_dir_flake) inventory = store.read() - modules = list_modules(str(clan_dir_flake.path)) + modules = list_service_modules(str(clan_dir_flake.path)) assert ( modules["modules"]["clan-core"]["admin"]["manifest"]["name"] == "clan-core/admin" diff --git a/pkgs/clan-cli/openapi.py b/pkgs/clan-cli/openapi.py index f903deda6..f052aebb1 100644 --- a/pkgs/clan-cli/openapi.py +++ b/pkgs/clan-cli/openapi.py @@ -41,7 +41,7 @@ TOP_LEVEL_RESOURCES = { "secret", # sops & key operations "log", # log operations "generator", # vars generators operations - "module", # module (clan.service) management + "service", # clan.service management "system", # system operations } From 6d2f522cbb9f82900517f6d54dd2c3f4ddde3d3e Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 13 Jul 2025 13:53:58 +0200 Subject: [PATCH 5/6] lib/modules: list modules consistent argument --- pkgs/clan-cli/clan_cli/tests/test_modules.py | 2 +- pkgs/clan-cli/clan_lib/services/modules.py | 3 +-- pkgs/clan-cli/clan_lib/tests/test_create.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/tests/test_modules.py b/pkgs/clan-cli/clan_cli/tests/test_modules.py index 339887883..22860a45a 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_modules.py +++ b/pkgs/clan-cli/clan_cli/tests/test_modules.py @@ -27,7 +27,7 @@ from clan_lib.machines.machines import Machine as MachineMachine @pytest.mark.with_core def test_list_modules(test_flake_with_core: FlakeForTest) -> None: base_path = test_flake_with_core.path - modules_info = list_service_modules(str(base_path)) + modules_info = list_service_modules(Flake(str(base_path))) assert "modules" in modules_info diff --git a/pkgs/clan-cli/clan_lib/services/modules.py b/pkgs/clan-cli/clan_lib/services/modules.py index ee058905b..468ede8e4 100644 --- a/pkgs/clan-cli/clan_lib/services/modules.py +++ b/pkgs/clan-cli/clan_lib/services/modules.py @@ -158,11 +158,10 @@ class ModuleList(TypedDict): @API.register -def list_service_modules(base_path: str) -> ModuleList: +def list_service_modules(flake: Flake) -> ModuleList: """ Show information about a module """ - flake = Flake(base_path) modules = flake.select("clanInternals.inventoryClass.modulesPerSource") return ModuleList({"modules": modules}) diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index 26ecaa9c8..d31fd88d5 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -206,7 +206,7 @@ def test_clan_create_api( store = InventoryStore(clan_dir_flake) inventory = store.read() - modules = list_service_modules(str(clan_dir_flake.path)) + modules = list_service_modules(clan_dir_flake) assert ( modules["modules"]["clan-core"]["admin"]["manifest"]["name"] == "clan-core/admin" From 9c7f684ec5f9e57ef7eb3f9cbfc34c3791bba5f5 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 13 Jul 2025 13:54:29 +0200 Subject: [PATCH 6/6] instances: create_service_instance init --- pkgs/clan-cli/clan_lib/services/instances.py | 133 +++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 pkgs/clan-cli/clan_lib/services/instances.py diff --git a/pkgs/clan-cli/clan_lib/services/instances.py b/pkgs/clan-cli/clan_lib/services/instances.py new file mode 100644 index 000000000..b1d2c35c5 --- /dev/null +++ b/pkgs/clan-cli/clan_lib/services/instances.py @@ -0,0 +1,133 @@ +from typing import TypedDict + +from clan_lib.api import API +from clan_lib.errors import ClanError +from clan_lib.flake.flake import Flake +from clan_lib.nix_models.clan import ( + InventoryInstanceModule, + InventoryInstanceRolesType, + InventoryInstancesType, + InventoryMachinesType, +) +from clan_lib.persist.inventory_store import InventoryStore +from clan_lib.persist.util import set_value_by_path +from clan_lib.services.modules import list_service_modules + +# TODO: move imports out of cli/__init__.py causing import cycles +# from clan_lib.machines.actions import list_machines + + +@API.register +def list_service_instances(flake: Flake) -> InventoryInstancesType: + """ + Returns all currently present service instances including their full configuration + """ + + inventory_store = InventoryStore(flake) + inventory = inventory_store.read() + instances = inventory.get("instances", {}) + return instances + + +def collect_tags(machines: InventoryMachinesType) -> set[str]: + res = set() + for _, machine in machines.items(): + res |= set(machine.get("tags", [])) + + return res + + +# Removed 'module' ref - Needs to be passed seperately +class InstanceConfig(TypedDict): + roles: InventoryInstanceRolesType + + +@API.register +def create_service_instance( + flake: Flake, + module_ref: InventoryInstanceModule, + instance_name: str, + instance_config: InstanceConfig, +) -> None: + # TODO: Should take a flake + avilable_modules = list_service_modules(flake) + + input_ref = module_ref.get("input", None) + if input_ref is None: + msg = "Setting module_ref.input is currently required" + raise ClanError(msg) + + module_set = avilable_modules.get("modules", {}).get(input_ref) + + if module_set is None: + msg = f"module set for input '{input_ref}' not found" + msg += f"\nAvilable input_refs: {avilable_modules.get('modules', {}).keys()}" + raise ClanError(msg) + + module_name = module_ref.get("name") + assert module_name + module = module_set.get(module_name) + if module is None: + msg = f"module with name '{module_name}' not found" + raise ClanError(msg) + + inventory_store = InventoryStore(flake) + inventory = inventory_store.read() + + instances = inventory.get("instances", {}) + if instance_name in instances: + msg = f"service instance '{instance_name}' already exists." + raise ClanError(msg) + + target_roles = instance_config.get("roles") + if not target_roles: + msg = "Creating a service instance requires adding roles" + raise ClanError(msg) + + available_roles = set(module.get("roles", {}).keys()) + + unavailable_roles = list(filter(lambda r: r not in available_roles, target_roles)) + if unavailable_roles: + msg = f"Unknown roles: {unavailable_roles}. Use one of {available_roles}" + raise ClanError(msg) + + role_configs = instance_config.get("roles") + if not role_configs: + return + + ## Validate machine references + all_machines = inventory.get("machines", {}) + available_machine_refs = set(all_machines.keys()) + available_tag_refs = collect_tags(all_machines) + + for role_name, role_members in role_configs.items(): + machine_refs = role_members.get("machines") + msg = f"Role: '{role_name}' - " + if machine_refs: + unavailable_machines = list( + filter(lambda m: m not in available_machine_refs, machine_refs) + ) + if unavailable_machines: + msg += f"Unknown machine reference: {unavailable_machines}. Use one of {available_machine_refs}" + raise ClanError(msg) + + tag_refs = role_members.get("tags") + if tag_refs: + unavailable_tags = list( + filter(lambda m: m not in available_tag_refs, tag_refs) + ) + + if unavailable_tags: + msg += ( + f"Unknown tags: {unavailable_tags}. Use one of {available_tag_refs}" + ) + raise ClanError(msg) + + # TODO: + # Validate instance_config roles settings against role schema + + set_value_by_path(inventory, f"instances.{instance_name}", instance_config) + set_value_by_path(inventory, f"instances.{instance_name}.module", module_ref) + inventory_store.write( + inventory, message=f"services: instance '{instance_name}' init" + )