Merge pull request 'api/modules: remove redundant localModules' (#4322) from api-modules into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4322

docs/nix/render_options/__init__.py

@@ -29,13 +29,13 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
 
-from clan_lib.api.modules import (
+from clan_lib.errors import ClanError
+from clan_lib.services.modules import (
     CategoryInfo,
     Frontmatter,
     extract_frontmatter,
     get_roles,
 )
-from clan_lib.errors import ClanError
 
 # Get environment variables
 CLAN_CORE_PATH = Path(os.environ["CLAN_CORE_PATH"])

lib/modules/inventoryClass/service-list-from-inputs.nix

@@ -35,11 +35,6 @@ in
         inputName: v: lib.mapAttrs (inspectModule inputName) v.clan.modules
       ) inputsWithModules;
   };
-  options.localModules = lib.mkOption {
-    readOnly = true;
-    type = lib.types.raw;
-    default = config.modulesPerSource.self;
-  };
   options.templatesPerSource = lib.mkOption {
     # { sourceName :: { moduleName :: {} }}
     readOnly = true;

pkgs/clan-cli/clan_cli/templates/apply_disk.py

@@ -3,8 +3,8 @@ import logging
 from collections.abc import Sequence
 from typing import Any
 
-from clan_lib.api.disk import set_machine_disk_schema
 from clan_lib.machines.machines import Machine
+from clan_lib.templates.disk import set_machine_disk_schema
 
 from clan_cli.completions import (
     add_dynamic_completer,

pkgs/clan-cli/clan_cli/tests/test_modules.py

@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING
 import pytest
 from clan_cli.machines.create import CreateOptions, create_machine
 from clan_cli.tests.fixtures_flakes import FlakeForTest
-from clan_lib.api.modules import list_modules
 from clan_lib.flake import Flake
 from clan_lib.nix import nix_eval, run
 from clan_lib.nix_models.clan import (
@@ -16,6 +15,7 @@ from clan_lib.nix_models.clan import (
 )
 from clan_lib.persist.inventory_store import InventoryStore
 from clan_lib.persist.util import set_value_by_path
+from clan_lib.services.modules import list_service_modules
 
 if TYPE_CHECKING:
     from .age_keys import KeyPair
@@ -27,10 +27,9 @@ from clan_lib.machines.machines import Machine as MachineMachine
 @pytest.mark.with_core
 def test_list_modules(test_flake_with_core: FlakeForTest) -> None:
     base_path = test_flake_with_core.path
-    modules_info = list_modules(str(base_path))
+    modules_info = list_service_modules(Flake(str(base_path)))
 
-    assert "localModules" in modules_info
-    assert "modulesPerSource" in modules_info
+    assert "modules" in modules_info
 
 
 @pytest.mark.impure

pkgs/clan-cli/clan_lib/machines/list.py

@@ -5,13 +5,13 @@ from dataclasses import dataclass
 from clan_cli.machines.hardware import HardwareConfig
 
 from clan_lib.api import API
-from clan_lib.api.disk import MachineDiskMatter
-from clan_lib.api.modules import parse_frontmatter
 from clan_lib.dirs import specific_machine_dir
 from clan_lib.flake import Flake
 from clan_lib.machines.actions import get_machine, list_machines
 from clan_lib.machines.machines import Machine
 from clan_lib.nix_models.clan import InventoryMachine
+from clan_lib.services.modules import parse_frontmatter
+from clan_lib.templates.disk import MachineDiskMatter
 
 log = logging.getLogger(__name__)
 

pkgs/clan-cli/clan_lib/services/instances.py

@@ -0,0 +1,133 @@
+from typing import TypedDict
+
+from clan_lib.api import API
+from clan_lib.errors import ClanError
+from clan_lib.flake.flake import Flake
+from clan_lib.nix_models.clan import (
+    InventoryInstanceModule,
+    InventoryInstanceRolesType,
+    InventoryInstancesType,
+    InventoryMachinesType,
+)
+from clan_lib.persist.inventory_store import InventoryStore
+from clan_lib.persist.util import set_value_by_path
+from clan_lib.services.modules import list_service_modules
+
+# TODO: move imports out of cli/__init__.py causing import cycles
+# from clan_lib.machines.actions import list_machines
+
+
+@API.register
+def list_service_instances(flake: Flake) -> InventoryInstancesType:
+    """
+    Returns all currently present service instances including their full configuration
+    """
+
+    inventory_store = InventoryStore(flake)
+    inventory = inventory_store.read()
+    instances = inventory.get("instances", {})
+    return instances
+
+
+def collect_tags(machines: InventoryMachinesType) -> set[str]:
+    res = set()
+    for _, machine in machines.items():
+        res |= set(machine.get("tags", []))
+
+    return res
+
+
+# Removed 'module' ref - Needs to be passed seperately
+class InstanceConfig(TypedDict):
+    roles: InventoryInstanceRolesType
+
+
+@API.register
+def create_service_instance(
+    flake: Flake,
+    module_ref: InventoryInstanceModule,
+    instance_name: str,
+    instance_config: InstanceConfig,
+) -> None:
+    # TODO: Should take a flake
+    avilable_modules = list_service_modules(flake)
+
+    input_ref = module_ref.get("input", None)
+    if input_ref is None:
+        msg = "Setting module_ref.input is currently required"
+        raise ClanError(msg)
+
+    module_set = avilable_modules.get("modules", {}).get(input_ref)
+
+    if module_set is None:
+        msg = f"module set for input '{input_ref}' not found"
+        msg += f"\nAvilable input_refs: {avilable_modules.get('modules', {}).keys()}"
+        raise ClanError(msg)
+
+    module_name = module_ref.get("name")
+    assert module_name
+    module = module_set.get(module_name)
+    if module is None:
+        msg = f"module with name '{module_name}' not found"
+        raise ClanError(msg)
+
+    inventory_store = InventoryStore(flake)
+    inventory = inventory_store.read()
+
+    instances = inventory.get("instances", {})
+    if instance_name in instances:
+        msg = f"service instance '{instance_name}' already exists."
+        raise ClanError(msg)
+
+    target_roles = instance_config.get("roles")
+    if not target_roles:
+        msg = "Creating a service instance requires adding roles"
+        raise ClanError(msg)
+
+    available_roles = set(module.get("roles", {}).keys())
+
+    unavailable_roles = list(filter(lambda r: r not in available_roles, target_roles))
+    if unavailable_roles:
+        msg = f"Unknown roles: {unavailable_roles}. Use one of {available_roles}"
+        raise ClanError(msg)
+
+    role_configs = instance_config.get("roles")
+    if not role_configs:
+        return
+
+    ## Validate machine references
+    all_machines = inventory.get("machines", {})
+    available_machine_refs = set(all_machines.keys())
+    available_tag_refs = collect_tags(all_machines)
+
+    for role_name, role_members in role_configs.items():
+        machine_refs = role_members.get("machines")
+        msg = f"Role: '{role_name}' - "
+        if machine_refs:
+            unavailable_machines = list(
+                filter(lambda m: m not in available_machine_refs, machine_refs)
+            )
+            if unavailable_machines:
+                msg += f"Unknown machine reference: {unavailable_machines}. Use one of {available_machine_refs}"
+                raise ClanError(msg)
+
+        tag_refs = role_members.get("tags")
+        if tag_refs:
+            unavailable_tags = list(
+                filter(lambda m: m not in available_tag_refs, tag_refs)
+            )
+
+            if unavailable_tags:
+                msg += (
+                    f"Unknown tags: {unavailable_tags}. Use one of {available_tag_refs}"
+                )
+                raise ClanError(msg)
+
+    # TODO:
+    # Validate instance_config roles settings against role schema
+
+    set_value_by_path(inventory, f"instances.{instance_name}", instance_config)
+    set_value_by_path(inventory, f"instances.{instance_name}.module", module_ref)
+    inventory_store.write(
+        inventory, message=f"services: instance '{instance_name}' init"
+    )

pkgs/clan-cli/clan_lib/services/modules.py

@@ -4,11 +4,10 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, TypedDict
 
+from clan_lib.api import API
 from clan_lib.errors import ClanError
 from clan_lib.flake import Flake
 
-from . import API
-
 
 class CategoryInfo(TypedDict):
     color: str
@@ -154,22 +153,18 @@ class ModuleInfo(TypedDict):
     roles: dict[str, None]
 
 
-class ModuleLists(TypedDict):
-    modulesPerSource: dict[str, dict[str, ModuleInfo]]
-    localModules: dict[str, ModuleInfo]
+class ModuleList(TypedDict):
+    modules: dict[str, dict[str, ModuleInfo]]
 
 
 @API.register
-def list_modules(base_path: str) -> ModuleLists:
+def list_service_modules(flake: Flake) -> ModuleList:
     """
     Show information about a module
     """
-    flake = Flake(base_path)
-    modules = flake.select(
-        "clanInternals.inventoryClass.{?modulesPerSource,?localModules}"
-    )
+    modules = flake.select("clanInternals.inventoryClass.modulesPerSource")
 
-    return modules
+    return ModuleList({"modules": modules})
 
 
 @dataclass

pkgs/clan-cli/clan_lib/templates/disk.py

@@ -6,12 +6,12 @@ from typing import Any, TypedDict
 from uuid import uuid4
 
 from clan_lib.api import API
-from clan_lib.api.modules import Frontmatter, extract_frontmatter
 from clan_lib.dirs import TemplateType, clan_templates
 from clan_lib.errors import ClanError
 from clan_lib.git import commit_file
 from clan_lib.machines.hardware import HardwareConfig, get_machine_hardware_config
 from clan_lib.machines.machines import Machine
+from clan_lib.services.modules import Frontmatter, extract_frontmatter
 
 log = logging.getLogger(__name__)
 

pkgs/clan-cli/clan_lib/tests/test_create.py

@@ -16,8 +16,6 @@ from clan_cli.secrets.sops import maybe_get_admin_public_keys
 from clan_cli.secrets.users import add_user
 from clan_cli.vars.generate import get_generators, run_generators
 
-from clan_lib.api.disk import hw_main_disk_options, set_machine_disk_schema
-from clan_lib.api.modules import list_modules
 from clan_lib.cmd import RunOpts, run
 from clan_lib.dirs import specific_machine_dir
 from clan_lib.errors import ClanError
@@ -33,7 +31,9 @@ from clan_lib.nix_models.clan import (
 from clan_lib.nix_models.clan import InventoryMachineDeploy as MachineDeploy
 from clan_lib.persist.inventory_store import InventoryStore
 from clan_lib.persist.util import set_value_by_path
+from clan_lib.services.modules import list_service_modules
 from clan_lib.ssh.remote import Remote, check_machine_ssh_login
+from clan_lib.templates.disk import hw_main_disk_options, set_machine_disk_schema
 
 log = logging.getLogger(__name__)
 
@@ -206,9 +206,9 @@ def test_clan_create_api(
     store = InventoryStore(clan_dir_flake)
     inventory = store.read()
 
-    modules = list_modules(str(clan_dir_flake.path))
+    modules = list_service_modules(clan_dir_flake)
     assert (
-        modules["modulesPerSource"]["clan-core"]["admin"]["manifest"]["name"]
+        modules["modules"]["clan-core"]["admin"]["manifest"]["name"]
         == "clan-core/admin"
     )
 

pkgs/clan-cli/openapi.py

@@ -41,7 +41,7 @@ TOP_LEVEL_RESOURCES = {
     "secret",  # sops & key operations
     "log",  # log operations
     "generator",  # vars generators operations
-    "module",  # module (clan.service) management
+    "service",  # clan.service management
     "system",  # system operations
 }