Inventory: fix persistence

pkgs/clan-cli/clan_cli/api/modules.py

@@ -3,16 +3,13 @@ import re
 import tomllib
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, TypedDict, get_args, get_type_hints
+from typing import Any, TypedDict
 
 from clan_cli.cmd import run_no_stdout
 from clan_cli.errors import ClanCmdError, ClanError
-from clan_cli.inventory import Inventory, load_inventory_json, set_inventory
-from clan_cli.inventory.classes import Service
 from clan_cli.nix import nix_eval
 
 from . import API
-from .serde import from_dict
 
 
 class CategoryInfo(TypedDict):
@@ -247,42 +244,3 @@ def get_module_info(
         features=["inventory"] if has_inventory_feature(module_path) else [],
         constraints=frontmatter.constraints,
     )
-
-
-@API.register
-def get_inventory(base_path: str) -> Inventory:
-    return load_inventory_json(base_path)
-
-
-@API.register
-def set_service_instance(
-    base_path: str, module_name: str, instance_name: str, config: dict[str, Any]
-) -> None:
-    """
-    A function that allows to set any service instance in the inventory.
-    Takes any untyped dict. The dict is then checked and converted into the correct type using the type hints of the service.
-    If any conversion error occurs, the function will raise an error.
-    """
-    service_keys = get_type_hints(Service).keys()
-
-    if module_name not in service_keys:
-        msg = f"{module_name} is not a valid Service attribute. Expected one of {', '.join(service_keys)}."
-        raise ClanError(msg)
-
-    inventory = load_inventory_json(base_path)
-    target_type = get_args(get_type_hints(Service)[module_name])[1]
-
-    module_instance_map: dict[str, Any] = inventory.get("services", {}).get(
-        module_name, {}
-    )
-
-    module_instance_map[instance_name] = from_dict(target_type, config)
-
-    inventory["services"] = inventory.get("services", {})
-    inventory["services"][module_name] = module_instance_map
-
-    set_inventory(
-        inventory, base_path, f"Update {module_name} instance {instance_name}"
-    )
-
-    # TODO: Add a check that rolls back the inventory if the service config is not valid or causes conflicts.

pkgs/clan-cli/clan_cli/inventory/__init__.py

@@ -48,16 +48,11 @@ __all__ = [
 ]
 
 
-def get_inventory_path(flake_dir: str | Path, create: bool = True) -> Path:
+def get_inventory_path(flake_dir: str | Path) -> Path:
     """
     Get the path to the inventory file in the flake directory
     """
     inventory_file = (Path(flake_dir) / "inventory.json").resolve()
-
-    if not inventory_file.exists() and create:
-        # Copy over the meta from the flake if the inventory is not initialized
-        init_inventory(str(flake_dir))
-
     return inventory_file
 
 
@@ -174,7 +169,7 @@ def calc_patches(
         key "machines.machine1.deploy.targetHost" is specified but writeability is only defined for "machines"
         We pop the last key and check if the parent key is writeable/non-writeable.
         """
-        remaining = key.split(".")[:-1]
+        remaining = key.split(".")
         while remaining:
             if ".".join(remaining) in writeables["writeable"]:
                 return True
@@ -182,7 +177,9 @@ def calc_patches(
                 return False
 
             remaining.pop()
-        raise ClanError(f"Cannot determine writeability for key '{key}'")
+
+        msg = f"Cannot determine writeability for key '{key}'"
+        raise ClanError(msg, description="F001")
 
     patchset = {}
     for update_key, update_data in update_flat.items():
@@ -342,12 +339,18 @@ def get_inventory_current_priority(flake_dir: str | Path) -> dict:
 @API.register
 def load_inventory_json(flake_dir: str | Path) -> Inventory:
     """
-    Load the inventory file from the flake directory
-    If no file is found, returns the default inventory
+    Load the inventory FILE from the flake directory
+    If no file is found, returns an empty dictionary
+
+    DO NOT USE THIS FUNCTION TO READ THE INVENTORY
+
+    Use load_inventory_eval instead
     """
 
     inventory_file = get_inventory_path(flake_dir)
 
+    if not inventory_file.exists():
+        return {}
     with inventory_file.open() as f:
         try:
             res: dict = json.load(f)
@@ -400,7 +403,7 @@ class WriteInfo:
 
 
 @API.register
-def load_inventory_with_writeable_keys(
+def get_inventory_with_writeable_keys(
     flake_dir: str | Path,
 ) -> WriteInfo:
     """
@@ -426,7 +429,12 @@ def set_inventory(inventory: Inventory, flake_dir: str | Path, message: str) ->
     and commit it to git with the given message
     """
 
-    write_info = load_inventory_with_writeable_keys(flake_dir)
+    write_info = get_inventory_with_writeable_keys(flake_dir)
+
+    # Remove internals from the inventory
+    inventory.pop("tags", None)  # type: ignore
+    inventory.pop("options", None)  # type: ignore
+    inventory.pop("assertions", None)  # type: ignore
 
     patchset = calc_patches(
         dict(write_info.data_disk),
@@ -435,12 +443,13 @@ def set_inventory(inventory: Inventory, flake_dir: str | Path, message: str) ->
         write_info.writeables,
     )
 
+    persisted = dict(write_info.data_disk)
     for patch_path, data in patchset.items():
-        patch(dict(write_info.data_disk), patch_path, data)
+        patch(persisted, patch_path, data)
 
     inventory_file = get_inventory_path(flake_dir)
     with inventory_file.open("w") as f:
-        json.dump(write_info.data_disk, f, indent=2)
+        json.dump(persisted, f, indent=2)
 
     commit_file(inventory_file, Path(flake_dir), commit_message=message)
 
@@ -462,40 +471,5 @@ def init_inventory(directory: str, init: Inventory | None = None) -> None:
 
 
 @API.register
-def merge_template_inventory(
-    inventory: Inventory, template_inventory: Inventory, machine_name: str
-) -> None:
-    """
-    Merge the template inventory into the current inventory
-    The template inventory is expected to be a subset of the current inventory
-    """
-    for service_name, instance in template_inventory.get("services", {}).items():
-        if len(instance.keys()) > 0:
-            msg = f"Service {service_name} in template inventory has multiple instances"
-            description = (
-                "Only one instance per service is allowed in a template inventory"
-            )
-            raise ClanError(msg, description=description)
-
-        # services.<service_name>.<instance_name>.config
-        config = next((v for v in instance.values() if "config" in v), None)
-        if not config:
-            msg = f"Service {service_name} in template inventory has no config"
-            description = "Invalid inventory configuration"
-            raise ClanError(msg, description=description)
-
-        # Disallow "config.machines" key
-        if "machines" in config:
-            msg = f"Service {service_name} in template inventory has machines"
-            description = "The 'machines' key is not allowed in template inventory"
-            raise ClanError(msg, description=description)
-
-        # Require "config.roles" key
-        if "roles" not in config:
-            msg = f"Service {service_name} in template inventory has no roles"
-            description = "roles key is required in template inventory"
-            raise ClanError(msg, description=description)
-
-        # TODO: Implement merging of template inventory
-        msg = "Merge template inventory is not implemented yet"
-        raise NotImplementedError(msg)
+def get_inventory(flake_dir: str | Path) -> Inventory:
+    return load_inventory_eval(flake_dir)

pkgs/clan-cli/clan_cli/machines/create.py

@@ -16,7 +16,7 @@ from clan_cli.git import commit_file
 from clan_cli.inventory import Machine as InventoryMachine
 from clan_cli.inventory import (
     MachineDeploy,
-    load_inventory_json,
+    get_inventory,
     set_inventory,
 )
 from clan_cli.machines.list import list_nixos_machines
@@ -121,7 +121,7 @@ def create_machine(opts: CreateOptions) -> None:
 
         shutil.copytree(src, dst, ignore_dangling_symlinks=True, copy_function=log_copy)
 
-    inventory = load_inventory_json(clan_dir)
+    inventory = get_inventory(clan_dir)
 
     target_host = opts.target_host
     # TODO: We should allow the template to specify machine metadata if not defined by user

pkgs/clan-cli/clan_cli/machines/delete.py

@@ -6,12 +6,12 @@ from clan_cli.clan_uri import FlakeId
 from clan_cli.completions import add_dynamic_completer, complete_machines
 from clan_cli.dirs import specific_machine_dir
 from clan_cli.errors import ClanError
-from clan_cli.inventory import load_inventory_json, set_inventory
+from clan_cli.inventory import get_inventory, set_inventory
 
 
 @API.register
 def delete_machine(flake: FlakeId, name: str) -> None:
-    inventory = load_inventory_json(flake.path)
+    inventory = get_inventory(flake.path)
 
     machine = inventory.get("machines", {}).pop(name, None)
     if machine is None:

pkgs/clan-cli/tests/test_modules.py

@@ -6,9 +6,9 @@ import pytest
 from clan_cli.api.modules import list_modules
 from clan_cli.clan_uri import FlakeId
 from clan_cli.inventory import (
+    Inventory,
     Machine,
     MachineDeploy,
-    load_inventory_json,
     set_inventory,
 )
 from clan_cli.machines.create import CreateOptions, create_machine
@@ -70,7 +70,7 @@ def test_add_module_to_inventory(
     )
     subprocess.run(["git", "add", "."], cwd=test_flake_with_core.path, check=True)
 
-    inventory = load_inventory_json(base_path)
+    inventory: Inventory = {}
 
     inventory["services"] = {
         "borgbackup": {