Merge pull request 'Inventory improvements' (#1795) from hsjobeki/clan-core:hsjobeki-main into main

pkgs/clan-cli/clan_cli/api/modules.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 from clan_cli.cmd import run_no_stdout
 from clan_cli.errors import ClanCmdError, ClanError
-from clan_cli.inventory import Inventory, load_inventory
+from clan_cli.inventory import Inventory, load_inventory_json
 from clan_cli.nix import nix_eval
 
 from . import API
@@ -152,4 +152,4 @@ def get_module_info(
 
 @API.register
 def get_inventory(base_path: str) -> Inventory:
-    return load_inventory(base_path)
+    return load_inventory_json(base_path)

pkgs/clan-cli/clan_cli/clan/create.py

@@ -1,12 +1,11 @@
 # !/usr/bin/env python3
 import argparse
 import os
-from dataclasses import dataclass, fields
+from dataclasses import dataclass
 from pathlib import Path
 
 from clan_cli.api import API
-from clan_cli.arg_actions import AppendOptionAction
-from clan_cli.inventory import Meta, load_inventory, save_inventory
+from clan_cli.inventory import Inventory, init_inventory
 
 from ..cmd import CmdOut, run
 from ..errors import ClanError
@@ -29,11 +28,9 @@ class CreateClanResponse:
 @dataclass
 class CreateOptions:
     directory: Path | str
-    # Metadata for the clan
-    # Metadata can be shown with `clan show`
-    meta: Meta | None = None
     # URL to the template to use. Defaults to the "minimal" template
     template_url: str = minimal_template_url
+    initial: Inventory | None = None
 
 
 def git_command(directory: Path, *args: str) -> list[str]:
@@ -88,17 +85,13 @@ def create_clan(options: CreateOptions) -> CreateClanResponse:
             git_command(directory, "config", "user.email", "clan@example.com")
         )
 
-    # Write inventory.json file
-    inventory = load_inventory(directory)
-    if options.meta is not None:
-        inventory.meta = options.meta
-    # Persist creates a commit message for each change
-    save_inventory(inventory, directory, "Init inventory")
-
     flake_update = run(
         nix_shell(["nixpkgs#nix"], ["nix", "flake", "update"]), cwd=directory
     )
 
+    if options.initial:
+        init_inventory(options.directory, init=options.initial)
+
     response = CreateClanResponse(
         flake_init=flake_init,
         git_init=git_init,
@@ -118,15 +111,6 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
         default=default_template_url,
     )
 
-    parser.add_argument(
-        "--meta",
-        help=f"""Metadata to set for the clan. Available options are: {", ".join([f.name for f in fields(Meta)]) }""",
-        nargs=2,
-        metavar=("name", "value"),
-        action=AppendOptionAction,
-        default=[],
-    )
-
     parser.add_argument(
         "path", type=Path, help="Path to the clan directory", default=Path(".")
     )

pkgs/clan-cli/clan_cli/clan/update.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 
 from clan_cli.api import API
-from clan_cli.inventory import Meta, load_inventory, save_inventory
+from clan_cli.inventory import Meta, load_inventory_json, save_inventory
 
 
 @dataclass
@@ -12,7 +12,7 @@ class UpdateOptions:
 
 @API.register
 def update_clan_meta(options: UpdateOptions) -> Meta:
-    inventory = load_inventory(options.directory)
+    inventory = load_inventory_json(options.directory)
     inventory.meta = options.meta
 
     save_inventory(inventory, options.directory, "Update clan metadata")

pkgs/clan-cli/clan_cli/clan_uri.py

@@ -9,32 +9,49 @@ from .errors import ClanError
 
 @dataclass
 class FlakeId:
-    # FIXME: this is such a footgun if you accidnetally pass a string
-    _value: str | Path
+    loc: str | Path
 
     def __post_init__(self) -> None:
         assert isinstance(
-            self._value, str | Path
-        ), f"Flake {self._value} has an invalid type: {type(self._value)}"
+            self.loc, str | Path
+        ), f"Flake {self.loc} has an invalid format: {type(self.loc)}"
 
     def __str__(self) -> str:
-        return str(self._value)
+        return str(self.loc)
 
     @property
     def path(self) -> Path:
-        assert isinstance(self._value, Path), f"Flake {self._value} is not a local path"
-        return self._value
+        assert self.is_local(), f"Flake {self.loc} is not a local path"
+        return Path(self.loc)
 
     @property
     def url(self) -> str:
-        assert isinstance(self._value, str), f"Flake {self._value} is not a remote url"
-        return self._value
+        assert self.is_remote(), f"Flake {self.loc} is not a remote url"
+        return str(self.loc)
 
     def is_local(self) -> bool:
-        return isinstance(self._value, Path)
+        """
+        https://nix.dev/manual/nix/2.22/language/builtins.html?highlight=urlS#source-types
+
+        Examples:
+
+        - file:///home/eelco/nix/README.md          file        LOCAL
+        - git+file://git:github.com:NixOS/nixpkgs   git+file    LOCAL
+        - https://example.com/index.html            https       REMOTE
+        - github:nixos/nixpkgs                      github      REMOTE
+        - ftp://serv.file                           ftp         REMOTE
+        - ./.                                       ''          LOCAL
+
+        """
+        x = urllib.parse.urlparse(str(self.loc))
+        if x.scheme == "" or "file" in x.scheme:
+            # See above *file* or empty are the only local schemas
+            return True
+
+        return False
 
     def is_remote(self) -> bool:
-        return isinstance(self._value, str)
+        return not self.is_local()
 
 
 # Define the ClanURI class

pkgs/clan-cli/clan_cli/git.py

@@ -93,6 +93,7 @@ def _commit_file_to_git(
                 "commit",
                 "-m",
                 commit_message,
+                "--no-verify",  # dont run pre-commit hooks
             ]
             + [str(file_path) for file_path in file_paths],
         )

pkgs/clan-cli/clan_cli/inventory/__init__.py

@@ -1,3 +1,17 @@
+"""
+All read/write operations MUST use the inventory.
+
+Machine data, clan data or service data can be accessed in a performant way.
+
+This file exports stable classnames for static & dynamic type safety.
+
+Utilize:
+
+- load_inventory_eval: To load the actual inventory with nix declarations merged.
+Operate on the returned inventory to make changes
+- save_inventory: To persist changes.
+"""
+
 import dataclasses
 import json
 from dataclasses import fields, is_dataclass
@@ -5,9 +19,12 @@ from pathlib import Path
 from types import UnionType
 from typing import Any, get_args, get_origin
 
-from clan_cli.errors import ClanError
+from clan_cli.api import API
+from clan_cli.errors import ClanCmdError, ClanError
 from clan_cli.git import commit_file
 
+from ..cmd import run_no_stdout
+from ..nix import nix_eval
 from .classes import (
     Inventory,
     Machine,
@@ -165,14 +182,42 @@ default_inventory = Inventory(
 )
 
 
-def load_inventory(
+def load_inventory_eval(flake_dir: str | Path) -> Inventory:
+    """
+    Loads the actual inventory.
+    After all merge operations with eventual nix code in buildClan.
+
+    Evaluates clanInternals.inventory with nix. Which is performant.
+
+    - Contains all clan metadata
+    - Contains all machines
+    - and more
+    """
+    cmd = nix_eval(
+        [
+            f"{flake_dir}#clanInternals.inventory",
+            "--json",
+        ]
+    )
+    proc = run_no_stdout(cmd)
+
+    try:
+        res = proc.stdout.strip()
+        data = json.loads(res)
+        inventory = from_dict(Inventory, data)
+        return inventory
+    except json.JSONDecodeError as e:
+        raise ClanError(f"Error decoding inventory from flake: {e}")
+
+
+def load_inventory_json(
     flake_dir: str | Path, default: Inventory = default_inventory
 ) -> Inventory:
     """
     Load the inventory file from the flake directory
     If not file is found, returns the default inventory
     """
-    inventory = default_inventory
+    inventory = default
 
     inventory_file = get_path(flake_dir)
     if inventory_file.exists():
@@ -184,6 +229,10 @@ def load_inventory(
                 # Error decoding the inventory file
                 raise ClanError(f"Error decoding inventory file: {e}")
 
+    if not inventory_file.exists():
+        # Copy over the meta from the flake if the inventory is not initialized
+        inventory.meta = load_inventory_eval(flake_dir).meta
+
     return inventory
 
 
@@ -198,3 +247,22 @@ def save_inventory(inventory: Inventory, flake_dir: str | Path, message: str) ->
         json.dump(dataclass_to_dict(inventory), f, indent=2)
 
     commit_file(inventory_file, Path(flake_dir), commit_message=message)
+
+
+@API.register
+def init_inventory(directory: str, init: Inventory | None = None) -> None:
+    inventory = None
+    # Try reading the current flake
+    if init is None:
+        try:
+            inventory = load_inventory_eval(directory)
+        except ClanCmdError:
+            pass
+
+    if init is not None:
+        inventory = init
+
+    # Write inventory.json file
+    if inventory is not None:
+        # Persist creates a commit message for each change
+        save_inventory(inventory, directory, "Init inventory")

pkgs/clan-cli/clan_cli/machines/create.py

@@ -1,13 +1,17 @@
 import argparse
 import logging
 import re
-from pathlib import Path
 
 from ..api import API
 from ..clan_uri import FlakeId
 from ..errors import ClanError
-from ..git import commit_file
-from ..inventory import Machine, MachineDeploy, get_path, load_inventory, save_inventory
+from ..inventory import (
+    Machine,
+    MachineDeploy,
+    load_inventory_eval,
+    load_inventory_json,
+    save_inventory,
+)
 
 log = logging.getLogger(__name__)
 
@@ -20,12 +24,16 @@ def create_machine(flake: FlakeId, machine: Machine) -> None:
             "Machine name must be a valid hostname", location="Create Machine"
         )
 
-    inventory = load_inventory(flake.path)
+    inventory = load_inventory_json(flake.path)
+
+    full_inventory = load_inventory_eval(flake.path)
+
+    if machine.name in full_inventory.machines.keys():
+        raise ClanError(f"Machine with the name {machine.name} already exists")
+
     inventory.machines.update({machine.name: machine})
     save_inventory(inventory, flake.path, f"Create machine {machine.name}")
 
-    commit_file(get_path(flake.path), Path(flake.path))
-
 
 def create_command(args: argparse.Namespace) -> None:
     create_machine(

pkgs/clan-cli/clan_cli/machines/delete.py

@@ -6,12 +6,12 @@ from ..clan_uri import FlakeId
 from ..completions import add_dynamic_completer, complete_machines
 from ..dirs import specific_machine_dir
 from ..errors import ClanError
-from ..inventory import load_inventory, save_inventory
+from ..inventory import load_inventory_json, save_inventory
 
 
 @API.register
 def delete_machine(flake: FlakeId, name: str) -> None:
-    inventory = load_inventory(flake.path)
+    inventory = load_inventory_json(flake.path)
 
     machine = inventory.machines.pop(name, None)
     if machine is None:

pkgs/clan-cli/clan_cli/machines/list.py

@@ -1,31 +1,17 @@
 import argparse
-import json
 import logging
 from pathlib import Path
 
 from clan_cli.api import API
-from clan_cli.inventory import Machine, from_dict
-
-from ..cmd import run_no_stdout
-from ..nix import nix_eval
+from clan_cli.inventory import Machine, load_inventory_eval
 
 log = logging.getLogger(__name__)
 
 
 @API.register
 def list_machines(flake_url: str | Path, debug: bool = False) -> dict[str, Machine]:
-    cmd = nix_eval(
-        [
-            f"{flake_url}#clanInternals.inventory.machines",
-            "--json",
-        ]
-    )
-
-    proc = run_no_stdout(cmd)
-
-    res = proc.stdout.strip()
-    data = {name: from_dict(Machine, v) for name, v in json.loads(res).items()}
-    return data
+    inventory = load_inventory_eval(flake_url)
+    return inventory.machines
 
 
 def list_command(args: argparse.Namespace) -> None: