Merge pull request 'clan-cli: Move clan machines import to clan machines create' (#2163) from Qubasa/clan-core:Qubasa-main into main

pkgs/clan-cli/clan_cli/__init__.py

@@ -415,10 +415,12 @@ def main() -> None:
     except ClanError as e:
         if isinstance(e, ClanCmdError):
             if e.cmd.msg:
-                log.exception(e.cmd.msg)
+                log.fatal(e.cmd.msg)
                 sys.exit(1)
 
-        log.exception("An error occurred")
+        log.fatal(e.msg)
+        if e.description:
+            print(f"========> {e.description}", file=sys.stderr)
         sys.exit(1)
     except KeyboardInterrupt:
         log.warning("Interrupted by user")

pkgs/clan-cli/clan_cli/config/__init__.py

@@ -44,6 +44,7 @@ def map_type(nix_type: str) -> Any:
 
 # merge two dicts recursively
 def merge(a: dict, b: dict, path: list[str] | None = None) -> dict:
+    a = a.copy()
     if path is None:
         path = []
     for key in b:

pkgs/clan-cli/clan_cli/inventory/__init__.py

@@ -15,6 +15,7 @@ Operate on the returned inventory to make changes
 import contextlib
 import json
 from pathlib import Path
+from typing import Any
 
 from clan_cli.api import API, dataclass_to_dict, from_dict
 from clan_cli.cmd import run_no_stdout
@@ -119,7 +120,9 @@ def load_inventory_json(
 
 
 @API.register
-def set_inventory(inventory: Inventory, flake_dir: str | Path, message: str) -> None:
+def set_inventory(
+    inventory: Inventory | dict[str, Any], flake_dir: str | Path, message: str
+) -> None:
     """ "
     Write the inventory to the flake directory
     and commit it to git with the given message
@@ -127,7 +130,10 @@ def set_inventory(inventory: Inventory, flake_dir: str | Path, message: str) ->
     inventory_file = get_path(flake_dir)
 
     with inventory_file.open("w") as f:
-        json.dump(dataclass_to_dict(inventory), f, indent=2)
+        if isinstance(inventory, Inventory):
+            json.dump(dataclass_to_dict(inventory), f, indent=2)
+        else:
+            json.dump(inventory, f, indent=2)
 
     commit_file(inventory_file, Path(flake_dir), commit_message=message)
 
@@ -147,3 +153,42 @@ def init_inventory(directory: str, init: Inventory | None = None) -> None:
     if inventory is not None:
         # Persist creates a commit message for each change
         set_inventory(inventory, directory, "Init inventory")
+
+
+@API.register
+def merge_template_inventory(
+    inventory: Inventory, template_inventory: Inventory, machine_name: str
+) -> None:
+    """
+    Merge the template inventory into the current inventory
+    The template inventory is expected to be a subset of the current inventory
+    """
+    for service_name, instance in template_inventory.services.items():
+        if len(instance.keys()) > 0:
+            msg = f"Service {service_name} in template inventory has multiple instances"
+            description = (
+                "Only one instance per service is allowed in a template inventory"
+            )
+            raise ClanError(msg, description=description)
+
+        # Get the service config without knowing instance name
+        service_conf = next((v for v in instance.values() if "config" in v), None)
+
+        if not service_conf:
+            msg = f"Service {service_name} in template inventory has no config"
+            description = "Invalid inventory configuration"
+            raise ClanError(msg, description=description)
+
+        if "machines" in service_conf:
+            msg = f"Service {service_name} in template inventory has machines"
+            description = "The 'machines' key is not allowed in template inventory"
+            raise ClanError(msg, description=description)
+
+        if "roles" not in service_conf:
+            msg = f"Service {service_name} in template inventory has no roles"
+            description = "roles key is required in template inventory"
+            raise ClanError(msg, description=description)
+
+        # TODO: We need a MachineReference type in nix before we can implement this properly
+        msg = "Merge template inventory is not implemented yet"
+        raise NotImplementedError(msg)

pkgs/clan-cli/clan_cli/machines/cli.py

@@ -4,7 +4,6 @@ import argparse
 from .create import register_create_parser
 from .delete import register_delete_parser
 from .hardware import register_update_hardware_config
-from .import_cmd import register_import_parser
 from .install import register_install_parser
 from .list import register_list_parser
 from .update import register_update_parser
@@ -114,22 +113,3 @@ For more detailed information, visit: https://docs.clan.lol/getting-started/depl
         formatter_class=argparse.RawTextHelpFormatter,
     )
     register_install_parser(install_parser)
-
-    import_parser = subparser.add_parser(
-        "import",
-        help="Import a machine",
-        description="Import a template machine from a local or remote source.",
-        epilog=(
-            """
-
-Examples:
-    $ clan machines import flash-installer
-    Will import a machine from the flash-installer template from clan-core.
-
-    $ clan machines import flash-installer --src https://git.clan.lol/clan/clan-core
-    Will import a machine from the flash-installer template from clan-core but with the source set to the provided URL.
-    """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    register_import_parser(import_parser)

pkgs/clan-cli/clan_cli/machines/create.py

@@ -1,79 +1,177 @@
 import argparse
 import logging
 import re
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+from tempfile import TemporaryDirectory
 
 from clan_cli.api import API
+from clan_cli.clan.create import git_command
 from clan_cli.clan_uri import FlakeId
+from clan_cli.cmd import Log, run
+from clan_cli.dirs import clan_templates, get_clan_flake_toplevel_or_env
 from clan_cli.errors import ClanError
+from clan_cli.inventory import Machine as InventoryMachine
 from clan_cli.inventory import (
-    Machine,
     MachineDeploy,
-    load_inventory_eval,
+    dataclass_to_dict,
     load_inventory_json,
+    merge_template_inventory,
     set_inventory,
 )
+from clan_cli.machines.list import list_nixos_machines
+from clan_cli.nix import nix_command
 
 log = logging.getLogger(__name__)
 
 
+def validate_directory(root_dir: Path) -> None:
+    machines_dir = root_dir / "machines"
+    for root, _, files in root_dir.walk():
+        for file in files:
+            file_path = Path(root) / file
+            if not file_path.is_relative_to(machines_dir):
+                msg = f"File {file_path} is not in the 'machines' directory."
+                log.error(msg)
+                description = "Template machines are only allowed to contain files in the 'machines' directory."
+                raise ClanError(msg, description=description)
+
+
+@dataclass
+class CreateOptions:
+    clan_dir: FlakeId
+    machine: InventoryMachine
+    template_src: FlakeId | None = None
+    template_name: str | None = None
+
+
 @API.register
-def create_machine(flake: FlakeId, machine: Machine) -> None:
+def create_machine(opts: CreateOptions) -> None:
+    if not opts.clan_dir.is_local():
+        msg = f"Clan {opts.clan_dir} is not a local clan."
+        description = "Import machine only works on local clans"
+        raise ClanError(msg, description=description)
+
+    if not opts.template_src:
+        opts.template_src = FlakeId(str(clan_templates()))
+
+    if not opts.template_name:
+        opts.template_name = "new-machine"
+
+    clan_dir = opts.clan_dir.path
+
+    log.debug(f"Importing machine '{opts.template_name}' from {opts.template_src}")
+
+    if opts.template_name in list_nixos_machines(clan_dir) and not opts.machine.name:
+        msg = f"{opts.template_name} is already defined in {clan_dir}"
+        description = (
+            "Please add the --rename option to import the machine with a different name"
+        )
+        raise ClanError(msg, description=description)
+
+    machine_name = opts.template_name if not opts.machine.name else opts.machine.name
+    dst = clan_dir / "machines" / machine_name
+
+    # TODO: Move this into nix code
     hostname_regex = r"^(?!-)[A-Za-z0-9-]{1,63}(?<!-)$"
-    if not re.match(hostname_regex, machine.name):
+    if not re.match(hostname_regex, machine_name):
         msg = "Machine name must be a valid hostname"
         raise ClanError(msg, location="Create Machine")
 
-    inventory = load_inventory_json(flake.path)
+    if dst.exists():
+        msg = f"Machine {machine_name} already exists in {clan_dir}"
+        description = (
+            "Please delete the existing machine or import with a different name"
+        )
+        raise ClanError(msg, description=description)
 
-    full_inventory = load_inventory_eval(flake.path)
+    with TemporaryDirectory() as tmpdir:
+        tmpdirp = Path(tmpdir)
+        command = nix_command(
+            [
+                "flake",
+                "init",
+                "-t",
+                f"{opts.template_src}#machineTemplates",
+            ]
+        )
 
-    if machine.name in full_inventory.machines:
+        # Check if debug logging is enabled
-        msg = f"Machine with the name {machine.name} already exists"
+        is_debug_enabled = log.isEnabledFor(logging.DEBUG)
-        raise ClanError(msg)
+        log_flag = Log.BOTH if is_debug_enabled else Log.NONE
+        run(command, log=log_flag, cwd=tmpdirp)
 
-    print(f"Define machine {machine.name}", machine)
+        validate_directory(tmpdirp)
 
-    inventory.machines.update({machine.name: machine})
+        src = tmpdirp / "machines" / opts.template_name
-    set_inventory(inventory, flake.path, f"Create machine {machine.name}")
+
+        if (
+            not (src / "configuration.nix").exists()
+            and not (src / "inventory.json").exists()
+        ):
+            msg = f"Template machine '{opts.template_name}' does not contain a configuration.nix or inventory.json"
+            description = (
+                "Template machine must contain a configuration.nix or inventory.json"
+            )
+            raise ClanError(msg, description=description)
+
+        def log_copy(src: str, dst: str) -> None:
+            relative_dst = dst.replace(f"{clan_dir}/", "")
+            log.info(f"Add file: {relative_dst}")
+            shutil.copy2(src, dst)
+
+        shutil.copytree(src, dst, ignore_dangling_symlinks=True, copy_function=log_copy)
+
+    run(git_command(clan_dir, "add", f"machines/{machine_name}"), cwd=clan_dir)
+
+    inventory = load_inventory_json(clan_dir)
+
+    # Merge the inventory from the template
+    if (dst / "inventory.json").exists():
+        template_inventory = load_inventory_json(dst)
+        merge_template_inventory(inventory, template_inventory, machine_name)
+
+    # TODO: We should allow the template to specify machine metadata if not defined by user
+    #
+    new_machine = InventoryMachine(name=machine_name, deploy=MachineDeploy())
+    inventory.machines.update({new_machine.name: dataclass_to_dict(new_machine)})
+    set_inventory(inventory, clan_dir, "Imported machine from template")
 
 
 def create_command(args: argparse.Namespace) -> None:
-    create_machine(
+    if args.flake:
-        args.flake,
+        clan_dir = args.flake
-        Machine(
+    else:
-            name=args.machine,
+        tmp = get_clan_flake_toplevel_or_env()
-            system=args.system,
+        clan_dir = FlakeId(str(tmp)) if tmp else None
-            description=args.description,
+
-            tags=args.tags,
+    if not clan_dir:
-            icon=args.icon,
+        msg = "No clan found."
-            deploy=MachineDeploy(),
+        description = (
-        ),
+            "Run this command in a clan directory or specify the --flake option"
+        )
+        raise ClanError(msg, description=description)
+
+    machine = InventoryMachine(
+        name=args.machine_name,
+        tags=args.tags,
+        deploy=MachineDeploy(),
     )
+    opts = CreateOptions(
+        clan_dir=clan_dir,
+        machine=machine,
+        template_name=args.template_name,
+    )
+    create_machine(opts)
 
 
 def register_create_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument("machine", type=str)
     parser.set_defaults(func=create_command)
-
     parser.add_argument(
-        "--system",
+        "machine_name",
         type=str,
-        default=None,
+        help="The name of the machine to import",
-        help="Host platform to use. i.e. 'x86_64-linux' or 'aarch64-darwin' etc.",
-        metavar="PLATFORM",
-    )
-    parser.add_argument(
-        "--description",
-        type=str,
-        default=None,
-        help="A description of the machine.",
-    )
-    parser.add_argument(
-        "--icon",
-        type=str,
-        default=None,
-        help="Path to an icon to use for the machine. - Must be a path to icon file relative to the flake directory, or a public url.",
-        metavar="PATH",
     )
     parser.add_argument(
         "--tags",
@@ -81,3 +179,8 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
         default=[],
         help="Tags to associate with the machine. Can be used to assign multiple machines to services.",
     )
+    parser.add_argument(
+        "--template-name",
+        type=str,
+        help="The name of the template machine to import",
+    )

pkgs/clan-cli/clan_cli/machines/import_cmd.py

@@ -1,160 +0,0 @@
-import argparse
-import logging
-import shutil
-from dataclasses import dataclass
-from pathlib import Path
-from tempfile import TemporaryDirectory
-
-from clan_cli.api import API
-from clan_cli.clan.create import git_command
-from clan_cli.clan_uri import FlakeId
-from clan_cli.cmd import Log, run
-from clan_cli.dirs import clan_templates, get_clan_flake_toplevel_or_env
-from clan_cli.errors import ClanError
-from clan_cli.machines.list import list_nixos_machines
-from clan_cli.machines.machines import Machine
-from clan_cli.nix import nix_command
-
-log = logging.getLogger(__name__)
-
-
-def validate_directory(root_dir: Path, machine_name: str) -> None:
-    machines_dir = root_dir / "machines" / machine_name
-    for root, _, files in root_dir.walk():
-        for file in files:
-            file_path = Path(root) / file
-            if not file_path.is_relative_to(machines_dir):
-                msg = f"File {file_path} is not in the 'machines/{machine_name}' directory."
-                description = "Template machines are only allowed to contain files in the 'machines/{machine_name}' directory."
-                raise ClanError(msg, description=description)
-
-
-@dataclass
-class ImportOptions:
-    target: FlakeId
-    src: Machine
-    rename: str | None = None
-
-
-@API.register
-def import_machine(opts: ImportOptions) -> None:
-    if not opts.target.is_local():
-        msg = f"Clan {opts.target} is not a local clan."
-        description = "Import machine only works on local clans"
-        raise ClanError(msg, description=description)
-    clan_dir = opts.target.path
-
-    log.debug(f"Importing machine '{opts.src.name}' from {opts.src.flake}")
-
-    if opts.src.name == opts.rename:
-        msg = "Rename option must be different from the template machine name"
-        raise ClanError(msg)
-
-    if opts.src.name in list_nixos_machines(clan_dir) and not opts.rename:
-        msg = f"{opts.src.name} is already defined in {clan_dir}"
-        description = (
-            "Please add the --rename option to import the machine with a different name"
-        )
-        raise ClanError(msg, description=description)
-
-    machine_name = opts.src.name if not opts.rename else opts.rename
-    dst = clan_dir / "machines" / machine_name
-
-    if dst.exists():
-        msg = f"Machine {machine_name} already exists in {clan_dir}"
-        description = (
-            "Please delete the existing machine or import with a different name"
-        )
-        raise ClanError(msg, description=description)
-
-    with TemporaryDirectory() as tmpdir:
-        tmpdirp = Path(tmpdir)
-        command = nix_command(
-            [
-                "flake",
-                "init",
-                "-t",
-                opts.src.get_id(),
-            ]
-        )
-        run(command, log=Log.NONE, cwd=tmpdirp)
-
-        validate_directory(tmpdirp, opts.src.name)
-        src = tmpdirp / "machines" / opts.src.name
-
-        if (
-            not (src / "configuration.nix").exists()
-            and not (src / "inventory.json").exists()
-        ):
-            msg = f"Template machine {opts.src.name} does not contain a configuration.nix or inventory.json"
-            description = (
-                "Template machine must contain a configuration.nix or inventory.json"
-            )
-            raise ClanError(msg, description=description)
-
-        def log_copy(src: str, dst: str) -> None:
-            relative_dst = dst.replace(f"{clan_dir}/", "")
-            log.info(f"Add file: {relative_dst}")
-            shutil.copy2(src, dst)
-
-        shutil.copytree(src, dst, ignore_dangling_symlinks=True, copy_function=log_copy)
-
-    run(git_command(clan_dir, "add", f"machines/{machine_name}"), cwd=clan_dir)
-
-    if (dst / "inventory.json").exists():
-        # TODO: Implement inventory import
-        msg = "Inventory import not implemented yet"
-        raise NotImplementedError(msg)
-        # inventory = load_inventory_json(clan_dir)
-
-        # inventory.machines[machine_name] = Inventory_Machine(
-        #     name=machine_name,
-        #     deploy=MachineDeploy(targetHost=None),
-        # )
-        # set_inventory(inventory, clan_dir, "Imported machine from template")
-
-
-def import_command(args: argparse.Namespace) -> None:
-    if args.flake:
-        target = args.flake
-    else:
-        tmp = get_clan_flake_toplevel_or_env()
-        target = FlakeId(str(tmp)) if tmp else None
-
-    if not target:
-        msg = "No clan found."
-        description = (
-            "Run this command in a clan directory or specify the --flake option"
-        )
-        raise ClanError(msg, description=description)
-
-    src_uri = args.src
-    if not src_uri:
-        src_uri = FlakeId(str(clan_templates()))
-
-    opts = ImportOptions(
-        target=target,
-        src=Machine(flake=src_uri, name=args.machine_name),
-        rename=args.rename,
-    )
-    import_machine(opts)
-
-
-def register_import_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
-        "machine_name",
-        type=str,
-        help="The name of the machine to import",
-    )
-    parser.add_argument(
-        "--src",
-        type=FlakeId,
-        help="The source flake to import the machine from",
-    )
-    parser.add_argument(
-        "--rename",
-        type=str,
-        help="Rename the imported machine",
-    )
-
-    parser.set_defaults(func=import_command)

pkgs/clan-cli/tests/test_modules.py

@@ -10,7 +10,7 @@ from clan_cli.inventory import (
     load_inventory_json,
     set_inventory,
 )
-from clan_cli.machines.create import create_machine
+from clan_cli.machines.create import CreateOptions, create_machine
 from clan_cli.nix import nix_eval, run_no_stdout
 from fixtures_flakes import FlakeForTest
 
@@ -53,13 +53,15 @@ def test_add_module_to_inventory(
             age_keys[0].pubkey,
         ]
     )
-    create_machine(
+    opts = CreateOptions(
-        FlakeId(str(base_path)),
+        clan_dir=FlakeId(str(base_path)),
-        Machine(
+        machine=Machine(
             name="machine1", tags=[], system="x86_64-linux", deploy=MachineDeploy()
         ),
     )
 
+    create_machine(opts)
+
     inventory = load_inventory_json(base_path)
 
     inventory.services = {

pkgs/webview-ui/app/src/routes/flash/view.tsx

@@ -160,6 +160,7 @@ export const Flash = () => {
         dry_run: false,
         write_efi_boot_entries: false,
         debug: false,
+        no_udev: true,
       });
     } catch (error) {
       toast.error(`Error could not flash disk: ${error}`);

pkgs/webview-ui/app/src/routes/machines/create.tsx

@@ -13,16 +13,18 @@ export function CreateMachine() {
   const navigate = useNavigate();
   const [formStore, { Form, Field }] = createForm<CreateMachineForm>({
     initialValues: {
-      flake: {
+      opts: {
-        loc: activeURI() || "",
+        clan_dir: {
-      },
+          loc: activeURI() || "",
-      machine: {
+        },
-        tags: ["all"],
+        machine: {
-        deploy: {
+          tags: ["all"],
-          targetHost: "",
+          deploy: {
+            targetHost: "",
+          },
+          name: "",
+          description: "",
         },
-        name: "",
-        description: "",
       },
     },
   });
@@ -39,14 +41,16 @@ export function CreateMachine() {
     console.log("submitting", values);
 
     const response = await callApi("create_machine", {
-      ...values,
+      opts: {
-      flake: {
+        ...values.opts,
-        loc: active_dir,
+        clan_dir: {
+          loc: active_dir,
+        },
       },
     });
 
     if (response.status === "success") {
-      toast.success(`Successfully created ${values.machine.name}`);
+      toast.success(`Successfully created ${values.opts.machine.name}`);
       reset(formStore);
 
       queryClient.invalidateQueries({
@@ -55,7 +59,7 @@ export function CreateMachine() {
       navigate("/machines");
     } else {
       toast.error(
-        `Error: ${response.errors[0].message}. Machine ${values.machine.name} could not be created`,
+        `Error: ${response.errors[0].message}. Machine ${values.opts.machine.name} could not be created`,
       );
     }
   };
@@ -65,7 +69,7 @@ export function CreateMachine() {
         <span class="px-2">Create new Machine</span>
         <Form onSubmit={handleSubmit}>
           <Field
-            name="machine.name"
+            name="opts.machine.name"
             validate={[required("This field is required")]}
           >
             {(field, props) => (
@@ -79,7 +83,7 @@ export function CreateMachine() {
               />
             )}
           </Field>
-          <Field name="machine.description">
+          <Field name="opts.machine.description">
             {(field, props) => (
               <TextInput
                 inputProps={props}
@@ -90,7 +94,7 @@ export function CreateMachine() {
               />
             )}
           </Field>
-          <Field name="machine.deploy.targetHost">
+          <Field name="opts.machine.deploy.targetHost">
             {(field, props) => (
               <>
                 <TextInput

templates/flake.nix

@@ -15,9 +15,9 @@
           description = "Flake-parts";
           path = ./flake-parts;
         };
-        flash-installer = {
+        machineTemplates = {
-          description = "Flash installer";
+          description = "Machine templates";
-          path = ./flash-installer;
+          path = ./machineTemplates;
         };
       };
     };

templates/machineTemplates/machines/flash-installer/configuration.nix

@@ -0,0 +1,7 @@
+{
+  imports = [
+
+  ];
+
+  # Flash machine template
+}

templates/machineTemplates/machines/new-machine/configuration.nix

@@ -3,5 +3,5 @@
 
   ];
 
-  # Hello World
+  # New machine!
 }