From a27880a65e490ef24c6236454868f1b6118870ed Mon Sep 17 00:00:00 2001 From: Qubasa Date: Mon, 23 Sep 2024 17:11:48 +0200 Subject: [PATCH] clan-cli: Move clan machines import to clan machines create --- pkgs/clan-cli/clan_cli/__init__.py | 6 +- pkgs/clan-cli/clan_cli/config/__init__.py | 1 + pkgs/clan-cli/clan_cli/inventory/__init__.py | 49 ++++- pkgs/clan-cli/clan_cli/machines/cli.py | 20 -- pkgs/clan-cli/clan_cli/machines/create.py | 185 ++++++++++++++---- pkgs/clan-cli/clan_cli/machines/import_cmd.py | 160 --------------- pkgs/clan-cli/tests/test_modules.py | 10 +- pkgs/webview-ui/app/src/routes/flash/view.tsx | 1 + .../app/src/routes/machines/create.tsx | 38 ++-- templates/flake.nix | 6 +- .../flash-installer/configuration.nix | 7 + .../machines/new-machine}/configuration.nix | 2 +- 12 files changed, 235 insertions(+), 250 deletions(-) delete mode 100644 pkgs/clan-cli/clan_cli/machines/import_cmd.py create mode 100644 templates/machineTemplates/machines/flash-installer/configuration.nix rename templates/{flash-installer/machines/flash-installer => machineTemplates/machines/new-machine}/configuration.nix (59%) diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index 097f87031..fa4463de8 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -415,10 +415,12 @@ def main() -> None: except ClanError as e: if isinstance(e, ClanCmdError): if e.cmd.msg: - log.exception(e.cmd.msg) + log.fatal(e.cmd.msg) sys.exit(1) - log.exception("An error occurred") + log.fatal(e.msg) + if e.description: + print(f"========> {e.description}", file=sys.stderr) sys.exit(1) except KeyboardInterrupt: log.warning("Interrupted by user") diff --git a/pkgs/clan-cli/clan_cli/config/__init__.py b/pkgs/clan-cli/clan_cli/config/__init__.py index edfeda23a..9715a84c6 100644 --- a/pkgs/clan-cli/clan_cli/config/__init__.py +++ b/pkgs/clan-cli/clan_cli/config/__init__.py @@ -44,6 +44,7 @@ def map_type(nix_type: str) -> Any: # merge two dicts recursively def merge(a: dict, b: dict, path: list[str] | None = None) -> dict: + a = a.copy() if path is None: path = [] for key in b: diff --git a/pkgs/clan-cli/clan_cli/inventory/__init__.py b/pkgs/clan-cli/clan_cli/inventory/__init__.py index 006036d5a..841a238f3 100644 --- a/pkgs/clan-cli/clan_cli/inventory/__init__.py +++ b/pkgs/clan-cli/clan_cli/inventory/__init__.py @@ -15,6 +15,7 @@ Operate on the returned inventory to make changes import contextlib import json from pathlib import Path +from typing import Any from clan_cli.api import API, dataclass_to_dict, from_dict from clan_cli.cmd import run_no_stdout @@ -119,7 +120,9 @@ def load_inventory_json( @API.register -def set_inventory(inventory: Inventory, flake_dir: str | Path, message: str) -> None: +def set_inventory( + inventory: Inventory | dict[str, Any], flake_dir: str | Path, message: str +) -> None: """ " Write the inventory to the flake directory and commit it to git with the given message @@ -127,7 +130,10 @@ def set_inventory(inventory: Inventory, flake_dir: str | Path, message: str) -> inventory_file = get_path(flake_dir) with inventory_file.open("w") as f: - json.dump(dataclass_to_dict(inventory), f, indent=2) + if isinstance(inventory, Inventory): + json.dump(dataclass_to_dict(inventory), f, indent=2) + else: + json.dump(inventory, f, indent=2) commit_file(inventory_file, Path(flake_dir), commit_message=message) @@ -147,3 +153,42 @@ def init_inventory(directory: str, init: Inventory | None = None) -> None: if inventory is not None: # Persist creates a commit message for each change set_inventory(inventory, directory, "Init inventory") + + +@API.register +def merge_template_inventory( + inventory: Inventory, template_inventory: Inventory, machine_name: str +) -> None: + """ + Merge the template inventory into the current inventory + The template inventory is expected to be a subset of the current inventory + """ + for service_name, instance in template_inventory.services.items(): + if len(instance.keys()) > 0: + msg = f"Service {service_name} in template inventory has multiple instances" + description = ( + "Only one instance per service is allowed in a template inventory" + ) + raise ClanError(msg, description=description) + + # Get the service config without knowing instance name + service_conf = next((v for v in instance.values() if "config" in v), None) + + if not service_conf: + msg = f"Service {service_name} in template inventory has no config" + description = "Invalid inventory configuration" + raise ClanError(msg, description=description) + + if "machines" in service_conf: + msg = f"Service {service_name} in template inventory has machines" + description = "The 'machines' key is not allowed in template inventory" + raise ClanError(msg, description=description) + + if "roles" not in service_conf: + msg = f"Service {service_name} in template inventory has no roles" + description = "roles key is required in template inventory" + raise ClanError(msg, description=description) + + # TODO: We need a MachineReference type in nix before we can implement this properly + msg = "Merge template inventory is not implemented yet" + raise NotImplementedError(msg) diff --git a/pkgs/clan-cli/clan_cli/machines/cli.py b/pkgs/clan-cli/clan_cli/machines/cli.py index 8bffbd8f1..a2f92e786 100644 --- a/pkgs/clan-cli/clan_cli/machines/cli.py +++ b/pkgs/clan-cli/clan_cli/machines/cli.py @@ -4,7 +4,6 @@ import argparse from .create import register_create_parser from .delete import register_delete_parser from .hardware import register_update_hardware_config -from .import_cmd import register_import_parser from .install import register_install_parser from .list import register_list_parser from .update import register_update_parser @@ -114,22 +113,3 @@ For more detailed information, visit: https://docs.clan.lol/getting-started/depl formatter_class=argparse.RawTextHelpFormatter, ) register_install_parser(install_parser) - - import_parser = subparser.add_parser( - "import", - help="Import a machine", - description="Import a template machine from a local or remote source.", - epilog=( - """ - -Examples: - $ clan machines import flash-installer - Will import a machine from the flash-installer template from clan-core. - - $ clan machines import flash-installer --src https://git.clan.lol/clan/clan-core - Will import a machine from the flash-installer template from clan-core but with the source set to the provided URL. - """ - ), - formatter_class=argparse.RawTextHelpFormatter, - ) - register_import_parser(import_parser) diff --git a/pkgs/clan-cli/clan_cli/machines/create.py b/pkgs/clan-cli/clan_cli/machines/create.py index 814d46416..8da4fe2c5 100644 --- a/pkgs/clan-cli/clan_cli/machines/create.py +++ b/pkgs/clan-cli/clan_cli/machines/create.py @@ -1,79 +1,177 @@ import argparse import logging import re +import shutil +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory from clan_cli.api import API +from clan_cli.clan.create import git_command from clan_cli.clan_uri import FlakeId +from clan_cli.cmd import Log, run +from clan_cli.dirs import clan_templates, get_clan_flake_toplevel_or_env from clan_cli.errors import ClanError +from clan_cli.inventory import Machine as InventoryMachine from clan_cli.inventory import ( - Machine, MachineDeploy, - load_inventory_eval, + dataclass_to_dict, load_inventory_json, + merge_template_inventory, set_inventory, ) +from clan_cli.machines.list import list_nixos_machines +from clan_cli.nix import nix_command log = logging.getLogger(__name__) +def validate_directory(root_dir: Path) -> None: + machines_dir = root_dir / "machines" + for root, _, files in root_dir.walk(): + for file in files: + file_path = Path(root) / file + if not file_path.is_relative_to(machines_dir): + msg = f"File {file_path} is not in the 'machines' directory." + log.error(msg) + description = "Template machines are only allowed to contain files in the 'machines' directory." + raise ClanError(msg, description=description) + + +@dataclass +class CreateOptions: + clan_dir: FlakeId + machine: InventoryMachine + template_src: FlakeId | None = None + template_name: str | None = None + + @API.register -def create_machine(flake: FlakeId, machine: Machine) -> None: +def create_machine(opts: CreateOptions) -> None: + if not opts.clan_dir.is_local(): + msg = f"Clan {opts.clan_dir} is not a local clan." + description = "Import machine only works on local clans" + raise ClanError(msg, description=description) + + if not opts.template_src: + opts.template_src = FlakeId(str(clan_templates())) + + if not opts.template_name: + opts.template_name = "new-machine" + + clan_dir = opts.clan_dir.path + + log.debug(f"Importing machine '{opts.template_name}' from {opts.template_src}") + + if opts.template_name in list_nixos_machines(clan_dir) and not opts.machine.name: + msg = f"{opts.template_name} is already defined in {clan_dir}" + description = ( + "Please add the --rename option to import the machine with a different name" + ) + raise ClanError(msg, description=description) + + machine_name = opts.template_name if not opts.machine.name else opts.machine.name + dst = clan_dir / "machines" / machine_name + + # TODO: Move this into nix code hostname_regex = r"^(?!-)[A-Za-z0-9-]{1,63}(? None: + relative_dst = dst.replace(f"{clan_dir}/", "") + log.info(f"Add file: {relative_dst}") + shutil.copy2(src, dst) + + shutil.copytree(src, dst, ignore_dangling_symlinks=True, copy_function=log_copy) + + run(git_command(clan_dir, "add", f"machines/{machine_name}"), cwd=clan_dir) + + inventory = load_inventory_json(clan_dir) + + # Merge the inventory from the template + if (dst / "inventory.json").exists(): + template_inventory = load_inventory_json(dst) + merge_template_inventory(inventory, template_inventory, machine_name) + + # TODO: We should allow the template to specify machine metadata if not defined by user + # + new_machine = InventoryMachine(name=machine_name, deploy=MachineDeploy()) + inventory.machines.update({new_machine.name: dataclass_to_dict(new_machine)}) + set_inventory(inventory, clan_dir, "Imported machine from template") def create_command(args: argparse.Namespace) -> None: - create_machine( - args.flake, - Machine( - name=args.machine, - system=args.system, - description=args.description, - tags=args.tags, - icon=args.icon, - deploy=MachineDeploy(), - ), + if args.flake: + clan_dir = args.flake + else: + tmp = get_clan_flake_toplevel_or_env() + clan_dir = FlakeId(str(tmp)) if tmp else None + + if not clan_dir: + msg = "No clan found." + description = ( + "Run this command in a clan directory or specify the --flake option" + ) + raise ClanError(msg, description=description) + + machine = InventoryMachine( + name=args.machine_name, + tags=args.tags, + deploy=MachineDeploy(), ) + opts = CreateOptions( + clan_dir=clan_dir, + machine=machine, + template_name=args.template_name, + ) + create_machine(opts) def register_create_parser(parser: argparse.ArgumentParser) -> None: - parser.add_argument("machine", type=str) parser.set_defaults(func=create_command) - parser.add_argument( - "--system", + "machine_name", type=str, - default=None, - help="Host platform to use. i.e. 'x86_64-linux' or 'aarch64-darwin' etc.", - metavar="PLATFORM", - ) - parser.add_argument( - "--description", - type=str, - default=None, - help="A description of the machine.", - ) - parser.add_argument( - "--icon", - type=str, - default=None, - help="Path to an icon to use for the machine. - Must be a path to icon file relative to the flake directory, or a public url.", - metavar="PATH", + help="The name of the machine to import", ) parser.add_argument( "--tags", @@ -81,3 +179,8 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None: default=[], help="Tags to associate with the machine. Can be used to assign multiple machines to services.", ) + parser.add_argument( + "--template-name", + type=str, + help="The name of the template machine to import", + ) diff --git a/pkgs/clan-cli/clan_cli/machines/import_cmd.py b/pkgs/clan-cli/clan_cli/machines/import_cmd.py deleted file mode 100644 index 4c32771b2..000000000 --- a/pkgs/clan-cli/clan_cli/machines/import_cmd.py +++ /dev/null @@ -1,160 +0,0 @@ -import argparse -import logging -import shutil -from dataclasses import dataclass -from pathlib import Path -from tempfile import TemporaryDirectory - -from clan_cli.api import API -from clan_cli.clan.create import git_command -from clan_cli.clan_uri import FlakeId -from clan_cli.cmd import Log, run -from clan_cli.dirs import clan_templates, get_clan_flake_toplevel_or_env -from clan_cli.errors import ClanError -from clan_cli.machines.list import list_nixos_machines -from clan_cli.machines.machines import Machine -from clan_cli.nix import nix_command - -log = logging.getLogger(__name__) - - -def validate_directory(root_dir: Path, machine_name: str) -> None: - machines_dir = root_dir / "machines" / machine_name - for root, _, files in root_dir.walk(): - for file in files: - file_path = Path(root) / file - if not file_path.is_relative_to(machines_dir): - msg = f"File {file_path} is not in the 'machines/{machine_name}' directory." - description = "Template machines are only allowed to contain files in the 'machines/{machine_name}' directory." - raise ClanError(msg, description=description) - - -@dataclass -class ImportOptions: - target: FlakeId - src: Machine - rename: str | None = None - - -@API.register -def import_machine(opts: ImportOptions) -> None: - if not opts.target.is_local(): - msg = f"Clan {opts.target} is not a local clan." - description = "Import machine only works on local clans" - raise ClanError(msg, description=description) - clan_dir = opts.target.path - - log.debug(f"Importing machine '{opts.src.name}' from {opts.src.flake}") - - if opts.src.name == opts.rename: - msg = "Rename option must be different from the template machine name" - raise ClanError(msg) - - if opts.src.name in list_nixos_machines(clan_dir) and not opts.rename: - msg = f"{opts.src.name} is already defined in {clan_dir}" - description = ( - "Please add the --rename option to import the machine with a different name" - ) - raise ClanError(msg, description=description) - - machine_name = opts.src.name if not opts.rename else opts.rename - dst = clan_dir / "machines" / machine_name - - if dst.exists(): - msg = f"Machine {machine_name} already exists in {clan_dir}" - description = ( - "Please delete the existing machine or import with a different name" - ) - raise ClanError(msg, description=description) - - with TemporaryDirectory() as tmpdir: - tmpdirp = Path(tmpdir) - command = nix_command( - [ - "flake", - "init", - "-t", - opts.src.get_id(), - ] - ) - run(command, log=Log.NONE, cwd=tmpdirp) - - validate_directory(tmpdirp, opts.src.name) - src = tmpdirp / "machines" / opts.src.name - - if ( - not (src / "configuration.nix").exists() - and not (src / "inventory.json").exists() - ): - msg = f"Template machine {opts.src.name} does not contain a configuration.nix or inventory.json" - description = ( - "Template machine must contain a configuration.nix or inventory.json" - ) - raise ClanError(msg, description=description) - - def log_copy(src: str, dst: str) -> None: - relative_dst = dst.replace(f"{clan_dir}/", "") - log.info(f"Add file: {relative_dst}") - shutil.copy2(src, dst) - - shutil.copytree(src, dst, ignore_dangling_symlinks=True, copy_function=log_copy) - - run(git_command(clan_dir, "add", f"machines/{machine_name}"), cwd=clan_dir) - - if (dst / "inventory.json").exists(): - # TODO: Implement inventory import - msg = "Inventory import not implemented yet" - raise NotImplementedError(msg) - # inventory = load_inventory_json(clan_dir) - - # inventory.machines[machine_name] = Inventory_Machine( - # name=machine_name, - # deploy=MachineDeploy(targetHost=None), - # ) - # set_inventory(inventory, clan_dir, "Imported machine from template") - - -def import_command(args: argparse.Namespace) -> None: - if args.flake: - target = args.flake - else: - tmp = get_clan_flake_toplevel_or_env() - target = FlakeId(str(tmp)) if tmp else None - - if not target: - msg = "No clan found." - description = ( - "Run this command in a clan directory or specify the --flake option" - ) - raise ClanError(msg, description=description) - - src_uri = args.src - if not src_uri: - src_uri = FlakeId(str(clan_templates())) - - opts = ImportOptions( - target=target, - src=Machine(flake=src_uri, name=args.machine_name), - rename=args.rename, - ) - import_machine(opts) - - -def register_import_parser(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "machine_name", - type=str, - help="The name of the machine to import", - ) - parser.add_argument( - "--src", - type=FlakeId, - help="The source flake to import the machine from", - ) - parser.add_argument( - "--rename", - type=str, - help="Rename the imported machine", - ) - - parser.set_defaults(func=import_command) diff --git a/pkgs/clan-cli/tests/test_modules.py b/pkgs/clan-cli/tests/test_modules.py index 16bbace6e..9d70b805f 100644 --- a/pkgs/clan-cli/tests/test_modules.py +++ b/pkgs/clan-cli/tests/test_modules.py @@ -10,7 +10,7 @@ from clan_cli.inventory import ( load_inventory_json, set_inventory, ) -from clan_cli.machines.create import create_machine +from clan_cli.machines.create import CreateOptions, create_machine from clan_cli.nix import nix_eval, run_no_stdout from fixtures_flakes import FlakeForTest @@ -53,13 +53,15 @@ def test_add_module_to_inventory( age_keys[0].pubkey, ] ) - create_machine( - FlakeId(str(base_path)), - Machine( + opts = CreateOptions( + clan_dir=FlakeId(str(base_path)), + machine=Machine( name="machine1", tags=[], system="x86_64-linux", deploy=MachineDeploy() ), ) + create_machine(opts) + inventory = load_inventory_json(base_path) inventory.services = { diff --git a/pkgs/webview-ui/app/src/routes/flash/view.tsx b/pkgs/webview-ui/app/src/routes/flash/view.tsx index 8eb07b8f6..aef6dd2ec 100644 --- a/pkgs/webview-ui/app/src/routes/flash/view.tsx +++ b/pkgs/webview-ui/app/src/routes/flash/view.tsx @@ -160,6 +160,7 @@ export const Flash = () => { dry_run: false, write_efi_boot_entries: false, debug: false, + no_udev: true, }); } catch (error) { toast.error(`Error could not flash disk: ${error}`); diff --git a/pkgs/webview-ui/app/src/routes/machines/create.tsx b/pkgs/webview-ui/app/src/routes/machines/create.tsx index 637e9fe34..5a2e9ea47 100644 --- a/pkgs/webview-ui/app/src/routes/machines/create.tsx +++ b/pkgs/webview-ui/app/src/routes/machines/create.tsx @@ -13,16 +13,18 @@ export function CreateMachine() { const navigate = useNavigate(); const [formStore, { Form, Field }] = createForm({ initialValues: { - flake: { - loc: activeURI() || "", - }, - machine: { - tags: ["all"], - deploy: { - targetHost: "", + opts: { + clan_dir: { + loc: activeURI() || "", + }, + machine: { + tags: ["all"], + deploy: { + targetHost: "", + }, + name: "", + description: "", }, - name: "", - description: "", }, }, }); @@ -39,14 +41,16 @@ export function CreateMachine() { console.log("submitting", values); const response = await callApi("create_machine", { - ...values, - flake: { - loc: active_dir, + opts: { + ...values.opts, + clan_dir: { + loc: active_dir, + }, }, }); if (response.status === "success") { - toast.success(`Successfully created ${values.machine.name}`); + toast.success(`Successfully created ${values.opts.machine.name}`); reset(formStore); queryClient.invalidateQueries({ @@ -55,7 +59,7 @@ export function CreateMachine() { navigate("/machines"); } else { toast.error( - `Error: ${response.errors[0].message}. Machine ${values.machine.name} could not be created`, + `Error: ${response.errors[0].message}. Machine ${values.opts.machine.name} could not be created`, ); } }; @@ -65,7 +69,7 @@ export function CreateMachine() { Create new Machine
{(field, props) => ( @@ -79,7 +83,7 @@ export function CreateMachine() { /> )} - + {(field, props) => ( )} - + {(field, props) => ( <>