yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'clan-cli: Move clan machines import to clan machines create' (#2163) from Qubasa/clan-core:Qubasa-main into main

Browse Source
This commit is contained in:
clan-bot
2024-09-23 15:18:21 +00:00
parent 26f8779787 d4e8f88185
commit f4949b26c4
12 changed files with 235 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
pkgs/clan-cli/clan_cli/__init__.py
View File

@@ -415,10 +415,12 @@ def main() -> None:
except ClanError as e:
if isinstance(e, ClanCmdError):
if e.cmd.msg:
log.exception(e.cmd.msg)
log.fatal(e.cmd.msg)
sys.exit(1)
log.exception("An error occurred")
log.fatal(e.msg)
if e.description:
print(f"========> {e.description}", file=sys.stderr)
sys.exit(1)
except KeyboardInterrupt:
log.warning("Interrupted by user")

1
pkgs/clan-cli/clan_cli/config/__init__.py
View File

@@ -44,6 +44,7 @@ def map_type(nix_type: str) -> Any:
# merge two dicts recursively
def merge(a: dict, b: dict, path: list[str] | None = None) -> dict:
a = a.copy()
if path is None:
path = []
for key in b:

49
pkgs/clan-cli/clan_cli/inventory/__init__.py
View File

@@ -15,6 +15,7 @@ Operate on the returned inventory to make changes
import contextlib
import json
from pathlib import Path
from typing import Any
from clan_cli.api import API, dataclass_to_dict, from_dict
from clan_cli.cmd import run_no_stdout
@@ -119,7 +120,9 @@ def load_inventory_json(
@API.register
def set_inventory(inventory: Inventory, flake_dir: str | Path, message: str) -> None:
def set_inventory(
inventory: Inventory | dict[str, Any], flake_dir: str | Path, message: str
) -> None:
""" "
Write the inventory to the flake directory
and commit it to git with the given message
@@ -127,7 +130,10 @@ def set_inventory(inventory: Inventory, flake_dir: str | Path, message: str) ->
inventory_file = get_path(flake_dir)
with inventory_file.open("w") as f:
json.dump(dataclass_to_dict(inventory), f, indent=2)
if isinstance(inventory, Inventory):
json.dump(dataclass_to_dict(inventory), f, indent=2)
else:
json.dump(inventory, f, indent=2)
commit_file(inventory_file, Path(flake_dir), commit_message=message)
@@ -147,3 +153,42 @@ def init_inventory(directory: str, init: Inventory | None = None) -> None:
if inventory is not None:
# Persist creates a commit message for each change
set_inventory(inventory, directory, "Init inventory")
@API.register
def merge_template_inventory(
inventory: Inventory, template_inventory: Inventory, machine_name: str
) -> None:
"""
Merge the template inventory into the current inventory
The template inventory is expected to be a subset of the current inventory
"""
for service_name, instance in template_inventory.services.items():
if len(instance.keys()) > 0:
msg = f"Service {service_name} in template inventory has multiple instances"
description = (
"Only one instance per service is allowed in a template inventory"
)
raise ClanError(msg, description=description)
# Get the service config without knowing instance name
service_conf = next((v for v in instance.values() if "config" in v), None)
if not service_conf:
msg = f"Service {service_name} in template inventory has no config"
description = "Invalid inventory configuration"
raise ClanError(msg, description=description)
if "machines" in service_conf:
msg = f"Service {service_name} in template inventory has machines"
description = "The 'machines' key is not allowed in template inventory"
raise ClanError(msg, description=description)
if "roles" not in service_conf:
msg = f"Service {service_name} in template inventory has no roles"
description = "roles key is required in template inventory"
raise ClanError(msg, description=description)
# TODO: We need a MachineReference type in nix before we can implement this properly
msg = "Merge template inventory is not implemented yet"
raise NotImplementedError(msg)

20
pkgs/clan-cli/clan_cli/machines/cli.py
View File

@@ -4,7 +4,6 @@ import argparse
from .create import register_create_parser
from .delete import register_delete_parser
from .hardware import register_update_hardware_config
from .import_cmd import register_import_parser
from .install import register_install_parser
from .list import register_list_parser
from .update import register_update_parser
@@ -114,22 +113,3 @@ For more detailed information, visit: https://docs.clan.lol/getting-started/depl
formatter_class=argparse.RawTextHelpFormatter,
)
register_install_parser(install_parser)
import_parser = subparser.add_parser(
"import",
help="Import a machine",
description="Import a template machine from a local or remote source.",
epilog=(
"""
Examples:
$ clan machines import flash-installer
Will import a machine from the flash-installer template from clan-core.
$ clan machines import flash-installer --src https://git.clan.lol/clan/clan-core
Will import a machine from the flash-installer template from clan-core but with the source set to the provided URL.
"""
),
formatter_class=argparse.RawTextHelpFormatter,
)
register_import_parser(import_parser)

185
pkgs/clan-cli/clan_cli/machines/create.py
View File

@@ -1,79 +1,177 @@
import argparse
import logging
import re
import shutil
from dataclasses import dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
from clan_cli.api import API
from clan_cli.clan.create import git_command
from clan_cli.clan_uri import FlakeId
from clan_cli.cmd import Log, run
from clan_cli.dirs import clan_templates, get_clan_flake_toplevel_or_env
from clan_cli.errors import ClanError
from clan_cli.inventory import Machine as InventoryMachine
from clan_cli.inventory import (
Machine,
MachineDeploy,
load_inventory_eval,
dataclass_to_dict,
load_inventory_json,
merge_template_inventory,
set_inventory,
)
from clan_cli.machines.list import list_nixos_machines
from clan_cli.nix import nix_command
log = logging.getLogger(__name__)
def validate_directory(root_dir: Path) -> None:
machines_dir = root_dir / "machines"
for root, _, files in root_dir.walk():
for file in files:
file_path = Path(root) / file
if not file_path.is_relative_to(machines_dir):
msg = f"File {file_path} is not in the 'machines' directory."
log.error(msg)
description = "Template machines are only allowed to contain files in the 'machines' directory."
raise ClanError(msg, description=description)
@dataclass
class CreateOptions:
clan_dir: FlakeId
machine: InventoryMachine
template_src: FlakeId | None = None
template_name: str | None = None
@API.register
def create_machine(flake: FlakeId, machine: Machine) -> None:
def create_machine(opts: CreateOptions) -> None:
if not opts.clan_dir.is_local():
msg = f"Clan {opts.clan_dir} is not a local clan."
description = "Import machine only works on local clans"
raise ClanError(msg, description=description)
if not opts.template_src:
opts.template_src = FlakeId(str(clan_templates()))
if not opts.template_name:
opts.template_name = "new-machine"
clan_dir = opts.clan_dir.path
log.debug(f"Importing machine '{opts.template_name}' from {opts.template_src}")
if opts.template_name in list_nixos_machines(clan_dir) and not opts.machine.name:
msg = f"{opts.template_name} is already defined in {clan_dir}"
description = (
"Please add the --rename option to import the machine with a different name"
)
raise ClanError(msg, description=description)
machine_name = opts.template_name if not opts.machine.name else opts.machine.name
dst = clan_dir / "machines" / machine_name
# TODO: Move this into nix code
hostname_regex = r"^(?!-)[A-Za-z0-9-]{1,63}(?<!-)$"
if not re.match(hostname_regex, machine.name):
if not re.match(hostname_regex, machine_name):
msg = "Machine name must be a valid hostname"
raise ClanError(msg, location="Create Machine")
inventory = load_inventory_json(flake.path)
if dst.exists():
msg = f"Machine {machine_name} already exists in {clan_dir}"
description = (
"Please delete the existing machine or import with a different name"
)
raise ClanError(msg, description=description)
full_inventory = load_inventory_eval(flake.path)
with TemporaryDirectory() as tmpdir:
tmpdirp = Path(tmpdir)
command = nix_command(
[
"flake",
"init",
"-t",
f"{opts.template_src}#machineTemplates",
]
)
if machine.name in full_inventory.machines:
msg = f"Machine with the name {machine.name} already exists"
raise ClanError(msg)
# Check if debug logging is enabled
is_debug_enabled = log.isEnabledFor(logging.DEBUG)
log_flag = Log.BOTH if is_debug_enabled else Log.NONE
run(command, log=log_flag, cwd=tmpdirp)
print(f"Define machine {machine.name}", machine)
validate_directory(tmpdirp)
inventory.machines.update({machine.name: machine})
set_inventory(inventory, flake.path, f"Create machine {machine.name}")
src = tmpdirp / "machines" / opts.template_name
if (
not (src / "configuration.nix").exists()
and not (src / "inventory.json").exists()
):
msg = f"Template machine '{opts.template_name}' does not contain a configuration.nix or inventory.json"
description = (
"Template machine must contain a configuration.nix or inventory.json"
)
raise ClanError(msg, description=description)
def log_copy(src: str, dst: str) -> None:
relative_dst = dst.replace(f"{clan_dir}/", "")
log.info(f"Add file: {relative_dst}")
shutil.copy2(src, dst)
shutil.copytree(src, dst, ignore_dangling_symlinks=True, copy_function=log_copy)
run(git_command(clan_dir, "add", f"machines/{machine_name}"), cwd=clan_dir)
inventory = load_inventory_json(clan_dir)
# Merge the inventory from the template
if (dst / "inventory.json").exists():
template_inventory = load_inventory_json(dst)
merge_template_inventory(inventory, template_inventory, machine_name)
# TODO: We should allow the template to specify machine metadata if not defined by user
#
new_machine = InventoryMachine(name=machine_name, deploy=MachineDeploy())
inventory.machines.update({new_machine.name: dataclass_to_dict(new_machine)})
set_inventory(inventory, clan_dir, "Imported machine from template")
def create_command(args: argparse.Namespace) -> None:
create_machine(
args.flake,
Machine(
name=args.machine,
system=args.system,
description=args.description,
tags=args.tags,
icon=args.icon,
deploy=MachineDeploy(),
),
if args.flake:
clan_dir = args.flake
else:
tmp = get_clan_flake_toplevel_or_env()
clan_dir = FlakeId(str(tmp)) if tmp else None
if not clan_dir:
msg = "No clan found."
description = (
"Run this command in a clan directory or specify the --flake option"
)
raise ClanError(msg, description=description)
machine = InventoryMachine(
name=args.machine_name,
tags=args.tags,
deploy=MachineDeploy(),
)
opts = CreateOptions(
clan_dir=clan_dir,
machine=machine,
template_name=args.template_name,
)
create_machine(opts)
def register_create_parser(parser: argparse.ArgumentParser) -> None:
parser.add_argument("machine", type=str)
parser.set_defaults(func=create_command)
parser.add_argument(
"--system",
"machine_name",
type=str,
default=None,
help="Host platform to use. i.e. 'x86_64-linux' or 'aarch64-darwin' etc.",
metavar="PLATFORM",
)
parser.add_argument(
"--description",
type=str,
default=None,
help="A description of the machine.",
)
parser.add_argument(
"--icon",
type=str,
default=None,
help="Path to an icon to use for the machine. - Must be a path to icon file relative to the flake directory, or a public url.",
metavar="PATH",
help="The name of the machine to import",
)
parser.add_argument(
"--tags",
@@ -81,3 +179,8 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
default=[],
help="Tags to associate with the machine. Can be used to assign multiple machines to services.",
)
parser.add_argument(
"--template-name",
type=str,
help="The name of the template machine to import",
)

160
pkgs/clan-cli/clan_cli/machines/import_cmd.py
View File

@@ -1,160 +0,0 @@
import argparse
import logging
import shutil
from dataclasses import dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
from clan_cli.api import API
from clan_cli.clan.create import git_command
from clan_cli.clan_uri import FlakeId
from clan_cli.cmd import Log, run
from clan_cli.dirs import clan_templates, get_clan_flake_toplevel_or_env
from clan_cli.errors import ClanError
from clan_cli.machines.list import list_nixos_machines
from clan_cli.machines.machines import Machine
from clan_cli.nix import nix_command
log = logging.getLogger(__name__)
def validate_directory(root_dir: Path, machine_name: str) -> None:
machines_dir = root_dir / "machines" / machine_name
for root, _, files in root_dir.walk():
for file in files:
file_path = Path(root) / file
if not file_path.is_relative_to(machines_dir):
msg = f"File {file_path} is not in the 'machines/{machine_name}' directory."
description = "Template machines are only allowed to contain files in the 'machines/{machine_name}' directory."
raise ClanError(msg, description=description)
@dataclass
class ImportOptions:
target: FlakeId
src: Machine
rename: str | None = None
@API.register
def import_machine(opts: ImportOptions) -> None:
if not opts.target.is_local():
msg = f"Clan {opts.target} is not a local clan."
description = "Import machine only works on local clans"
raise ClanError(msg, description=description)
clan_dir = opts.target.path
log.debug(f"Importing machine '{opts.src.name}' from {opts.src.flake}")
if opts.src.name == opts.rename:
msg = "Rename option must be different from the template machine name"
raise ClanError(msg)
if opts.src.name in list_nixos_machines(clan_dir) and not opts.rename:
msg = f"{opts.src.name} is already defined in {clan_dir}"
description = (
"Please add the --rename option to import the machine with a different name"
)
raise ClanError(msg, description=description)
machine_name = opts.src.name if not opts.rename else opts.rename
dst = clan_dir / "machines" / machine_name
if dst.exists():
msg = f"Machine {machine_name} already exists in {clan_dir}"
description = (
"Please delete the existing machine or import with a different name"
)
raise ClanError(msg, description=description)
with TemporaryDirectory() as tmpdir:
tmpdirp = Path(tmpdir)
command = nix_command(
[
"flake",
"init",
"-t",
opts.src.get_id(),
]
)
run(command, log=Log.NONE, cwd=tmpdirp)
validate_directory(tmpdirp, opts.src.name)
src = tmpdirp / "machines" / opts.src.name
if (
not (src / "configuration.nix").exists()
and not (src / "inventory.json").exists()
):
msg = f"Template machine {opts.src.name} does not contain a configuration.nix or inventory.json"
description = (
"Template machine must contain a configuration.nix or inventory.json"
)
raise ClanError(msg, description=description)
def log_copy(src: str, dst: str) -> None:
relative_dst = dst.replace(f"{clan_dir}/", "")
log.info(f"Add file: {relative_dst}")
shutil.copy2(src, dst)
shutil.copytree(src, dst, ignore_dangling_symlinks=True, copy_function=log_copy)
run(git_command(clan_dir, "add", f"machines/{machine_name}"), cwd=clan_dir)
if (dst / "inventory.json").exists():
# TODO: Implement inventory import
msg = "Inventory import not implemented yet"
raise NotImplementedError(msg)
# inventory = load_inventory_json(clan_dir)
# inventory.machines[machine_name] = Inventory_Machine(
# name=machine_name,
# deploy=MachineDeploy(targetHost=None),
# )
# set_inventory(inventory, clan_dir, "Imported machine from template")
def import_command(args: argparse.Namespace) -> None:
if args.flake:
target = args.flake
else:
tmp = get_clan_flake_toplevel_or_env()
target = FlakeId(str(tmp)) if tmp else None
if not target:
msg = "No clan found."
description = (
"Run this command in a clan directory or specify the --flake option"
)
raise ClanError(msg, description=description)
src_uri = args.src
if not src_uri:
src_uri = FlakeId(str(clan_templates()))
opts = ImportOptions(
target=target,
src=Machine(flake=src_uri, name=args.machine_name),
rename=args.rename,
)
import_machine(opts)
def register_import_parser(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
"machine_name",
type=str,
help="The name of the machine to import",
)
parser.add_argument(
"--src",
type=FlakeId,
help="The source flake to import the machine from",
)
parser.add_argument(
"--rename",
type=str,
help="Rename the imported machine",
)
parser.set_defaults(func=import_command)

10
pkgs/clan-cli/tests/test_modules.py
View File

@@ -10,7 +10,7 @@ from clan_cli.inventory import (
load_inventory_json,
set_inventory,
)
from clan_cli.machines.create import create_machine
from clan_cli.machines.create import CreateOptions, create_machine
from clan_cli.nix import nix_eval, run_no_stdout
from fixtures_flakes import FlakeForTest
@@ -53,13 +53,15 @@ def test_add_module_to_inventory(
age_keys[0].pubkey,
]
)
create_machine(
FlakeId(str(base_path)),
Machine(
opts = CreateOptions(
clan_dir=FlakeId(str(base_path)),
machine=Machine(
name="machine1", tags=[], system="x86_64-linux", deploy=MachineDeploy()
),
)
create_machine(opts)
inventory = load_inventory_json(base_path)
inventory.services = {

1
pkgs/webview-ui/app/src/routes/flash/view.tsx
View File

@@ -160,6 +160,7 @@ export const Flash = () => {
dry_run: false,
write_efi_boot_entries: false,
debug: false,
no_udev: true,
});
} catch (error) {
toast.error(`Error could not flash disk: ${error}`);

38
pkgs/webview-ui/app/src/routes/machines/create.tsx
View File

@@ -13,16 +13,18 @@ export function CreateMachine() {
const navigate = useNavigate();
const [formStore, { Form, Field }] = createForm<CreateMachineForm>({
initialValues: {
flake: {
loc: activeURI() || "",
},
machine: {
tags: ["all"],
deploy: {
targetHost: "",
opts: {
clan_dir: {
loc: activeURI() || "",
},
machine: {
tags: ["all"],
deploy: {
targetHost: "",
},
name: "",
description: "",
},
name: "",
description: "",
},
},
});
@@ -39,14 +41,16 @@ export function CreateMachine() {
console.log("submitting", values);
const response = await callApi("create_machine", {
...values,
flake: {
loc: active_dir,
opts: {
...values.opts,
clan_dir: {
loc: active_dir,
},
},
});
if (response.status === "success") {
toast.success(`Successfully created ${values.machine.name}`);
toast.success(`Successfully created ${values.opts.machine.name}`);
reset(formStore);
queryClient.invalidateQueries({
@@ -55,7 +59,7 @@ export function CreateMachine() {
navigate("/machines");
} else {
toast.error(
`Error: ${response.errors[0].message}. Machine ${values.machine.name} could not be created`,
`Error: ${response.errors[0].message}. Machine ${values.opts.machine.name} could not be created`,
);
}
};
@@ -65,7 +69,7 @@ export function CreateMachine() {
<span class="px-2">Create new Machine</span>
<Form onSubmit={handleSubmit}>
<Field
name="machine.name"
name="opts.machine.name"
validate={[required("This field is required")]}
>
{(field, props) => (
@@ -79,7 +83,7 @@ export function CreateMachine() {
/>
)}
</Field>
<Field name="machine.description">
<Field name="opts.machine.description">
{(field, props) => (
<TextInput
inputProps={props}
@@ -90,7 +94,7 @@ export function CreateMachine() {
/>
)}
</Field>
<Field name="machine.deploy.targetHost">
<Field name="opts.machine.deploy.targetHost">
{(field, props) => (
<>
<TextInput

6
templates/flake.nix
View File

@@ -15,9 +15,9 @@
description = "Flake-parts";
path = ./flake-parts;
};
flash-installer = {
description = "Flash installer";
path = ./flash-installer;
machineTemplates = {
description = "Machine templates";
path = ./machineTemplates;
};
};
};

7
templates/machineTemplates/machines/flash-installer/configuration.nix Normal file
View File

@@ -0,0 +1,7 @@
{
imports = [
];
# Flash machine template
}

2
templates/flash-installer/machines/flash-installer/configuration.nix → templates/machineTemplates/machines/new-machine/configuration.nix
View File

@@ -3,5 +3,5 @@
];
# Hello World
# New machine!
}