diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index 144e9ec28..52b69b75f 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -57,21 +57,15 @@ def create_parser(prog: str | None = None) -> argparse.ArgumentParser: default=[], ) - def flake_path(arg: str) -> Path: + def flake_path(arg: str) -> str | Path: flake_dir = Path(arg).resolve() - if not flake_dir.exists(): - raise argparse.ArgumentTypeError( - f"flake directory {flake_dir} does not exist" - ) - if not flake_dir.is_dir(): - raise argparse.ArgumentTypeError( - f"flake directory {flake_dir} is not a directory" - ) - return flake_dir + if flake_dir.exists() and flake_dir.is_dir(): + return flake_dir + return arg parser.add_argument( "--flake", - help="path to the flake where the clan resides in", + help="path to the flake where the clan resides in, can be a remote flake or local", default=get_clan_flake_toplevel(), type=flake_path, ) diff --git a/pkgs/clan-cli/clan_cli/backups/create.py b/pkgs/clan-cli/clan_cli/backups/create.py index cf6ac38a5..dbfa7b899 100644 --- a/pkgs/clan-cli/clan_cli/backups/create.py +++ b/pkgs/clan-cli/clan_cli/backups/create.py @@ -31,7 +31,7 @@ def create_backup(machine: Machine, provider: str | None = None) -> None: def create_command(args: argparse.Namespace) -> None: - machine = Machine(name=args.machine, flake_dir=args.flake) + machine = Machine(name=args.machine, flake=args.flake) create_backup(machine=machine, provider=args.provider) diff --git a/pkgs/clan-cli/clan_cli/backups/list.py b/pkgs/clan-cli/clan_cli/backups/list.py index dc863b926..8a679ce0a 100644 --- a/pkgs/clan-cli/clan_cli/backups/list.py +++ b/pkgs/clan-cli/clan_cli/backups/list.py @@ -45,9 +45,7 @@ def list_provider(machine: Machine, provider: str) -> list[Backup]: def list_backups(machine: Machine, provider: str | None = None) -> list[Backup]: - backup_metadata = json.loads( - machine.eval_nix(f"nixosConfigurations.{machine.name}.config.clanCore.backups") - ) + backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups")) results = [] if provider is None: for _provider in backup_metadata["providers"]: @@ -60,7 +58,7 @@ def list_backups(machine: Machine, provider: str | None = None) -> list[Backup]: def list_command(args: argparse.Namespace) -> None: - machine = Machine(name=args.machine, flake_dir=args.flake) + machine = Machine(name=args.machine, flake=args.flake) backups = list_backups(machine=machine, provider=args.provider) print(backups) diff --git a/pkgs/clan-cli/clan_cli/backups/restore.py b/pkgs/clan-cli/clan_cli/backups/restore.py index 2a1c5fc47..d9e377fe1 100644 --- a/pkgs/clan-cli/clan_cli/backups/restore.py +++ b/pkgs/clan-cli/clan_cli/backups/restore.py @@ -91,7 +91,7 @@ def restore_backup( def restore_command(args: argparse.Namespace) -> None: - machine = Machine(name=args.machine, flake_dir=args.flake) + machine = Machine(name=args.machine, flake=args.flake) backups = list_backups(machine=machine, provider=args.provider) restore_backup( machine=machine, diff --git a/pkgs/clan-cli/clan_cli/dirs.py b/pkgs/clan-cli/clan_cli/dirs.py index e0a000dd6..fd75bf297 100644 --- a/pkgs/clan-cli/clan_cli/dirs.py +++ b/pkgs/clan-cli/clan_cli/dirs.py @@ -47,7 +47,7 @@ def user_gcroot_dir() -> Path: return p -def specific_groot_dir(*, clan_name: str, flake_url: str) -> Path: +def machine_gcroot(*, clan_name: str, flake_url: str) -> Path: # Always build icon so that we can symlink it to the gcroot gcroot_dir = user_gcroot_dir() clan_gcroot = gcroot_dir / clan_key_safe(clan_name, flake_url) diff --git a/pkgs/clan-cli/clan_cli/flakes/inspect.py b/pkgs/clan-cli/clan_cli/flakes/inspect.py index aa0a875b9..8578d1873 100644 --- a/pkgs/clan-cli/clan_cli/flakes/inspect.py +++ b/pkgs/clan-cli/clan_cli/flakes/inspect.py @@ -3,9 +3,10 @@ from dataclasses import dataclass from pathlib import Path from ..cmd import run -from ..dirs import specific_groot_dir +from ..dirs import machine_gcroot from ..errors import ClanError from ..machines.list import list_machines +from ..machines.machines import Machine from ..nix import nix_build, nix_config, nix_eval, nix_metadata from ..vms.inspect import VmConfig, inspect_vm @@ -29,23 +30,24 @@ def run_cmd(cmd: list[str]) -> str: return proc.stdout.strip() -def inspect_flake(flake_url: str | Path, flake_attr: str) -> FlakeConfig: +def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig: config = nix_config() system = config["system"] # Check if the machine exists machines = list_machines(flake_url) - if flake_attr not in machines: + if machine_name not in machines: raise ClanError( - f"Machine {flake_attr} not found in {flake_url}. Available machines: {', '.join(machines)}" + f"Machine {machine_name} not found in {flake_url}. Available machines: {', '.join(machines)}" ) - vm = inspect_vm(flake_url, flake_attr) + machine = Machine(machine_name, flake_url) + vm = inspect_vm(machine) # Get the cLAN name cmd = nix_eval( [ - f'{flake_url}#clanInternals.machines."{system}"."{flake_attr}".config.clanCore.clanName' + f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clanCore.clanName' ] ) res = run_cmd(cmd) @@ -54,7 +56,7 @@ def inspect_flake(flake_url: str | Path, flake_attr: str) -> FlakeConfig: # Get the clan icon path cmd = nix_eval( [ - f'{flake_url}#clanInternals.machines."{system}"."{flake_attr}".config.clanCore.clanIcon' + f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clanCore.clanIcon' ] ) res = run_cmd(cmd) @@ -67,10 +69,9 @@ def inspect_flake(flake_url: str | Path, flake_attr: str) -> FlakeConfig: cmd = nix_build( [ - f'{flake_url}#clanInternals.machines."{system}"."{flake_attr}".config.clanCore.clanIcon' + f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clanCore.clanIcon' ], - specific_groot_dir(clan_name=clan_name, flake_url=str(flake_url)) - / "clanIcon", + machine_gcroot(clan_name=clan_name, flake_url=str(flake_url)) / "clanIcon", ) run_cmd(cmd) @@ -81,7 +82,7 @@ def inspect_flake(flake_url: str | Path, flake_attr: str) -> FlakeConfig: vm=vm, flake_url=flake_url, clan_name=clan_name, - flake_attr=flake_attr, + flake_attr=machine_name, nar_hash=meta["locked"]["narHash"], icon=icon_path, description=meta.get("description"), @@ -102,7 +103,7 @@ def inspect_command(args: argparse.Namespace) -> None: flake=args.flake or Path.cwd(), ) res = inspect_flake( - flake_url=inspect_options.flake, flake_attr=inspect_options.machine + flake_url=inspect_options.flake, machine_name=inspect_options.machine ) print("cLAN name:", res.clan_name) print("Icon:", res.icon) diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index 330a4f1f9..973c3457d 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -34,7 +34,7 @@ def install_nixos(machine: Machine, kexec: str | None = None) -> None: cmd = [ "nixos-anywhere", "-f", - f"{machine.flake_dir}#{flake_attr}", + f"{machine.flake}#{flake_attr}", "-t", "--no-reboot", "--extra-files", @@ -68,7 +68,7 @@ def install_command(args: argparse.Namespace) -> None: target_host=args.target_host, kexec=args.kexec, ) - machine = Machine(opts.machine, flake_dir=opts.flake) + machine = Machine(opts.machine, flake=opts.flake) machine.deployment_address = opts.target_host install_nixos(machine, kexec=opts.kexec) diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 4d022c76d..235afa030 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -1,33 +1,16 @@ import json from pathlib import Path -from ..cmd import Log, run +from ..cmd import run from ..nix import nix_build, nix_config, nix_eval from ..ssh import Host, parse_deployment_address -def build_machine_data(machine_name: str, clan_dir: Path) -> dict: - config = nix_config() - system = config["system"] - - proc = run( - nix_build( - [ - f'{clan_dir}#clanInternals.machines."{system}"."{machine_name}".config.system.clan.deployment.file' - ] - ), - log=Log.BOTH, - error_msg="failed to build machine data", - ) - - return json.loads(Path(proc.stdout.strip()).read_text()) - - class Machine: def __init__( self, name: str, - flake_dir: Path, + flake: Path | str, machine_data: dict | None = None, ) -> None: """ @@ -36,22 +19,39 @@ class Machine: @clan_dir: the directory of the clan, optional, if not set it will be determined from the current working directory @machine_json: can be optionally used to skip evaluation of the machine, location of the json file with machine data """ - self.name = name - self.flake_dir = flake_dir + self.name: str = name + self.flake: str | Path = flake + self.eval_cache: dict[str, str] = {} + self.build_cache: dict[str, Path] = {} + + # TODO do this lazily if machine_data is None: - self.machine_data = build_machine_data(name, self.flake_dir) + self.machine_data = json.loads( + self.build_nix("config.system.clan.deployment.file").read_text() + ) else: self.machine_data = machine_data self.deployment_address = self.machine_data["deploymentAddress"] - self.secrets_module = self.machine_data["secretsModule"] - self.secrets_data = json.loads( - Path(self.machine_data["secretsData"]).read_text() - ) + self.secrets_module = self.machine_data.get("secretsModule", None) + if "secretsData" in self.machine_data: + self.secrets_data = json.loads( + Path(self.machine_data["secretsData"]).read_text() + ) self.secrets_upload_directory = self.machine_data["secretsUploadDirectory"] - self.eval_cache: dict[str, str] = {} - self.build_cache: dict[str, Path] = {} + + @property + def flake_dir(self) -> Path: + if isinstance(self.flake, Path): + return self.flake + + if hasattr(self, "flake_path"): + return Path(self.flake_path) + + print(nix_eval([f"{self.flake}"])) + self.flake_path = run(nix_eval([f"{self.flake}"])).stdout.strip() + return Path(self.flake_path) @property def host(self) -> Host: @@ -67,9 +67,25 @@ class Machine: if attr in self.eval_cache and not refresh: return self.eval_cache[attr] - output = run( - nix_eval([f"path:{self.flake_dir}#{attr}"]), - ).stdout.strip() + config = nix_config() + system = config["system"] + + if isinstance(self.flake, Path): + output = run( + nix_eval( + [ + f'path:{self.flake}#clanInternals.machines."{system}"."{self.name}".{attr}' + ] + ), + ).stdout.strip() + else: + output = run( + nix_eval( + [ + f'{self.flake}#clanInternals.machines."{system}"."{self.name}".{attr}' + ] + ), + ).stdout.strip() self.eval_cache[attr] = output return output @@ -80,8 +96,25 @@ class Machine: """ if attr in self.build_cache and not refresh: return self.build_cache[attr] - outpath = run( - nix_build([f"path:{self.flake_dir}#{attr}"]), - ).stdout.strip() + + config = nix_config() + system = config["system"] + + if isinstance(self.flake, Path): + outpath = run( + nix_build( + [ + f'path:{self.flake}#clanInternals.machines."{system}"."{self.name}".{attr}' + ] + ), + ).stdout.strip() + else: + outpath = run( + nix_build( + [ + f'{self.flake}#clanInternals.machines."{system}"."{self.name}".{attr}' + ] + ), + ).stdout.strip() self.build_cache[attr] = Path(outpath) return Path(outpath) diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 9710e8cb5..378803aab 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -93,9 +93,7 @@ def get_all_machines(clan_dir: Path) -> HostGroup: name, machine_data["deploymentAddress"], meta={ - "machine": Machine( - name=name, flake_dir=clan_dir, machine_data=machine_data - ) + "machine": Machine(name=name, flake=clan_dir, machine_data=machine_data) }, ) hosts.append(host) @@ -105,7 +103,7 @@ def get_all_machines(clan_dir: Path) -> HostGroup: def get_selected_machines(machine_names: list[str], flake_dir: Path) -> HostGroup: hosts = [] for name in machine_names: - machine = Machine(name=name, flake_dir=flake_dir) + machine = Machine(name=name, flake=flake_dir) hosts.append(machine.host) return HostGroup(hosts) @@ -115,7 +113,7 @@ def update(args: argparse.Namespace) -> None: if args.flake is None: raise ClanError("Could not find clan flake toplevel directory") if len(args.machines) == 1 and args.target_host is not None: - machine = Machine(name=args.machines[0], flake_dir=args.flake) + machine = Machine(name=args.machines[0], flake=args.flake) machine.deployment_address = args.target_host host = parse_deployment_address( args.machines[0], diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index ca49defe7..f5d90f40b 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -28,11 +28,11 @@ def generate_secrets(machine: Machine) -> None: not secret_store.exists(service, secret) for secret in machine.secrets_data[service]["secrets"] ) or any( - not (machine.flake_dir / fact).exists() + not (machine.flake / fact).exists() for fact in machine.secrets_data[service]["facts"].values() ) for fact in machine.secrets_data[service]["facts"].values(): - if not (machine.flake_dir / fact).exists(): + if not (machine.flake / fact).exists(): print(f"fact {fact} is missing") if needs_regeneration: env = os.environ.copy() @@ -66,7 +66,7 @@ def generate_secrets(machine: Machine) -> None: msg = f"did not generate a file for '{name}' when running the following command:\n" msg += machine.secrets_data[service]["generator"] raise ClanError(msg) - fact_path = machine.flake_dir / fact_path + fact_path = machine.flake / fact_path fact_path.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(fact_file, fact_path) @@ -74,7 +74,7 @@ def generate_secrets(machine: Machine) -> None: def generate_command(args: argparse.Namespace) -> None: - machine = Machine(name=args.machine, flake_dir=args.flake) + machine = Machine(name=args.machine, flake=args.flake) generate_secrets(machine) diff --git a/pkgs/clan-cli/clan_cli/secrets/upload.py b/pkgs/clan-cli/clan_cli/secrets/upload.py index f2f91ef91..796b73f06 100644 --- a/pkgs/clan-cli/clan_cli/secrets/upload.py +++ b/pkgs/clan-cli/clan_cli/secrets/upload.py @@ -43,7 +43,7 @@ def upload_secrets(machine: Machine) -> None: def upload_command(args: argparse.Namespace) -> None: - machine = Machine(name=args.machine, flake_dir=args.flake) + machine = Machine(name=args.machine, flake=args.flake) upload_secrets(machine) diff --git a/pkgs/clan-cli/clan_cli/vms/inspect.py b/pkgs/clan-cli/clan_cli/vms/inspect.py index 2cfd231e8..4b286bb88 100644 --- a/pkgs/clan-cli/clan_cli/vms/inspect.py +++ b/pkgs/clan-cli/clan_cli/vms/inspect.py @@ -3,15 +3,14 @@ import json from dataclasses import dataclass from pathlib import Path -from ..cmd import run -from ..nix import nix_config, nix_eval +from ..machines.machines import Machine @dataclass class VmConfig: - clan_name: str + machine_name: str flake_url: str | Path - flake_attr: str + clan_name: str cores: int memory_size: int @@ -19,21 +18,9 @@ class VmConfig: wayland: bool = False -def inspect_vm(flake_url: str | Path, flake_attr: str) -> VmConfig: - config = nix_config() - system = config["system"] - - cmd = nix_eval( - [ - f'{flake_url}#clanInternals.machines."{system}"."{flake_attr}".config.clanCore.vm.inspect', - "--refresh", - ] - ) - - proc = run(cmd) - - data = json.loads(proc.stdout) - return VmConfig(flake_url=flake_url, flake_attr=flake_attr, **data) +def inspect_vm(machine: Machine) -> VmConfig: + data = json.loads(machine.eval_nix("config.clanCore.vm.inspect")) + return VmConfig(machine_name=machine.name, flake_url=machine.flake, **data) @dataclass @@ -47,9 +34,9 @@ def inspect_command(args: argparse.Namespace) -> None: machine=args.machine, flake=args.flake or Path.cwd(), ) - res = inspect_vm( - flake_url=inspect_options.flake, flake_attr=inspect_options.machine - ) + + machine = Machine(inspect_options.machine, inspect_options.flake) + res = inspect_vm(machine) print("Cores:", res.cores) print("Memory size:", res.memory_size) print("Graphics:", res.graphics) diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index 5ce432c72..42a304006 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -1,20 +1,20 @@ import argparse +import importlib import json import logging import os -import sys import tempfile -import importlib from dataclasses import dataclass, field from pathlib import Path from typing import IO from ..cmd import Log, run -from ..dirs import module_root, specific_groot_dir, vm_state_dir +from ..dirs import machine_gcroot, module_root, vm_state_dir from ..errors import ClanError -from ..nix import nix_build, nix_config, nix_shell -from .inspect import VmConfig, inspect_vm from ..machines.machines import Machine +from ..nix import nix_build, nix_config, nix_shell +from ..secrets.generate import generate_secrets +from .inspect import VmConfig, inspect_vm log = logging.getLogger(__name__) @@ -75,7 +75,7 @@ def qemu_command( # fmt: off command = [ "qemu-kvm", - "-name", vm.flake_attr, + "-name", vm.machine_name, "-m", f'{nixos_config["memorySize"]}M', "-smp", str(nixos_config["cores"]), "-cpu", "max", @@ -104,32 +104,34 @@ def qemu_command( return command -def get_vm_create_info(vm: VmConfig, nix_options: list[str]) -> dict[str, str]: +# TODO move this to the Machines class +def get_vm_create_info( + machine: Machine, vm: VmConfig, nix_options: list[str] +) -> dict[str, str]: config = nix_config() system = config["system"] - clan_dir = vm.flake_url - machine = vm.flake_attr + clan_dir = machine.flake cmd = nix_build( [ - f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.system.clan.vm.create', + f'{clan_dir}#clanInternals.machines."{system}"."{machine.name}".config.system.clan.vm.create', *nix_options, ], - specific_groot_dir(clan_name=vm.clan_name, flake_url=str(vm.flake_url)) - / f"vm-{machine}", + machine_gcroot(clan_name=vm.clan_name, flake_url=str(vm.flake_url)) + / f"vm-{machine.name}", + ) + proc = run( + cmd, log=Log.BOTH, error_msg=f"Could not build vm config for {machine.name}" ) - proc = run(cmd, log=Log.BOTH, error_msg=f"Could not build vm config for {machine}") try: return json.loads(Path(proc.stdout.strip()).read_text()) except json.JSONDecodeError as e: raise ClanError(f"Failed to parse vm config: {e}") -def generate_secrets( - vm: VmConfig, - nixos_config: dict[str, str], +def get_secrets( + machine: Machine, tmpdir: Path, - log_fd: IO[str] | None, ) -> Path: secrets_dir = tmpdir / "secrets" secrets_dir.mkdir(exist_ok=True) @@ -138,19 +140,12 @@ def generate_secrets( secret_store = secrets_module.SecretStore(machine=machine) # Only generate secrets for local clans - if isinstance(vm.flake_url, Path) and vm.flake_url.is_dir(): - if Path(vm.flake_url).is_dir(): - run([nixos_config["generateSecrets"], vm.clan_name], env=env) - else: - log.warning("won't generate secrets for non local clan") + if isinstance(machine.flake, Path) and machine.flake.is_dir(): + generate_secrets(machine) + else: + log.warning("won't generate secrets for non local clan") - cmd = [nixos_config["uploadSecrets"]] - run( - cmd, - env=env, - log=Log.BOTH, - error_msg=f"Could not upload secrets for {vm.flake_attr}", - ) + secret_store.upload(secrets_dir) return secrets_dir @@ -191,26 +186,28 @@ def prepare_disk(tmpdir: Path, log_fd: IO[str] | None) -> Path: def run_vm( - vm: VmConfig, nix_options: list[str] = [], log_fd: IO[str] | None = None + vm: VmConfig, + nix_options: list[str] = [], + log_fd: IO[str] | None = None, ) -> None: """ log_fd can be used to stream the output of all commands to a UI """ - machine = vm.flake_attr + machine = Machine(vm.machine_name, vm.flake_url) log.debug(f"Creating VM for {machine}") # TODO: We should get this from the vm argument - nixos_config = get_vm_create_info(vm, nix_options) + nixos_config = get_vm_create_info(machine, vm, nix_options) with tempfile.TemporaryDirectory() as tmpdir_: tmpdir = Path(tmpdir_) xchg_dir = tmpdir / "xchg" xchg_dir.mkdir(exist_ok=True) - secrets_dir = generate_secrets(vm, nixos_config, tmpdir, log_fd) + secrets_dir = get_secrets(machine, tmpdir) disk_img = prepare_disk(tmpdir, log_fd) - state_dir = vm_state_dir(vm.clan_name, str(vm.flake_url), machine) + state_dir = vm_state_dir(vm.clan_name, str(machine.flake), machine.name) state_dir.mkdir(parents=True, exist_ok=True) qemu_cmd = qemu_command( @@ -243,7 +240,6 @@ def run_vm( @dataclass class RunOptions: machine: str - flake_url: str | None flake: Path nix_options: list[str] = field(default_factory=list) wayland: bool = False @@ -252,14 +248,14 @@ class RunOptions: def run_command(args: argparse.Namespace) -> None: run_options = RunOptions( machine=args.machine, - flake_url=args.flake_url, - flake=args.flake or Path.cwd(), + flake=args.flake, nix_options=args.option, wayland=args.wayland, ) - flake_url = run_options.flake_url or run_options.flake - vm = inspect_vm(flake_url=flake_url, flake_attr=run_options.machine) + machine = Machine(run_options.machine, run_options.flake) + + vm = inspect_vm(machine=machine) # TODO: allow to set this in the config vm.wayland = run_options.wayland