From ec54a6a97828c7d44f0a2ddeabac833ecb4e664b Mon Sep 17 00:00:00 2001 From: Qubasa Date: Thu, 12 Dec 2024 15:35:26 +0100 Subject: [PATCH] clan-cli: Replace log.info to machine.info if applicable --- pkgs/clan-cli/clan_cli/backups/create.py | 2 +- pkgs/clan-cli/clan_cli/facts/check.py | 8 +- pkgs/clan-cli/clan_cli/facts/generate.py | 2 +- .../clan_cli/facts/public_modules/vm.py | 2 +- pkgs/clan-cli/clan_cli/facts/upload.py | 2 +- pkgs/clan-cli/clan_cli/flash/automount.py | 18 ++- pkgs/clan-cli/clan_cli/flash/flash.py | 2 +- pkgs/clan-cli/clan_cli/machines/hardware.py | 8 +- pkgs/clan-cli/clan_cli/machines/install.py | 110 ++++++++++-------- pkgs/clan-cli/clan_cli/machines/machines.py | 18 ++- pkgs/clan-cli/clan_cli/machines/update.py | 4 +- pkgs/clan-cli/clan_cli/secrets/sops.py | 2 +- pkgs/clan-cli/clan_cli/state/list.py | 13 ++- pkgs/clan-cli/clan_cli/vars/check.py | 16 +-- pkgs/clan-cli/clan_cli/vars/generate.py | 4 +- .../clan_cli/vars/public_modules/vm.py | 2 +- pkgs/clan-cli/clan_cli/vms/run.py | 6 +- 17 files changed, 126 insertions(+), 93 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/backups/create.py b/pkgs/clan-cli/clan_cli/backups/create.py index f2513e102..b9104d012 100644 --- a/pkgs/clan-cli/clan_cli/backups/create.py +++ b/pkgs/clan-cli/clan_cli/backups/create.py @@ -14,7 +14,7 @@ log = logging.getLogger(__name__) def create_backup(machine: Machine, provider: str | None = None) -> None: - log.info(f"creating backup for {machine.name}") + machine.info(f"creating backup for {machine.name}") backup_scripts = json.loads(machine.eval_nix("config.clan.core.backups")) if provider is None: for provider in backup_scripts["providers"]: diff --git a/pkgs/clan-cli/clan_cli/facts/check.py b/pkgs/clan-cli/clan_cli/facts/check.py index 892e234f8..8a37091c0 100644 --- a/pkgs/clan-cli/clan_cli/facts/check.py +++ b/pkgs/clan-cli/clan_cli/facts/check.py @@ -24,20 +24,20 @@ def check_secrets(machine: Machine, service: None | str = None) -> bool: else: secret_name = secret_fact["name"] if not secret_facts_store.exists(service, secret_name): - log.info( + machine.info( f"Secret fact '{secret_fact}' for service '{service}' in machine {machine.name} is missing." ) missing_secret_facts.append((service, secret_name)) for public_fact in machine.facts_data[service]["public"]: if not public_facts_store.exists(service, public_fact): - log.info( + machine.info( f"Public fact '{public_fact}' for service '{service}' in machine {machine.name} is missing." ) missing_public_facts.append((service, public_fact)) - log.debug(f"missing_secret_facts: {missing_secret_facts}") - log.debug(f"missing_public_facts: {missing_public_facts}") + machine.debug(f"missing_secret_facts: {missing_secret_facts}") + machine.debug(f"missing_public_facts: {missing_public_facts}") return not (missing_secret_facts or missing_public_facts) diff --git a/pkgs/clan-cli/clan_cli/facts/generate.py b/pkgs/clan-cli/clan_cli/facts/generate.py index dbba5350d..ed202aa5a 100644 --- a/pkgs/clan-cli/clan_cli/facts/generate.py +++ b/pkgs/clan-cli/clan_cli/facts/generate.py @@ -73,7 +73,7 @@ def generate_service_facts( service_dir = tmpdir / service # check if all secrets exist and generate them if at least one is missing needs_regeneration = not check_secrets(machine, service=service) - log.debug(f"{service} needs_regeneration: {needs_regeneration}") + machine.debug(f"{service} needs_regeneration: {needs_regeneration}") if not (needs_regeneration or regenerate): return False if not isinstance(machine.flake, Path): diff --git a/pkgs/clan-cli/clan_cli/facts/public_modules/vm.py b/pkgs/clan-cli/clan_cli/facts/public_modules/vm.py index adfe75370..ae458ff5d 100644 --- a/pkgs/clan-cli/clan_cli/facts/public_modules/vm.py +++ b/pkgs/clan-cli/clan_cli/facts/public_modules/vm.py @@ -15,7 +15,7 @@ class FactStore(FactStoreBase): self.machine = machine self.works_remotely = False self.dir = vm_state_dir(machine.flake, machine.name) / "facts" - log.debug(f"FactStore initialized with dir {self.dir}") + machine.debug(f"FactStore initialized with dir {self.dir}") def exists(self, service: str, name: str) -> bool: fact_path = self.dir / service / name diff --git a/pkgs/clan-cli/clan_cli/facts/upload.py b/pkgs/clan-cli/clan_cli/facts/upload.py index 47bda5e74..aff152d53 100644 --- a/pkgs/clan-cli/clan_cli/facts/upload.py +++ b/pkgs/clan-cli/clan_cli/facts/upload.py @@ -16,7 +16,7 @@ def upload_secrets(machine: Machine) -> None: secret_facts_store = secret_facts_module.SecretStore(machine=machine) if not secret_facts_store.needs_upload(): - log.info("Secrets already uploaded") + machine.info("Secrets already uploaded") return with TemporaryDirectory(prefix="facts-upload-") as tempdir: diff --git a/pkgs/clan-cli/clan_cli/flash/automount.py b/pkgs/clan-cli/clan_cli/flash/automount.py index 7fdcdc5d3..af2295f0d 100644 --- a/pkgs/clan-cli/clan_cli/flash/automount.py +++ b/pkgs/clan-cli/clan_cli/flash/automount.py @@ -6,12 +6,15 @@ from pathlib import Path from clan_cli.cmd import Log, RunOpts, run from clan_cli.errors import ClanError +from clan_cli.machines.machines import Machine log = logging.getLogger(__name__) @contextmanager -def pause_automounting(devices: list[Path]) -> Generator[None, None, None]: +def pause_automounting( + devices: list[Path], machine: Machine +) -> Generator[None, None, None]: """ Pause automounting on the device for the duration of this context manager @@ -30,11 +33,16 @@ def pause_automounting(devices: list[Path]) -> Generator[None, None, None]: str_devs = [str(dev) for dev in devices] cmd = ["sudo", str(inhibit_path), "enable", *str_devs] - result = run(cmd, RunOpts(log=Log.BOTH, check=False, needs_user_terminal=True)) + result = run( + cmd, + RunOpts( + log=Log.BOTH, check=False, needs_user_terminal=True, prefix=machine.name + ), + ) if result.returncode != 0: - log.error("Failed to inhibit automounting") + machine.error("Failed to inhibit automounting") yield None cmd = ["sudo", str(inhibit_path), "disable", *str_devs] - result = run(cmd, RunOpts(log=Log.BOTH, check=False)) + result = run(cmd, RunOpts(log=Log.BOTH, check=False, prefix=machine.name)) if result.returncode != 0: - log.error("Failed to re-enable automounting") + machine.error("Failed to re-enable automounting") diff --git a/pkgs/clan-cli/clan_cli/flash/flash.py b/pkgs/clan-cli/clan_cli/flash/flash.py index 2c8f19c67..8b0002372 100644 --- a/pkgs/clan-cli/clan_cli/flash/flash.py +++ b/pkgs/clan-cli/clan_cli/flash/flash.py @@ -49,7 +49,7 @@ def flash_machine( extra_args: list[str] | None = None, ) -> None: devices = [Path(disk.device) for disk in disks] - with pause_automounting(devices): + with pause_automounting(devices, machine): if extra_args is None: extra_args = [] system_config_nix: dict[str, Any] = {} diff --git a/pkgs/clan-cli/clan_cli/machines/hardware.py b/pkgs/clan-cli/clan_cli/machines/hardware.py index 7e8d719ba..28c7b3c7c 100644 --- a/pkgs/clan-cli/clan_cli/machines/hardware.py +++ b/pkgs/clan-cli/clan_cli/machines/hardware.py @@ -71,7 +71,7 @@ def show_machine_deployment_target(clan_dir: Path, machine_name: str) -> str | N "--json", ] ) - proc = run_no_stdout(cmd) + proc = run_no_stdout(cmd, RunOpts(prefix=machine_name)) res = proc.stdout.strip() target_host = json.loads(res) @@ -93,7 +93,7 @@ def show_machine_hardware_platform(clan_dir: Path, machine_name: str) -> str | N "--json", ] ) - proc = run_no_stdout(cmd) + proc = run_no_stdout(cmd, RunOpts(prefix=machine_name)) res = proc.stdout.strip() host_platform = json.loads(res) @@ -160,9 +160,9 @@ def generate_machine_hardware_info(opts: HardwareGenerateOptions) -> HardwareCon *config_command, ], ) - out = run(cmd, RunOpts(needs_user_terminal=True)) + out = run(cmd, RunOpts(needs_user_terminal=True, prefix=machine.name)) if out.returncode != 0: - log.error(out) + machine.error(str(out)) msg = f"Failed to inspect {opts.machine}. Address: {opts.target_host}" raise ClanError(msg) diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index bf3c7e37b..3a6049911 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -3,6 +3,7 @@ import importlib import json import logging import os +import sys from dataclasses import dataclass, field from pathlib import Path from tempfile import TemporaryDirectory @@ -48,12 +49,12 @@ def install_machine(opts: InstallOptions) -> None: machine.override_target_host = opts.target_host secret_facts_module = importlib.import_module(machine.secret_facts_module) - log.info(f"installing {machine.name}") + machine.info(f"installing {machine.name}") secret_facts_store = secret_facts_module.SecretStore(machine=machine) h = machine.target_host target_host = f"{h.user or 'root'}@{h.host}" - log.info(f"target host: {target_host}") + machine.info(f"target host: {target_host}") generate_facts([machine]) generate_vars([machine]) @@ -103,7 +104,7 @@ def install_machine(opts: InstallOptions) -> None: ] if not machine.can_build_locally or opts.build_on_remote: - log.info("Architecture mismatch. Building on remote machine") + machine.info("Architecture mismatch. Building on remote machine") cmd.append("--build-on-remote") if machine.target_host.port: @@ -119,62 +120,70 @@ def install_machine(opts: InstallOptions) -> None: ["nixpkgs#nixos-anywhere"], cmd, ), - RunOpts(log=Log.BOTH), + RunOpts(log=Log.BOTH, prefix=machine.name, needs_user_terminal=True), ) def install_command(args: argparse.Namespace) -> None: - if args.flake is None: - msg = "Could not find clan flake toplevel directory" - raise ClanError(msg) - json_ssh_deploy = None - if args.json: - json_file = Path(args.json) - if json_file.is_file(): - json_ssh_deploy = json.loads(json_file.read_text()) + try: + if args.flake is None: + msg = "Could not find clan flake toplevel directory" + raise ClanError(msg) + json_ssh_deploy = None + if args.json: + json_file = Path(args.json) + if json_file.is_file(): + json_ssh_deploy = json.loads(json_file.read_text()) + else: + json_ssh_deploy = json.loads(args.json) + elif args.png: + json_ssh_deploy = json.loads(qrcode_scan(args.png)) + + if json_ssh_deploy: + target_host = ( + f"root@{find_reachable_host_from_deploy_json(json_ssh_deploy)}" + ) + password = json_ssh_deploy["pass"] + elif args.target_host: + target_host = args.target_host + password = None else: - json_ssh_deploy = json.loads(args.json) - elif args.png: - json_ssh_deploy = json.loads(qrcode_scan(args.png)) + machine = Machine( + name=args.machine, flake=args.flake, nix_options=args.option + ) + target_host = str(machine.target_host) + password = None - if json_ssh_deploy: - target_host = f"root@{find_reachable_host_from_deploy_json(json_ssh_deploy)}" - password = json_ssh_deploy["pass"] - elif args.target_host: - target_host = args.target_host - password = None - else: - machine = Machine(name=args.machine, flake=args.flake, nix_options=args.option) - target_host = str(machine.target_host) - password = None + if args.password: + password = args.password - if args.password: - password = args.password + if not target_host: + msg = "No target host provided, please provide a target host." + raise ClanError(msg) - if not target_host: - msg = "No target host provided, please provide a target host." - raise ClanError(msg) + if not args.yes: + ask = input(f"Install {args.machine} to {target_host}? [y/N] ") + if ask != "y": + return None - if not args.yes: - ask = input(f"Install {args.machine} to {target_host}? [y/N] ") - if ask != "y": - return None - - return install_machine( - InstallOptions( - flake=args.flake, - machine=args.machine, - target_host=target_host, - kexec=args.kexec, - debug=args.debug, - no_reboot=args.no_reboot, - json_ssh_deploy=json_ssh_deploy, - nix_options=args.option, - build_on_remote=args.build_on_remote, - update_hardware_config=HardwareConfig(args.update_hardware_config), - password=password, - ), - ) + return install_machine( + InstallOptions( + flake=args.flake, + machine=args.machine, + target_host=target_host, + kexec=args.kexec, + debug=args.debug, + no_reboot=args.no_reboot, + json_ssh_deploy=json_ssh_deploy, + nix_options=args.option, + build_on_remote=args.build_on_remote, + update_hardware_config=HardwareConfig(args.update_hardware_config), + password=password, + ), + ) + except KeyboardInterrupt: + log.warning("Interrupted by user") + sys.exit(1) def find_reachable_host_from_deploy_json(deploy_json: dict[str, str]) -> str: @@ -252,4 +261,5 @@ def register_install_parser(parser: argparse.ArgumentParser) -> None: "--png", help="specify the json file for ssh data as the qrcode image (generated by starting the clan installer)", ) + parser.set_defaults(func=install_command) diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index f5749b038..725901f8b 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -8,7 +8,7 @@ from tempfile import NamedTemporaryFile from typing import TYPE_CHECKING, Any, Literal from clan_cli.clan_uri import FlakeId -from clan_cli.cmd import run_no_stdout +from clan_cli.cmd import RunOpts, run_no_stdout from clan_cli.errors import ClanError from clan_cli.facts import public_modules as facts_public_modules from clan_cli.facts import secret_modules as facts_secret_modules @@ -70,7 +70,8 @@ class Machine: output = self._eval_cache.get(attr) if output is None: output = run_no_stdout( - nix_eval(["--impure", "--expr", attr]) + nix_eval(["--impure", "--expr", attr]), + opts=RunOpts(prefix=self.name), ).stdout.strip() self._eval_cache[attr] = output return json.loads(output) @@ -239,7 +240,8 @@ class Machine: "--expr", f'let x = (builtins.fetchTree {{ type = "file"; url = "file://{config_json.name}"; }}); in {{ narHash = x.narHash; path = x.outPath; }}', ] - ) + ), + opts=RunOpts(prefix=self.name), ).stdout.strip() ) @@ -277,9 +279,15 @@ class Machine: args += nix_options + self.nix_options if method == "eval": - output = run_no_stdout(nix_eval(args)).stdout.strip() + output = run_no_stdout( + nix_eval(args), opts=RunOpts(prefix=self.name) + ).stdout.strip() return output - return Path(run_no_stdout(nix_build(args)).stdout.strip()) + return Path( + run_no_stdout( + nix_build(args), opts=RunOpts(prefix=self.name) + ).stdout.strip() + ) def eval_nix( self, diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 5f2fcee93..9f931dffd 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -170,7 +170,9 @@ def deploy_machine(machines: list[Machine]) -> None: # if the machine is mobile, we retry to deploy with the mobile workaround method is_mobile = machine.deployment.get("nixosMobileWorkaround", False) if is_mobile and ret.returncode != 0: - log.info("Mobile machine detected, applying workaround deployment method") + machine.info( + "Mobile machine detected, applying workaround deployment method" + ) ret = host.run( test_cmd, RunOpts(msg_color=MsgColor(stderr=AnsiColor.DEFAULT)), diff --git a/pkgs/clan-cli/clan_cli/secrets/sops.py b/pkgs/clan-cli/clan_cli/secrets/sops.py index 246ad5556..d0d5f5a1d 100644 --- a/pkgs/clan-cli/clan_cli/secrets/sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/sops.py @@ -77,7 +77,7 @@ class KeyType(enum.Enum): except FileNotFoundError: return except Exception as ex: - log.warn(f"Could not read age keys from {key_path}: {ex}") + log.warning(f"Could not read age keys from {key_path}: {ex}") # Sops will try every location, see age/keysource.go if key_path := os.environ.get("SOPS_AGE_KEY_FILE"): diff --git a/pkgs/clan-cli/clan_cli/state/list.py b/pkgs/clan-cli/clan_cli/state/list.py index e2e8e3b7b..59c306e9f 100644 --- a/pkgs/clan-cli/clan_cli/state/list.py +++ b/pkgs/clan-cli/clan_cli/state/list.py @@ -3,7 +3,7 @@ import json import logging from pathlib import Path -from clan_cli.cmd import run_no_stdout +from clan_cli.cmd import RunOpts, run_no_stdout from clan_cli.completions import ( add_dynamic_completer, complete_machines, @@ -11,12 +11,13 @@ from clan_cli.completions import ( ) from clan_cli.dirs import get_clan_flake_toplevel_or_env from clan_cli.errors import ClanCmdError, ClanError +from clan_cli.machines.machines import Machine from clan_cli.nix import nix_eval log = logging.getLogger(__name__) -def list_state_folders(machine: str, service: None | str = None) -> None: +def list_state_folders(machine: Machine, service: None | str = None) -> None: uri = "TODO" if (clan_dir_result := get_clan_flake_toplevel_or_env()) is not None: flake = clan_dir_result @@ -31,7 +32,7 @@ def list_state_folders(machine: str, service: None | str = None) -> None: res = "{}" try: - proc = run_no_stdout(cmd) + proc = run_no_stdout(cmd, opts=RunOpts(prefix=machine.name)) res = proc.stdout.strip() except ClanCmdError as e: msg = "Clan might not have meta attributes" @@ -49,7 +50,7 @@ def list_state_folders(machine: str, service: None | str = None) -> None: msg = f"Service {service} isn't configured for this machine." raise ClanError( msg, - location=f"clan state list {machine} --service {service}", + location=f"clan state list {machine.name} --service {service}", description=f"The service: {service} needs to be configured for the machine.", ) @@ -69,7 +70,9 @@ def list_state_folders(machine: str, service: None | str = None) -> None: def list_command(args: argparse.Namespace) -> None: - list_state_folders(machine=args.machine, service=args.service) + list_state_folders( + Machine(name=args.machine, flake=args.flake), service=args.service + ) def register_state_parser(parser: argparse.ArgumentParser) -> None: diff --git a/pkgs/clan-cli/clan_cli/vars/check.py b/pkgs/clan-cli/clan_cli/vars/check.py index 5718ee01b..b74f1ff91 100644 --- a/pkgs/clan-cli/clan_cli/vars/check.py +++ b/pkgs/clan-cli/clan_cli/vars/check.py @@ -60,7 +60,7 @@ def vars_status(machine: Machine, generator_name: None | str = None) -> VarStatu if file.secret: if not secret_vars_store.exists(generator, file.name): - log.info( + machine.info( f"Secret var '{file.name}' for service '{generator.name}' in machine {machine.name} is missing." ) missing_secret_vars.append(file) @@ -70,13 +70,13 @@ def vars_status(machine: Machine, generator_name: None | str = None) -> VarStatu file_name=file.name, ) if msg: - log.info( + machine.info( f"Secret var '{file.name}' for service '{generator.name}' in machine {machine.name} needs update: {msg}" ) unfixed_secret_vars.append(file) elif not public_vars_store.exists(generator, file.name): - log.info( + machine.info( f"Public var '{file.name}' for service '{generator.name}' in machine {machine.name} is missing." ) missing_public_vars.append(file) @@ -86,13 +86,13 @@ def vars_status(machine: Machine, generator_name: None | str = None) -> VarStatu and public_vars_store.hash_is_valid(generator) ): invalid_generators.append(generator.name) - log.info( + machine.info( f"Generator '{generator.name}' in machine {machine.name} has outdated invalidation hash." ) - log.debug(f"missing_secret_vars: {missing_secret_vars}") - log.debug(f"missing_public_vars: {missing_public_vars}") - log.debug(f"unfixed_secret_vars: {unfixed_secret_vars}") - log.debug(f"invalid_generators: {invalid_generators}") + machine.debug(f"missing_secret_vars: {missing_secret_vars}") + machine.debug(f"missing_public_vars: {missing_public_vars}") + machine.debug(f"unfixed_secret_vars: {unfixed_secret_vars}") + machine.debug(f"invalid_generators: {invalid_generators}") return VarStatus( missing_secret_vars, missing_public_vars, diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 6fb0d366a..f1830a02b 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -283,13 +283,13 @@ def _migration_file_exists( if is_secret: if machine.secret_facts_store.exists(generator.name, fact_name): return True - log.debug( + machine.debug( f"Cannot migrate fact {fact_name} for service {generator.name}, as it does not exist in the secret fact store" ) if not is_secret: if machine.public_facts_store.exists(generator.name, fact_name): return True - log.debug( + machine.debug( f"Cannot migrate fact {fact_name} for service {generator.name}, as it does not exist in the public fact store" ) return False diff --git a/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py b/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py index 4a6962e26..d693da8dd 100644 --- a/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py +++ b/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py @@ -19,7 +19,7 @@ class FactStore(StoreBase): self.machine = machine self.works_remotely = False self.dir = vm_state_dir(machine.flake, machine.name) / "facts" - log.debug(f"FactStore initialized with dir {self.dir}") + machine.debug(f"FactStore initialized with dir {self.dir}") @property def store_name(self) -> str: diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index 18f48e89b..768688acd 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -117,6 +117,7 @@ def prepare_disk( @contextmanager def start_vm( + machine: Machine, args: list[str], packages: list[str], extra_env: dict[str, str], @@ -127,7 +128,7 @@ def start_vm( env = os.environ.copy() env.update(extra_env) cmd = nix_shell(packages, args) - log.debug(f"Starting VM with command: {cmd}") + machine.debug(f"Starting VM with command: {cmd}") with subprocess.Popen( cmd, env=env, stdout=stdout, stderr=stderr, stdin=stdin ) as process: @@ -215,7 +216,7 @@ def spawn_vm( nix_options = [] with ExitStack() as stack: machine = Machine(name=vm.machine_name, flake=vm.flake_url) - log.debug(f"Creating VM for {machine}") + machine.debug(f"Creating VM for {machine}") # store the temporary rootfs inside XDG_CACHE_HOME on the host # otherwise, when using /tmp, we risk running out of memory @@ -292,6 +293,7 @@ def spawn_vm( start_waypipe(qemu_cmd.vsock_cid, f"[{vm.machine_name}] "), start_virtiofsd(virtiofsd_socket), start_vm( + machine, qemu_cmd.args, packages, extra_env,