From d65bdb30ebff77ec7560e583bfdd3b0007bf6706 Mon Sep 17 00:00:00 2001 From: Qubasa Date: Tue, 24 Jun 2025 11:20:49 +0200 Subject: [PATCH 1/5] clan-cli: Move morph.py to clan_lib/machines --- pkgs/clan-cli/clan_cli/machines/morph.py | 130 +--------------------- pkgs/clan-cli/clan_lib/machines/morph.py | 133 +++++++++++++++++++++++ 2 files changed, 135 insertions(+), 128 deletions(-) create mode 100644 pkgs/clan-cli/clan_lib/machines/morph.py diff --git a/pkgs/clan-cli/clan_cli/machines/morph.py b/pkgs/clan-cli/clan_cli/machines/morph.py index 3e1e8e1ef..d820fb211 100644 --- a/pkgs/clan-cli/clan_cli/machines/morph.py +++ b/pkgs/clan-cli/clan_cli/machines/morph.py @@ -1,140 +1,14 @@ import argparse -import json import logging -import os -import random -import re -from pathlib import Path -from tempfile import TemporaryDirectory -from clan_lib.cmd import Log, RunOpts, run -from clan_lib.dirs import get_clan_flake_toplevel_or_env, specific_machine_dir +from clan_lib.dirs import get_clan_flake_toplevel_or_env from clan_lib.errors import ClanError from clan_lib.flake import Flake -from clan_lib.machines.actions import list_machines -from clan_lib.machines.machines import Machine -from clan_lib.nix import nix_build, nix_command -from clan_lib.nix_models.clan import InventoryMachine - -from clan_cli.machines.create import CreateOptions, create_machine -from clan_cli.vars.generate import generate_vars +from clan_lib.machines.morph import morph_machine log = logging.getLogger(__name__) -def is_local_input(node: dict[str, dict[str, str]]) -> bool: - locked = node.get("locked") - if not locked: - return False - # matches path and git+file:// - return ( - locked["type"] == "path" - or re.match(r"^\w+\+file://", locked.get("url", "")) is not None - ) - - -def random_hostname() -> str: - adjectives = ["wacky", "happy", "fluffy", "silly", "quirky", "zany", "bouncy"] - nouns = ["unicorn", "penguin", "goose", "ninja", "octopus", "hamster", "robot"] - adjective = random.choice(adjectives) - noun = random.choice(nouns) - return f"{adjective}-{noun}" - - -def morph_machine( - flake: Flake, template: str, ask_confirmation: bool, name: str | None = None -) -> None: - cmd = nix_command( - [ - "flake", - "archive", - "--json", - f"{flake}", - ] - ) - - archive_json = run( - cmd, RunOpts(error_msg="Failed to archive flake for morphing") - ).stdout.rstrip() - archive_path = json.loads(archive_json)["path"] - - with TemporaryDirectory(prefix="morph-") as _temp_dir: - flakedir = Path(_temp_dir).resolve() / "flake" - - flakedir.mkdir(parents=True, exist_ok=True) - run(["cp", "-r", archive_path + "/.", str(flakedir)]) - run(["chmod", "-R", "+w", str(flakedir)]) - - os.chdir(flakedir) - - if name is None: - name = random_hostname() - - if name not in list_machines(flake): - create_opts = CreateOptions( - template=template, - machine=InventoryMachine(name=name), - clan_dir=Flake(str(flakedir)), - ) - create_machine(create_opts, commit=False) - - machine = Machine(name=name, flake=Flake(str(flakedir))) - - generate_vars([machine], generator_name=None, regenerate=False) - - machine.secret_vars_store.populate_dir( - output_dir=Path("/run/secrets"), phases=["activation", "users", "services"] - ) - - # run(["nixos-facter", "-o", f"{flakedir}/machines/{name}/facter.json"]).stdout - - # facter_json = run(["nixos-facter"]).stdout - # run(["cp", "facter.json", f"{flakedir}/machines/{name}/facter.json"]).stdout - - machine_dir = specific_machine_dir(machine) - machine_dir.mkdir(parents=True, exist_ok=True) - Path(f"{machine_dir}/facter.json").write_text('{"system": "x86_64-linux"}') - result_path = run( - nix_build( - [f"{flakedir}#nixosConfigurations.{name}.config.system.build.toplevel"] - ) - ).stdout.rstrip() - - ropts = RunOpts(log=Log.BOTH) - - run( - [ - f"{result_path}/sw/bin/nixos-rebuild", - "dry-activate", - "--flake", - f"{flakedir}#{name}", - ], - ropts, - ).stdout.rstrip() - - if ask_confirmation: - log.warning("ARE YOU SURE YOU WANT TO DO THIS?") - log.warning( - "You should have read and understood all of the above and know what you are doing." - ) - - ask = input( - f"Do you really want convert this machine into {name}? If to continue, type in the new machine name: " - ) - if ask != name: - return - - run( - [ - f"{result_path}/sw/bin/nixos-rebuild", - "test", - "--flake", - f"{flakedir}#{name}", - ], - ropts, - ).stdout.rstrip() - - def morph_command(args: argparse.Namespace) -> None: if args.flake: clan_dir = args.flake diff --git a/pkgs/clan-cli/clan_lib/machines/morph.py b/pkgs/clan-cli/clan_lib/machines/morph.py new file mode 100644 index 000000000..7855ee67d --- /dev/null +++ b/pkgs/clan-cli/clan_lib/machines/morph.py @@ -0,0 +1,133 @@ +import json +import logging +import os +import random +import re +from pathlib import Path +from tempfile import TemporaryDirectory + +from clan_cli.machines.create import CreateOptions, create_machine +from clan_cli.vars.generate import generate_vars + +from clan_lib.cmd import Log, RunOpts, run +from clan_lib.dirs import specific_machine_dir +from clan_lib.flake import Flake +from clan_lib.machines.actions import list_machines +from clan_lib.machines.machines import Machine +from clan_lib.nix import nix_build, nix_command +from clan_lib.nix_models.clan import InventoryMachine + +log = logging.getLogger(__name__) + + +def is_local_input(node: dict[str, dict[str, str]]) -> bool: + locked = node.get("locked") + if not locked: + return False + # matches path and git+file:// + return ( + locked["type"] == "path" + or re.match(r"^\w+\+file://", locked.get("url", "")) is not None + ) + + +def random_hostname() -> str: + adjectives = ["wacky", "happy", "fluffy", "silly", "quirky", "zany", "bouncy"] + nouns = ["unicorn", "penguin", "goose", "ninja", "octopus", "hamster", "robot"] + adjective = random.choice(adjectives) + noun = random.choice(nouns) + return f"{adjective}-{noun}" + + +def morph_machine( + flake: Flake, template: str, ask_confirmation: bool, name: str | None = None +) -> None: + cmd = nix_command( + [ + "flake", + "archive", + "--json", + f"{flake}", + ] + ) + + archive_json = run( + cmd, RunOpts(error_msg="Failed to archive flake for morphing") + ).stdout.rstrip() + archive_path = json.loads(archive_json)["path"] + + with TemporaryDirectory(prefix="morph-") as _temp_dir: + flakedir = Path(_temp_dir).resolve() / "flake" + + flakedir.mkdir(parents=True, exist_ok=True) + run(["cp", "-r", archive_path + "/.", str(flakedir)]) + run(["chmod", "-R", "+w", str(flakedir)]) + + os.chdir(flakedir) + + if name is None: + name = random_hostname() + + if name not in list_machines(flake): + create_opts = CreateOptions( + template=template, + machine=InventoryMachine(name=name), + clan_dir=Flake(str(flakedir)), + ) + create_machine(create_opts, commit=False) + + machine = Machine(name=name, flake=Flake(str(flakedir))) + + generate_vars([machine], generator_name=None, regenerate=False) + + machine.secret_vars_store.populate_dir( + output_dir=Path("/run/secrets"), phases=["activation", "users", "services"] + ) + + # run(["nixos-facter", "-o", f"{flakedir}/machines/{name}/facter.json"]).stdout + + # facter_json = run(["nixos-facter"]).stdout + # run(["cp", "facter.json", f"{flakedir}/machines/{name}/facter.json"]).stdout + + machine_dir = specific_machine_dir(machine) + machine_dir.mkdir(parents=True, exist_ok=True) + Path(f"{machine_dir}/facter.json").write_text('{"system": "x86_64-linux"}') + result_path = run( + nix_build( + [f"{flakedir}#nixosConfigurations.{name}.config.system.build.toplevel"] + ) + ).stdout.rstrip() + + ropts = RunOpts(log=Log.BOTH) + + run( + [ + f"{result_path}/sw/bin/nixos-rebuild", + "dry-activate", + "--flake", + f"{flakedir}#{name}", + ], + ropts, + ).stdout.rstrip() + + if ask_confirmation: + log.warning("ARE YOU SURE YOU WANT TO DO THIS?") + log.warning( + "You should have read and understood all of the above and know what you are doing." + ) + + ask = input( + f"Do you really want convert this machine into {name}? If to continue, type in the new machine name: " + ) + if ask != name: + return + + run( + [ + f"{result_path}/sw/bin/nixos-rebuild", + "test", + "--flake", + f"{flakedir}#{name}", + ], + ropts, + ).stdout.rstrip() From 21b0c00c05367347566ef70cf9ea4ba178a3c96d Mon Sep 17 00:00:00 2001 From: Qubasa Date: Tue, 24 Jun 2025 11:32:42 +0200 Subject: [PATCH 2/5] clan-cli: Move list.py to clan_lib/machines --- pkgs/clan-cli/clan_cli/machines/list.py | 101 +--------------------- pkgs/clan-cli/clan_lib/machines/list.py | 106 ++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 100 deletions(-) create mode 100644 pkgs/clan-cli/clan_lib/machines/list.py diff --git a/pkgs/clan-cli/clan_cli/machines/list.py b/pkgs/clan-cli/clan_cli/machines/list.py index 582dd6f0d..84ed970cb 100644 --- a/pkgs/clan-cli/clan_cli/machines/list.py +++ b/pkgs/clan-cli/clan_cli/machines/list.py @@ -1,113 +1,14 @@ import argparse import logging -import re -from dataclasses import dataclass -from clan_lib.api import API -from clan_lib.api.disk import MachineDiskMatter -from clan_lib.api.modules import parse_frontmatter -from clan_lib.dirs import specific_machine_dir -from clan_lib.errors import ClanError from clan_lib.flake import Flake -from clan_lib.machines.actions import get_machine, list_machines -from clan_lib.machines.machines import Machine -from clan_lib.nix_models.clan import InventoryMachine +from clan_lib.machines.list import list_full_machines, query_machines_by_tags from clan_cli.completions import add_dynamic_completer, complete_tags -from clan_cli.machines.hardware import HardwareConfig log = logging.getLogger(__name__) -def list_full_machines( - flake: Flake, nix_options: list[str] | None = None -) -> dict[str, Machine]: - """ - Like `list_machines`, but returns a full 'machine' instance for each machine. - """ - machines = list_machines(flake) - - res: dict[str, Machine] = {} - - if nix_options is None: - nix_options = [] - - for inv_machine in machines.values(): - name = inv_machine.get("name") - # Technically, this should not happen, but we are defensive here. - if name is None: - msg = "InternalError: Machine name is required. But got a machine without a name." - raise ClanError(msg) - - machine = Machine( - name=name, - flake=flake, - nix_options=nix_options, - ) - res[machine.name] = machine - - return res - - -def query_machines_by_tags(flake: Flake, tags: list[str]) -> dict[str, Machine]: - """ - Query machines by their respective tags, if multiple tags are specified - then only machines that have those respective tags specified will be listed. - It is an intersection of the tags and machines. - """ - machines = list_full_machines(flake) - - filtered_machines = {} - for machine in machines.values(): - inv_machine = get_machine(machine.flake, machine.name) - machine_tags = inv_machine.get("tags", []) - if all(tag in machine_tags for tag in tags): - filtered_machines[machine.name] = machine - - return filtered_machines - - -@dataclass -class MachineDetails: - machine: InventoryMachine - hw_config: HardwareConfig | None = None - disk_schema: MachineDiskMatter | None = None - - -def extract_header(c: str) -> str: - header_lines = [] - for line in c.splitlines(): - match = re.match(r"^\s*#(.*)", line) - if match: - header_lines.append(match.group(1).strip()) - else: - break # Stop once the header ends - return "\n".join(header_lines) - - -@API.register -def get_machine_details(machine: Machine) -> MachineDetails: - machine_inv = get_machine(machine.flake, machine.name) - hw_config = HardwareConfig.detect_type(machine) - - machine_dir = specific_machine_dir(machine) - disk_schema: MachineDiskMatter | None = None - disk_path = machine_dir / "disko.nix" - if disk_path.exists(): - with disk_path.open() as f: - content = f.read() - header = extract_header(content) - data, _rest = parse_frontmatter(header) - if data: - disk_schema = data # type: ignore - - return MachineDetails( - machine=machine_inv, - hw_config=hw_config, - disk_schema=disk_schema, - ) - - def list_command(args: argparse.Namespace) -> None: flake: Flake = args.flake diff --git a/pkgs/clan-cli/clan_lib/machines/list.py b/pkgs/clan-cli/clan_lib/machines/list.py new file mode 100644 index 000000000..6232e3017 --- /dev/null +++ b/pkgs/clan-cli/clan_lib/machines/list.py @@ -0,0 +1,106 @@ +import logging +import re +from dataclasses import dataclass + +from clan_cli.machines.hardware import HardwareConfig + +from clan_lib.api import API +from clan_lib.api.disk import MachineDiskMatter +from clan_lib.api.modules import parse_frontmatter +from clan_lib.dirs import specific_machine_dir +from clan_lib.errors import ClanError +from clan_lib.flake import Flake +from clan_lib.machines.actions import get_machine, list_machines +from clan_lib.machines.machines import Machine +from clan_lib.nix_models.clan import InventoryMachine + +log = logging.getLogger(__name__) + + +def list_full_machines( + flake: Flake, nix_options: list[str] | None = None +) -> dict[str, Machine]: + """ + Like `list_machines`, but returns a full 'machine' instance for each machine. + """ + machines = list_machines(flake) + + res: dict[str, Machine] = {} + + if nix_options is None: + nix_options = [] + + for inv_machine in machines.values(): + name = inv_machine.get("name") + # Technically, this should not happen, but we are defensive here. + if name is None: + msg = "InternalError: Machine name is required. But got a machine without a name." + raise ClanError(msg) + + machine = Machine( + name=name, + flake=flake, + nix_options=nix_options, + ) + res[machine.name] = machine + + return res + + +def query_machines_by_tags(flake: Flake, tags: list[str]) -> dict[str, Machine]: + """ + Query machines by their respective tags, if multiple tags are specified + then only machines that have those respective tags specified will be listed. + It is an intersection of the tags and machines. + """ + machines = list_full_machines(flake) + + filtered_machines = {} + for machine in machines.values(): + inv_machine = get_machine(machine.flake, machine.name) + machine_tags = inv_machine.get("tags", []) + if all(tag in machine_tags for tag in tags): + filtered_machines[machine.name] = machine + + return filtered_machines + + +@dataclass +class MachineDetails: + machine: InventoryMachine + hw_config: HardwareConfig | None = None + disk_schema: MachineDiskMatter | None = None + + +def extract_header(c: str) -> str: + header_lines = [] + for line in c.splitlines(): + match = re.match(r"^\s*#(.*)", line) + if match: + header_lines.append(match.group(1).strip()) + else: + break # Stop once the header ends + return "\n".join(header_lines) + + +@API.register +def get_machine_details(machine: Machine) -> MachineDetails: + machine_inv = get_machine(machine.flake, machine.name) + hw_config = HardwareConfig.detect_type(machine) + + machine_dir = specific_machine_dir(machine) + disk_schema: MachineDiskMatter | None = None + disk_path = machine_dir / "disko.nix" + if disk_path.exists(): + with disk_path.open() as f: + content = f.read() + header = extract_header(content) + data, _rest = parse_frontmatter(header) + if data: + disk_schema = data # type: ignore + + return MachineDetails( + machine=machine_inv, + hw_config=hw_config, + disk_schema=disk_schema, + ) From 2bfc33bc312ff00d991946a262ac9371ee61d602 Mon Sep 17 00:00:00 2001 From: Qubasa Date: Tue, 24 Jun 2025 11:37:21 +0200 Subject: [PATCH 3/5] clan-cli: Move delete.py to clan_lib/machines --- pkgs/clan-cli/clan_cli/machines/delete.py | 50 +-------------------- pkgs/clan-cli/clan_lib/machines/delete.py | 55 +++++++++++++++++++++++ 2 files changed, 56 insertions(+), 49 deletions(-) create mode 100644 pkgs/clan-cli/clan_lib/machines/delete.py diff --git a/pkgs/clan-cli/clan_cli/machines/delete.py b/pkgs/clan-cli/clan_cli/machines/delete.py index 47f3c91bf..680b20b4b 100644 --- a/pkgs/clan-cli/clan_cli/machines/delete.py +++ b/pkgs/clan-cli/clan_cli/machines/delete.py @@ -1,62 +1,14 @@ import argparse import logging -import shutil -from pathlib import Path -from clan_lib import inventory -from clan_lib.api import API -from clan_lib.dirs import specific_machine_dir +from clan_lib.machines.delete import delete_machine from clan_lib.machines.machines import Machine from clan_cli.completions import add_dynamic_completer, complete_machines -from clan_cli.secrets.folders import sops_secrets_folder -from clan_cli.secrets.machines import has_machine as secrets_has_machine -from clan_cli.secrets.machines import remove_machine as secrets_machine_remove -from clan_cli.secrets.secrets import ( - list_secrets, -) log = logging.getLogger(__name__) -@API.register -def delete_machine(machine: Machine) -> None: - inventory_store = inventory.InventoryStore(machine.flake) - try: - inventory_store.delete( - {f"machines.{machine.name}"}, - ) - except KeyError as exc: - # louis@(2025-03-09): test infrastructure does not seem to set the - # inventory properly, but more importantly only one machine in my - # personal clan ended up in the inventory for some reason, so I think - # it makes sense to eat the exception here. - log.warning( - f"{machine.name} was missing or already deleted from the machines inventory: {exc}" - ) - - changed_paths: list[Path] = [] - - folder = specific_machine_dir(machine) - if folder.exists(): - changed_paths.append(folder) - shutil.rmtree(folder) - - # louis@(2025-02-04): clean-up legacy (pre-vars) secrets: - sops_folder = sops_secrets_folder(machine.flake.path) - filter_fn = lambda secret_name: secret_name.startswith(f"{machine.name}-") - for secret_name in list_secrets(machine.flake.path, filter_fn): - secret_path = sops_folder / secret_name - changed_paths.append(secret_path) - shutil.rmtree(secret_path) - - changed_paths.extend(machine.public_vars_store.delete_store()) - changed_paths.extend(machine.secret_vars_store.delete_store()) - # Remove the machine's key, and update secrets & vars that referenced it: - if secrets_has_machine(machine.flake.path, machine.name): - secrets_machine_remove(machine.flake.path, machine.name) - - def delete_command(args: argparse.Namespace) -> None: delete_machine(Machine(flake=args.flake, name=args.name)) diff --git a/pkgs/clan-cli/clan_lib/machines/delete.py b/pkgs/clan-cli/clan_lib/machines/delete.py new file mode 100644 index 000000000..29bf5c473 --- /dev/null +++ b/pkgs/clan-cli/clan_lib/machines/delete.py @@ -0,0 +1,55 @@ +import logging +import shutil +from pathlib import Path + +from clan_cli.secrets.folders import sops_secrets_folder +from clan_cli.secrets.machines import has_machine as secrets_has_machine +from clan_cli.secrets.machines import remove_machine as secrets_machine_remove +from clan_cli.secrets.secrets import ( + list_secrets, +) + +from clan_lib import inventory +from clan_lib.api import API +from clan_lib.dirs import specific_machine_dir +from clan_lib.machines.machines import Machine + +log = logging.getLogger(__name__) + + +@API.register +def delete_machine(machine: Machine) -> None: + inventory_store = inventory.InventoryStore(machine.flake) + try: + inventory_store.delete( + {f"machines.{machine.name}"}, + ) + except KeyError as exc: + # louis@(2025-03-09): test infrastructure does not seem to set the + # inventory properly, but more importantly only one machine in my + # personal clan ended up in the inventory for some reason, so I think + # it makes sense to eat the exception here. + log.warning( + f"{machine.name} was missing or already deleted from the machines inventory: {exc}" + ) + + changed_paths: list[Path] = [] + + folder = specific_machine_dir(machine) + if folder.exists(): + changed_paths.append(folder) + shutil.rmtree(folder) + + # louis@(2025-02-04): clean-up legacy (pre-vars) secrets: + sops_folder = sops_secrets_folder(machine.flake.path) + filter_fn = lambda secret_name: secret_name.startswith(f"{machine.name}-") + for secret_name in list_secrets(machine.flake.path, filter_fn): + secret_path = sops_folder / secret_name + changed_paths.append(secret_path) + shutil.rmtree(secret_path) + + changed_paths.extend(machine.public_vars_store.delete_store()) + changed_paths.extend(machine.secret_vars_store.delete_store()) + # Remove the machine's key, and update secrets & vars that referenced it: + if secrets_has_machine(machine.flake.path, machine.name): + secrets_machine_remove(machine.flake.path, machine.name) From f89b5063afcfa6c5590ecf115a123b4d9352bb87 Mon Sep 17 00:00:00 2001 From: Qubasa Date: Tue, 24 Jun 2025 12:03:56 +0200 Subject: [PATCH 4/5] clan-vm-manager: Fix list_machines import --- pkgs/clan-vm-manager/clan_vm_manager/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-vm-manager/clan_vm_manager/history.py b/pkgs/clan-vm-manager/clan_vm_manager/history.py index d8725bf42..1594e8fcd 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/history.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/history.py @@ -6,11 +6,11 @@ import logging from typing import Any from clan_cli.clan.inspect import FlakeConfig, inspect_flake -from clan_cli.machines.list import list_machines from clan_lib.dirs import user_history_file from clan_lib.errors import ClanError from clan_lib.flake import Flake from clan_lib.locked_open import read_history_file, write_history_file +from clan_lib.machines.list import list_machines from clan_vm_manager.clan_uri import ClanURI From 6e4dc03b1db8bb09f3c4a0a1588332f83d89c70b Mon Sep 17 00:00:00 2001 From: Qubasa Date: Tue, 24 Jun 2025 12:15:58 +0200 Subject: [PATCH 5/5] clan-cli: Move update.py to clan_lib/machines --- pkgs/clan-cli/clan_cli/machines/update.py | 194 +-------------------- pkgs/clan-cli/clan_lib/machines/update.py | 199 ++++++++++++++++++++++ 2 files changed, 202 insertions(+), 191 deletions(-) create mode 100644 pkgs/clan-cli/clan_lib/machines/update.py diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index f39753681..8573fcd83 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -1,19 +1,12 @@ import argparse -import json import logging -import os -import re -import shlex import sys -from contextlib import ExitStack -from clan_lib.api import API -from clan_lib.async_run import AsyncContext, AsyncOpts, AsyncRuntime, is_async_cancelled -from clan_lib.cmd import Log, MsgColor, RunOpts, run -from clan_lib.colors import AnsiColor +from clan_lib.async_run import AsyncContext, AsyncOpts, AsyncRuntime from clan_lib.errors import ClanError from clan_lib.machines.machines import Machine -from clan_lib.nix import nix_command, nix_config, nix_metadata +from clan_lib.machines.update import deploy_machine +from clan_lib.nix import nix_config from clan_lib.ssh.remote import Remote from clan_cli.completions import ( @@ -21,192 +14,11 @@ from clan_cli.completions import ( complete_machines, complete_tags, ) -from clan_cli.facts.generate import generate_facts -from clan_cli.facts.upload import upload_secrets from clan_cli.machines.list import list_full_machines, query_machines_by_tags -from clan_cli.vars.generate import generate_vars -from clan_cli.vars.upload import upload_secret_vars log = logging.getLogger(__name__) -def is_local_input(node: dict[str, dict[str, str]]) -> bool: - locked = node.get("locked") - if not locked: - return False - # matches path and git+file:// - local = ( - locked["type"] == "path" - # local vcs inputs i.e. git+file:/// - or re.match(r"^file://", locked.get("url", "")) is not None - ) - if local: - print(f"[WARN] flake input has local node: {json.dumps(node)}") - return local - - -def upload_sources(machine: Machine, ssh: Remote) -> str: - env = ssh.nix_ssh_env(os.environ.copy()) - - flake_url = ( - str(machine.flake.path) if machine.flake.is_local else machine.flake.identifier - ) - flake_data = nix_metadata(flake_url) - has_path_inputs = any( - is_local_input(node) for node in flake_data["locks"]["nodes"].values() - ) - - # Construct the remote URL with proper parameters for Darwin - remote_url = f"ssh-ng://{ssh.target}" - # MacOS doesn't come with a proper login shell for ssh and therefore doesn't have nix in $PATH as it doesn't source /etc/profile - if machine._class_ == "darwin": - remote_url += "?remote-program=bash -lc 'exec nix-daemon --stdio'" - - if not has_path_inputs: - # Just copy the flake to the remote machine, we can substitute other inputs there. - path = flake_data["path"] - cmd = nix_command( - [ - "copy", - "--to", - remote_url, - "--no-check-sigs", - path, - ] - ) - run( - cmd, - RunOpts( - env=env, - needs_user_terminal=True, - error_msg="failed to upload sources", - prefix=machine.name, - ), - ) - return path - - # Slow path: we need to upload all sources to the remote machine - cmd = nix_command( - [ - "flake", - "archive", - "--to", - remote_url, - "--json", - flake_url, - ] - ) - proc = run( - cmd, - RunOpts( - env=env, needs_user_terminal=True, error_msg="failed to upload sources" - ), - ) - - try: - return json.loads(proc.stdout)["path"] - except (json.JSONDecodeError, OSError) as e: - msg = f"failed to parse output of {shlex.join(cmd)}: {e}\nGot: {proc.stdout}" - raise ClanError(msg) from e - - -@API.register -def deploy_machine( - machine: Machine, target_host: Remote, build_host: Remote | None -) -> None: - with ExitStack() as stack: - target_host = stack.enter_context(target_host.ssh_control_master()) - - if build_host is not None: - build_host = stack.enter_context(build_host.ssh_control_master()) - - host = build_host or target_host - - sudo_host = stack.enter_context(target_host.become_root()) - - generate_facts([machine], service=None, regenerate=False) - generate_vars([machine], generator_name=None, regenerate=False) - - upload_secrets(machine, sudo_host) - upload_secret_vars(machine, sudo_host) - - path = upload_sources(machine, sudo_host) - - nix_options = [ - "--show-trace", - "--option", - "keep-going", - "true", - "--option", - "accept-flake-config", - "true", - "-L", - *machine.nix_options, - "--flake", - f"{path}#{machine.name}", - ] - - become_root = True - - if machine._class_ == "nixos": - nix_options += [ - "--fast", - "--build-host", - "", - ] - - if build_host: - become_root = False - nix_options += ["--target-host", target_host.target] - - if target_host.user != "root": - nix_options += ["--use-remote-sudo"] - switch_cmd = ["nixos-rebuild", "switch", *nix_options] - elif machine._class_ == "darwin": - # use absolute path to darwin-rebuild - switch_cmd = [ - "/run/current-system/sw/bin/darwin-rebuild", - "switch", - *nix_options, - ] - - if become_root: - host = sudo_host - - remote_env = host.nix_ssh_env(control_master=False) - ret = host.run( - switch_cmd, - RunOpts( - check=False, - log=Log.BOTH, - msg_color=MsgColor(stderr=AnsiColor.DEFAULT), - needs_user_terminal=True, - ), - extra_env=remote_env, - ) - - if is_async_cancelled(): - return - - # retry nixos-rebuild switch if the first attempt failed - if ret.returncode != 0: - is_mobile = machine.deployment.get("nixosMobileWorkaround", False) - # if the machine is mobile, we retry to deploy with the mobile workaround method - if is_mobile: - machine.info( - "Mobile machine detected, applying workaround deployment method" - ) - ret = host.run( - ["nixos--rebuild", "test", *nix_options] if is_mobile else switch_cmd, - RunOpts( - log=Log.BOTH, - msg_color=MsgColor(stderr=AnsiColor.DEFAULT), - needs_user_terminal=True, - ), - extra_env=remote_env, - ) - - def update_command(args: argparse.Namespace) -> None: try: if args.flake is None: diff --git a/pkgs/clan-cli/clan_lib/machines/update.py b/pkgs/clan-cli/clan_lib/machines/update.py new file mode 100644 index 000000000..e85f9f946 --- /dev/null +++ b/pkgs/clan-cli/clan_lib/machines/update.py @@ -0,0 +1,199 @@ +import json +import logging +import os +import re +import shlex +from contextlib import ExitStack + +from clan_cli.facts.generate import generate_facts +from clan_cli.facts.upload import upload_secrets +from clan_cli.vars.generate import generate_vars +from clan_cli.vars.upload import upload_secret_vars + +from clan_lib.api import API +from clan_lib.async_run import is_async_cancelled +from clan_lib.cmd import Log, MsgColor, RunOpts, run +from clan_lib.colors import AnsiColor +from clan_lib.errors import ClanError +from clan_lib.machines.machines import Machine +from clan_lib.nix import nix_command, nix_metadata +from clan_lib.ssh.remote import Remote + +log = logging.getLogger(__name__) + + +def is_local_input(node: dict[str, dict[str, str]]) -> bool: + locked = node.get("locked") + if not locked: + return False + # matches path and git+file:// + local = ( + locked["type"] == "path" + # local vcs inputs i.e. git+file:/// + or re.match(r"^file://", locked.get("url", "")) is not None + ) + if local: + print(f"[WARN] flake input has local node: {json.dumps(node)}") + return local + + +def upload_sources(machine: Machine, ssh: Remote) -> str: + env = ssh.nix_ssh_env(os.environ.copy()) + + flake_url = ( + str(machine.flake.path) if machine.flake.is_local else machine.flake.identifier + ) + flake_data = nix_metadata(flake_url) + has_path_inputs = any( + is_local_input(node) for node in flake_data["locks"]["nodes"].values() + ) + + # Construct the remote URL with proper parameters for Darwin + remote_url = f"ssh-ng://{ssh.target}" + # MacOS doesn't come with a proper login shell for ssh and therefore doesn't have nix in $PATH as it doesn't source /etc/profile + if machine._class_ == "darwin": + remote_url += "?remote-program=bash -lc 'exec nix-daemon --stdio'" + + if not has_path_inputs: + # Just copy the flake to the remote machine, we can substitute other inputs there. + path = flake_data["path"] + cmd = nix_command( + [ + "copy", + "--to", + remote_url, + "--no-check-sigs", + path, + ] + ) + run( + cmd, + RunOpts( + env=env, + needs_user_terminal=True, + error_msg="failed to upload sources", + prefix=machine.name, + ), + ) + return path + + # Slow path: we need to upload all sources to the remote machine + cmd = nix_command( + [ + "flake", + "archive", + "--to", + remote_url, + "--json", + flake_url, + ] + ) + proc = run( + cmd, + RunOpts( + env=env, needs_user_terminal=True, error_msg="failed to upload sources" + ), + ) + + try: + return json.loads(proc.stdout)["path"] + except (json.JSONDecodeError, OSError) as e: + msg = f"failed to parse output of {shlex.join(cmd)}: {e}\nGot: {proc.stdout}" + raise ClanError(msg) from e + + +@API.register +def deploy_machine( + machine: Machine, target_host: Remote, build_host: Remote | None +) -> None: + with ExitStack() as stack: + target_host = stack.enter_context(target_host.ssh_control_master()) + + if build_host is not None: + build_host = stack.enter_context(build_host.ssh_control_master()) + + host = build_host or target_host + + sudo_host = stack.enter_context(target_host.become_root()) + + generate_facts([machine], service=None, regenerate=False) + generate_vars([machine], generator_name=None, regenerate=False) + + upload_secrets(machine, sudo_host) + upload_secret_vars(machine, sudo_host) + + path = upload_sources(machine, sudo_host) + + nix_options = [ + "--show-trace", + "--option", + "keep-going", + "true", + "--option", + "accept-flake-config", + "true", + "-L", + *machine.nix_options, + "--flake", + f"{path}#{machine.name}", + ] + + become_root = True + + if machine._class_ == "nixos": + nix_options += [ + "--fast", + "--build-host", + "", + ] + + if build_host: + become_root = False + nix_options += ["--target-host", target_host.target] + + if target_host.user != "root": + nix_options += ["--use-remote-sudo"] + switch_cmd = ["nixos-rebuild", "switch", *nix_options] + elif machine._class_ == "darwin": + # use absolute path to darwin-rebuild + switch_cmd = [ + "/run/current-system/sw/bin/darwin-rebuild", + "switch", + *nix_options, + ] + + if become_root: + host = sudo_host + + remote_env = host.nix_ssh_env(control_master=False) + ret = host.run( + switch_cmd, + RunOpts( + check=False, + log=Log.BOTH, + msg_color=MsgColor(stderr=AnsiColor.DEFAULT), + needs_user_terminal=True, + ), + extra_env=remote_env, + ) + + if is_async_cancelled(): + return + + # retry nixos-rebuild switch if the first attempt failed + if ret.returncode != 0: + is_mobile = machine.deployment.get("nixosMobileWorkaround", False) + # if the machine is mobile, we retry to deploy with the mobile workaround method + if is_mobile: + machine.info( + "Mobile machine detected, applying workaround deployment method" + ) + ret = host.run( + ["nixos--rebuild", "test", *nix_options] if is_mobile else switch_cmd, + RunOpts( + log=Log.BOTH, + msg_color=MsgColor(stderr=AnsiColor.DEFAULT), + needs_user_terminal=True, + ), + extra_env=remote_env, + )