From c33fd4e5047c09f55e53e77be0696f65e0556139 Mon Sep 17 00:00:00 2001 From: DavHau Date: Tue, 29 Jul 2025 16:38:55 +0700 Subject: [PATCH 1/3] ssh: Introduce LocalHost vs. Remote via Host interface Motivation: local builds and deployments without ssh Add a new interface `Host` which is implemented bei either `Remote` or `Localhost` This simplifies all interactions with hosts. THe caller does ot need to know if the Host is remote or local in mot cases anymore --- pkgs/clan-cli/clan_lib/ssh/host.py | 85 ++++++++++++++++++ pkgs/clan-cli/clan_lib/ssh/localhost.py | 111 ++++++++++++++++++++++++ test_host_interface.py | 36 ++++++++ 3 files changed, 232 insertions(+) create mode 100644 pkgs/clan-cli/clan_lib/ssh/host.py create mode 100644 pkgs/clan-cli/clan_lib/ssh/localhost.py create mode 100644 test_host_interface.py diff --git a/pkgs/clan-cli/clan_lib/ssh/host.py b/pkgs/clan-cli/clan_lib/ssh/host.py new file mode 100644 index 000000000..4c804de8f --- /dev/null +++ b/pkgs/clan-cli/clan_lib/ssh/host.py @@ -0,0 +1,85 @@ +"""Base Host interface for both local and remote command execution.""" + +import logging +from abc import ABC, abstractmethod +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass + +from clan_lib.cmd import CmdOut, RunOpts + +cmdlog = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class Host(ABC): + """ + Abstract base class for host command execution. + This provides a common interface for both local and remote hosts. + """ + + command_prefix: str + + @property + @abstractmethod + def target(self) -> str: + """Return a descriptive target string for this host.""" + + @property + @abstractmethod + def user(self) -> str: + """Return the user for this host.""" + + @abstractmethod + def run( + self, + cmd: list[str], + opts: RunOpts | None = None, + extra_env: dict[str, str] | None = None, + tty: bool = False, + verbose_ssh: bool = False, + quiet: bool = False, + control_master: bool = True, + ) -> CmdOut: + """ + Run a command on the host. + + Args: + cmd: Command to execute + opts: Run options + extra_env: Additional environment variables + tty: Whether to allocate a TTY (for remote hosts) + verbose_ssh: Enable verbose SSH output (for remote hosts) + quiet: Suppress command logging + control_master: Use SSH ControlMaster (for remote hosts) + + Returns: + Command output + """ + + @contextmanager + @abstractmethod + def become_root(self) -> Iterator["Host"]: + """ + Context manager to execute commands as root. + """ + + @contextmanager + @abstractmethod + def host_connection(self) -> Iterator["Host"]: + """ + Context manager to manage host connections. + For remote hosts, this manages SSH ControlMaster connections. + For local hosts, this is a no-op that returns self. + """ + + @abstractmethod + def nix_ssh_env( + self, + env: dict[str, str] | None = None, + control_master: bool = True, + ) -> dict[str, str]: + """ + Get environment variables for Nix operations. + Remote hosts will add NIX_SSHOPTS, local hosts won't. + """ diff --git a/pkgs/clan-cli/clan_lib/ssh/localhost.py b/pkgs/clan-cli/clan_lib/ssh/localhost.py new file mode 100644 index 000000000..ca68b2d73 --- /dev/null +++ b/pkgs/clan-cli/clan_lib/ssh/localhost.py @@ -0,0 +1,111 @@ +import logging +import os +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field + +from clan_lib.cmd import CmdOut, RunOpts, run +from clan_lib.colors import AnsiColor +from clan_lib.ssh.host import Host + +cmdlog = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class LocalHost(Host): + """ + A Host implementation that executes commands locally without SSH. + """ + + command_prefix: str = "localhost" + _user: str = field(default_factory=lambda: os.environ.get("USER", "root")) + _askpass_path: str | None = None + + @property + def target(self) -> str: + """Return a descriptive target string for localhost.""" + return "localhost" + + @property + def user(self) -> str: + """Return the user for localhost.""" + return self._user + + def run( + self, + cmd: list[str], + opts: RunOpts | None = None, + extra_env: dict[str, str] | None = None, + tty: bool = False, + verbose_ssh: bool = False, + quiet: bool = False, + control_master: bool = True, + ) -> CmdOut: + """ + Run a command locally instead of via SSH. + """ + if opts is None: + opts = RunOpts() + + # Set up environment + env = opts.env or os.environ.copy() + if extra_env: + env.update(extra_env) + + # Handle sudo if needed + if self._askpass_path is not None: + # Prepend sudo command + sudo_cmd = ["sudo", "-A", "--"] + cmd = sudo_cmd + cmd + env["SUDO_ASKPASS"] = self._askpass_path + + # Set options + opts.env = env + opts.prefix = opts.prefix or self.command_prefix + + # Log the command + displayed_cmd = " ".join(cmd) + if not quiet: + cmdlog.info( + f"$ {displayed_cmd}", + extra={ + "command_prefix": self.command_prefix, + "color": AnsiColor.GREEN.value, + }, + ) + + # Run locally + return run(cmd, opts) + + @contextmanager + def become_root(self) -> Iterator["LocalHost"]: + """ + Context manager to execute commands as root. + """ + if self._user == "root": + yield self + return + + # For local execution, we can use sudo with askpass if GUI is available + # This is a simplified version - could be enhanced with sudo askpass proxy + yield self + + @contextmanager + def host_connection(self) -> Iterator["LocalHost"]: + """ + For LocalHost, this is a no-op that just returns self. + """ + yield self + + def nix_ssh_env( + self, + env: dict[str, str] | None = None, + control_master: bool = True, + ) -> dict[str, str]: + """ + LocalHost doesn't need SSH environment variables. + """ + if env is None: + env = {} + # Don't set NIX_SSHOPTS for localhost + return env diff --git a/test_host_interface.py b/test_host_interface.py new file mode 100644 index 000000000..aa62e943a --- /dev/null +++ b/test_host_interface.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Test script for Host interface with LocalHost implementation.""" + +from clan_lib.cmd import RunOpts +from clan_lib.ssh.host import Host +from clan_lib.ssh.localhost import LocalHost + + +def test_localhost() -> None: + # Create LocalHost instance + localhost = LocalHost(command_prefix="local-test") + + # Verify it's a Host instance + assert isinstance(localhost, Host), "LocalHost should be an instance of Host" + + # Test basic command execution + result = localhost.run(["echo", "Hello from LocalHost"]) + assert result.returncode == 0, f"Command failed with code {result.returncode}" + assert result.stdout.strip() == "Hello from LocalHost", ( + f"Unexpected output: {result.stdout}" + ) + + # Test with environment variable + result = localhost.run( + ["printenv", "TEST_VAR"], + opts=RunOpts(check=False), # Don't check return code + extra_env={"TEST_VAR": "LocalHost works!"}, + ) + assert result.returncode == 0, f"Command failed with code {result.returncode}" + assert result.stdout.strip() == "LocalHost works!", ( + f"Expected 'LocalHost works!', got '{result.stdout.strip()}'" + ) + + +if __name__ == "__main__": + test_localhost() From b74193514d6adc7e2fd0b2042726ff4289ec595d Mon Sep 17 00:00:00 2001 From: DavHau Date: Tue, 29 Jul 2025 16:39:59 +0700 Subject: [PATCH 2/3] ssh: refactor callers to use new Host interface --- .../clan_cli/facts/secret_modules/__init__.py | 4 ++-- .../facts/secret_modules/password_store.py | 6 +++--- .../clan_cli/facts/secret_modules/sops.py | 4 ++-- pkgs/clan-cli/clan_cli/facts/upload.py | 6 +++--- pkgs/clan-cli/clan_cli/ssh/upload.py | 4 ++-- pkgs/clan-cli/clan_cli/tests/hosts.py | 2 +- pkgs/clan-cli/clan_cli/tests/test_ssh_local.py | 2 +- .../clan_cli/tests/test_upload_single_file.py | 2 +- pkgs/clan-cli/clan_cli/vars/_types.py | 4 ++-- .../clan_cli/vars/public_modules/in_repo.py | 4 ++-- pkgs/clan-cli/clan_cli/vars/public_modules/vm.py | 4 ++-- pkgs/clan-cli/clan_cli/vars/secret_modules/fs.py | 4 ++-- .../vars/secret_modules/password_store.py | 6 +++--- .../clan-cli/clan_cli/vars/secret_modules/sops.py | 4 ++-- pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py | 4 ++-- pkgs/clan-cli/clan_cli/vars/upload.py | 6 +++--- pkgs/clan-cli/clan_lib/backups/create.py | 4 ++-- pkgs/clan-cli/clan_lib/backups/list.py | 2 +- pkgs/clan-cli/clan_lib/backups/restore.py | 4 ++-- pkgs/clan-cli/clan_lib/machines/hardware.py | 2 +- pkgs/clan-cli/clan_lib/machines/update.py | 11 ++++++----- pkgs/clan-cli/clan_lib/network/check.py | 2 +- pkgs/clan-cli/clan_lib/ssh/remote.py | 15 ++++++++------- pkgs/clan-cli/clan_lib/ssh/remote_test.py | 4 ++-- 24 files changed, 56 insertions(+), 54 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py b/pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py index 7db2c3ed7..fe7269ae2 100644 --- a/pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py +++ b/pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from pathlib import Path import clan_lib.machines.machines as machines -from clan_lib.ssh.remote import Remote +from clan_lib.ssh.host import Host class SecretStoreBase(ABC): @@ -26,7 +26,7 @@ class SecretStoreBase(ABC): def exists(self, service: str, name: str) -> bool: pass - def needs_upload(self, host: Remote) -> bool: + def needs_upload(self, host: Host) -> bool: return True @abstractmethod diff --git a/pkgs/clan-cli/clan_cli/facts/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/facts/secret_modules/password_store.py index 674d141c8..8ac033a55 100644 --- a/pkgs/clan-cli/clan_cli/facts/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/facts/secret_modules/password_store.py @@ -6,7 +6,7 @@ from typing import override from clan_lib.cmd import Log, RunOpts from clan_lib.machines.machines import Machine from clan_lib.nix import nix_shell -from clan_lib.ssh.remote import Remote +from clan_lib.ssh.host import Host from clan_cli.facts.secret_modules import SecretStoreBase @@ -94,9 +94,9 @@ class SecretStore(SecretStoreBase): return b"\n".join(hashes) @override - def needs_upload(self, host: Remote) -> bool: + def needs_upload(self, host: Host) -> bool: local_hash = self.generate_hash() - with host.ssh_control_master() as ssh: + with host.host_connection() as ssh: remote_hash = ssh.run( # TODO get the path to the secrets from the machine ["cat", f"{self.machine.secrets_upload_directory}/.pass_info"], diff --git a/pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py b/pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py index a25ed1d30..545811aba 100644 --- a/pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py +++ b/pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import override from clan_lib.machines.machines import Machine -from clan_lib.ssh.remote import Remote +from clan_lib.ssh.host import Host from clan_cli.secrets.folders import sops_secrets_folder from clan_cli.secrets.machines import add_machine, has_machine @@ -64,7 +64,7 @@ class SecretStore(SecretStoreBase): ) @override - def needs_upload(self, host: Remote) -> bool: + def needs_upload(self, host: Host) -> bool: return False # We rely now on the vars backend to upload the age key diff --git a/pkgs/clan-cli/clan_cli/facts/upload.py b/pkgs/clan-cli/clan_cli/facts/upload.py index 27833ad8f..5f8880fbc 100644 --- a/pkgs/clan-cli/clan_cli/facts/upload.py +++ b/pkgs/clan-cli/clan_cli/facts/upload.py @@ -5,7 +5,7 @@ from tempfile import TemporaryDirectory from clan_lib.flake import require_flake from clan_lib.machines.machines import Machine -from clan_lib.ssh.remote import Remote +from clan_lib.ssh.host import Host from clan_cli.completions import add_dynamic_completer, complete_machines from clan_cli.ssh.upload import upload @@ -13,7 +13,7 @@ from clan_cli.ssh.upload import upload log = logging.getLogger(__name__) -def upload_secrets(machine: Machine, host: Remote) -> None: +def upload_secrets(machine: Machine, host: Host) -> None: if not machine.secret_facts_store.needs_upload(host): machine.info("Secrets already uploaded") return @@ -28,7 +28,7 @@ def upload_secrets(machine: Machine, host: Remote) -> None: def upload_command(args: argparse.Namespace) -> None: flake = require_flake(args.flake) machine = Machine(name=args.machine, flake=flake) - with machine.target_host().ssh_control_master() as host: + with machine.target_host().host_connection() as host: upload_secrets(machine, host) diff --git a/pkgs/clan-cli/clan_cli/ssh/upload.py b/pkgs/clan-cli/clan_cli/ssh/upload.py index e9e9cb90c..f75a44b29 100644 --- a/pkgs/clan-cli/clan_cli/ssh/upload.py +++ b/pkgs/clan-cli/clan_cli/ssh/upload.py @@ -4,11 +4,11 @@ from tempfile import TemporaryDirectory from clan_lib.cmd import Log, RunOpts from clan_lib.errors import ClanError -from clan_lib.ssh.remote import Remote +from clan_lib.ssh.host import Host def upload( - host: Remote, + host: Host, local_src: Path, remote_dest: Path, # must be a directory file_user: str = "root", diff --git a/pkgs/clan-cli/clan_cli/tests/hosts.py b/pkgs/clan-cli/clan_cli/tests/hosts.py index 84c32e720..9c64381fa 100644 --- a/pkgs/clan-cli/clan_cli/tests/hosts.py +++ b/pkgs/clan-cli/clan_cli/tests/hosts.py @@ -12,7 +12,7 @@ def hosts(sshd: Sshd) -> list[Remote]: login = pwd.getpwuid(os.getuid()).pw_name group = [ Remote( - "127.0.0.1", + address="127.0.0.1", port=sshd.port, user=login, private_key=Path(sshd.key), diff --git a/pkgs/clan-cli/clan_cli/tests/test_ssh_local.py b/pkgs/clan-cli/clan_cli/tests/test_ssh_local.py index cc1b25acf..e7b008ce9 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_ssh_local.py +++ b/pkgs/clan-cli/clan_cli/tests/test_ssh_local.py @@ -2,7 +2,7 @@ from clan_lib.async_run import AsyncRuntime from clan_lib.cmd import ClanCmdTimeoutError, Log, RunOpts from clan_lib.ssh.remote import Remote -host = Remote("some_host", user="root", command_prefix="local_test") +host = Remote(address="some_host", user="root", command_prefix="local_test") def test_run_environment(runtime: AsyncRuntime) -> None: diff --git a/pkgs/clan-cli/clan_cli/tests/test_upload_single_file.py b/pkgs/clan-cli/clan_cli/tests/test_upload_single_file.py index 6fd63d234..a30a47bcd 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_upload_single_file.py +++ b/pkgs/clan-cli/clan_cli/tests/test_upload_single_file.py @@ -15,7 +15,7 @@ def test_upload_single_file( src_file = temporary_home / "test.txt" src_file.write_text("test") dest_file = temporary_home / "test_dest.txt" - with host.ssh_control_master() as host: + with host.host_connection() as host: upload(host, src_file, dest_file) assert dest_file.exists() diff --git a/pkgs/clan-cli/clan_cli/vars/_types.py b/pkgs/clan-cli/clan_cli/vars/_types.py index 5d835c82b..459591c7f 100644 --- a/pkgs/clan-cli/clan_cli/vars/_types.py +++ b/pkgs/clan-cli/clan_cli/vars/_types.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING from clan_lib.errors import ClanError from clan_lib.flake import Flake -from clan_lib.ssh.remote import Remote +from clan_lib.ssh.host import Host if TYPE_CHECKING: from .generate import Generator, Var @@ -200,5 +200,5 @@ class StoreBase(ABC): pass @abstractmethod - def upload(self, machine: str, host: Remote, phases: list[str]) -> None: + def upload(self, machine: str, host: Host, phases: list[str]) -> None: pass diff --git a/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py b/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py index 9b32a5f50..ce214daa5 100644 --- a/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py +++ b/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py @@ -6,7 +6,7 @@ from clan_cli.vars._types import StoreBase from clan_cli.vars.generate import Generator, Var from clan_lib.errors import ClanError from clan_lib.flake import Flake -from clan_lib.ssh.remote import Remote +from clan_lib.ssh.host import Host class FactStore(StoreBase): @@ -73,6 +73,6 @@ class FactStore(StoreBase): msg = "populate_dir is not implemented for public vars stores" raise NotImplementedError(msg) - def upload(self, machine: str, host: Remote, phases: list[str]) -> None: + def upload(self, machine: str, host: Host, phases: list[str]) -> None: msg = "upload is not implemented for public vars stores" raise NotImplementedError(msg) diff --git a/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py b/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py index b76840c68..774ced48d 100644 --- a/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py +++ b/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py @@ -8,7 +8,7 @@ from clan_cli.vars.generate import Generator, Var from clan_lib.dirs import vm_state_dir from clan_lib.errors import ClanError from clan_lib.flake import Flake -from clan_lib.ssh.remote import Remote +from clan_lib.ssh.host import Host log = logging.getLogger(__name__) @@ -82,6 +82,6 @@ class FactStore(StoreBase): msg = "populate_dir is not implemented for public vars stores" raise NotImplementedError(msg) - def upload(self, machine: str, host: Remote, phases: list[str]) -> None: + def upload(self, machine: str, host: Host, phases: list[str]) -> None: msg = "upload is not implemented for public vars stores" raise NotImplementedError(msg) diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/fs.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/fs.py index 2251d7bf1..bafad9b9b 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/fs.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/fs.py @@ -5,7 +5,7 @@ from pathlib import Path from clan_cli.vars._types import StoreBase from clan_cli.vars.generate import Generator, Var from clan_lib.flake import Flake -from clan_lib.ssh.remote import Remote +from clan_lib.ssh.host import Host class SecretStore(StoreBase): @@ -57,6 +57,6 @@ class SecretStore(StoreBase): shutil.rmtree(self.dir) return [] - def upload(self, machine: str, host: Remote, phases: list[str]) -> None: + def upload(self, machine: str, host: Host, phases: list[str]) -> None: msg = "Cannot upload secrets with FS backend" raise NotImplementedError(msg) diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py index 851dab41c..01d242b6d 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py @@ -10,7 +10,7 @@ from clan_cli.ssh.upload import upload from clan_cli.vars._types import StoreBase from clan_cli.vars.generate import Generator, Var from clan_lib.flake import Flake -from clan_lib.ssh.remote import Remote +from clan_lib.ssh.host import Host log = logging.getLogger(__name__) @@ -157,7 +157,7 @@ class SecretStore(StoreBase): manifest.append(git_hash) return b"\n".join(manifest) - def needs_upload(self, machine: str, host: Remote) -> bool: + def needs_upload(self, machine: str, host: Host) -> bool: local_hash = self.generate_hash(machine) if not local_hash: return True @@ -243,7 +243,7 @@ class SecretStore(StoreBase): if hash_data: (output_dir / ".pass_info").write_bytes(hash_data) - def upload(self, machine: str, host: Remote, phases: list[str]) -> None: + def upload(self, machine: str, host: Host, phases: list[str]) -> None: if "partitioning" in phases: msg = "Cannot upload partitioning secrets" raise NotImplementedError(msg) diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py index 9188dedbf..5d25a5a87 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py @@ -28,7 +28,7 @@ from clan_cli.vars.generate import Generator from clan_cli.vars.var import Var from clan_lib.errors import ClanError from clan_lib.flake import Flake -from clan_lib.ssh.remote import Remote +from clan_lib.ssh.host import Host @dataclass @@ -246,7 +246,7 @@ class SecretStore(StoreBase): target_path.chmod(file.mode) @override - def upload(self, machine: str, host: Remote, phases: list[str]) -> None: + def upload(self, machine: str, host: Host, phases: list[str]) -> None: if "partitioning" in phases: msg = "Cannot upload partitioning secrets" raise NotImplementedError(msg) diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py index 995c8903b..bc9f3aca7 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py @@ -6,7 +6,7 @@ from clan_cli.vars._types import StoreBase from clan_cli.vars.generate import Generator, Var from clan_lib.dirs import vm_state_dir from clan_lib.flake import Flake -from clan_lib.ssh.remote import Remote +from clan_lib.ssh.host import Host class SecretStore(StoreBase): @@ -71,6 +71,6 @@ class SecretStore(StoreBase): shutil.rmtree(output_dir) shutil.copytree(vars_dir, output_dir) - def upload(self, machine: str, host: Remote, phases: list[str]) -> None: + def upload(self, machine: str, host: Host, phases: list[str]) -> None: msg = "Cannot upload secrets to VMs" raise NotImplementedError(msg) diff --git a/pkgs/clan-cli/clan_cli/vars/upload.py b/pkgs/clan-cli/clan_cli/vars/upload.py index 5b2f8af5b..700b5e385 100644 --- a/pkgs/clan-cli/clan_cli/vars/upload.py +++ b/pkgs/clan-cli/clan_cli/vars/upload.py @@ -5,12 +5,12 @@ from pathlib import Path from clan_cli.completions import add_dynamic_completer, complete_machines from clan_lib.flake import require_flake from clan_lib.machines.machines import Machine -from clan_lib.ssh.remote import Remote +from clan_lib.ssh.host import Host log = logging.getLogger(__name__) -def upload_secret_vars(machine: Machine, host: Remote) -> None: +def upload_secret_vars(machine: Machine, host: Host) -> None: machine.secret_vars_store.upload( machine.name, host, phases=["activation", "users", "services"] ) @@ -32,7 +32,7 @@ def upload_command(args: argparse.Namespace) -> None: populate_secret_vars(machine, directory) return - with machine.target_host().ssh_control_master() as host, host.become_root() as host: + with machine.target_host().host_connection() as host, host.become_root() as host: upload_secret_vars(machine, host) diff --git a/pkgs/clan-cli/clan_lib/backups/create.py b/pkgs/clan-cli/clan_lib/backups/create.py index 879c15466..428744bdf 100644 --- a/pkgs/clan-cli/clan_lib/backups/create.py +++ b/pkgs/clan-cli/clan_lib/backups/create.py @@ -10,7 +10,7 @@ def create_backup(machine: Machine, provider: str | None = None) -> None: if not backup_scripts["providers"]: msg = "No providers specified" raise ClanError(msg) - with host.ssh_control_master() as ssh: + with host.host_connection() as ssh: for provider in backup_scripts["providers"]: proc = ssh.run( [backup_scripts["providers"][provider]["create"]], @@ -23,7 +23,7 @@ def create_backup(machine: Machine, provider: str | None = None) -> None: if provider not in backup_scripts["providers"]: msg = f"provider {provider} not found" raise ClanError(msg) - with host.ssh_control_master() as ssh: + with host.host_connection() as ssh: proc = ssh.run( [backup_scripts["providers"][provider]["create"]], ) diff --git a/pkgs/clan-cli/clan_lib/backups/list.py b/pkgs/clan-cli/clan_lib/backups/list.py index c68c13a9c..7360959cb 100644 --- a/pkgs/clan-cli/clan_lib/backups/list.py +++ b/pkgs/clan-cli/clan_lib/backups/list.py @@ -43,7 +43,7 @@ def list_provider(machine: Machine, host: Remote, provider: str) -> list[Backup] def list_backups(machine: Machine, provider: str | None = None) -> list[Backup]: backup_metadata = machine.select("config.clan.core.backups") results = [] - with machine.target_host().ssh_control_master() as host: + with machine.target_host().host_connection() as host: if provider is None: for _provider in backup_metadata["providers"]: results += list_provider(machine, host, _provider) diff --git a/pkgs/clan-cli/clan_lib/backups/restore.py b/pkgs/clan-cli/clan_lib/backups/restore.py index 38fce7751..fe109415b 100644 --- a/pkgs/clan-cli/clan_lib/backups/restore.py +++ b/pkgs/clan-cli/clan_lib/backups/restore.py @@ -20,7 +20,7 @@ def restore_service( # FIXME: If we have too many folder this might overflow the stack. env["FOLDERS"] = ":".join(set(folders)) - with host.ssh_control_master() as ssh: + with host.host_connection() as ssh: if pre_restore := backup_folders[service]["preRestoreCommand"]: proc = ssh.run( [pre_restore], @@ -58,7 +58,7 @@ def restore_backup( service: str | None = None, ) -> None: errors = [] - with machine.target_host().ssh_control_master() as host: + with machine.target_host().host_connection() as host: if service is None: backup_folders = machine.select("config.clan.core.state") for _service in backup_folders: diff --git a/pkgs/clan-cli/clan_lib/machines/hardware.py b/pkgs/clan-cli/clan_lib/machines/hardware.py index 6bf051eae..5c39ed6fb 100644 --- a/pkgs/clan-cli/clan_lib/machines/hardware.py +++ b/pkgs/clan-cli/clan_lib/machines/hardware.py @@ -90,7 +90,7 @@ def run_machine_hardware_info( "--show-hardware-config", ] - with target_host.ssh_control_master() as ssh, ssh.become_root() as sudo_ssh: + with target_host.host_connection() as ssh, ssh.become_root() as sudo_ssh: out = sudo_ssh.run(config_command, opts=RunOpts(check=False)) if out.returncode != 0: if "nixos-facter" in out.stderr and "not found" in out.stderr: diff --git a/pkgs/clan-cli/clan_lib/machines/update.py b/pkgs/clan-cli/clan_lib/machines/update.py index 3feff4986..40aada4f7 100644 --- a/pkgs/clan-cli/clan_lib/machines/update.py +++ b/pkgs/clan-cli/clan_lib/machines/update.py @@ -17,6 +17,7 @@ from clan_lib.colors import AnsiColor from clan_lib.errors import ClanError from clan_lib.machines.machines import Machine from clan_lib.nix import nix_command, nix_metadata +from clan_lib.ssh.host import Host from clan_lib.ssh.remote import Remote log = logging.getLogger(__name__) @@ -37,7 +38,7 @@ def is_local_input(node: dict[str, dict[str, str]]) -> bool: return local -def upload_sources(machine: Machine, ssh: Remote, force_fetch_local: bool) -> str: +def upload_sources(machine: Machine, ssh: Host, force_fetch_local: bool) -> str: env = ssh.nix_ssh_env(os.environ.copy()) flake_url = ( @@ -110,8 +111,8 @@ def upload_sources(machine: Machine, ssh: Remote, force_fetch_local: bool) -> st @API.register def run_machine_update( machine: Machine, - target_host: Remote, - build_host: Remote | None, + target_host: Host, + build_host: Host | None, force_fetch_local: bool = False, ) -> None: """Update an existing machine using nixos-rebuild or darwin-rebuild. @@ -126,13 +127,13 @@ def run_machine_update( """ with ExitStack() as stack: - target_host = stack.enter_context(target_host.ssh_control_master()) + target_host = stack.enter_context(target_host.host_connection()) # If no build host is specified, use the target host as the build host. if build_host is None: build_host = target_host else: - build_host = stack.enter_context(build_host.ssh_control_master()) + stack.enter_context(build_host.host_connection()) # Some operations require root privileges on the target host. target_host_root = stack.enter_context(target_host.become_root()) diff --git a/pkgs/clan-cli/clan_lib/network/check.py b/pkgs/clan-cli/clan_lib/network/check.py index 44af02fce..768a21b0a 100644 --- a/pkgs/clan-cli/clan_lib/network/check.py +++ b/pkgs/clan-cli/clan_lib/network/check.py @@ -35,7 +35,7 @@ def check_machine_ssh_login( if opts is None: opts = ConnectionOptions() - with remote.ssh_control_master() as ssh: + with remote.host_connection() as ssh: try: ssh.run( ["true"], diff --git a/pkgs/clan-cli/clan_lib/ssh/remote.py b/pkgs/clan-cli/clan_lib/ssh/remote.py index 30d1fa2ce..720453fc8 100644 --- a/pkgs/clan-cli/clan_lib/ssh/remote.py +++ b/pkgs/clan-cli/clan_lib/ssh/remote.py @@ -16,6 +16,7 @@ from clan_lib.cmd import CmdOut, RunOpts, run from clan_lib.colors import AnsiColor from clan_lib.errors import ClanError, indent_command # Assuming these are available from clan_lib.nix import nix_shell +from clan_lib.ssh.host import Host from clan_lib.ssh.host_key import HostKeyCheck, hostkey_to_ssh_opts from clan_lib.ssh.parse import parse_ssh_uri from clan_lib.ssh.sudo_askpass_proxy import SudoAskpassProxy @@ -30,7 +31,7 @@ NO_OUTPUT_TIMEOUT = 20 @dataclass(frozen=True) -class Remote: +class Remote(Host): address: str command_prefix: str user: str = "root" @@ -136,7 +137,7 @@ class Remote: return run(cmd, opts) @contextmanager - def ssh_control_master(self) -> Iterator["Remote"]: + def host_connection(self) -> Iterator["Remote"]: """ Context manager to manage SSH ControlMaster connections. This will create a temporary directory for the control socket. @@ -318,11 +319,11 @@ class Remote: if env is None: env = {} env["NIX_SSHOPTS"] = " ".join( - self.ssh_cmd_opts(control_master=control_master) # Renamed + self._ssh_cmd_opts(control_master=control_master) # Renamed ) return env - def ssh_cmd_opts( + def _ssh_cmd_opts( self, control_master: bool = True, ) -> list[str]: @@ -373,7 +374,7 @@ class Remote: packages.append("sshpass") password_args = ["sshpass", "-p", self.password] - current_ssh_opts = self.ssh_cmd_opts(control_master=control_master) + current_ssh_opts = self._ssh_cmd_opts(control_master=control_master) if verbose_ssh or self.verbose_ssh: current_ssh_opts.extend(["-v"]) if tty: @@ -396,7 +397,7 @@ class Remote: ] return nix_shell(packages, cmd) - def check_sshpass_errorcode(self, res: subprocess.CompletedProcess) -> None: + def _check_sshpass_errorcode(self, res: subprocess.CompletedProcess) -> None: """ Check the return code of the sshpass command and raise an error if it indicates a failure. Error codes are based on man sshpass(1) and may vary by version. @@ -454,7 +455,7 @@ class Remote: # We only check the error code if a password is set, as sshpass is used. # AS sshpass swallows all output. if self.password: - self.check_sshpass_errorcode(res) + self._check_sshpass_errorcode(res) def check_machine_ssh_reachable( self, opts: "ConnectionOptions | None" = None diff --git a/pkgs/clan-cli/clan_lib/ssh/remote_test.py b/pkgs/clan-cli/clan_lib/ssh/remote_test.py index e3fdd71f1..b2e3ef6ad 100644 --- a/pkgs/clan-cli/clan_lib/ssh/remote_test.py +++ b/pkgs/clan-cli/clan_lib/ssh/remote_test.py @@ -180,7 +180,7 @@ def test_run_no_shell(hosts: list[Remote], runtime: AsyncRuntime) -> None: def test_sudo_ask_proxy(hosts: list[Remote]) -> None: host = hosts[0] - with host.ssh_control_master() as host: + with host.host_connection() as host: proxy = SudoAskpassProxy(host, prompt_command=["bash", "-c", "echo yes"]) try: @@ -197,7 +197,7 @@ def test_sudo_ask_proxy(hosts: list[Remote]) -> None: def test_run_function(hosts: list[Remote], runtime: AsyncRuntime) -> None: def some_func(h: Remote) -> bool: - with h.ssh_control_master() as ssh: + with h.host_connection() as ssh: p = ssh.run(["echo", "hello"]) return p.stdout == "hello\n" From af7ce9b8edfbdbaeccea7ec969df63f6bf93bd22 Mon Sep 17 00:00:00 2001 From: DavHau Date: Tue, 29 Jul 2025 18:44:25 +0700 Subject: [PATCH 3/3] machines update: support local build Now the user can pass `--build-host local`, to select the local machine as a build host, in which case no ssh is used. This means the admin machine does not necessarily have ssh set up to itself, which was confusing for many users. Also this makes it easier to re-use a well configured nix remote build setup which is only available on the local machine. Eg if `--build-host local` nix' defaults for remote builds on that machine will be utilized. --- checks/update/flake-module.nix | 129 +++++++++++++++++----- pkgs/clan-cli/clan_cli/machines/update.py | 23 +++- pkgs/clan-cli/clan_lib/machines/update.py | 3 +- 3 files changed, 121 insertions(+), 34 deletions(-) diff --git a/checks/update/flake-module.nix b/checks/update/flake-module.nix index 0c8b5906a..c2f8af6f0 100644 --- a/checks/update/flake-module.nix +++ b/checks/update/flake-module.nix @@ -35,6 +35,13 @@ services.openssh.enable = true; services.openssh.settings.PasswordAuthentication = false; users.users.root.openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ]; + services.openssh.knownHosts.localhost.publicKeyFile = ../assets/ssh/pubkey; + services.openssh.hostKeys = [ + { + path = ../assets/ssh/privkey; + type = "ed25519"; + } + ]; security.sudo.wheelNeedsPassword = false; boot.consoleLogLevel = lib.mkForce 100; @@ -99,7 +106,8 @@ let closureInfo = pkgs.closureInfo { rootPaths = [ - self.checks.x86_64-linux.clan-core-for-checks + self.packages.${pkgs.system}.clan-cli + self.checks.${pkgs.system}.clan-core-for-checks self.clanInternals.machines.${pkgs.hostPlatform.system}.test-update-machine.config.system.build.toplevel pkgs.stdenv.drvPath pkgs.bash.drvPath @@ -150,15 +158,7 @@ # Update the machine configuration to add a new file machine_config_path = os.path.join(flake_dir, "machines", "test-update-machine", "configuration.nix") os.makedirs(os.path.dirname(machine_config_path), exist_ok=True) - - with open(machine_config_path, "w") as f: - f.write(""" - { - environment.etc."update-successful".text = "ok"; - } - """) - # Run clan update command # Note: update command doesn't accept -i flag, SSH key must be in ssh-agent # Start ssh-agent and add the key agent_output = subprocess.check_output(["${pkgs.openssh}/bin/ssh-agent", "-s"], text=True) @@ -167,11 +167,95 @@ os.environ["SSH_AUTH_SOCK"] = line.split("=", 1)[1].split(";")[0] elif line.startswith("SSH_AGENT_PID="): os.environ["SSH_AGENT_PID"] = line.split("=", 1)[1].split(";")[0] - + # Add the SSH key to the agent subprocess.run(["${pkgs.openssh}/bin/ssh-add", ssh_conn.ssh_key], check=True) + ############## + print("TEST: update with --build-host local") + with open(machine_config_path, "w") as f: + f.write(""" + { + environment.etc."update-build-local-successful".text = "ok"; + } + """) + + # rsync the flake into the container + os.environ["PATH"] = f"{os.environ['PATH']}:${pkgs.openssh}/bin" + subprocess.run( + [ + "${pkgs.rsync}/bin/rsync", + "-a", + "--delete", + "-e", + "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", + f"{str(flake_dir)}/", + f"root@192.168.1.1:/flake", + ], + check=True + ) + + # allow machine to ssh into itself + subprocess.run([ + "ssh", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "StrictHostKeyChecking=no", + f"root@192.168.1.1", + "mkdir -p /root/.ssh && chmod 700 /root/.ssh && echo \"$(cat \"${../assets/ssh/privkey}\")\" > /root/.ssh/id_ed25519 && chmod 600 /root/.ssh/id_ed25519", + ], check=True) + + # install the clan-cli package into the container's Nix store + subprocess.run( + [ + "${pkgs.nix}/bin/nix", + "copy", + "--to", + "ssh://root@192.168.1.1", + "--no-check-sigs", + f"${self.packages.${pkgs.system}.clan-cli}", + "--extra-experimental-features", "nix-command flakes", + "--from", f"{os.environ["TMPDIR"]}/store" + ], + check=True, + env={ + **os.environ, + "NIX_SSHOPTS": "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", + }, + ) + + # Run ssh on the host to run the clan update command via --build-host local + subprocess.run([ + "ssh", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "StrictHostKeyChecking=no", + f"root@192.168.1.1", + "${self.packages.${pkgs.system}.clan-cli}/bin/clan", + "machines", + "update", + "--debug", + "--flake", "/flake", + "--host-key-check", "none", + "--fetch-local", # Use local store instead of fetching from network + "--build-host", "local", + "test-update-machine", + "--target-host", f"root@localhost", + ], check=True) + + # Verify the update was successful + machine.succeed("test -f /etc/update-build-local-successful") + + + ############## + print("TEST: update with --fetch-local") + + with open(machine_config_path, "w") as f: + f.write(""" + { + environment.etc."update-fetch-local-successful".text = "ok"; + } + """) + # Run clan update command subprocess.run([ "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan", @@ -186,10 +270,12 @@ ], check=True) # Verify the update was successful - machine.succeed("test -f /etc/update-successful") + machine.succeed("test -f /etc/update-fetch-local-successful") - # Test update with --build-host - # Update configuration again to test build-host functionality + + ############## + print("TEST: update with --build-host 192.168.1.1") + # Update configuration again with open(machine_config_path, "w") as f: f.write(""" { @@ -213,23 +299,6 @@ # Verify the second update was successful machine.succeed("test -f /etc/build-host-update-successful") - - # Run clan update command with --build-host - subprocess.run([ - "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan", - "machines", - "update", - "--debug", - "--flake", flake_dir, - "--host-key-check", "none", - "--fetch-local", # Use local store instead of fetching from network - "--build-host", f"root@192.168.1.1:{ssh_conn.host_port}", - "test-update-machine", - "--target-host", f"root@192.168.1.1:{ssh_conn.host_port}", - ], check=True) - - # Verify the second update was successful - machine.succeed("test -f /etc/build-host-update-successful") ''; } { inherit pkgs self; }; }; diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 0064e1ba7..da19c0932 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -13,7 +13,9 @@ from clan_lib.machines.machines import Machine from clan_lib.machines.suggestions import validate_machine_names from clan_lib.machines.update import run_machine_update from clan_lib.nix import nix_config +from clan_lib.ssh.host import Host from clan_lib.ssh.host_key import HostKeyCheck +from clan_lib.ssh.localhost import LocalHost from clan_lib.ssh.remote import Remote from clan_cli.completions import ( @@ -128,6 +130,19 @@ def update_command(args: argparse.Namespace) -> None: host_key_check = args.host_key_check with AsyncRuntime() as runtime: for machine in machines_to_update: + # figure out on which machine to build on + build_host: Host | None = None + if args.build_host: + if args.build_host == "local": + build_host = LocalHost() + else: + build_host = Remote.from_ssh_uri( + machine_name=machine.name, + address=args.build_host, + ).override(host_key_check=host_key_check) + else: + build_host = machine.build_host() + # Figure out the target host if args.target_host: target_host = Remote.from_ssh_uri( machine_name=machine.name, @@ -137,6 +152,7 @@ def update_command(args: argparse.Namespace) -> None: target_host = machine.target_host().override( host_key_check=host_key_check ) + # run the update runtime.async_run( AsyncOpts( tid=machine.name, @@ -145,7 +161,7 @@ def update_command(args: argparse.Namespace) -> None: run_machine_update, machine=machine, target_host=target_host, - build_host=machine.build_host(), + build_host=build_host, force_fetch_local=args.fetch_local, ) runtime.join_all() @@ -189,7 +205,10 @@ def register_update_parser(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--build-host", type=str, - help="Address of the machine to build the flake, in the format of user@host:1234.", + help=( + "The machine on which to build the machine configuration.\n" + "Pass 'local' to build on the local machine, or an ssh address like user@host:1234\n" + ), ) parser.add_argument( "--fetch-local", diff --git a/pkgs/clan-cli/clan_lib/machines/update.py b/pkgs/clan-cli/clan_lib/machines/update.py index 40aada4f7..b11498237 100644 --- a/pkgs/clan-cli/clan_lib/machines/update.py +++ b/pkgs/clan-cli/clan_lib/machines/update.py @@ -18,7 +18,6 @@ from clan_lib.errors import ClanError from clan_lib.machines.machines import Machine from clan_lib.nix import nix_command, nix_metadata from clan_lib.ssh.host import Host -from clan_lib.ssh.remote import Remote log = logging.getLogger(__name__) @@ -133,7 +132,7 @@ def run_machine_update( if build_host is None: build_host = target_host else: - stack.enter_context(build_host.host_connection()) + build_host = stack.enter_context(build_host.host_connection()) # Some operations require root privileges on the target host. target_host_root = stack.enter_context(target_host.become_root())