From 2c66b36931070053362617895da08e832b9bf66b Mon Sep 17 00:00:00 2001 From: lassulus Date: Tue, 12 Nov 2024 15:56:11 +0100 Subject: [PATCH 1/4] core vars: remove default for dirs --- nixosModules/clanCore/vars/settings-opts.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nixosModules/clanCore/vars/settings-opts.nix b/nixosModules/clanCore/vars/settings-opts.nix index fce8bc693..b94894c2c 100644 --- a/nixosModules/clanCore/vars/settings-opts.nix +++ b/nixosModules/clanCore/vars/settings-opts.nix @@ -23,10 +23,10 @@ }; secretUploadDirectory = lib.mkOption { - type = lib.types.nullOr lib.types.path; - default = null; + type = lib.types.path; description = '' The directory where secrets are uploaded into, This is backend specific. + This is usally set by the secret store backend. ''; }; @@ -63,10 +63,10 @@ }; publicDirectory = lib.mkOption { - type = lib.types.nullOr lib.types.path; - default = null; + type = lib.types.path; description = '' The directory where public facts are stored. + This is usally set by the public store backend. ''; }; } From ebfc8ecfd0c275f2ff00e74c6829a815a6ab5ff0 Mon Sep 17 00:00:00 2001 From: lassulus Date: Tue, 12 Nov 2024 16:04:59 +0100 Subject: [PATCH 2/4] cli machines update: run deploy directly if deploying single machine --- pkgs/clan-cli/clan_cli/machines/machine_group.py | 1 + pkgs/clan-cli/clan_cli/machines/update.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_cli/machines/machine_group.py b/pkgs/clan-cli/clan_cli/machines/machine_group.py index 43853c019..9ce7c98fb 100644 --- a/pkgs/clan-cli/clan_cli/machines/machine_group.py +++ b/pkgs/clan-cli/clan_cli/machines/machine_group.py @@ -10,6 +10,7 @@ T = TypeVar("T") class MachineGroup: def __init__(self, machines: list[Machine]) -> None: + self.machines = machines self.group = HostGroup([m.target_host for m in machines]) def __repr__(self) -> str: diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index d7d4524f2..a3bd49bac 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -120,6 +120,7 @@ def deploy_machine(machines: MachineGroup) -> None: generate_vars([machine], None, False) upload_secrets(machine) + path = upload_sources( machine, ) @@ -151,7 +152,10 @@ def deploy_machine(machines: MachineGroup) -> None: if ret.returncode != 0: ret = host.run(cmd, extra_env=env) - machines.run_function(deploy) + if len(machines.group.hosts) > 1: + machines.run_function(deploy) + else: + deploy(machines.machines[0]) def update(args: argparse.Namespace) -> None: From 616ddca734b43e1c1e7f81230618ab0cd3673d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 13 Nov 2024 12:25:20 +0100 Subject: [PATCH 3/4] cmd: also process stdin --- pkgs/clan-cli/clan_cli/cmd.py | 40 ++++++++++++++++++++++++------- pkgs/clan-cli/clan_cli/vms/run.py | 4 ++-- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/cmd.py b/pkgs/clan-cli/clan_cli/cmd.py index a50e217ec..064e754d8 100644 --- a/pkgs/clan-cli/clan_cli/cmd.py +++ b/pkgs/clan-cli/clan_cli/cmd.py @@ -29,14 +29,17 @@ class Log(Enum): NONE = 4 -def handle_output(process: subprocess.Popen, log: Log) -> tuple[str, str]: +def handle_io( + process: subprocess.Popen, input_bytes: bytes | None, log: Log +) -> tuple[str, str]: rlist = [process.stdout, process.stderr] + wlist = [process.stdin] if input_bytes is not None else [] stdout_buf = b"" stderr_buf = b"" - while len(rlist) != 0: - readlist, _, _ = select.select(rlist, [], [], 0.1) - if len(readlist) == 0: # timeout in select + while len(rlist) != 0 or len(wlist) != 0: + readlist, writelist, _ = select.select(rlist, wlist, [], 0.1) + if len(readlist) == 0 and len(writelist) == 0: if process.poll() is None: continue # Process has exited @@ -62,12 +65,31 @@ def handle_output(process: subprocess.Popen, log: Log) -> tuple[str, str]: sys.stderr.buffer.write(ret) sys.stderr.flush() stderr_buf += ret + + if process.stdin in writelist: + if input_bytes: + try: + assert process.stdin is not None + written = os.write(process.stdin.fileno(), input_bytes) + except BrokenPipeError: + wlist.remove(process.stdin) + else: + input_bytes = input_bytes[written:] + if len(input_bytes) == 0: + wlist.remove(process.stdin) + process.stdin.flush() + process.stdin.close() + else: + wlist.remove(process.stdin) return stdout_buf.decode("utf-8", "replace"), stderr_buf.decode("utf-8", "replace") @contextmanager def terminate_process_group(process: subprocess.Popen) -> Iterator[None]: - process_group = os.getpgid(process.pid) + try: + process_group = os.getpgid(process.pid) + except ProcessLookupError: + return if process_group == os.getpgid(os.getpid()): msg = "Bug! Refusing to terminate the current process group" raise ClanError(msg) @@ -183,11 +205,13 @@ def run( start = timeit.default_timer() with ExitStack() as stack: + stdin = subprocess.PIPE if input is not None else None process = stack.enter_context( subprocess.Popen( cmd, cwd=str(cwd), env=env, + stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=not needs_user_terminal, @@ -200,10 +224,8 @@ def run( else: stack.enter_context(terminate_process_group(process)) - stdout_buf, stderr_buf = handle_output(process, log) - - if input: - process.communicate(input) + stdout_buf, stderr_buf = handle_io(process, input, log) + process.wait() global TIME_TABLE if TIME_TABLE: diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index f43613349..1af362be5 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -12,7 +12,7 @@ from dataclasses import dataclass from pathlib import Path from tempfile import TemporaryDirectory -from clan_cli.cmd import CmdOut, Log, handle_output, run +from clan_cli.cmd import CmdOut, Log, handle_io, run from clan_cli.completions import add_dynamic_completer, complete_machines from clan_cli.dirs import module_root, user_cache_dir, vm_state_dir from clan_cli.errors import ClanCmdError, ClanError @@ -339,7 +339,7 @@ def run_vm( ) as vm, ThreadPoolExecutor(max_workers=1) as executor, ): - future = executor.submit(handle_output, vm.process, Log.BOTH) + future = executor.submit(handle_io, vm.process, input_bytes=None, log=Log.BOTH) args: list[str] = vm.process.args # type: ignore[assignment] if runtime_config.command is not None: From ddc7afd67daa24326895b41e0d1e927d82ca9964 Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 11 Nov 2024 16:07:00 +0100 Subject: [PATCH 4/4] clan_cli vars: actually upload --- pkgs/clan-cli/clan_cli/machines/machines.py | 4 ++ pkgs/clan-cli/clan_cli/machines/update.py | 2 + .../vars/secret_modules/password_store.py | 40 ++++++++++--------- pkgs/clan-cli/clan_cli/vars/upload.py | 8 ++-- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 8cee21106..3100997ef 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -151,6 +151,10 @@ class Machine: def secrets_upload_directory(self) -> str: return self.deployment["facts"]["secretUploadDirectory"] + @property + def secret_vars_upload_directory(self) -> str: + return self.deployment["vars"]["secretUploadDirectory"] + @property def flake_dir(self) -> Path: if self.flake.is_local(): diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index a3bd49bac..3994d37cd 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -21,6 +21,7 @@ from clan_cli.machines.machines import Machine from clan_cli.nix import nix_command, nix_metadata from clan_cli.ssh import HostKeyCheck from clan_cli.vars.generate import generate_vars +from clan_cli.vars.upload import upload_secret_vars from .inventory import get_all_machines, get_selected_machines from .machine_group import MachineGroup @@ -120,6 +121,7 @@ def deploy_machine(machines: MachineGroup) -> None: generate_vars([machine], None, False) upload_secrets(machine) + upload_secret_vars(machine) path = upload_sources( machine, diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py index 4667be714..73d9b1622 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py @@ -1,9 +1,9 @@ import os -import subprocess from itertools import chain from pathlib import Path from typing import override +from clan_cli.cmd import run from clan_cli.machines.machines import Machine from clan_cli.nix import nix_shell @@ -36,7 +36,7 @@ class SecretStore(SecretStoreBase): shared: bool = False, deployed: bool = True, ) -> Path | None: - subprocess.run( + run( nix_shell( ["nixpkgs#pass"], [ @@ -52,7 +52,7 @@ class SecretStore(SecretStoreBase): return None # we manage the files outside of the git repo def get(self, generator_name: str, name: str, shared: bool = False) -> bytes: - return subprocess.run( + return run( nix_shell( ["nixpkgs#pass"], [ @@ -61,9 +61,7 @@ class SecretStore(SecretStoreBase): str(self.entry_dir(generator_name, name, shared)), ], ), - check=True, - stdout=subprocess.PIPE, - ).stdout + ).stdout.encode() def exists(self, generator_name: str, name: str, shared: bool = False) -> bool: return ( @@ -74,7 +72,7 @@ class SecretStore(SecretStoreBase): def generate_hash(self) -> bytes: hashes = [] hashes.append( - subprocess.run( + run( nix_shell( ["nixpkgs#git"], [ @@ -87,9 +85,10 @@ class SecretStore(SecretStoreBase): self.entry_prefix, ], ), - stdout=subprocess.PIPE, check=False, - ).stdout.strip() + ) + .stdout.strip() + .encode() ) shared_dir = Path(self._password_store_dir) / self.entry_prefix / "shared" machine_dir = ( @@ -101,7 +100,7 @@ class SecretStore(SecretStoreBase): for symlink in chain(shared_dir.glob("**/*"), machine_dir.glob("**/*")): if symlink.is_symlink(): hashes.append( - subprocess.run( + run( nix_shell( ["nixpkgs#git"], [ @@ -114,9 +113,10 @@ class SecretStore(SecretStoreBase): str(symlink), ], ), - stdout=subprocess.PIPE, check=False, - ).stdout.strip() + ) + .stdout.strip() + .encode() ) # we sort the hashes to make sure that the order is always the same @@ -128,9 +128,8 @@ class SecretStore(SecretStoreBase): local_hash = self.generate_hash() remote_hash = self.machine.target_host.run( # TODO get the path to the secrets from the machine - ["cat", f"{self.machine.secrets_upload_directory}/.pass_info"], + ["cat", f"{self.machine.secret_vars_upload_directory}/.pass_info"], check=False, - stdout=subprocess.PIPE, ).stdout.strip() if not remote_hash: @@ -143,10 +142,15 @@ class SecretStore(SecretStoreBase): for secret_var in self.get_all(): if not secret_var.deployed: continue - rel_dir = self.rel_dir( - secret_var.generator, secret_var.name, secret_var.shared - ) - with (output_dir / rel_dir).open("wb") as f: + if secret_var.shared: + output_file = ( + output_dir / "shared" / secret_var.generator / secret_var.name + ) + else: + output_file = output_dir / secret_var.generator / secret_var.name + + output_file.parent.mkdir(parents=True, exist_ok=True) + with (output_file).open("wb") as f: f.write( self.get(secret_var.generator, secret_var.name, secret_var.shared) ) diff --git a/pkgs/clan-cli/clan_cli/vars/upload.py b/pkgs/clan-cli/clan_cli/vars/upload.py index 83f2542e6..a75df063a 100644 --- a/pkgs/clan-cli/clan_cli/vars/upload.py +++ b/pkgs/clan-cli/clan_cli/vars/upload.py @@ -12,8 +12,8 @@ from clan_cli.nix import nix_shell log = logging.getLogger(__name__) -def upload_secrets(machine: Machine) -> None: - secret_store_module = importlib.import_module(machine.secret_facts_module) +def upload_secret_vars(machine: Machine) -> None: + secret_store_module = importlib.import_module(machine.secret_vars_module) secret_store = secret_store_module.SecretStore(machine=machine) if not secret_store.needs_upload(): @@ -38,7 +38,7 @@ def upload_secrets(machine: Machine) -> None: "--delete", "--chmod=D700,F600", f"{tempdir!s}/", - f"{host.target_for_rsync}:{machine.secrets_upload_directory}/", + f"{host.target_for_rsync}:{machine.secret_vars_upload_directory}/", ], ), log=Log.BOTH, @@ -48,7 +48,7 @@ def upload_secrets(machine: Machine) -> None: def upload_command(args: argparse.Namespace) -> None: machine = Machine(name=args.machine, flake=args.flake) - upload_secrets(machine) + upload_secret_vars(machine) def register_upload_parser(parser: argparse.ArgumentParser) -> None: