yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'clan_cli vars: actually upload' (#2378) from lassulus/clan-core:vars-fix into main

Browse Source
This commit is contained in:
clan-bot
2024-11-13 12:31:43 +00:00
parent dea0cd5844 ddc7afd67d
commit 3b4046b207
8 changed files with 75 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
nixosModules/clanCore/vars/settings-opts.nix
View File

@@ -23,10 +23,10 @@
};
secretUploadDirectory = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
type = lib.types.path;
description = ''
The directory where secrets are uploaded into, This is backend specific.
This is usally set by the secret store backend.
'';
};
@@ -63,10 +63,10 @@
};
publicDirectory = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
type = lib.types.path;
description = ''
The directory where public facts are stored.
This is usally set by the public store backend.
'';
};
}

40
pkgs/clan-cli/clan_cli/cmd.py
View File

@@ -29,14 +29,17 @@ class Log(Enum):
NONE = 4
def handle_output(process: subprocess.Popen, log: Log) -> tuple[str, str]:
def handle_io(
process: subprocess.Popen, input_bytes: bytes | None, log: Log
) -> tuple[str, str]:
rlist = [process.stdout, process.stderr]
wlist = [process.stdin] if input_bytes is not None else []
stdout_buf = b""
stderr_buf = b""
while len(rlist) != 0:
readlist, _, _ = select.select(rlist, [], [], 0.1)
if len(readlist) == 0: # timeout in select
while len(rlist) != 0 or len(wlist) != 0:
readlist, writelist, _ = select.select(rlist, wlist, [], 0.1)
if len(readlist) == 0 and len(writelist) == 0:
if process.poll() is None:
continue
# Process has exited
@@ -62,12 +65,31 @@ def handle_output(process: subprocess.Popen, log: Log) -> tuple[str, str]:
sys.stderr.buffer.write(ret)
sys.stderr.flush()
stderr_buf += ret
if process.stdin in writelist:
if input_bytes:
try:
assert process.stdin is not None
written = os.write(process.stdin.fileno(), input_bytes)
except BrokenPipeError:
wlist.remove(process.stdin)
else:
input_bytes = input_bytes[written:]
if len(input_bytes) == 0:
wlist.remove(process.stdin)
process.stdin.flush()
process.stdin.close()
else:
wlist.remove(process.stdin)
return stdout_buf.decode("utf-8", "replace"), stderr_buf.decode("utf-8", "replace")
@contextmanager
def terminate_process_group(process: subprocess.Popen) -> Iterator[None]:
process_group = os.getpgid(process.pid)
try:
process_group = os.getpgid(process.pid)
except ProcessLookupError:
return
if process_group == os.getpgid(os.getpid()):
msg = "Bug! Refusing to terminate the current process group"
raise ClanError(msg)
@@ -183,11 +205,13 @@ def run(
start = timeit.default_timer()
with ExitStack() as stack:
stdin = subprocess.PIPE if input is not None else None
process = stack.enter_context(
subprocess.Popen(
cmd,
cwd=str(cwd),
env=env,
stdin=stdin,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
start_new_session=not needs_user_terminal,
@@ -200,10 +224,8 @@ def run(
else:
stack.enter_context(terminate_process_group(process))
stdout_buf, stderr_buf = handle_output(process, log)
if input:
process.communicate(input)
stdout_buf, stderr_buf = handle_io(process, input, log)
process.wait()
global TIME_TABLE
if TIME_TABLE:

1
pkgs/clan-cli/clan_cli/machines/machine_group.py
View File

@@ -10,6 +10,7 @@ T = TypeVar("T")
class MachineGroup:
def __init__(self, machines: list[Machine]) -> None:
self.machines = machines
self.group = HostGroup([m.target_host for m in machines])
def __repr__(self) -> str:

4
pkgs/clan-cli/clan_cli/machines/machines.py
View File

@@ -151,6 +151,10 @@ class Machine:
def secrets_upload_directory(self) -> str:
return self.deployment["facts"]["secretUploadDirectory"]
@property
def secret_vars_upload_directory(self) -> str:
return self.deployment["vars"]["secretUploadDirectory"]
@property
def flake_dir(self) -> Path:
if self.flake.is_local():

8
pkgs/clan-cli/clan_cli/machines/update.py
View File

@@ -21,6 +21,7 @@ from clan_cli.machines.machines import Machine
from clan_cli.nix import nix_command, nix_metadata
from clan_cli.ssh import HostKeyCheck
from clan_cli.vars.generate import generate_vars
from clan_cli.vars.upload import upload_secret_vars
from .inventory import get_all_machines, get_selected_machines
from .machine_group import MachineGroup
@@ -120,6 +121,8 @@ def deploy_machine(machines: MachineGroup) -> None:
generate_vars([machine], None, False)
upload_secrets(machine)
upload_secret_vars(machine)
path = upload_sources(
machine,
)
@@ -151,7 +154,10 @@ def deploy_machine(machines: MachineGroup) -> None:
if ret.returncode != 0:
ret = host.run(cmd, extra_env=env)
machines.run_function(deploy)
if len(machines.group.hosts) > 1:
machines.run_function(deploy)
else:
deploy(machines.machines[0])
def update(args: argparse.Namespace) -> None:

40
pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py
View File

@@ -1,9 +1,9 @@
import os
import subprocess
from itertools import chain
from pathlib import Path
from typing import override
from clan_cli.cmd import run
from clan_cli.machines.machines import Machine
from clan_cli.nix import nix_shell
@@ -36,7 +36,7 @@ class SecretStore(SecretStoreBase):
shared: bool = False,
deployed: bool = True,
) -> Path | None:
subprocess.run(
run(
nix_shell(
["nixpkgs#pass"],
[
@@ -52,7 +52,7 @@ class SecretStore(SecretStoreBase):
return None # we manage the files outside of the git repo
def get(self, generator_name: str, name: str, shared: bool = False) -> bytes:
return subprocess.run(
return run(
nix_shell(
["nixpkgs#pass"],
[
@@ -61,9 +61,7 @@ class SecretStore(SecretStoreBase):
str(self.entry_dir(generator_name, name, shared)),
],
),
check=True,
stdout=subprocess.PIPE,
).stdout
).stdout.encode()
def exists(self, generator_name: str, name: str, shared: bool = False) -> bool:
return (
@@ -74,7 +72,7 @@ class SecretStore(SecretStoreBase):
def generate_hash(self) -> bytes:
hashes = []
hashes.append(
subprocess.run(
run(
nix_shell(
["nixpkgs#git"],
[
@@ -87,9 +85,10 @@ class SecretStore(SecretStoreBase):
self.entry_prefix,
],
),
stdout=subprocess.PIPE,
check=False,
).stdout.strip()
)
.stdout.strip()
.encode()
)
shared_dir = Path(self._password_store_dir) / self.entry_prefix / "shared"
machine_dir = (
@@ -101,7 +100,7 @@ class SecretStore(SecretStoreBase):
for symlink in chain(shared_dir.glob("**/*"), machine_dir.glob("**/*")):
if symlink.is_symlink():
hashes.append(
subprocess.run(
run(
nix_shell(
["nixpkgs#git"],
[
@@ -114,9 +113,10 @@ class SecretStore(SecretStoreBase):
str(symlink),
],
),
stdout=subprocess.PIPE,
check=False,
).stdout.strip()
)
.stdout.strip()
.encode()
)
# we sort the hashes to make sure that the order is always the same
@@ -128,9 +128,8 @@ class SecretStore(SecretStoreBase):
local_hash = self.generate_hash()
remote_hash = self.machine.target_host.run(
# TODO get the path to the secrets from the machine
["cat", f"{self.machine.secrets_upload_directory}/.pass_info"],
["cat", f"{self.machine.secret_vars_upload_directory}/.pass_info"],
check=False,
stdout=subprocess.PIPE,
).stdout.strip()
if not remote_hash:
@@ -143,10 +142,15 @@ class SecretStore(SecretStoreBase):
for secret_var in self.get_all():
if not secret_var.deployed:
continue
rel_dir = self.rel_dir(
secret_var.generator, secret_var.name, secret_var.shared
)
with (output_dir / rel_dir).open("wb") as f:
if secret_var.shared:
output_file = (
output_dir / "shared" / secret_var.generator / secret_var.name
)
else:
output_file = output_dir / secret_var.generator / secret_var.name
output_file.parent.mkdir(parents=True, exist_ok=True)
with (output_file).open("wb") as f:
f.write(
self.get(secret_var.generator, secret_var.name, secret_var.shared)
)

8
pkgs/clan-cli/clan_cli/vars/upload.py
View File

@@ -12,8 +12,8 @@ from clan_cli.nix import nix_shell
log = logging.getLogger(__name__)
def upload_secrets(machine: Machine) -> None:
secret_store_module = importlib.import_module(machine.secret_facts_module)
def upload_secret_vars(machine: Machine) -> None:
secret_store_module = importlib.import_module(machine.secret_vars_module)
secret_store = secret_store_module.SecretStore(machine=machine)
if not secret_store.needs_upload():
@@ -38,7 +38,7 @@ def upload_secrets(machine: Machine) -> None:
"--delete",
"--chmod=D700,F600",
f"{tempdir!s}/",
f"{host.target_for_rsync}:{machine.secrets_upload_directory}/",
f"{host.target_for_rsync}:{machine.secret_vars_upload_directory}/",
],
),
log=Log.BOTH,
@@ -48,7 +48,7 @@ def upload_secrets(machine: Machine) -> None:
def upload_command(args: argparse.Namespace) -> None:
machine = Machine(name=args.machine, flake=args.flake)
upload_secrets(machine)
upload_secret_vars(machine)
def register_upload_parser(parser: argparse.ArgumentParser) -> None:

4
pkgs/clan-cli/clan_cli/vms/run.py
View File

@@ -12,7 +12,7 @@ from dataclasses import dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
from clan_cli.cmd import CmdOut, Log, handle_output, run
from clan_cli.cmd import CmdOut, Log, handle_io, run
from clan_cli.completions import add_dynamic_completer, complete_machines
from clan_cli.dirs import module_root, user_cache_dir, vm_state_dir
from clan_cli.errors import ClanCmdError, ClanError
@@ -339,7 +339,7 @@ def run_vm(
) as vm,
ThreadPoolExecutor(max_workers=1) as executor,
):
future = executor.submit(handle_output, vm.process, Log.BOTH)
future = executor.submit(handle_io, vm.process, input_bytes=None, log=Log.BOTH)
args: list[str] = vm.process.args # type: ignore[assignment]
if runtime_config.command is not None: