From de3a08ab63ee8544edf5f91a59883018e136c98c Mon Sep 17 00:00:00 2001 From: DavHau Date: Sat, 31 May 2025 11:11:19 +0700 Subject: [PATCH] sops: initialize age_plugins early This avoids re-initializing the Flake object deep in the tree, which in turn leads to issue when overriding the Flake for testing, eg the URl would reset. --- .../clan_cli/facts/secret_modules/sops.py | 6 ++- pkgs/clan-cli/clan_cli/secrets/groups.py | 29 ++++++++++--- pkgs/clan-cli/clan_cli/secrets/import_sops.py | 2 + pkgs/clan-cli/clan_cli/secrets/machines.py | 25 ++++++++--- pkgs/clan-cli/clan_cli/secrets/secrets.py | 42 +++++++++++-------- pkgs/clan-cli/clan_cli/secrets/sops.py | 35 +++++++--------- pkgs/clan-cli/clan_cli/secrets/users.py | 28 +++++++++---- .../clan_cli/vars/secret_modules/sops.py | 21 +++++++--- 8 files changed, 124 insertions(+), 64 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py b/pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py index cf602c985..4d45da35d 100644 --- a/pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py +++ b/pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py @@ -7,7 +7,7 @@ from clan_lib.ssh.remote import Remote from clan_cli.secrets.folders import sops_secrets_folder from clan_cli.secrets.machines import add_machine, has_machine from clan_cli.secrets.secrets import decrypt_secret, encrypt_secret, has_secret -from clan_cli.secrets.sops import generate_private_key +from clan_cli.secrets.sops import generate_private_key, load_age_plugins from . import SecretStoreBase @@ -32,6 +32,7 @@ class SecretStore(SecretStoreBase): / f"{self.machine.name}-age.key", priv_key, add_groups=self.machine.deployment["sops"]["defaultGroups"], + age_plugins=load_age_plugins(self.machine.flake), ) add_machine(self.machine.flake_dir, self.machine.name, pub_key, False) @@ -47,13 +48,14 @@ class SecretStore(SecretStoreBase): value, add_machines=[self.machine.name], add_groups=groups, + age_plugins=load_age_plugins(self.machine.flake), ) return path def get(self, service: str, name: str) -> bytes: return decrypt_secret( - self.machine.flake_dir, sops_secrets_folder(self.machine.flake_dir) / f"{self.machine.name}-{name}", + age_plugins=load_age_plugins(self.machine.flake), ).encode("utf-8") def exists(self, service: str, name: str) -> bool: diff --git a/pkgs/clan-cli/clan_cli/secrets/groups.py b/pkgs/clan-cli/clan_cli/secrets/groups.py index 494053ff6..967b89c14 100644 --- a/pkgs/clan-cli/clan_cli/secrets/groups.py +++ b/pkgs/clan-cli/clan_cli/secrets/groups.py @@ -14,6 +14,7 @@ from clan_cli.completions import ( complete_users, ) from clan_cli.machines.types import machine_name_type, validate_hostname +from clan_cli.secrets.sops import load_age_plugins from . import secrets from .folders import ( @@ -239,12 +240,14 @@ def add_group_argument(parser: argparse.ArgumentParser) -> None: add_dynamic_completer(group_action, complete_groups) -def add_secret(flake_dir: Path, group: str, name: str) -> None: +def add_secret( + flake_dir: Path, group: str, name: str, age_plugins: list[str] | None +) -> None: secrets.allow_member( - flake_dir, secrets.groups_folder(sops_secrets_folder(flake_dir) / name), sops_groups_folder(flake_dir), group, + age_plugins=age_plugins, ) @@ -264,12 +267,21 @@ def get_groups(flake_dir: Path, what: str, name: str) -> list[str]: def add_secret_command(args: argparse.Namespace) -> None: - add_secret(args.flake.path, args.group, args.secret) + add_secret( + args.flake.path, + args.group, + args.secret, + age_plugins=load_age_plugins(args.flake), + ) -def remove_secret(flake_dir: Path, group: str, name: str) -> None: +def remove_secret( + flake_dir: Path, group: str, name: str, age_plugins: list[str] +) -> None: updated_paths = secrets.disallow_member( - flake_dir, secrets.groups_folder(sops_secrets_folder(flake_dir) / name), group + secrets.groups_folder(sops_secrets_folder(flake_dir) / name), + group, + age_plugins=age_plugins, ) commit_files( updated_paths, @@ -279,7 +291,12 @@ def remove_secret(flake_dir: Path, group: str, name: str) -> None: def remove_secret_command(args: argparse.Namespace) -> None: - remove_secret(args.flake.path, args.group, args.secret) + remove_secret( + args.flake.path, + args.group, + args.secret, + age_plugins=load_age_plugins(args.flake), + ) def register_groups_parser(parser: argparse.ArgumentParser) -> None: diff --git a/pkgs/clan-cli/clan_cli/secrets/import_sops.py b/pkgs/clan-cli/clan_cli/secrets/import_sops.py index 7264ac3f0..9c2f3cf15 100644 --- a/pkgs/clan-cli/clan_cli/secrets/import_sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/import_sops.py @@ -13,6 +13,7 @@ from clan_cli.completions import ( complete_machines, complete_users, ) +from clan_cli.secrets.sops import load_age_plugins from .secrets import encrypt_secret, sops_secrets_folder @@ -56,6 +57,7 @@ def import_sops(args: argparse.Namespace) -> None: add_groups=args.group, add_machines=args.machine, add_users=args.user, + age_plugins=load_age_plugins(args.flake), ) diff --git a/pkgs/clan-cli/clan_cli/secrets/machines.py b/pkgs/clan-cli/clan_cli/secrets/machines.py index 909fd7274..209fbc715 100644 --- a/pkgs/clan-cli/clan_cli/secrets/machines.py +++ b/pkgs/clan-cli/clan_cli/secrets/machines.py @@ -20,7 +20,7 @@ from .folders import ( sops_secrets_folder, ) from .secrets import update_secrets -from .sops import read_key, write_key +from .sops import load_age_plugins, read_key, write_key from .types import public_or_private_age_key_type, secret_name_type @@ -73,12 +73,17 @@ def list_sops_machines(flake_dir: Path) -> list[str]: return list_objects(path, validate) -def add_secret(flake_dir: Path, machine: str, secret_path: Path) -> None: +def add_secret( + flake_dir: Path, + machine: str, + secret_path: Path, + age_plugins: list[str] | None, +) -> None: paths = secrets.allow_member( - flake_dir, secrets.machines_folder(secret_path), sops_machines_folder(flake_dir), machine, + age_plugins=age_plugins, ) commit_files( paths, @@ -87,11 +92,13 @@ def add_secret(flake_dir: Path, machine: str, secret_path: Path) -> None: ) -def remove_secret(flake_dir: Path, machine: str, secret: str) -> None: +def remove_secret( + flake_dir: Path, machine: str, secret: str, age_plugins: list[str] | None +) -> None: updated_paths = secrets.disallow_member( - flake_dir, secrets.machines_folder(sops_secrets_folder(flake_dir) / secret), machine, + age_plugins=age_plugins, ) commit_files( updated_paths, @@ -138,6 +145,7 @@ def add_secret_command(args: argparse.Namespace) -> None: args.flake.path, args.machine, sops_secrets_folder(args.flake.path) / args.secret, + age_plugins=load_age_plugins(args.flake), ) @@ -145,7 +153,12 @@ def remove_secret_command(args: argparse.Namespace) -> None: if args.flake is None: msg = "Could not find clan flake toplevel directory" raise ClanError(msg) - remove_secret(args.flake.path, args.machine, args.secret) + remove_secret( + args.flake.path, + args.machine, + args.secret, + age_plugins=load_age_plugins(args.flake), + ) def register_machines_parser(parser: argparse.ArgumentParser) -> None: diff --git a/pkgs/clan-cli/clan_cli/secrets/secrets.py b/pkgs/clan-cli/clan_cli/secrets/secrets.py index b73f10b52..ed23657f7 100644 --- a/pkgs/clan-cli/clan_cli/secrets/secrets.py +++ b/pkgs/clan-cli/clan_cli/secrets/secrets.py @@ -31,6 +31,7 @@ from .folders import ( from .sops import ( decrypt_file, encrypt_file, + load_age_plugins, read_keys, update_keys, ) @@ -71,7 +72,9 @@ def list_vars_secrets(flake_dir: Path) -> list[Path]: def update_secrets( - flake_dir: Path, filter_secrets: Callable[[Path], bool] = lambda _: True + flake_dir: Path, + filter_secrets: Callable[[Path], bool] = lambda _: True, + age_plugins: list[str] | None = None, ) -> list[Path]: changed_files = [] secret_paths = [sops_secrets_folder(flake_dir) / s for s in list_secrets(flake_dir)] @@ -86,11 +89,7 @@ def update_secrets( changed_files.extend(cleanup_dangling_symlinks(path / "groups")) changed_files.extend(cleanup_dangling_symlinks(path / "machines")) changed_files.extend( - update_keys( - flake_dir, - path, - collect_keys_for_path(path), - ) + update_keys(path, collect_keys_for_path(path), age_plugins=age_plugins) ) return changed_files @@ -149,6 +148,7 @@ def encrypt_secret( add_machines: list[str] | None = None, add_groups: list[str] | None = None, git_commit: bool = True, + age_plugins: list[str] | None = None, ) -> None: if add_groups is None: add_groups = [] @@ -174,33 +174,33 @@ def encrypt_secret( for user in add_users: files_to_commit.extend( allow_member( - flake_dir, users_folder(secret_path), sops_users_folder(flake_dir), user, do_update_keys, + age_plugins=age_plugins, ) ) for machine in add_machines: files_to_commit.extend( allow_member( - flake_dir, machines_folder(secret_path), sops_machines_folder(flake_dir), machine, do_update_keys, + age_plugins=age_plugins, ) ) for group in add_groups: files_to_commit.extend( allow_member( - flake_dir, groups_folder(secret_path), sops_groups_folder(flake_dir), group, do_update_keys, + age_plugins=age_plugins, ) ) @@ -211,16 +211,16 @@ def encrypt_secret( files_to_commit.extend( allow_member( - flake_dir, users_folder(secret_path), sops_users_folder(flake_dir), username, do_update_keys, + age_plugins=age_plugins, ) ) secret_path = secret_path / "secret" - encrypt_file(flake_dir, secret_path, value, sorted(recipient_keys)) + encrypt_file(secret_path, value, sorted(recipient_keys), age_plugins) files_to_commit.append(secret_path) if git_commit: commit_files( @@ -280,11 +280,11 @@ def list_directory(directory: Path) -> str: def allow_member( - flake_dir: str | Path, group_folder: Path, source_folder: Path, name: str, do_update_keys: bool = True, + age_plugins: list[str] | None = None, ) -> list[Path]: source = source_folder / name if not source.exists(): @@ -307,15 +307,17 @@ def allow_member( if do_update_keys: changed.extend( update_keys( - flake_dir, group_folder.parent, collect_keys_for_path(group_folder.parent), + age_plugins=age_plugins, ) ) return changed -def disallow_member(flake_dir: str | Path, group_folder: Path, name: str) -> list[Path]: +def disallow_member( + group_folder: Path, name: str, age_plugins: list[str] | None +) -> list[Path]: target = group_folder / name if not target.exists(): msg = f"{name} does not exist in group in {group_folder}: " @@ -336,7 +338,9 @@ def disallow_member(flake_dir: str | Path, group_folder: Path, name: str) -> lis group_folder.parent.rmdir() return update_keys( - flake_dir, target.parent.parent, collect_keys_for_path(group_folder.parent) + target.parent.parent, + collect_keys_for_path(group_folder.parent), + age_plugins=age_plugins, ) @@ -368,7 +372,7 @@ def list_command(args: argparse.Namespace) -> None: print("\n".join(lst)) -def decrypt_secret(flake_dir: Path, secret_path: Path) -> str: +def decrypt_secret(secret_path: Path, age_plugins: list[str] | None) -> str: # lopter(2024-10): I can't think of a good way to ensure that we have the # private key for the secret. I mean we could collect all private keys we # could find and then make sure we have the one for the secret, but that @@ -377,13 +381,14 @@ def decrypt_secret(flake_dir: Path, secret_path: Path) -> str: if not path.exists(): msg = f"Secret '{secret_path!s}' does not exist" raise ClanError(msg) - return decrypt_file(flake_dir, path) + return decrypt_file(path, age_plugins=age_plugins) def get_command(args: argparse.Namespace) -> None: print( decrypt_secret( - args.flake.path, sops_secrets_folder(args.flake.path) / args.secret + sops_secrets_folder(args.flake.path) / args.secret, + age_plugins=load_age_plugins(args.flake), ), end="", ) @@ -410,6 +415,7 @@ def set_command(args: argparse.Namespace) -> None: args.user, args.machine, args.group, + age_plugins=load_age_plugins(args.flake), ) diff --git a/pkgs/clan-cli/clan_cli/secrets/sops.py b/pkgs/clan-cli/clan_cli/secrets/sops.py index d89d8a0a8..a76910c4a 100644 --- a/pkgs/clan-cli/clan_cli/secrets/sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/sops.py @@ -191,12 +191,7 @@ class Operation(enum.StrEnum): UPDATE_KEYS = "updatekeys" -def load_age_plugins(flake_dir: str | Path) -> list[str]: - if not flake_dir: - msg = "Missing flake directory" - raise ClanError(msg) - - flake = Flake(str(flake_dir)) +def load_age_plugins(flake: Flake) -> list[str]: result = flake.select("clanInternals.?secrets.?age.?plugins") plugins = result["secrets"]["age"]["plugins"] if plugins == {}: @@ -210,10 +205,10 @@ def load_age_plugins(flake_dir: str | Path) -> list[str]: def sops_run( - flake_dir: str | Path, call: Operation, secret_path: Path, public_keys: Iterable[SopsKey], + age_plugins: list[str] | None, run_opts: RunOpts | None = None, ) -> tuple[int, str]: """Call the sops binary for the given operation.""" @@ -221,6 +216,8 @@ def sops_run( # one place because calling into sops needs to be done with a carefully # setup context, and I don't feel good about the idea of having that logic # exist in multiple places. + if age_plugins is None: + age_plugins = [] sops_cmd = ["sops"] environ = os.environ.copy() with NamedTemporaryFile(delete=False, mode="w") as manifest: @@ -268,8 +265,6 @@ def sops_run( raise ClanError(msg) sops_cmd.append(str(secret_path)) - age_plugins = load_age_plugins(flake_dir) - cmd = nix_shell(["sops", "gnupg", *age_plugins], sops_cmd) opts = ( dataclasses.replace(run_opts, env=environ) @@ -440,27 +435,27 @@ def ensure_admin_public_keys(flake_dir: Path) -> set[SopsKey]: def update_keys( - flake_dir: str | Path, secret_path: Path, keys: Iterable[SopsKey] + secret_path: Path, keys: Iterable[SopsKey], age_plugins: list[str] | None = None ) -> list[Path]: secret_path = secret_path / "secret" error_msg = f"Could not update keys for {secret_path}" rc, _ = sops_run( - flake_dir, Operation.UPDATE_KEYS, secret_path, keys, - RunOpts(log=Log.BOTH, error_msg=error_msg), + run_opts=RunOpts(log=Log.BOTH, error_msg=error_msg), + age_plugins=age_plugins, ) was_modified = ExitStatus.parse(rc) != ExitStatus.FILE_HAS_NOT_BEEN_MODIFIED return [secret_path] if was_modified else [] def encrypt_file( - flake_dir: str | Path, secret_path: Path, content: str | IO[bytes] | bytes | None, pubkeys: list[SopsKey], + age_plugins: list[str] | None = None, ) -> None: folder = secret_path.parent folder.mkdir(parents=True, exist_ok=True) @@ -468,11 +463,11 @@ def encrypt_file( if not content: # This will spawn an editor to edit the file. rc, _ = sops_run( - flake_dir, Operation.EDIT, secret_path, pubkeys, - RunOpts(), + run_opts=RunOpts(), + age_plugins=age_plugins, ) status = ExitStatus.parse(rc) if rc == 0 or status == ExitStatus.FILE_HAS_NOT_BEEN_MODIFIED: @@ -507,11 +502,11 @@ def encrypt_file( msg = f"Invalid content type: {type(content)}" raise ClanError(msg) sops_run( - flake_dir, Operation.ENCRYPT, Path(source.name), pubkeys, - RunOpts(log=Log.BOTH), + run_opts=RunOpts(log=Log.BOTH), + age_plugins=age_plugins, ) # atomic copy of the encrypted file with NamedTemporaryFile(dir=folder, delete=False) as dest: @@ -522,16 +517,16 @@ def encrypt_file( Path(source.name).unlink() -def decrypt_file(flake_dir: str | Path, secret_path: Path) -> str: +def decrypt_file(secret_path: Path, age_plugins: list[str] | None = None) -> str: # decryption uses private keys from the environment or default paths: no_public_keys_needed: list[SopsKey] = [] _, stdout = sops_run( - flake_dir, Operation.DECRYPT, secret_path, no_public_keys_needed, - RunOpts(error_msg=f"Could not decrypt {secret_path}"), + run_opts=RunOpts(error_msg=f"Could not decrypt {secret_path}"), + age_plugins=age_plugins, ) return stdout diff --git a/pkgs/clan-cli/clan_cli/secrets/users.py b/pkgs/clan-cli/clan_cli/secrets/users.py index 48375aeac..03a570cab 100644 --- a/pkgs/clan-cli/clan_cli/secrets/users.py +++ b/pkgs/clan-cli/clan_cli/secrets/users.py @@ -20,7 +20,7 @@ from .folders import ( sops_users_folder, ) from .secrets import update_secrets -from .sops import append_keys, read_keys, remove_keys, write_keys +from .sops import append_keys, load_age_plugins, read_keys, remove_keys, write_keys from .types import ( VALID_USER_NAME, public_or_private_age_key_type, @@ -92,12 +92,14 @@ def list_users(flake_dir: Path) -> list[str]: return list_objects(path, validate) -def add_secret(flake_dir: Path, user: str, secret: str) -> None: +def add_secret( + flake_dir: Path, user: str, secret: str, age_plugins: list[str] | None +) -> None: updated_paths = secrets.allow_member( - flake_dir, secrets.users_folder(sops_secrets_folder(flake_dir) / secret), sops_users_folder(flake_dir), user, + age_plugins=age_plugins, ) commit_files( updated_paths, @@ -106,9 +108,11 @@ def add_secret(flake_dir: Path, user: str, secret: str) -> None: ) -def remove_secret(flake_dir: Path, user: str, secret: str) -> None: +def remove_secret( + flake_dir: Path, user: str, secret: str, age_plugins: list[str] | None +) -> None: updated_paths = secrets.disallow_member( - flake_dir, secrets.users_folder(sops_secrets_folder(flake_dir) / secret), user + secrets.users_folder(sops_secrets_folder(flake_dir) / secret), user, age_plugins ) commit_files( updated_paths, @@ -215,14 +219,24 @@ def add_secret_command(args: argparse.Namespace) -> None: if args.flake is None: msg = "Could not find clan flake toplevel directory" raise ClanError(msg) - add_secret(args.flake.path, args.user, args.secret) + add_secret( + args.flake.path, + args.user, + args.secret, + age_plugins=load_age_plugins(args.flake), + ) def remove_secret_command(args: argparse.Namespace) -> None: if args.flake is None: msg = "Could not find clan flake toplevel directory" raise ClanError(msg) - remove_secret(args.flake.path, args.user, args.secret) + remove_secret( + args.flake.path, + args.user, + args.secret, + age_plugins=load_age_plugins(args.flake), + ) def add_key_command(args: argparse.Namespace) -> None: diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py index 04a3bb485..d71cf3c6f 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py @@ -21,6 +21,7 @@ from clan_cli.secrets.secrets import ( groups_folder, has_secret, ) +from clan_cli.secrets.sops import load_age_plugins from clan_cli.ssh.upload import upload from clan_cli.vars._types import StoreBase from clan_cli.vars.generate import Generator @@ -71,6 +72,7 @@ class SecretStore(StoreBase): / f"{self.machine.name}-age.key", priv_key, add_groups=self.machine.deployment["sops"]["defaultGroups"], + age_plugins=load_age_plugins(self.machine.flake), ) add_machine(self.machine.flake_dir, self.machine.name, pub_key, False) @@ -158,12 +160,14 @@ class SecretStore(StoreBase): add_machines=[self.machine.name] if var.deploy else [], add_groups=self.machine.deployment["sops"]["defaultGroups"], git_commit=False, + age_plugins=load_age_plugins(self.machine.flake), ) return secret_folder def get(self, generator: Generator, name: str) -> bytes: return decrypt_secret( - self.machine.flake_dir, self.secret_path(generator, name) + self.secret_path(generator, name), + age_plugins=load_age_plugins(self.machine.flake), ).encode("utf-8") def delete(self, generator: "Generator", name: str) -> Iterable[Path]: @@ -187,8 +191,8 @@ class SecretStore(StoreBase): # skip uploading the secret, not managed by us return key = decrypt_secret( - self.machine.flake_dir, sops_secrets_folder(self.machine.flake_dir) / key_name, + age_plugins=load_age_plugins(self.machine.flake), ) (output_dir / "key.txt").touch(mode=0o600) (output_dir / "key.txt").write_text(key) @@ -241,7 +245,12 @@ class SecretStore(StoreBase): if self.machine_has_access(generator, name): return secret_folder = self.secret_path(generator, name) - add_secret(self.machine.flake_dir, self.machine.name, secret_folder) + add_secret( + self.machine.flake_dir, + self.machine.name, + secret_folder, + age_plugins=load_age_plugins(self.machine.flake), + ) def collect_keys_for_secret(self, path: Path) -> set[sops.SopsKey]: from clan_cli.secrets.secrets import ( @@ -303,20 +312,22 @@ class SecretStore(StoreBase): secret_path = self.secret_path(generator, file.name) + age_plugins = load_age_plugins(self.machine.flake) + for group in self.machine.deployment["sops"]["defaultGroups"]: allow_member( - self.machine.flake_dir, groups_folder(secret_path), sops_groups_folder(self.machine.flake_dir), group, # we just want to create missing symlinks, we call update_keys below: do_update_keys=False, + age_plugins=age_plugins, ) update_keys( - self.machine.flake_dir, secret_path, collect_keys_for_path(secret_path), + age_plugins=age_plugins, ) if file_name and not file_found: msg = f"file {file_name} was not found"