sops: initialize age_plugins early

This avoids re-initializing the Flake object deep in the tree, which in turn leads to issue when overriding the Flake for testing, eg the URl would reset.

pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py

@@ -7,7 +7,7 @@ from clan_lib.ssh.remote import Remote
 from clan_cli.secrets.folders import sops_secrets_folder
 from clan_cli.secrets.machines import add_machine, has_machine
 from clan_cli.secrets.secrets import decrypt_secret, encrypt_secret, has_secret
-from clan_cli.secrets.sops import generate_private_key
+from clan_cli.secrets.sops import generate_private_key, load_age_plugins
 
 from . import SecretStoreBase
 
@@ -32,6 +32,7 @@ class SecretStore(SecretStoreBase):
             / f"{self.machine.name}-age.key",
             priv_key,
             add_groups=self.machine.deployment["sops"]["defaultGroups"],
+            age_plugins=load_age_plugins(self.machine.flake),
         )
         add_machine(self.machine.flake_dir, self.machine.name, pub_key, False)
 
@@ -47,13 +48,14 @@ class SecretStore(SecretStoreBase):
             value,
             add_machines=[self.machine.name],
             add_groups=groups,
+            age_plugins=load_age_plugins(self.machine.flake),
         )
         return path
 
     def get(self, service: str, name: str) -> bytes:
         return decrypt_secret(
-            self.machine.flake_dir,
             sops_secrets_folder(self.machine.flake_dir) / f"{self.machine.name}-{name}",
+            age_plugins=load_age_plugins(self.machine.flake),
         ).encode("utf-8")
 
     def exists(self, service: str, name: str) -> bool:

pkgs/clan-cli/clan_cli/secrets/groups.py

@@ -14,6 +14,7 @@ from clan_cli.completions import (
     complete_users,
 )
 from clan_cli.machines.types import machine_name_type, validate_hostname
+from clan_cli.secrets.sops import load_age_plugins
 
 from . import secrets
 from .folders import (
@@ -239,12 +240,14 @@ def add_group_argument(parser: argparse.ArgumentParser) -> None:
     add_dynamic_completer(group_action, complete_groups)
 
 
-def add_secret(flake_dir: Path, group: str, name: str) -> None:
+def add_secret(
+    flake_dir: Path, group: str, name: str, age_plugins: list[str] | None
+) -> None:
     secrets.allow_member(
-        flake_dir,
         secrets.groups_folder(sops_secrets_folder(flake_dir) / name),
         sops_groups_folder(flake_dir),
         group,
+        age_plugins=age_plugins,
     )
 
 
@@ -264,12 +267,21 @@ def get_groups(flake_dir: Path, what: str, name: str) -> list[str]:
 
 
 def add_secret_command(args: argparse.Namespace) -> None:
-    add_secret(args.flake.path, args.group, args.secret)
+    add_secret(
+        args.flake.path,
+        args.group,
+        args.secret,
+        age_plugins=load_age_plugins(args.flake),
+    )
 
 
-def remove_secret(flake_dir: Path, group: str, name: str) -> None:
+def remove_secret(
+    flake_dir: Path, group: str, name: str, age_plugins: list[str]
+) -> None:
     updated_paths = secrets.disallow_member(
-        flake_dir, secrets.groups_folder(sops_secrets_folder(flake_dir) / name), group
+        secrets.groups_folder(sops_secrets_folder(flake_dir) / name),
+        group,
+        age_plugins=age_plugins,
     )
     commit_files(
         updated_paths,
@@ -279,7 +291,12 @@ def remove_secret(flake_dir: Path, group: str, name: str) -> None:
 
 
 def remove_secret_command(args: argparse.Namespace) -> None:
-    remove_secret(args.flake.path, args.group, args.secret)
+    remove_secret(
+        args.flake.path,
+        args.group,
+        args.secret,
+        age_plugins=load_age_plugins(args.flake),
+    )
 
 
 def register_groups_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/secrets/import_sops.py

@@ -13,6 +13,7 @@ from clan_cli.completions import (
     complete_machines,
     complete_users,
 )
+from clan_cli.secrets.sops import load_age_plugins
 
 from .secrets import encrypt_secret, sops_secrets_folder
 
@@ -56,6 +57,7 @@ def import_sops(args: argparse.Namespace) -> None:
                 add_groups=args.group,
                 add_machines=args.machine,
                 add_users=args.user,
+                age_plugins=load_age_plugins(args.flake),
             )
 
 

pkgs/clan-cli/clan_cli/secrets/machines.py

@@ -20,7 +20,7 @@ from .folders import (
     sops_secrets_folder,
 )
 from .secrets import update_secrets
-from .sops import read_key, write_key
+from .sops import load_age_plugins, read_key, write_key
 from .types import public_or_private_age_key_type, secret_name_type
 
 
@@ -73,12 +73,17 @@ def list_sops_machines(flake_dir: Path) -> list[str]:
     return list_objects(path, validate)
 
 
-def add_secret(flake_dir: Path, machine: str, secret_path: Path) -> None:
+def add_secret(
+    flake_dir: Path,
+    machine: str,
+    secret_path: Path,
+    age_plugins: list[str] | None,
+) -> None:
     paths = secrets.allow_member(
-        flake_dir,
         secrets.machines_folder(secret_path),
         sops_machines_folder(flake_dir),
         machine,
+        age_plugins=age_plugins,
     )
     commit_files(
         paths,
@@ -87,11 +92,13 @@ def add_secret(flake_dir: Path, machine: str, secret_path: Path) -> None:
     )
 
 
-def remove_secret(flake_dir: Path, machine: str, secret: str) -> None:
+def remove_secret(
+    flake_dir: Path, machine: str, secret: str, age_plugins: list[str] | None
+) -> None:
     updated_paths = secrets.disallow_member(
-        flake_dir,
         secrets.machines_folder(sops_secrets_folder(flake_dir) / secret),
         machine,
+        age_plugins=age_plugins,
     )
     commit_files(
         updated_paths,
@@ -138,6 +145,7 @@ def add_secret_command(args: argparse.Namespace) -> None:
         args.flake.path,
         args.machine,
         sops_secrets_folder(args.flake.path) / args.secret,
+        age_plugins=load_age_plugins(args.flake),
     )
 
 
@@ -145,7 +153,12 @@ def remove_secret_command(args: argparse.Namespace) -> None:
     if args.flake is None:
         msg = "Could not find clan flake toplevel directory"
         raise ClanError(msg)
-    remove_secret(args.flake.path, args.machine, args.secret)
+    remove_secret(
+        args.flake.path,
+        args.machine,
+        args.secret,
+        age_plugins=load_age_plugins(args.flake),
+    )
 
 
 def register_machines_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/secrets/secrets.py

@@ -31,6 +31,7 @@ from .folders import (
 from .sops import (
     decrypt_file,
     encrypt_file,
+    load_age_plugins,
     read_keys,
     update_keys,
 )
@@ -71,7 +72,9 @@ def list_vars_secrets(flake_dir: Path) -> list[Path]:
 
 
 def update_secrets(
-    flake_dir: Path, filter_secrets: Callable[[Path], bool] = lambda _: True
+    flake_dir: Path,
+    filter_secrets: Callable[[Path], bool] = lambda _: True,
+    age_plugins: list[str] | None = None,
 ) -> list[Path]:
     changed_files = []
     secret_paths = [sops_secrets_folder(flake_dir) / s for s in list_secrets(flake_dir)]
@@ -86,11 +89,7 @@ def update_secrets(
         changed_files.extend(cleanup_dangling_symlinks(path / "groups"))
         changed_files.extend(cleanup_dangling_symlinks(path / "machines"))
         changed_files.extend(
-            update_keys(
+            update_keys(path, collect_keys_for_path(path), age_plugins=age_plugins)
-                flake_dir,
-                path,
-                collect_keys_for_path(path),
-            )
         )
     return changed_files
 
@@ -149,6 +148,7 @@ def encrypt_secret(
     add_machines: list[str] | None = None,
     add_groups: list[str] | None = None,
     git_commit: bool = True,
+    age_plugins: list[str] | None = None,
 ) -> None:
     if add_groups is None:
         add_groups = []
@@ -174,33 +174,33 @@ def encrypt_secret(
     for user in add_users:
         files_to_commit.extend(
             allow_member(
-                flake_dir,
                 users_folder(secret_path),
                 sops_users_folder(flake_dir),
                 user,
                 do_update_keys,
+                age_plugins=age_plugins,
             )
         )
 
     for machine in add_machines:
         files_to_commit.extend(
             allow_member(
-                flake_dir,
                 machines_folder(secret_path),
                 sops_machines_folder(flake_dir),
                 machine,
                 do_update_keys,
+                age_plugins=age_plugins,
             )
         )
 
     for group in add_groups:
         files_to_commit.extend(
             allow_member(
-                flake_dir,
                 groups_folder(secret_path),
                 sops_groups_folder(flake_dir),
                 group,
                 do_update_keys,
+                age_plugins=age_plugins,
             )
         )
 
@@ -211,16 +211,16 @@ def encrypt_secret(
 
         files_to_commit.extend(
             allow_member(
-                flake_dir,
                 users_folder(secret_path),
                 sops_users_folder(flake_dir),
                 username,
                 do_update_keys,
+                age_plugins=age_plugins,
             )
         )
 
     secret_path = secret_path / "secret"
-    encrypt_file(flake_dir, secret_path, value, sorted(recipient_keys))
+    encrypt_file(secret_path, value, sorted(recipient_keys), age_plugins)
     files_to_commit.append(secret_path)
     if git_commit:
         commit_files(
@@ -280,11 +280,11 @@ def list_directory(directory: Path) -> str:
 
 
 def allow_member(
-    flake_dir: str | Path,
     group_folder: Path,
     source_folder: Path,
     name: str,
     do_update_keys: bool = True,
+    age_plugins: list[str] | None = None,
 ) -> list[Path]:
     source = source_folder / name
     if not source.exists():
@@ -307,15 +307,17 @@ def allow_member(
     if do_update_keys:
         changed.extend(
             update_keys(
-                flake_dir,
                 group_folder.parent,
                 collect_keys_for_path(group_folder.parent),
+                age_plugins=age_plugins,
             )
         )
     return changed
 
 
-def disallow_member(flake_dir: str | Path, group_folder: Path, name: str) -> list[Path]:
+def disallow_member(
+    group_folder: Path, name: str, age_plugins: list[str] | None
+) -> list[Path]:
     target = group_folder / name
     if not target.exists():
         msg = f"{name} does not exist in group in {group_folder}: "
@@ -336,7 +338,9 @@ def disallow_member(flake_dir: str | Path, group_folder: Path, name: str) -> lis
         group_folder.parent.rmdir()
 
     return update_keys(
-        flake_dir, target.parent.parent, collect_keys_for_path(group_folder.parent)
+        target.parent.parent,
+        collect_keys_for_path(group_folder.parent),
+        age_plugins=age_plugins,
     )
 
 
@@ -368,7 +372,7 @@ def list_command(args: argparse.Namespace) -> None:
         print("\n".join(lst))
 
 
-def decrypt_secret(flake_dir: Path, secret_path: Path) -> str:
+def decrypt_secret(secret_path: Path, age_plugins: list[str] | None) -> str:
     # lopter(2024-10): I can't think of a good way to ensure that we have the
     # private key for the secret. I mean we could collect all private keys we
     # could find and then make sure we have the one for the secret, but that
@@ -377,13 +381,14 @@ def decrypt_secret(flake_dir: Path, secret_path: Path) -> str:
     if not path.exists():
         msg = f"Secret '{secret_path!s}' does not exist"
         raise ClanError(msg)
-    return decrypt_file(flake_dir, path)
+    return decrypt_file(path, age_plugins=age_plugins)
 
 
 def get_command(args: argparse.Namespace) -> None:
     print(
         decrypt_secret(
-            args.flake.path, sops_secrets_folder(args.flake.path) / args.secret
+            sops_secrets_folder(args.flake.path) / args.secret,
+            age_plugins=load_age_plugins(args.flake),
         ),
         end="",
     )
@@ -410,6 +415,7 @@ def set_command(args: argparse.Namespace) -> None:
         args.user,
         args.machine,
         args.group,
+        age_plugins=load_age_plugins(args.flake),
     )
 
 

pkgs/clan-cli/clan_cli/secrets/sops.py

@@ -191,12 +191,7 @@ class Operation(enum.StrEnum):
     UPDATE_KEYS = "updatekeys"
 
 
-def load_age_plugins(flake_dir: str | Path) -> list[str]:
+def load_age_plugins(flake: Flake) -> list[str]:
-    if not flake_dir:
-        msg = "Missing flake directory"
-        raise ClanError(msg)
-
-    flake = Flake(str(flake_dir))
     result = flake.select("clanInternals.?secrets.?age.?plugins")
     plugins = result["secrets"]["age"]["plugins"]
     if plugins == {}:
@@ -210,10 +205,10 @@ def load_age_plugins(flake_dir: str | Path) -> list[str]:
 
 
 def sops_run(
-    flake_dir: str | Path,
     call: Operation,
     secret_path: Path,
     public_keys: Iterable[SopsKey],
+    age_plugins: list[str] | None,
     run_opts: RunOpts | None = None,
 ) -> tuple[int, str]:
     """Call the sops binary for the given operation."""
@@ -221,6 +216,8 @@ def sops_run(
     # one place because calling into sops needs to be done with a carefully
     # setup context, and I don't feel good about the idea of having that logic
     # exist in multiple places.
+    if age_plugins is None:
+        age_plugins = []
     sops_cmd = ["sops"]
     environ = os.environ.copy()
     with NamedTemporaryFile(delete=False, mode="w") as manifest:
@@ -268,8 +265,6 @@ def sops_run(
                 raise ClanError(msg)
         sops_cmd.append(str(secret_path))
 
-        age_plugins = load_age_plugins(flake_dir)
-
         cmd = nix_shell(["sops", "gnupg", *age_plugins], sops_cmd)
         opts = (
             dataclasses.replace(run_opts, env=environ)
@@ -440,27 +435,27 @@ def ensure_admin_public_keys(flake_dir: Path) -> set[SopsKey]:
 
 
 def update_keys(
-    flake_dir: str | Path, secret_path: Path, keys: Iterable[SopsKey]
+    secret_path: Path, keys: Iterable[SopsKey], age_plugins: list[str] | None = None
 ) -> list[Path]:
     secret_path = secret_path / "secret"
     error_msg = f"Could not update keys for {secret_path}"
 
     rc, _ = sops_run(
-        flake_dir,
         Operation.UPDATE_KEYS,
         secret_path,
         keys,
-        RunOpts(log=Log.BOTH, error_msg=error_msg),
+        run_opts=RunOpts(log=Log.BOTH, error_msg=error_msg),
+        age_plugins=age_plugins,
     )
     was_modified = ExitStatus.parse(rc) != ExitStatus.FILE_HAS_NOT_BEEN_MODIFIED
     return [secret_path] if was_modified else []
 
 
 def encrypt_file(
-    flake_dir: str | Path,
     secret_path: Path,
     content: str | IO[bytes] | bytes | None,
     pubkeys: list[SopsKey],
+    age_plugins: list[str] | None = None,
 ) -> None:
     folder = secret_path.parent
     folder.mkdir(parents=True, exist_ok=True)
@@ -468,11 +463,11 @@ def encrypt_file(
     if not content:
         # This will spawn an editor to edit the file.
         rc, _ = sops_run(
-            flake_dir,
             Operation.EDIT,
             secret_path,
             pubkeys,
-            RunOpts(),
+            run_opts=RunOpts(),
+            age_plugins=age_plugins,
         )
         status = ExitStatus.parse(rc)
         if rc == 0 or status == ExitStatus.FILE_HAS_NOT_BEEN_MODIFIED:
@@ -507,11 +502,11 @@ def encrypt_file(
                 msg = f"Invalid content type: {type(content)}"
                 raise ClanError(msg)
         sops_run(
-            flake_dir,
             Operation.ENCRYPT,
             Path(source.name),
             pubkeys,
-            RunOpts(log=Log.BOTH),
+            run_opts=RunOpts(log=Log.BOTH),
+            age_plugins=age_plugins,
         )
         # atomic copy of the encrypted file
         with NamedTemporaryFile(dir=folder, delete=False) as dest:
@@ -522,16 +517,16 @@ def encrypt_file(
             Path(source.name).unlink()
 
 
-def decrypt_file(flake_dir: str | Path, secret_path: Path) -> str:
+def decrypt_file(secret_path: Path, age_plugins: list[str] | None = None) -> str:
     # decryption uses private keys from the environment or default paths:
     no_public_keys_needed: list[SopsKey] = []
 
     _, stdout = sops_run(
-        flake_dir,
         Operation.DECRYPT,
         secret_path,
         no_public_keys_needed,
-        RunOpts(error_msg=f"Could not decrypt {secret_path}"),
+        run_opts=RunOpts(error_msg=f"Could not decrypt {secret_path}"),
+        age_plugins=age_plugins,
     )
     return stdout
 

pkgs/clan-cli/clan_cli/secrets/users.py

@@ -20,7 +20,7 @@ from .folders import (
     sops_users_folder,
 )
 from .secrets import update_secrets
-from .sops import append_keys, read_keys, remove_keys, write_keys
+from .sops import append_keys, load_age_plugins, read_keys, remove_keys, write_keys
 from .types import (
     VALID_USER_NAME,
     public_or_private_age_key_type,
@@ -92,12 +92,14 @@ def list_users(flake_dir: Path) -> list[str]:
     return list_objects(path, validate)
 
 
-def add_secret(flake_dir: Path, user: str, secret: str) -> None:
+def add_secret(
+    flake_dir: Path, user: str, secret: str, age_plugins: list[str] | None
+) -> None:
     updated_paths = secrets.allow_member(
-        flake_dir,
         secrets.users_folder(sops_secrets_folder(flake_dir) / secret),
         sops_users_folder(flake_dir),
         user,
+        age_plugins=age_plugins,
     )
     commit_files(
         updated_paths,
@@ -106,9 +108,11 @@ def add_secret(flake_dir: Path, user: str, secret: str) -> None:
     )
 
 
-def remove_secret(flake_dir: Path, user: str, secret: str) -> None:
+def remove_secret(
+    flake_dir: Path, user: str, secret: str, age_plugins: list[str] | None
+) -> None:
     updated_paths = secrets.disallow_member(
-        flake_dir, secrets.users_folder(sops_secrets_folder(flake_dir) / secret), user
+        secrets.users_folder(sops_secrets_folder(flake_dir) / secret), user, age_plugins
     )
     commit_files(
         updated_paths,
@@ -215,14 +219,24 @@ def add_secret_command(args: argparse.Namespace) -> None:
     if args.flake is None:
         msg = "Could not find clan flake toplevel directory"
         raise ClanError(msg)
-    add_secret(args.flake.path, args.user, args.secret)
+    add_secret(
+        args.flake.path,
+        args.user,
+        args.secret,
+        age_plugins=load_age_plugins(args.flake),
+    )
 
 
 def remove_secret_command(args: argparse.Namespace) -> None:
     if args.flake is None:
         msg = "Could not find clan flake toplevel directory"
         raise ClanError(msg)
-    remove_secret(args.flake.path, args.user, args.secret)
+    remove_secret(
+        args.flake.path,
+        args.user,
+        args.secret,
+        age_plugins=load_age_plugins(args.flake),
+    )
 
 
 def add_key_command(args: argparse.Namespace) -> None:

pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py

@@ -21,6 +21,7 @@ from clan_cli.secrets.secrets import (
     groups_folder,
     has_secret,
 )
+from clan_cli.secrets.sops import load_age_plugins
 from clan_cli.ssh.upload import upload
 from clan_cli.vars._types import StoreBase
 from clan_cli.vars.generate import Generator
@@ -71,6 +72,7 @@ class SecretStore(StoreBase):
             / f"{self.machine.name}-age.key",
             priv_key,
             add_groups=self.machine.deployment["sops"]["defaultGroups"],
+            age_plugins=load_age_plugins(self.machine.flake),
         )
         add_machine(self.machine.flake_dir, self.machine.name, pub_key, False)
 
@@ -158,12 +160,14 @@ class SecretStore(StoreBase):
             add_machines=[self.machine.name] if var.deploy else [],
             add_groups=self.machine.deployment["sops"]["defaultGroups"],
             git_commit=False,
+            age_plugins=load_age_plugins(self.machine.flake),
         )
         return secret_folder
 
     def get(self, generator: Generator, name: str) -> bytes:
         return decrypt_secret(
-            self.machine.flake_dir, self.secret_path(generator, name)
+            self.secret_path(generator, name),
+            age_plugins=load_age_plugins(self.machine.flake),
         ).encode("utf-8")
 
     def delete(self, generator: "Generator", name: str) -> Iterable[Path]:
@@ -187,8 +191,8 @@ class SecretStore(StoreBase):
                 # skip uploading the secret, not managed by us
                 return
             key = decrypt_secret(
-                self.machine.flake_dir,
                 sops_secrets_folder(self.machine.flake_dir) / key_name,
+                age_plugins=load_age_plugins(self.machine.flake),
             )
             (output_dir / "key.txt").touch(mode=0o600)
             (output_dir / "key.txt").write_text(key)
@@ -241,7 +245,12 @@ class SecretStore(StoreBase):
         if self.machine_has_access(generator, name):
             return
         secret_folder = self.secret_path(generator, name)
-        add_secret(self.machine.flake_dir, self.machine.name, secret_folder)
+        add_secret(
+            self.machine.flake_dir,
+            self.machine.name,
+            secret_folder,
+            age_plugins=load_age_plugins(self.machine.flake),
+        )
 
     def collect_keys_for_secret(self, path: Path) -> set[sops.SopsKey]:
         from clan_cli.secrets.secrets import (
@@ -303,20 +312,22 @@ class SecretStore(StoreBase):
 
                 secret_path = self.secret_path(generator, file.name)
 
+                age_plugins = load_age_plugins(self.machine.flake)
+
                 for group in self.machine.deployment["sops"]["defaultGroups"]:
                     allow_member(
-                        self.machine.flake_dir,
                         groups_folder(secret_path),
                         sops_groups_folder(self.machine.flake_dir),
                         group,
                         # we just want to create missing symlinks, we call update_keys below:
                         do_update_keys=False,
+                        age_plugins=age_plugins,
                     )
 
                 update_keys(
-                    self.machine.flake_dir,
                     secret_path,
                     collect_keys_for_path(secret_path),
+                    age_plugins=age_plugins,
                 )
         if file_name and not file_found:
             msg = f"file {file_name} was not found"