feat: support age plugins

Extends how we parse the contents of `SOPS_AGE_KEY` / `SOPS_AGE_KEY_FILE` / `keys.txt`, allowing a user to prepend a comment before any `AGE-PLUGIN-` secret key entry to indicate its corresponding public key.

For example:

```
AGE-PLUGIN-FIDO2-HMAC-xxxxxxxxxxxxx
```

The comment can use any prefix (e.g. `# public key: age1xxxx`, `# recipient: age1xxx`) as we are looking directly for `age1xxxx` within the line.

This change is necessary to support `age` plugins as there is no unified mechanism to recover the public key from a plugin's secret key.

If a plugin secret key does not have a preceding public key comment, an error will be thrown when attempting to set a secret.

docs/site/getting-started/secrets.md

@@ -1,18 +1,19 @@
 
 Clan enables encryption of secrets (such as passwords & keys) ensuring security and ease-of-use among users.
 
-By default Clan utilizes the [sops](https://github.com/getsops/sops) format and integrates with [sops-nix](https://github.com/Mic92/sops-nix) on NixOS machines.
+By default, Clan uses the [sops](https://github.com/getsops/sops) format
+and integrates with [sops-nix](https://github.com/Mic92/sops-nix) on NixOS machines.
 Clan can also be configured to be used with other secret store [backends](https://docs.clan.lol/reference/clan-core/vars/#clan.core.vars.settings.secretStore).
 
 This guide will walk you through:
 
-- **Creating a Keypair for Your User**: Learn how to generate a keypair for $USER to securely control all secrets.
+- **Creating a Keypair for Your User**: Learn how to generate a keypair for `$USER` to securely control all secrets.
 - **Creating Your First Secret**: Step-by-step instructions on creating your initial secret.
 - **Assigning Machine Access to the Secret**: Understand how to grant a machine access to the newly created secret.
 
 ## Create Your Admin Keypair
 
-To get started, you'll need to create **Your admin keypair**.
+To get started, you'll need to create **your admin keypair**.
 
 !!! info
     Don't worry — if you've already made one before, this step won't change or overwrite it.
@@ -34,9 +35,42 @@ Also add your age public key to the repository with 'clan secrets users add YOUR
     Make sure to keep a safe backup of the private key you've just created.
     If it's lost, you won't be able to get to your secrets anymore because they all need the admin key to be unlocked.
 
+If you already have an [age] secret key and want to use that instead, you can simply edit `~/.config/sops/age/keys.txt`:
+
+```title="~/.config/sops/age/keys.txt"
+AGE-SECRET-KEY-13GWMK0KNNKXPTJ8KQ9LPSQZU7G3KU8LZDW474NX3D956GGVFAZRQTAE3F4
+```
+
+Alternatively, you can provide your [age] secret key as an environment variable `SOPS_AGE_KEY`, or in a different file
+using `SOPS_AGE_KEY_FILE`.
+For more information see the [SOPS] guide on [encrypting with age].
+
 !!! note
     It's safe to add any secrets created by the clan CLI and placed in your repository to version control systems like `git`.
 
+### Using Age Plugins
+
+If you wish to use a key generated using an [age plugin] as your admin key, extra care is needed.
+
+You must **precede your secret key with a comment that contains its corresponding recipient**.
+
+This is usually output as part of the generation process
+and is only required because there is no unified mechanism for recovering a recipient from a plugin secret key.
+
+Here is an example:
+
+```title="~/.config/sops/age/keys.txt"
+# public key: age1zdy49ek6z60q9r34vf5mmzkx6u43pr9haqdh5lqdg7fh5tpwlfwqea356l
+AGE-PLUGIN-FIDO2-HMAC-1QQPQZRFR7ZZ2WCV...
+```
+
+!!! note
+    The comment that precedes the plugin secret key need only contain the recipient.
+    Any other text is ignored.
+
+    In the example above, you can specify `# recipient: age1zdy...`, `# public: age1zdy....` or even
+    just `# age1zdy....`
+
 ### Add Your Public Key(s)
 
 ```console
@@ -70,7 +104,7 @@ If you followed the quickstart tutorial all necessary secrets are initialized at
 You can list keys for your user with `clan secrets users get $USER`:
 
 ```console
-❯ bin/clan secrets users get alice
+clan secrets users get alice
 
 [
   {
@@ -97,3 +131,8 @@ To remove a key from your user:
 ```console
 clan secrets users remove-key $USER --age-key <your_public_key>
 ```
+
+[age]: https://github.com/FiloSottile/age
+[age plugin]: https://github.com/FiloSottile/awesome-age?tab=readme-ov-file#plugins
+[sops]: https://github.com/getsops/sops
+[encrypting with age]: https://github.com/getsops/sops?tab=readme-ov-file#encrypting-using-age

pkgs/clan-cli/clan_cli/secrets/secrets.py

@@ -155,16 +155,14 @@ def encrypt_secret(
     if add_users is None:
         add_users = []
 
-    keys = sops.ensure_admin_public_key(flake_dir)
+    admin_keys = sops.ensure_admin_public_keys(flake_dir)
 
-    if not keys:
+    if not admin_keys:
         # todo double check the correct command to run
         msg = "No keys found. Please run 'clan secrets add-key' to add a key."
         raise ClanError(msg)
 
-    username = next(iter(keys)).username
+    username = next(iter(admin_keys)).username
-
-    recipient_keys = set()
 
     # encrypt_secret can be called before the secret has been created
     # so don't try to call sops.update_keys on a non-existent file:
@@ -203,8 +201,8 @@ def encrypt_secret(
 
     recipient_keys = collect_keys_for_path(secret_path)
 
-    if not keys.intersection(recipient_keys):
+    if not admin_keys.intersection(recipient_keys):
-        recipient_keys.update(keys)
+        recipient_keys.update(admin_keys)
 
         files_to_commit.extend(
             allow_member(

pkgs/clan-cli/clan_cli/secrets/sops.py

@@ -4,6 +4,7 @@ import io
 import json
 import logging
 import os
+import re
 import shutil
 import subprocess
 from collections.abc import Iterable, Sequence
@@ -19,7 +20,9 @@ from clan_cli.dirs import user_config_dir
 from clan_cli.errors import ClanError
 from clan_cli.nix import nix_shell
 
-from .folders import sops_machines_folder, sops_users_folder
+from .folders import sops_users_folder
+
+AGE_RECIPIENT_REGEX = re.compile(r"^.*((age1|ssh-(rsa|ed25519) ).*?)(\s|$)")
 
 log = logging.getLogger(__name__)
 
@@ -55,14 +58,20 @@ class KeyType(enum.Enum):
             def maybe_read_from_path(key_path: Path) -> None:
                 try:
                     # as in parse.go in age:
-                    lines = Path(key_path).read_text().strip().splitlines()
+                    content = Path(key_path).read_text().strip()
-                    for private_key in filter(lambda ln: not ln.startswith("#"), lines):
+
-                        public_key = get_public_age_key(private_key)
+                    try:
+                        for public_key in get_public_age_keys(content):
                             log.info(
                                 f"Found age public key from a private key "
                                 f"in {key_path}: {public_key}"
                             )
+
                             keyring.append(public_key)
+                    except ClanError as e:
+                        error_msg = f"Failed to read age keys from {key_path}"
+                        raise ClanError(error_msg) from e
+
                 except FileNotFoundError:
                     return
                 except Exception as ex:
@@ -72,13 +81,19 @@ class KeyType(enum.Enum):
                 # SOPS_AGE_KEY is fed into age.ParseIdentities by Sops, and
                 # reads identities line by line. See age/keysource.go in
                 # Sops, and age/parse.go in Age.
-                for private_key in keys.strip().splitlines():
+                content = keys.strip()
-                    public_key = get_public_age_key(private_key)
+
+                try:
+                    for public_key in get_public_age_keys(content):
                         log.info(
                             f"Found age public key from a private key "
                             f"in the environment (SOPS_AGE_KEY): {public_key}"
                         )
+
                         keyring.append(public_key)
+                except ClanError as e:
+                    error_msg = "Failed to read age keys from SOPS_AGE_KEY"
+                    raise ClanError(error_msg) from e
 
             # Sops will try every location, see age/keysource.go
             elif key_path := os.environ.get("SOPS_AGE_KEY_FILE"):
@@ -249,7 +264,44 @@ def sops_run(
         return p.returncode, p.stdout
 
 
-def get_public_age_key(privkey: str) -> str:
+def get_public_age_keys(contents: str) -> set[str]:
+    # we use a set as it's possible we may detect the same key twice, once in a `# comment` and once by recovering it
+    # from AGE-SECRET-KEY
+    keys: set[str] = set()
+    recipient: str | None = None
+
+    for line_number, line in enumerate(contents.splitlines()):
+        match = AGE_RECIPIENT_REGEX.match(line)
+
+        if match:
+            recipient = match[1]
+            keys.add(recipient)
+
+        if line.startswith("#"):
+            continue
+
+        if line.startswith("AGE-PLUGIN-"):
+            if not recipient:
+                msg = f"Did you forget to precede line {line_number} with it's corresponding `# recipient: age1xxxxxxxx` entry?"
+                raise ClanError(msg)
+
+            # reset recipient
+            recipient = None
+
+        if line.startswith("AGE-SECRET-KEY-"):
+            try:
+                keys.add(get_public_age_key_from_private_key(line))
+            except Exception as e:
+                msg = "Failed to get public key for age private key. Is the key malformed?"
+                raise ClanError(msg) from e
+
+            # reset recipient
+            recipient = None
+
+    return keys
+
+
+def get_public_age_key_from_private_key(privkey: str) -> str:
     cmd = nix_shell(["age"], ["age-keygen", "-y"])
 
     error_msg = "Failed to get public key for age private key. Is the key malformed?"
@@ -298,23 +350,7 @@ def get_user_name(flake_dir: Path, user: str) -> str:
         print(f"{flake_dir / user} already exists")
 
 
-def maybe_get_user_or_machine(flake_dir: Path, key: SopsKey) -> set[SopsKey] | None:
+def maybe_get_user(flake_dir: Path, key: SopsKey) -> set[SopsKey] | None:
-    folders = [sops_users_folder(flake_dir), sops_machines_folder(flake_dir)]
-
-    for folder in folders:
-        if folder.exists():
-            for user in folder.iterdir():
-                if not (user / "key.json").exists():
-                    continue
-
-                keys = read_keys(user)
-                if key in keys:
-                    return {SopsKey(key.pubkey, user.name, key.key_type)}
-
-    return None
-
-
-def get_user(flake_dir: Path, key: SopsKey) -> set[SopsKey] | None:
     folder = sops_users_folder(flake_dir)
 
     if folder.exists():
@@ -329,15 +365,6 @@ def get_user(flake_dir: Path, key: SopsKey) -> set[SopsKey] | None:
     return None
 
 
-@API.register
-def ensure_user_or_machine(flake_dir: Path, key: SopsKey) -> set[SopsKey]:
-    maybe_keys = maybe_get_user_or_machine(flake_dir, key)
-    if maybe_keys:
-        return maybe_keys
-    msg = f"A sops key is not yet added to the repository. Please add it with 'clan secrets users add youruser {key.pubkey}' (replace youruser with your user name)"
-    raise ClanError(msg)
-
-
 def default_admin_private_key_path() -> Path:
     raw_path = os.environ.get("SOPS_AGE_KEY_FILE")
     if raw_path:
@@ -346,8 +373,9 @@ def default_admin_private_key_path() -> Path:
 
 
 @API.register
-def maybe_get_admin_public_key() -> None | SopsKey:
+def maybe_get_admin_public_key() -> SopsKey | None:
     keyring = SopsKey.collect_public_keys()
+
     if len(keyring) == 0:
         return None
 
@@ -366,13 +394,22 @@ def maybe_get_admin_public_key() -> None | SopsKey:
     return keyring[0]
 
 
-def ensure_admin_public_key(flake_dir: Path) -> set[SopsKey]:
+def ensure_admin_public_keys(flake_dir: Path) -> set[SopsKey]:
     key = maybe_get_admin_public_key()
-    if key:
+
-        return ensure_user_or_machine(flake_dir, key)
+    if not key:
-    msg = "No sops key found. Please generate one with 'clan secrets key generate'."
+        msg = "No SOPS key found. Please generate one with `clan secrets key generate`."
         raise ClanError(msg)
 
+    user_keys = maybe_get_user(flake_dir, key)
+
+    if not user_keys:
+        # todo improve error message
+        msg = f"We could not figure out which Clan secrets user you are with the SOPS key we found: {key.pubkey}"
+        raise ClanError(msg)
+
+    return user_keys
+
 
 def update_keys(secret_path: Path, keys: Iterable[SopsKey]) -> list[Path]:
     secret_path = secret_path / "secret"

pkgs/clan-cli/clan_cli/secrets/types.py

@@ -5,7 +5,7 @@ from pathlib import Path
 
 from clan_cli.errors import ClanError
 
-from .sops import get_public_age_key
+from .sops import get_public_age_keys
 
 VALID_SECRET_NAME = re.compile(r"^[a-zA-Z0-9._-]+$")
 VALID_USER_NAME = re.compile(r"^[a-z_]([a-z0-9_-]{0,31})?$")
@@ -21,14 +21,19 @@ def secret_name_type(arg_value: str) -> str:
 def public_or_private_age_key_type(arg_value: str) -> str:
     if Path(arg_value).is_file():
         arg_value = Path(arg_value).read_text().strip()
-    for line in arg_value.splitlines():
+
-        if line.startswith("#"):
+    public_keys = get_public_age_keys(arg_value)
-            continue
+
-        if line.startswith("age1"):
+    match len(public_keys):
-            return line.strip()
+        case 0:
-        if line.startswith("AGE-SECRET-KEY-"):
+            msg = f"Please provide an age public key starting with age1 or an age private key starting with AGE-SECRET-KEY- or AGE-PLUGIN-, got: '{arg_value}'"
-            return get_public_age_key(line)
+            raise ClanError(msg)
-    msg = f"Please provide an age public key starting with age1 or an age private key AGE-SECRET-KEY-, got: '{arg_value}'"
+
+        case 1:
+            return next(iter(public_keys))
+
+        case n:
+            msg = f"{n} age keys were provided, please provide only 1: '{arg_value}'"
             raise ClanError(msg)
 
 

pkgs/clan-cli/clan_cli/tests/test_import_sops_cli.py

@@ -91,6 +91,7 @@ def test_import_sops(
     cli.run(cmd)
     with capture_output as output:
         cli.run(["secrets", "users", "list", "--flake", str(test_flake.path)])
+
     users = sorted(output.out.rstrip().split())
     assert users == ["user1", "user2"]
 

pkgs/clan-cli/clan_cli/tests/test_modules.py

@@ -40,6 +40,8 @@ def test_add_module_to_inventory(
     age_keys: list["KeyPair"],
 ) -> None:
     base_path = test_flake_with_core.path
+
+    with monkeypatch.context():
         monkeypatch.chdir(test_flake_with_core.path)
         monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
 
@@ -60,7 +62,9 @@ def test_add_module_to_inventory(
         )
 
         create_machine(opts)
-    (test_flake_with_core.path / "machines" / "machine1" / "facter.json").write_text(
+        (
+            test_flake_with_core.path / "machines" / "machine1" / "facter.json"
+        ).write_text(
             json.dumps(
                 {
                     "version": 1,
@@ -87,7 +91,13 @@ def test_add_module_to_inventory(
         set_inventory(inventory, base_path, "Add borgbackup service")
 
         # cmd = ["facts", "generate", "--flake", str(test_flake_with_core.path), "machine1"]
-    cmd = ["vars", "generate", "--flake", str(test_flake_with_core.path), "machine1"]
+        cmd = [
+            "vars",
+            "generate",
+            "--flake",
+            str(test_flake_with_core.path),
+            "machine1",
+        ]
 
         cli.run(cmd)
 

pkgs/clan-cli/clan_cli/tests/test_secrets_cli.py

@@ -75,9 +75,10 @@ def _test_identities(
             ]
         )
 
-    with monkeypatch.context():
+    with monkeypatch.context() as m:
-        monkeypatch.setenv("SOPS_NIX_SECRET", "deadfeed")
+        m.setenv("SOPS_NIX_SECRET", "deadfeed")
-        monkeypatch.setenv("SOPS_AGE_KEY", admin_age_key.privkey)
+        m.setenv("SOPS_AGE_KEY", admin_age_key.privkey)
+
         cli.run(
             [
                 "secrets",
@@ -89,6 +90,7 @@ def _test_identities(
                 test_secret_name,
             ]
         )
+
     assert_secrets_file_recipients(
         test_flake.path,
         test_secret_name,
@@ -157,6 +159,7 @@ def test_users(
     age_keys: list["KeyPair"],
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
+    with monkeypatch.context():
         _test_identities("users", test_flake, capture_output, age_keys, monkeypatch)
 
         # some additional user-specific tests
@@ -170,6 +173,8 @@ def test_users(
             "charlie": [age_keys[3], age_keys[4]],
         }
 
+        monkeypatch.setenv("SOPS_NIX_SECRET", "deadfeed")
+
         for user, keys in user_keys.items():
             key_args = [f"--age-key={key.pubkey}" for key in keys]
 
@@ -189,7 +194,9 @@ def test_users(
 
             # check they are returned in get
             with capture_output as output:
-            cli.run(["secrets", "users", "get", "--flake", str(test_flake.path), user])
+                cli.run(
+                    ["secrets", "users", "get", "--flake", str(test_flake.path), user]
+                )
 
             for key in keys:
                 assert key.pubkey in output.out
@@ -368,6 +375,7 @@ def test_groups(
     with monkeypatch.context():
         monkeypatch.setenv("SOPS_NIX_SECRET", "deafbeef")
         monkeypatch.setenv("SOPS_AGE_KEY", admin_age_key.privkey)
+
         cli.run(
             [
                 "secrets",
@@ -539,11 +547,20 @@ def test_secrets(
     # Read the key that was generated
     with capture_output as output:
         cli.run(["secrets", "key", "show", "--flake", str(test_flake.path)])
-    key = json.loads(output.out)["publickey"]
+
-    assert key.startswith("age1")
+    key = json.loads(output.out)
+    assert key["publickey"].startswith("age1")
     # Add testuser with the key that was generated for the clan
     cli.run(
-        ["secrets", "users", "add", "--flake", str(test_flake.path), "testuser", key]
+        [
+            "secrets",
+            "users",
+            "add",
+            "--flake",
+            str(test_flake.path),
+            "testuser",
+            key["publickey"],
+        ]
     )
 
     with pytest.raises(ClanError):  # does not exist yet
@@ -855,6 +872,7 @@ def test_secrets_key_generate_gpg(
                 "testuser",
             ]
         )
+
         with capture_output as output:
             cli.run(
                 [
@@ -873,8 +891,12 @@ def test_secrets_key_generate_gpg(
         assert key["type"] == "pgp"
         assert key["publickey"] == gpg_key.fingerprint
 
-        monkeypatch.setenv("SOPS_NIX_SECRET", "secret-value")
+        with monkeypatch.context() as m:
+            m.setenv("SOPS_NIX_SECRET", "secret-value")
+
             cli.run(["secrets", "set", "--flake", str(test_flake.path), "secret-name"])
             with capture_output as output:
-            cli.run(["secrets", "get", "--flake", str(test_flake.path), "secret-name"])
+                cli.run(
+                    ["secrets", "get", "--flake", str(test_flake.path), "secret-name"]
+                )
             assert output.out == "secret-value"

pkgs/clan-cli/clan_cli/tests/test_secrets_upload.py

@@ -44,6 +44,8 @@ def test_secrets_upload(
     config["clan"]["networking"]["targetHost"] = addr
     config["clan"]["core"]["facts"]["secretUploadDirectory"] = str(flake.path / "facts")
     flake.refresh()
+
+    with monkeypatch.context():
         monkeypatch.chdir(str(flake.path))
         monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
 
@@ -81,7 +83,10 @@ def test_secrets_upload(
                 age_keys[1].pubkey,
             ]
         )
-    monkeypatch.setenv("SOPS_NIX_SECRET", age_keys[0].privkey)
+
+        with monkeypatch.context() as m:
+            m.setenv("SOPS_NIX_SECRET", age_keys[0].privkey)
+
             cli.run(["secrets", "set", "--flake", str(flake.path), "vm1-age.key"])
 
             flake_path = flake.path.joinpath("flake.nix")

pkgs/clan-cli/clan_cli/tests/test_vms_cli.py

@@ -32,8 +32,10 @@ def test_run(
     test_flake_with_core: FlakeForTest,
     age_keys: list["KeyPair"],
 ) -> None:
+    with monkeypatch.context():
         monkeypatch.chdir(test_flake_with_core.path)
         monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
+
         cli.run(
             [
                 "secrets",