Merge pull request 'vars: implement secret generation' (#1731) from DavHau/clan-core:DavHau-vars into main

nixosModules/clanCore/vars/interface.nix

@@ -36,128 +36,138 @@ in
         Each generator is expected to produce a set of files under a directory.
       '';
       default = { };
-      type = attrsOf (submodule {
+      type = attrsOf (
-        imports = [ ./generator.nix ];
+        submodule (generator: {
-        options = options {
+          imports = [ ./generator.nix ];
-          dependencies = {
+          options = options {
-            description = ''
+            dependencies = {
-              A list of other generators that this generator depends on.
+              description = ''
-              The output values of these generators will be available to the generator script as files.
+                A list of other generators that this generator depends on.
-              For example, the file 'file1' of a dependency named 'dep1' will be available via $dependencies/dep1/file1.
+                The output values of these generators will be available to the generator script as files.
-            '';
+                For example, the file 'file1' of a dependency named 'dep1' will be available via $dependencies/dep1/file1.
-            type = listOf str;
+              '';
-            default = [ ];
+              type = listOf str;
-          };
+              default = [ ];
-          files = {
+            };
-            description = ''
+            files = {
-              A set of files to generate.
+              description = ''
-              The generator 'script' is expected to produce exactly these files under $out.
+                A set of files to generate.
-            '';
+                The generator 'script' is expected to produce exactly these files under $out.
-            type = attrsOf (
+              '';
-              submodule (file: {
+              type = attrsOf (
-                imports = [ config.settings.fileModule ];
+                submodule (file: {
-                options = options {
+                  imports = [ config.settings.fileModule ];
-                  name = {
+                  options = options {
-                    type = lib.types.str;
+                    name = {
-                    description = ''
+                      type = lib.types.str;
-                      name of the public fact
+                      description = ''
-                    '';
+                        name of the public fact
-                    readOnly = true;
+                      '';
-                    default = file.config._module.args.name;
+                      readOnly = true;
+                      default = file.config._module.args.name;
+                    };
+                    generatorName = {
+                      type = lib.types.str;
+                      description = ''
+                        name of the generator
+                      '';
+                      readOnly = true;
+                      default = generator.name;
+                    };
+                    secret = {
+                      description = ''
+                        Whether the file should be treated as a secret.
+                      '';
+                      type = bool;
+                      default = true;
+                    };
+                    path = {
+                      description = ''
+                        The path to the file containing the content of the generated value.
+                        This will be set automatically
+                      '';
+                      type = str;
+                      readOnly = true;
+                    };
+                    value = {
+                      description = ''
+                        The content of the generated value.
+                        Only available if the file is not secret.
+                      '';
+                      type = str;
+                      default = throw "Cannot access value of secret file";
+                      defaultText = "Throws error because the value of a secret file is not accessible";
+                    };
                   };
-                  secret = {
+                })
+              );
+            };
+            prompts = {
+              description = ''
+                A set of prompts to ask the user for values.
+                Prompts are available to the generator script as files.
+                For example, a prompt named 'prompt1' will be available via $prompts/prompt1
+              '';
+              type = attrsOf (submodule {
+                options = {
+                  description = {
                     description = ''
-                      Whether the file should be treated as a secret.
+                      The description of the prompted value
-                    '';
-                    type = bool;
-                    default = true;
-                  };
-                  path = {
-                    description = ''
-                      The path to the file containing the content of the generated value.
-                      This will be set automatically
                     '';
                     type = str;
-                    readOnly = true;
+                    example = "SSH private key";
                   };
-                  value = {
+                  type = {
                     description = ''
-                      The content of the generated value.
+                      The input type of the prompt.
-                      Only available if the file is not secret.
+                      The following types are available:
+                        - hidden: A hidden text (e.g. password)
+                        - line: A single line of text
+                        - multiline: A multiline text
                     '';
-                    type = str;
+                    type = enum [
-                    default = throw "Cannot access value of secret file";
+                      "hidden"
-                    defaultText = "Throws error because the value of a secret file is not accessible";
+                      "line"
+                      "multiline"
+                    ];
+                    default = "line";
                   };
                 };
-              })
+              });
-            );
+            };
+            runtimeInputs = {
+              description = ''
+                A list of packages that the generator script requires.
+                These packages will be available in the PATH when the script is run.
+              '';
+              type = listOf package;
+              default = [ ];
+            };
+            script = {
+              description = ''
+                The script to run to generate the files.
+                The script will be run with the following environment variables:
+                  - $dependencies: The directory containing the output values of all declared dependencies
+                  - $out: The output directory to put the generated files
+                  - $prompts: The directory containing the prompted values as files
+                The script should produce the files specified in the 'files' attribute under $out.
+              '';
+              type = either str path;
+            };
+            finalScript = {
+              description = ''
+                The final generator script, wrapped, so:
+                  - all required programs are in PATH
+                  - sandbox is set up correctly
+              '';
+              type = lib.types.str;
+              readOnly = true;
+              internal = true;
+              visible = false;
+            };
           };
-          prompts = {
+        })
-            description = ''
+      );
-              A set of prompts to ask the user for values.
-              Prompts are available to the generator script as files.
-              For example, a prompt named 'prompt1' will be available via $prompts/prompt1
-            '';
-            type = attrsOf (submodule {
-              options = {
-                description = {
-                  description = ''
-                    The description of the prompted value
-                  '';
-                  type = str;
-                  example = "SSH private key";
-                };
-                type = {
-                  description = ''
-                    The input type of the prompt.
-                    The following types are available:
-                      - hidden: A hidden text (e.g. password)
-                      - line: A single line of text
-                      - multiline: A multiline text
-                  '';
-                  type = enum [
-                    "hidden"
-                    "line"
-                    "multiline"
-                  ];
-                  default = "line";
-                };
-              };
-            });
-          };
-          runtimeInputs = {
-            description = ''
-              A list of packages that the generator script requires.
-              These packages will be available in the PATH when the script is run.
-            '';
-            type = listOf package;
-            default = [ ];
-          };
-          script = {
-            description = ''
-              The script to run to generate the files.
-              The script will be run with the following environment variables:
-                - $dependencies: The directory containing the output values of all declared dependencies
-                - $out: The output directory to put the generated files
-                - $prompts: The directory containing the prompted values as files
-              The script should produce the files specified in the 'files' attribute under $out.
-            '';
-            type = either str path;
-          };
-          finalScript = {
-            description = ''
-              The final generator script, wrapped, so:
-                - all required programs are in PATH
-                - sandbox is set up correctly
-            '';
-            type = lib.types.str;
-            readOnly = true;
-            internal = true;
-            visible = false;
-          };
-        };
-      });
     };
   };
 }

nixosModules/clanCore/vars/secret/sops.nix

@@ -38,7 +38,7 @@ in
     fileModule = file: {
       path =
         lib.mkIf file.secret
-          config.sops.secrets.${"${config.clan.core.machineName}-${file.config.name}"}.path
+          config.sops.secrets.${"vars-${config.clan.core.machineName}-${file.config.generatorName}-${file.config.name}"}.path
             or "/no-such-path";
     };
     secretModule = "clan_cli.vars.secret_modules.sops";

pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py

@@ -14,13 +14,16 @@ class SecretStore(SecretStoreBase):
         self.machine = machine
 
         # no need to generate keys if we don't manage secrets
-        if not hasattr(self.machine, "vars_data") or not self.machine.vars_generators:
+        if not self.machine.vars_generators:
             return
+        has_secrets = False
         for generator in self.machine.vars_generators.values():
             if "files" in generator:
                 for file in generator["files"].values():
                     if file["secret"]:
-                        return
+                        has_secrets = True
+        if not has_secrets:
+            return
 
         if has_machine(self.machine.flake_dir, self.machine.name):
             return
@@ -38,7 +41,7 @@ class SecretStore(SecretStoreBase):
     ) -> Path | None:
         path = (
             sops_secrets_folder(self.machine.flake_dir)
-            / f"{self.machine.name}-{generator_name}-{name}"
+            / f"vars-{self.machine.name}-{generator_name}-{name}"
         )
         encrypt_secret(
             self.machine.flake_dir,
@@ -49,15 +52,15 @@ class SecretStore(SecretStoreBase):
         )
         return path
 
-    def get(self, service: str, name: str) -> bytes:
+    def get(self, generator_name: str, name: str) -> bytes:
         return decrypt_secret(
-            self.machine.flake_dir, f"{self.machine.name}-{name}"
+            self.machine.flake_dir, f"vars-{self.machine.name}-{generator_name}-{name}"
         ).encode("utf-8")
 
-    def exists(self, service: str, name: str) -> bool:
+    def exists(self, generator_name: str, name: str) -> bool:
         return has_secret(
             self.machine.flake_dir,
-            f"{self.machine.name}-{name}",
+            f"vars-{self.machine.name}-{generator_name}-{name}",
         )
 
     def upload(self, output_dir: Path) -> None:

pkgs/clan-cli/tests/age_keys.py

@@ -7,6 +7,11 @@ class KeyPair:
         self.privkey = privkey
 
 
+class SopsSetup:
+    def __init__(self, keys: list[KeyPair]) -> None:
+        self.keys = keys
+
+
 KEYS = [
     KeyPair(
         "age1dhwqzkah943xzc34tc3dlmfayyevcmdmxzjezdgdy33euxwf59vsp3vk3c",
@@ -29,3 +34,14 @@ def age_keys() -> list[KeyPair]:
     Root directory of the tests
     """
     return KEYS
+
+
+@pytest.fixture
+def sops_setup(
+    monkeypatch: pytest.MonkeyPatch,
+) -> SopsSetup:
+    """
+    Root directory of the tests
+    """
+    monkeypatch.setenv("SOPS_AGE_KEY", KEYS[0].privkey)
+    return SopsSetup(KEYS)

pkgs/clan-cli/tests/test_vars.py

@@ -1,17 +1,15 @@
+import os
 from pathlib import Path
-from typing import TYPE_CHECKING
 
 import pytest
+from age_keys import SopsSetup
 from fixtures_flakes import generate_flake
 from helpers import cli
 from root import CLAN_CORE
 
-if TYPE_CHECKING:
-    pass
-
 
 @pytest.mark.impure
-def test_generate_secret(
+def test_generate_public_var(
     monkeypatch: pytest.MonkeyPatch,
     temporary_home: Path,
     # age_keys: list["KeyPair"],
@@ -41,8 +39,58 @@ def test_generate_secret(
         ),
     )
     monkeypatch.chdir(flake.path)
-    cmd = ["vars", "generate", "--flake", str(flake.path), "my_machine"]
+    cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
-    cli.run(cmd)
     assert (
         flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret"
     ).is_file()
+
+
+@pytest.mark.impure
+def test_generate_secret_var(
+    monkeypatch: pytest.MonkeyPatch,
+    temporary_home: Path,
+    sops_setup: SopsSetup,
+) -> None:
+    flake = generate_flake(
+        temporary_home,
+        flake_template=CLAN_CORE / "templates" / "minimal",
+        machine_configs=dict(
+            my_machine=dict(
+                clan=dict(
+                    core=dict(
+                        vars=dict(
+                            generators=dict(
+                                my_generator=dict(
+                                    files=dict(
+                                        my_secret=dict(
+                                            secret=True,
+                                        )
+                                    ),
+                                    script="echo hello > $out/my_secret",
+                                )
+                            )
+                        )
+                    )
+                )
+            )
+        ),
+    )
+    monkeypatch.chdir(flake.path)
+    cli.run(
+        [
+            "secrets",
+            "users",
+            "add",
+            "--flake",
+            str(flake.path),
+            os.environ.get("USER", "user"),
+            sops_setup.keys[0].pubkey,
+        ]
+    )
+    cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
+    assert not (
+        flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret"
+    ).is_file()
+    assert (
+        flake.path / "sops" / "secrets" / "vars-my_machine-my_generator-my_secret"
+    ).is_dir()