Merge pull request 'vars: implement invalidation mechanism' (#2445) from DavHau/clan-core:DavHau-dave into main

nixosModules/clanCore/vars/default.nix

@@ -42,6 +42,7 @@ in
           inherit (generator)
             dependencies
             finalScript
+            invalidationHash
             migrateFact
             prompts
             share

nixosModules/clanCore/vars/interface.nix

@@ -6,13 +6,19 @@
 }:
 let
   inherit (lib) mkOption;
+  inherit (builtins)
+    hashString
+    toJSON
+    ;
   inherit (lib.types)
     attrsOf
     bool
     either
     enum
+    int
     listOf
     nullOr
+    oneOf
     package
     path
     str
@@ -59,6 +65,40 @@ in
               example = "my_service";
               default = null;
             };
+            invalidationData = lib.mkOption {
+              description = ''
+                A set of values that invalidate the generated values.
+                If any of these values change, the generated values will be re-generated.
+              '';
+              default = null;
+              type =
+                let
+                  data = nullOr (oneOf [
+                    bool
+                    int
+                    str
+                    (attrsOf data)
+                    # lists are not allowed as of now due to potential ordering issues
+                  ]);
+                in
+                data;
+            };
+            # the invalidationHash is the validation interface to the outside world
+            invalidationHash = lib.mkOption {
+              internal = true;
+              description = ''
+                A hash of the invalidation data.
+                If the hash changes, the generated values will be re-generated.
+              '';
+              type = nullOr str;
+              # TODO: recursively traverse the structure and sort all lists in order to support lists
+              default =
+                # For backwards compat, the hash is null by default in which case the check is omitted
+                if generator.config.invalidationData == null then
+                  null
+                else
+                  hashString "sha256" (toJSON generator.config.invalidationData);
+            };
             files = lib.mkOption {
               description = ''
                 A set of files to generate.
@@ -138,7 +178,6 @@ in
                       '';
                       type = str;
                     };
-
                     owner = lib.mkOption {
                       description = "The user name or id that will own the secret file.";
                       default = "root";
@@ -147,7 +186,6 @@ in
                       description = "The group name or id that will own the secret file.";
                       default = "root";
                     };
-
                     value =
                       lib.mkOption {
                         description = ''

pkgs/clan-cli/clan_cli/vars/_types.py

@@ -191,3 +191,38 @@ class StoreBase(ABC):
                     )
                 )
         return all_vars
+
+    def get_invalidation_hash(self, generator_name: str) -> str | None:
+        """
+        Return the invalidation hash that indicates if a generator needs to be re-run
+        due to a change in its definition
+        """
+        hash_file = (
+            self.machine.flake_dir / "vars" / generator_name / "invalidation_hash"
+        )
+        if not hash_file.exists():
+            return None
+        return hash_file.read_text().strip()
+
+    def set_invalidation_hash(self, generator_name: str, hash_str: str) -> None:
+        """
+        Store the invalidation hash that indicates if a generator needs to be re-run
+        """
+        hash_file = (
+            self.machine.flake_dir / "vars" / generator_name / "invalidation_hash"
+        )
+        hash_file.parent.mkdir(parents=True, exist_ok=True)
+        hash_file.write_text(hash_str)
+
+    def hash_is_valid(self, generator_name: str) -> bool:
+        """
+        Check if the invalidation hash is up to date
+        If the hash is not set in nix and hasn't been stored before, it is considered valid
+            -> this provides backward and forward compatibility
+        """
+        stored_hash = self.get_invalidation_hash(generator_name)
+        target_hash = self.machine.vars_generators[generator_name]["invalidationHash"]
+        # if the hash is neither set in nix nor on disk, it is considered valid (provides backwards compat)
+        if target_hash is None and stored_hash is None:
+            return True
+        return stored_hash == target_hash

pkgs/clan-cli/clan_cli/vars/check.py

@@ -10,7 +10,9 @@ log = logging.getLogger(__name__)
 
 def vars_status(
     machine: Machine, generator_name: None | str = None
-) -> tuple[list[tuple[str, str]], list[tuple[str, str]], list[tuple[str, str]]]:
+) -> tuple[
+    list[tuple[str, str]], list[tuple[str, str]], list[tuple[str, str]], list[str]
+]:
     secret_vars_module = importlib.import_module(machine.secret_vars_module)
     secret_vars_store = secret_vars_module.SecretStore(machine=machine)
     public_vars_module = importlib.import_module(machine.public_vars_module)
@@ -19,7 +21,8 @@ def vars_status(
     missing_secret_vars = []
     missing_public_vars = []
     # signals if a var needs to be updated (eg. needs re-encryption due to new users added)
-    outdated_secret_vars = []
+    unfixed_secret_vars = []
+    invalid_generators = []
     if generator_name:
         generators = [generator_name]
     else:
@@ -34,32 +37,54 @@ def vars_status(
                     )
                     missing_secret_vars.append((generator_name, name))
                 else:
-                    needs_update, msg = secret_vars_store.needs_fix(
+                    needs_fix, msg = secret_vars_store.needs_fix(
                         generator_name, name, shared=shared
                     )
-                    if needs_update:
+                    if needs_fix:
                         log.info(
                             f"Secret var '{name}' for service '{generator_name}' in machine {machine.name} needs update: {msg}"
                         )
-                        outdated_secret_vars.append((generator_name, name))
+                        unfixed_secret_vars.append((generator_name, name))
 
             elif not public_vars_store.exists(generator_name, name, shared=shared):
                 log.info(
                     f"Public var '{name}' for service '{generator_name}' in machine {machine.name} is missing."
                 )
                 missing_public_vars.append((generator_name, name))
-
+        # check if invalidation hash is up to date
+        if not (
+            secret_vars_store.hash_is_valid(generator_name)
+            and public_vars_store.hash_is_valid(generator_name)
+        ):
+            invalid_generators.append(generator_name)
+            log.info(
+                f"Generator '{generator_name}' in machine {machine.name} has outdated invalidation hash."
+            )
     log.debug(f"missing_secret_vars: {missing_secret_vars}")
     log.debug(f"missing_public_vars: {missing_public_vars}")
-    log.debug(f"outdated_secret_vars: {outdated_secret_vars}")
+    log.debug(f"unfixed_secret_vars: {unfixed_secret_vars}")
-    return missing_secret_vars, missing_public_vars, outdated_secret_vars
+    log.debug(f"invalid_generators: {invalid_generators}")
+    return (
+        missing_secret_vars,
+        missing_public_vars,
+        unfixed_secret_vars,
+        invalid_generators,
+    )
 
 
 def check_vars(machine: Machine, generator_name: None | str = None) -> bool:
-    missing_secret_vars, missing_public_vars, outdated_secret_vars = vars_status(
+    (
-        machine, generator_name=generator_name
+        missing_secret_vars,
+        missing_public_vars,
+        unfixed_secret_vars,
+        invalid_generators,
+    ) = vars_status(machine, generator_name=generator_name)
+    return not (
+        missing_secret_vars
+        or missing_public_vars
+        or unfixed_secret_vars
+        or invalid_generators
     )
-    return not (missing_secret_vars or missing_public_vars or outdated_secret_vars)
 
 
 def check_command(args: argparse.Namespace) -> None:

pkgs/clan-cli/clan_cli/vars/generate.py

@@ -105,8 +105,9 @@ def execute_generator(
         msg = f"flake is not a Path: {machine.flake}"
         msg += "fact/secret generation is only supported for local flakes"
 
-    generator = machine.vars_generators[generator_name]["finalScript"]
+    generator = machine.vars_generators[generator_name]
-    is_shared = machine.vars_generators[generator_name]["share"]
+    script = generator["finalScript"]
+    is_shared = generator["share"]
 
     # build temporary file tree of dependencies
     decrypted_dependencies = decrypt_dependencies(
@@ -137,32 +138,34 @@ def execute_generator(
         dependencies_as_dir(decrypted_dependencies, tmpdir_in)
         # populate prompted values
         # TODO: make prompts rest API friendly
-        if machine.vars_generators[generator_name]["prompts"]:
+        if generator["prompts"]:
             tmpdir_prompts.mkdir()
             env["prompts"] = str(tmpdir_prompts)
-            for prompt_name in machine.vars_generators[generator_name]["prompts"]:
+            for prompt_name in generator["prompts"]:
                 prompt_file = tmpdir_prompts / prompt_name
                 value = get_prompt_value(prompt_name)
                 prompt_file.write_text(value)
 
         if sys.platform == "linux":
-            cmd = bubblewrap_cmd(generator, tmpdir)
+            cmd = bubblewrap_cmd(script, tmpdir)
         else:
-            cmd = ["bash", "-c", generator]
+            cmd = ["bash", "-c", script]
         run(
             cmd,
             env=env,
         )
         files_to_commit = []
         # store secrets
-        files = machine.vars_generators[generator_name]["files"]
+        files = generator["files"]
+        public_changed = False
+        secret_changed = False
         for file_name, file in files.items():
             is_deployed = file["deploy"]
 
             secret_file = tmpdir_out / file_name
             if not secret_file.is_file():
                 msg = f"did not generate a file for '{file_name}' when running the following command:\n"
-                msg += generator
+                msg += script
                 raise ClanError(msg)
             if file["secret"]:
                 file_path = secret_vars_store.set(
@@ -172,6 +175,7 @@ def execute_generator(
                     shared=is_shared,
                     deployed=is_deployed,
                 )
+                secret_changed = True
             else:
                 file_path = public_vars_store.set(
                     generator_name,
@@ -179,8 +183,18 @@ def execute_generator(
                     secret_file.read_bytes(),
                     shared=is_shared,
                 )
+                public_changed = True
             if file_path:
                 files_to_commit.append(file_path)
+            if generator["invalidationHash"] is not None:
+                if public_changed:
+                    public_vars_store.set_invalidation_hash(
+                        generator_name, generator["invalidationHash"]
+                    )
+                if secret_changed:
+                    secret_vars_store.set_invalidation_hash(
+                        generator_name, generator["invalidationHash"]
+                    )
     commit_files(
         files_to_commit,
         machine.flake_dir,

pkgs/clan-cli/tests/test_vars.py

@@ -886,3 +886,34 @@ def test_vars_get(
         get_var(machine, "my_shared_generator/my_shared_value").printable_value
         == "hello"
     )
+
+
+@pytest.mark.impure
+def test_invalidation(
+    monkeypatch: pytest.MonkeyPatch,
+    flake: ClanFlake,
+) -> None:
+    config = flake.machines["my_machine"]
+    my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
+    my_generator["files"]["my_value"]["secret"] = False
+    my_generator["script"] = "echo -n $RANDOM > $out/my_value"
+    flake.refresh()
+    monkeypatch.chdir(flake.path)
+    cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
+    machine = Machine(name="my_machine", flake=FlakeId(str(flake.path)))
+    value1 = get_var(machine, "my_generator/my_value").printable_value
+    # generate again and make sure nothing changes without the invalidation data being set
+    cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
+    value1_new = get_var(machine, "my_generator/my_value").printable_value
+    assert value1 == value1_new
+    # set the invalidation data of the generator
+    my_generator["invalidationData"] = 1
+    flake.refresh()
+    # generate again and make sure the value changes
+    cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
+    value2 = get_var(machine, "my_generator/my_value").printable_value
+    assert value1 != value2
+    # generate again without changing invalidation data -> value should not change
+    cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
+    value2_new = get_var(machine, "my_generator/my_value").printable_value
+    assert value2 == value2_new