From 3f62e143ec91c864caada247f5b1151c293d5bc5 Mon Sep 17 00:00:00 2001 From: DavHau Date: Wed, 20 Nov 2024 16:10:59 +0700 Subject: [PATCH] vars: implement invalidation mechanism This adds options `invalidationData` to generators. `invalidationData` can be used by an author of a generator to signal if a re-generation is required after updating the logic. Whenever a generator with invalidation data is executed, a hash of that data is stored by the respective public and/or secret backends. The stored hashes will be checked on future deployments, and a re-generation is triggered whenever a hash doesn't match what's defined in nix. --- nixosModules/clanCore/vars/default.nix | 1 + nixosModules/clanCore/vars/interface.nix | 42 ++++++++++++++++++++- pkgs/clan-cli/clan_cli/vars/_types.py | 35 ++++++++++++++++++ pkgs/clan-cli/clan_cli/vars/check.py | 47 ++++++++++++++++++------ pkgs/clan-cli/clan_cli/vars/generate.py | 30 +++++++++++---- pkgs/clan-cli/tests/test_vars.py | 31 ++++++++++++++++ 6 files changed, 165 insertions(+), 21 deletions(-) diff --git a/nixosModules/clanCore/vars/default.nix b/nixosModules/clanCore/vars/default.nix index d8a667ab4..ecdd65879 100644 --- a/nixosModules/clanCore/vars/default.nix +++ b/nixosModules/clanCore/vars/default.nix @@ -42,6 +42,7 @@ in inherit (generator) dependencies finalScript + invalidationHash migrateFact prompts share diff --git a/nixosModules/clanCore/vars/interface.nix b/nixosModules/clanCore/vars/interface.nix index 67cca84f0..f28cdc0fc 100644 --- a/nixosModules/clanCore/vars/interface.nix +++ b/nixosModules/clanCore/vars/interface.nix @@ -6,13 +6,19 @@ }: let inherit (lib) mkOption; + inherit (builtins) + hashString + toJSON + ; inherit (lib.types) attrsOf bool either enum + int listOf nullOr + oneOf package path str @@ -59,6 +65,40 @@ in example = "my_service"; default = null; }; + invalidationData = lib.mkOption { + description = '' + A set of values that invalidate the generated values. + If any of these values change, the generated values will be re-generated. + ''; + default = null; + type = + let + data = nullOr (oneOf [ + bool + int + str + (attrsOf data) + # lists are not allowed as of now due to potential ordering issues + ]); + in + data; + }; + # the invalidationHash is the validation interface to the outside world + invalidationHash = lib.mkOption { + internal = true; + description = '' + A hash of the invalidation data. + If the hash changes, the generated values will be re-generated. + ''; + type = nullOr str; + # TODO: recursively traverse the structure and sort all lists in order to support lists + default = + # For backwards compat, the hash is null by default in which case the check is omitted + if generator.config.invalidationData == null then + null + else + hashString "sha256" (toJSON generator.config.invalidationData); + }; files = lib.mkOption { description = '' A set of files to generate. @@ -138,7 +178,6 @@ in ''; type = str; }; - owner = lib.mkOption { description = "The user name or id that will own the secret file."; default = "root"; @@ -147,7 +186,6 @@ in description = "The group name or id that will own the secret file."; default = "root"; }; - value = lib.mkOption { description = '' diff --git a/pkgs/clan-cli/clan_cli/vars/_types.py b/pkgs/clan-cli/clan_cli/vars/_types.py index 58c05c519..6e76dbe90 100644 --- a/pkgs/clan-cli/clan_cli/vars/_types.py +++ b/pkgs/clan-cli/clan_cli/vars/_types.py @@ -191,3 +191,38 @@ class StoreBase(ABC): ) ) return all_vars + + def get_invalidation_hash(self, generator_name: str) -> str | None: + """ + Return the invalidation hash that indicates if a generator needs to be re-run + due to a change in its definition + """ + hash_file = ( + self.machine.flake_dir / "vars" / generator_name / "invalidation_hash" + ) + if not hash_file.exists(): + return None + return hash_file.read_text().strip() + + def set_invalidation_hash(self, generator_name: str, hash_str: str) -> None: + """ + Store the invalidation hash that indicates if a generator needs to be re-run + """ + hash_file = ( + self.machine.flake_dir / "vars" / generator_name / "invalidation_hash" + ) + hash_file.parent.mkdir(parents=True, exist_ok=True) + hash_file.write_text(hash_str) + + def hash_is_valid(self, generator_name: str) -> bool: + """ + Check if the invalidation hash is up to date + If the hash is not set in nix and hasn't been stored before, it is considered valid + -> this provides backward and forward compatibility + """ + stored_hash = self.get_invalidation_hash(generator_name) + target_hash = self.machine.vars_generators[generator_name]["invalidationHash"] + # if the hash is neither set in nix nor on disk, it is considered valid (provides backwards compat) + if target_hash is None and stored_hash is None: + return True + return stored_hash == target_hash diff --git a/pkgs/clan-cli/clan_cli/vars/check.py b/pkgs/clan-cli/clan_cli/vars/check.py index 38d10663e..8d82328bd 100644 --- a/pkgs/clan-cli/clan_cli/vars/check.py +++ b/pkgs/clan-cli/clan_cli/vars/check.py @@ -10,7 +10,9 @@ log = logging.getLogger(__name__) def vars_status( machine: Machine, generator_name: None | str = None -) -> tuple[list[tuple[str, str]], list[tuple[str, str]], list[tuple[str, str]]]: +) -> tuple[ + list[tuple[str, str]], list[tuple[str, str]], list[tuple[str, str]], list[str] +]: secret_vars_module = importlib.import_module(machine.secret_vars_module) secret_vars_store = secret_vars_module.SecretStore(machine=machine) public_vars_module = importlib.import_module(machine.public_vars_module) @@ -19,7 +21,8 @@ def vars_status( missing_secret_vars = [] missing_public_vars = [] # signals if a var needs to be updated (eg. needs re-encryption due to new users added) - outdated_secret_vars = [] + unfixed_secret_vars = [] + invalid_generators = [] if generator_name: generators = [generator_name] else: @@ -34,32 +37,54 @@ def vars_status( ) missing_secret_vars.append((generator_name, name)) else: - needs_update, msg = secret_vars_store.needs_fix( + needs_fix, msg = secret_vars_store.needs_fix( generator_name, name, shared=shared ) - if needs_update: + if needs_fix: log.info( f"Secret var '{name}' for service '{generator_name}' in machine {machine.name} needs update: {msg}" ) - outdated_secret_vars.append((generator_name, name)) + unfixed_secret_vars.append((generator_name, name)) elif not public_vars_store.exists(generator_name, name, shared=shared): log.info( f"Public var '{name}' for service '{generator_name}' in machine {machine.name} is missing." ) missing_public_vars.append((generator_name, name)) - + # check if invalidation hash is up to date + if not ( + secret_vars_store.hash_is_valid(generator_name) + and public_vars_store.hash_is_valid(generator_name) + ): + invalid_generators.append(generator_name) + log.info( + f"Generator '{generator_name}' in machine {machine.name} has outdated invalidation hash." + ) log.debug(f"missing_secret_vars: {missing_secret_vars}") log.debug(f"missing_public_vars: {missing_public_vars}") - log.debug(f"outdated_secret_vars: {outdated_secret_vars}") - return missing_secret_vars, missing_public_vars, outdated_secret_vars + log.debug(f"unfixed_secret_vars: {unfixed_secret_vars}") + log.debug(f"invalid_generators: {invalid_generators}") + return ( + missing_secret_vars, + missing_public_vars, + unfixed_secret_vars, + invalid_generators, + ) def check_vars(machine: Machine, generator_name: None | str = None) -> bool: - missing_secret_vars, missing_public_vars, outdated_secret_vars = vars_status( - machine, generator_name=generator_name + ( + missing_secret_vars, + missing_public_vars, + unfixed_secret_vars, + invalid_generators, + ) = vars_status(machine, generator_name=generator_name) + return not ( + missing_secret_vars + or missing_public_vars + or unfixed_secret_vars + or invalid_generators ) - return not (missing_secret_vars or missing_public_vars or outdated_secret_vars) def check_command(args: argparse.Namespace) -> None: diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 9d1604247..01051faae 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -105,8 +105,9 @@ def execute_generator( msg = f"flake is not a Path: {machine.flake}" msg += "fact/secret generation is only supported for local flakes" - generator = machine.vars_generators[generator_name]["finalScript"] - is_shared = machine.vars_generators[generator_name]["share"] + generator = machine.vars_generators[generator_name] + script = generator["finalScript"] + is_shared = generator["share"] # build temporary file tree of dependencies decrypted_dependencies = decrypt_dependencies( @@ -137,32 +138,34 @@ def execute_generator( dependencies_as_dir(decrypted_dependencies, tmpdir_in) # populate prompted values # TODO: make prompts rest API friendly - if machine.vars_generators[generator_name]["prompts"]: + if generator["prompts"]: tmpdir_prompts.mkdir() env["prompts"] = str(tmpdir_prompts) - for prompt_name in machine.vars_generators[generator_name]["prompts"]: + for prompt_name in generator["prompts"]: prompt_file = tmpdir_prompts / prompt_name value = get_prompt_value(prompt_name) prompt_file.write_text(value) if sys.platform == "linux": - cmd = bubblewrap_cmd(generator, tmpdir) + cmd = bubblewrap_cmd(script, tmpdir) else: - cmd = ["bash", "-c", generator] + cmd = ["bash", "-c", script] run( cmd, env=env, ) files_to_commit = [] # store secrets - files = machine.vars_generators[generator_name]["files"] + files = generator["files"] + public_changed = False + secret_changed = False for file_name, file in files.items(): is_deployed = file["deploy"] secret_file = tmpdir_out / file_name if not secret_file.is_file(): msg = f"did not generate a file for '{file_name}' when running the following command:\n" - msg += generator + msg += script raise ClanError(msg) if file["secret"]: file_path = secret_vars_store.set( @@ -172,6 +175,7 @@ def execute_generator( shared=is_shared, deployed=is_deployed, ) + secret_changed = True else: file_path = public_vars_store.set( generator_name, @@ -179,8 +183,18 @@ def execute_generator( secret_file.read_bytes(), shared=is_shared, ) + public_changed = True if file_path: files_to_commit.append(file_path) + if generator["invalidationHash"] is not None: + if public_changed: + public_vars_store.set_invalidation_hash( + generator_name, generator["invalidationHash"] + ) + if secret_changed: + secret_vars_store.set_invalidation_hash( + generator_name, generator["invalidationHash"] + ) commit_files( files_to_commit, machine.flake_dir, diff --git a/pkgs/clan-cli/tests/test_vars.py b/pkgs/clan-cli/tests/test_vars.py index c7580540d..4de343b9c 100644 --- a/pkgs/clan-cli/tests/test_vars.py +++ b/pkgs/clan-cli/tests/test_vars.py @@ -886,3 +886,34 @@ def test_vars_get( get_var(machine, "my_shared_generator/my_shared_value").printable_value == "hello" ) + + +@pytest.mark.impure +def test_invalidation( + monkeypatch: pytest.MonkeyPatch, + flake: ClanFlake, +) -> None: + config = flake.machines["my_machine"] + my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"] + my_generator["files"]["my_value"]["secret"] = False + my_generator["script"] = "echo -n $RANDOM > $out/my_value" + flake.refresh() + monkeypatch.chdir(flake.path) + cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) + machine = Machine(name="my_machine", flake=FlakeId(str(flake.path))) + value1 = get_var(machine, "my_generator/my_value").printable_value + # generate again and make sure nothing changes without the invalidation data being set + cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) + value1_new = get_var(machine, "my_generator/my_value").printable_value + assert value1 == value1_new + # set the invalidation data of the generator + my_generator["invalidationData"] = 1 + flake.refresh() + # generate again and make sure the value changes + cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) + value2 = get_var(machine, "my_generator/my_value").printable_value + assert value1 != value2 + # generate again without changing invalidation data -> value should not change + cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) + value2_new = get_var(machine, "my_generator/my_value").printable_value + assert value2 == value2_new