yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

vars: implement invalidation mechanism

Browse Source 
This adds options `invalidationData` to generators.

`invalidationData` can be used by an author of a generator to signal if a re-generation is required after updating the logic.

Whenever a generator with invalidation data is executed, a hash of that data is stored by the respective public and/or secret backends.

The stored hashes will be checked on future deployments, and a re-generation is triggered whenever a hash doesn't match what's defined in nix.
This commit is contained in:
DavHau
2024-11-20 16:10:59 +07:00
parent adff2c8460
commit 3f62e143ec
6 changed files with 165 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
nixosModules/clanCore/vars/default.nix
View File

@@ -42,6 +42,7 @@ in
inherit (generator)
dependencies
finalScript
invalidationHash
migrateFact
prompts
share

42
nixosModules/clanCore/vars/interface.nix
View File

@@ -6,13 +6,19 @@
}:
let
inherit (lib) mkOption;
inherit (builtins)
hashString
toJSON
;
inherit (lib.types)
attrsOf
bool
either
enum
int
listOf
nullOr
oneOf
package
path
str
@@ -59,6 +65,40 @@ in
example = "my_service";
default = null;
};
invalidationData = lib.mkOption {
description = ''
A set of values that invalidate the generated values.
If any of these values change, the generated values will be re-generated.
'';
default = null;
type =
let
data = nullOr (oneOf [
bool
int
str
(attrsOf data)
# lists are not allowed as of now due to potential ordering issues
]);
in
data;
};
# the invalidationHash is the validation interface to the outside world
invalidationHash = lib.mkOption {
internal = true;
description = ''
A hash of the invalidation data.
If the hash changes, the generated values will be re-generated.
'';
type = nullOr str;
# TODO: recursively traverse the structure and sort all lists in order to support lists
default =
# For backwards compat, the hash is null by default in which case the check is omitted
if generator.config.invalidationData == null then
null
else
hashString "sha256" (toJSON generator.config.invalidationData);
};
files = lib.mkOption {
description = ''
A set of files to generate.
@@ -138,7 +178,6 @@ in
'';
type = str;
};
owner = lib.mkOption {
description = "The user name or id that will own the secret file.";
default = "root";
@@ -147,7 +186,6 @@ in
description = "The group name or id that will own the secret file.";
default = "root";
};
value =
lib.mkOption {
description = ''

35
pkgs/clan-cli/clan_cli/vars/_types.py
View File

@@ -191,3 +191,38 @@ class StoreBase(ABC):
)
)
return all_vars
def get_invalidation_hash(self, generator_name: str) -> str | None:
"""
Return the invalidation hash that indicates if a generator needs to be re-run
due to a change in its definition
"""
hash_file = (
self.machine.flake_dir / "vars" / generator_name / "invalidation_hash"
)
if not hash_file.exists():
return None
return hash_file.read_text().strip()
def set_invalidation_hash(self, generator_name: str, hash_str: str) -> None:
"""
Store the invalidation hash that indicates if a generator needs to be re-run
"""
hash_file = (
self.machine.flake_dir / "vars" / generator_name / "invalidation_hash"
)
hash_file.parent.mkdir(parents=True, exist_ok=True)
hash_file.write_text(hash_str)
def hash_is_valid(self, generator_name: str) -> bool:
"""
Check if the invalidation hash is up to date
If the hash is not set in nix and hasn't been stored before, it is considered valid
-> this provides backward and forward compatibility
"""
stored_hash = self.get_invalidation_hash(generator_name)
target_hash = self.machine.vars_generators[generator_name]["invalidationHash"]
# if the hash is neither set in nix nor on disk, it is considered valid (provides backwards compat)
if target_hash is None and stored_hash is None:
return True
return stored_hash == target_hash

47
pkgs/clan-cli/clan_cli/vars/check.py
View File

@@ -10,7 +10,9 @@ log = logging.getLogger(__name__)
def vars_status(
machine: Machine, generator_name: None | str = None
) -> tuple[list[tuple[str, str]], list[tuple[str, str]], list[tuple[str, str]]]:
) -> tuple[
list[tuple[str, str]], list[tuple[str, str]], list[tuple[str, str]], list[str]
]:
secret_vars_module = importlib.import_module(machine.secret_vars_module)
secret_vars_store = secret_vars_module.SecretStore(machine=machine)
public_vars_module = importlib.import_module(machine.public_vars_module)
@@ -19,7 +21,8 @@ def vars_status(
missing_secret_vars = []
missing_public_vars = []
# signals if a var needs to be updated (eg. needs re-encryption due to new users added)
outdated_secret_vars = []
unfixed_secret_vars = []
invalid_generators = []
if generator_name:
generators = [generator_name]
else:
@@ -34,32 +37,54 @@ def vars_status(
)
missing_secret_vars.append((generator_name, name))
else:
needs_update, msg = secret_vars_store.needs_fix(
needs_fix, msg = secret_vars_store.needs_fix(
generator_name, name, shared=shared
)
if needs_update:
if needs_fix:
log.info(
f"Secret var '{name}' for service '{generator_name}' in machine {machine.name} needs update: {msg}"
)
outdated_secret_vars.append((generator_name, name))
unfixed_secret_vars.append((generator_name, name))
elif not public_vars_store.exists(generator_name, name, shared=shared):
log.info(
f"Public var '{name}' for service '{generator_name}' in machine {machine.name} is missing."
)
missing_public_vars.append((generator_name, name))
# check if invalidation hash is up to date
if not (
secret_vars_store.hash_is_valid(generator_name)
and public_vars_store.hash_is_valid(generator_name)
):
invalid_generators.append(generator_name)
log.info(
f"Generator '{generator_name}' in machine {machine.name} has outdated invalidation hash."
)
log.debug(f"missing_secret_vars: {missing_secret_vars}")
log.debug(f"missing_public_vars: {missing_public_vars}")
log.debug(f"outdated_secret_vars: {outdated_secret_vars}")
return missing_secret_vars, missing_public_vars, outdated_secret_vars
log.debug(f"unfixed_secret_vars: {unfixed_secret_vars}")
log.debug(f"invalid_generators: {invalid_generators}")
return (
missing_secret_vars,
missing_public_vars,
unfixed_secret_vars,
invalid_generators,
)
def check_vars(machine: Machine, generator_name: None | str = None) -> bool:
missing_secret_vars, missing_public_vars, outdated_secret_vars = vars_status(
machine, generator_name=generator_name
(
missing_secret_vars,
missing_public_vars,
unfixed_secret_vars,
invalid_generators,
) = vars_status(machine, generator_name=generator_name)
return not (
missing_secret_vars
or missing_public_vars
or unfixed_secret_vars
or invalid_generators
)
return not (missing_secret_vars or missing_public_vars or outdated_secret_vars)
def check_command(args: argparse.Namespace) -> None:

30
pkgs/clan-cli/clan_cli/vars/generate.py
View File

@@ -105,8 +105,9 @@ def execute_generator(
msg = f"flake is not a Path: {machine.flake}"
msg += "fact/secret generation is only supported for local flakes"
generator = machine.vars_generators[generator_name]["finalScript"]
is_shared = machine.vars_generators[generator_name]["share"]
generator = machine.vars_generators[generator_name]
script = generator["finalScript"]
is_shared = generator["share"]
# build temporary file tree of dependencies
decrypted_dependencies = decrypt_dependencies(
@@ -137,32 +138,34 @@ def execute_generator(
dependencies_as_dir(decrypted_dependencies, tmpdir_in)
# populate prompted values
# TODO: make prompts rest API friendly
if machine.vars_generators[generator_name]["prompts"]:
if generator["prompts"]:
tmpdir_prompts.mkdir()
env["prompts"] = str(tmpdir_prompts)
for prompt_name in machine.vars_generators[generator_name]["prompts"]:
for prompt_name in generator["prompts"]:
prompt_file = tmpdir_prompts / prompt_name
value = get_prompt_value(prompt_name)
prompt_file.write_text(value)
if sys.platform == "linux":
cmd = bubblewrap_cmd(generator, tmpdir)
cmd = bubblewrap_cmd(script, tmpdir)
else:
cmd = ["bash", "-c", generator]
cmd = ["bash", "-c", script]
run(
cmd,
env=env,
)
files_to_commit = []
# store secrets
files = machine.vars_generators[generator_name]["files"]
files = generator["files"]
public_changed = False
secret_changed = False
for file_name, file in files.items():
is_deployed = file["deploy"]
secret_file = tmpdir_out / file_name
if not secret_file.is_file():
msg = f"did not generate a file for '{file_name}' when running the following command:\n"
msg += generator
msg += script
raise ClanError(msg)
if file["secret"]:
file_path = secret_vars_store.set(
@@ -172,6 +175,7 @@ def execute_generator(
shared=is_shared,
deployed=is_deployed,
)
secret_changed = True
else:
file_path = public_vars_store.set(
generator_name,
@@ -179,8 +183,18 @@ def execute_generator(
secret_file.read_bytes(),
shared=is_shared,
)
public_changed = True
if file_path:
files_to_commit.append(file_path)
if generator["invalidationHash"] is not None:
if public_changed:
public_vars_store.set_invalidation_hash(
generator_name, generator["invalidationHash"]
)
if secret_changed:
secret_vars_store.set_invalidation_hash(
generator_name, generator["invalidationHash"]
)
commit_files(
files_to_commit,
machine.flake_dir,

31
pkgs/clan-cli/tests/test_vars.py
View File

@@ -886,3 +886,34 @@ def test_vars_get(
get_var(machine, "my_shared_generator/my_shared_value").printable_value
== "hello"
)
@pytest.mark.impure
def test_invalidation(
monkeypatch: pytest.MonkeyPatch,
flake: ClanFlake,
) -> None:
config = flake.machines["my_machine"]
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
my_generator["files"]["my_value"]["secret"] = False
my_generator["script"] = "echo -n $RANDOM > $out/my_value"
flake.refresh()
monkeypatch.chdir(flake.path)
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
machine = Machine(name="my_machine", flake=FlakeId(str(flake.path)))
value1 = get_var(machine, "my_generator/my_value").printable_value
# generate again and make sure nothing changes without the invalidation data being set
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
value1_new = get_var(machine, "my_generator/my_value").printable_value
assert value1 == value1_new
# set the invalidation data of the generator
my_generator["invalidationData"] = 1
flake.refresh()
# generate again and make sure the value changes
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
value2 = get_var(machine, "my_generator/my_value").printable_value
assert value1 != value2
# generate again without changing invalidation data -> value should not change
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
value2_new = get_var(machine, "my_generator/my_value").printable_value
assert value2 == value2_new