diff --git a/nixosModules/clanCore/vars/default.nix b/nixosModules/clanCore/vars/default.nix index aff6be588..c253908c0 100644 --- a/nixosModules/clanCore/vars/default.nix +++ b/nixosModules/clanCore/vars/default.nix @@ -39,7 +39,7 @@ in vars = { generators = lib.flip lib.mapAttrs config.clan.core.vars.generators ( _name: generator: { - inherit (generator) finalScript; + inherit (generator) dependencies finalScript; files = lib.flip lib.mapAttrs generator.files (_name: file: { inherit (file) secret; }); } ); diff --git a/nixosModules/clanCore/vars/interface.nix b/nixosModules/clanCore/vars/interface.nix index edf80f501..36996702c 100644 --- a/nixosModules/clanCore/vars/interface.nix +++ b/nixosModules/clanCore/vars/interface.nix @@ -44,7 +44,7 @@ in description = '' A list of other generators that this generator depends on. The output values of these generators will be available to the generator script as files. - For example, the file 'file1' of a dependency named 'dep1' will be available via $dependencies/dep1/file1. + For example, the file 'file1' of a dependency named 'dep1' will be available via $in/dep1/file1. ''; type = listOf str; default = [ ]; @@ -147,7 +147,7 @@ in description = '' The script to run to generate the files. The script will be run with the following environment variables: - - $dependencies: The directory containing the output values of all declared dependencies + - $in: The directory containing the output values of all declared dependencies - $out: The output directory to put the generated files - $prompts: The directory containing the prompted values as files The script should produce the files specified in the 'files' attribute under $out. diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index cc46d24a5..57113be32 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -5,6 +5,7 @@ import os import subprocess import sys from collections.abc import Callable +from graphlib import TopologicalSorter from pathlib import Path from tempfile import TemporaryDirectory @@ -37,7 +38,7 @@ def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str: return proc.stdout -def bubblewrap_cmd(generator: str, generator_dir: Path) -> list[str]: +def bubblewrap_cmd(generator: str, generator_dir: Path, dep_tmpdir: Path) -> list[str]: # fmt: off return run_cmd( [ @@ -50,6 +51,7 @@ def bubblewrap_cmd(generator: str, generator_dir: Path) -> list[str]: "--tmpfs", "/usr/lib/systemd", "--dev", "/dev", "--bind", str(generator_dir), str(generator_dir), + "--ro-bind", str(dep_tmpdir), str(dep_tmpdir), "--unshare-all", "--unshare-user", "--uid", "1000", @@ -60,16 +62,58 @@ def bubblewrap_cmd(generator: str, generator_dir: Path) -> list[str]: # fmt: on +# TODO: implement caching to not decrypt the same secret multiple times +def decrypt_dependencies( + machine: Machine, + generator_name: str, + secret_vars_store: SecretStoreBase, + public_vars_store: FactStoreBase, +) -> dict[str, dict[str, bytes]]: + generator = machine.vars_generators[generator_name] + dependencies = set(generator["dependencies"]) + decrypted_dependencies = {} + for dep_generator in dependencies: + decrypted_dependencies[dep_generator] = {} + dep_files = machine.vars_generators[dep_generator]["files"] + for file_name, file in dep_files.items(): + if file["secret"]: + decrypted_dependencies[dep_generator][file_name] = ( + secret_vars_store.get(dep_generator, file_name) + ) + else: + decrypted_dependencies[dep_generator][file_name] = ( + public_vars_store.get(dep_generator, file_name) + ) + return decrypted_dependencies + + +# decrypt dependencies and return temporary file tree +def dependencies_as_dir( + decrypted_dependencies: dict[str, dict[str, bytes]], + tmpdir: Path, +) -> Path: + for dep_generator, files in decrypted_dependencies.items(): + dep_generator_dir = tmpdir / dep_generator + dep_generator_dir.mkdir() + dep_generator_dir.chmod(0o700) + for file_name, file in files.items(): + file_path = dep_generator_dir / file_name + file_path.touch() + file_path.chmod(0o600) + file_path.write_bytes(file) + return tmpdir + + def execute_generator( machine: Machine, generator_name: str, regenerate: bool, secret_vars_store: SecretStoreBase, public_vars_store: FactStoreBase, - tmpdir: Path, + dep_tmpdir: Path, prompt: Callable[[str], str], ) -> bool: - generator_dir = tmpdir / generator_name + generator_dir = dep_tmpdir / generator_name # check if all secrets exist and generate them if at least one is missing needs_regeneration = not check_secrets(machine, generator_name=generator_name) log.debug(f"{generator_name} needs_regeneration: {needs_regeneration}") @@ -79,22 +123,30 @@ def execute_generator( msg = f"flake is not a Path: {machine.flake}" msg += "fact/secret generation is only supported for local flakes" - env = os.environ.copy() - generator_dir.mkdir(parents=True) - env["out"] = str(generator_dir) # compatibility for old outputs.nix users generator = machine.vars_generators[generator_name]["finalScript"] # if machine.vars_data[generator_name]["generator"]["prompt"]: # prompt_value = prompt(machine.vars_data[generator_name]["generator"]["prompt"]) # env["prompt_value"] = prompt_value - if sys.platform == "linux": - cmd = bubblewrap_cmd(generator, generator_dir) - else: - cmd = ["bash", "-c", generator] - run( - cmd, - env=env, + + # build temporary file tree of dependencies + decrypted_dependencies = decrypt_dependencies( + machine, generator_name, secret_vars_store, public_vars_store ) + env = os.environ.copy() + generator_dir.mkdir(parents=True) + env["out"] = str(generator_dir) + with TemporaryDirectory() as tmp: + dep_tmpdir = dependencies_as_dir(decrypted_dependencies, Path(tmp)) + env["in"] = str(dep_tmpdir) + if sys.platform == "linux": + cmd = bubblewrap_cmd(generator, generator_dir, dep_tmpdir=dep_tmpdir) + else: + cmd = ["bash", "-c", generator] + run( + cmd, + env=env, + ) files_to_commit = [] # store secrets files = machine.vars_generators[generator_name]["files"] @@ -129,6 +181,17 @@ def prompt_func(text: str) -> str: return read_multiline_input() +def _get_subgraph(graph: dict[str, set], vertex: str) -> dict[str, set]: + visited = set() + queue = [vertex] + while queue: + vertex = queue.pop(0) + if vertex not in visited: + visited.add(vertex) + queue.extend(graph[vertex] - visited) + return {k: v for k, v in graph.items() if k in visited} + + def _generate_vars_for_machine( machine: Machine, generator_name: str | None, @@ -152,21 +215,40 @@ def _generate_vars_for_machine( f"Could not find generator with name: {generator_name}. The following generators are available: {generators}" ) - if generator_name: - machine_generator_facts = { - generator_name: machine.vars_generators[generator_name] - } - else: - machine_generator_facts = machine.vars_generators + # if generator_name: + # machine_generator_facts = { + # generator_name: machine.vars_generators[generator_name] + # } + # else: + # machine_generator_facts = machine.vars_generators - for generator_name in machine_generator_facts: + graph = { + gen_name: set(generator["dependencies"]) + for gen_name, generator in machine.vars_generators.items() + } + + # extract sub-graph if specific generator selected + if generator_name: + graph = _get_subgraph(graph, generator_name) + + # check if all dependencies actually exist + for gen_name, dependencies in graph.items(): + for dep in dependencies: + if dep not in graph: + raise ClanError( + f"Generator {gen_name} has a dependency on {dep}, which does not exist" + ) + + # process generators in topological order + sorter = TopologicalSorter(graph) + for generator_name in sorter.static_order(): machine_updated |= execute_generator( machine=machine, generator_name=generator_name, regenerate=regenerate, secret_vars_store=secret_vars_store, public_vars_store=public_vars_store, - tmpdir=local_temp, + dep_tmpdir=local_temp, prompt=prompt, ) if machine_updated: diff --git a/pkgs/clan-cli/tests/test_vars.py b/pkgs/clan-cli/tests/test_vars.py index a87162b5e..e79cfb207 100644 --- a/pkgs/clan-cli/tests/test_vars.py +++ b/pkgs/clan-cli/tests/test_vars.py @@ -2,6 +2,7 @@ import os from collections import defaultdict from collections.abc import Callable from pathlib import Path +from tempfile import TemporaryDirectory from typing import Any import pytest @@ -23,6 +24,50 @@ def def_value() -> defaultdict: nested_dict: Callable[[], dict[str, Any]] = lambda: defaultdict(def_value) +def test_get_subgraph() -> None: + from clan_cli.vars.generate import _get_subgraph + + graph = dict( + a={"b", "c"}, + b={"c"}, + c=set(), + d=set(), + ) + assert _get_subgraph(graph, "a") == { + "a": {"b", "c"}, + "b": {"c"}, + "c": set(), + } + assert _get_subgraph(graph, "b") == {"b": {"c"}, "c": set()} + + +def test_dependencies_as_files() -> None: + from clan_cli.vars.generate import dependencies_as_dir + + decrypted_dependencies = dict( + gen_1=dict( + var_1a=b"var_1a", + var_1b=b"var_1b", + ), + gen_2=dict( + var_2a=b"var_2a", + var_2b=b"var_2b", + ), + ) + with TemporaryDirectory() as tmpdir: + dep_tmpdir = dependencies_as_dir(decrypted_dependencies, Path(tmpdir)) + assert dep_tmpdir.is_dir() + assert (dep_tmpdir / "gen_1" / "var_1a").read_bytes() == b"var_1a" + assert (dep_tmpdir / "gen_1" / "var_1b").read_bytes() == b"var_1b" + assert (dep_tmpdir / "gen_2" / "var_2a").read_bytes() == b"var_2a" + assert (dep_tmpdir / "gen_2" / "var_2b").read_bytes() == b"var_2b" + # ensure the files are not world readable + assert (dep_tmpdir / "gen_1" / "var_1a").stat().st_mode & 0o777 == 0o600 + assert (dep_tmpdir / "gen_1" / "var_1b").stat().st_mode & 0o777 == 0o600 + assert (dep_tmpdir / "gen_2" / "var_2a").stat().st_mode & 0o777 == 0o600 + assert (dep_tmpdir / "gen_2" / "var_2b").stat().st_mode & 0o777 == 0o600 + + @pytest.mark.impure def test_generate_public_var( monkeypatch: pytest.MonkeyPatch, @@ -155,3 +200,35 @@ def test_generate_secret_for_multiple_machines( assert sops_store2.exists("my_generator", "my_secret") assert sops_store1.get("my_generator", "my_secret").decode() == "machine1\n" assert sops_store2.get("my_generator", "my_secret").decode() == "machine2\n" + + +@pytest.mark.impure +def test_dependant_generators( + monkeypatch: pytest.MonkeyPatch, + temporary_home: Path, +) -> None: + config = nested_dict() + parent_gen = config["clan"]["core"]["vars"]["generators"]["parent_generator"] + parent_gen["files"]["my_value"]["secret"] = False + parent_gen["script"] = "echo hello > $out/my_value" + child_gen = config["clan"]["core"]["vars"]["generators"]["child_generator"] + child_gen["files"]["my_value"]["secret"] = False + child_gen["dependencies"] = ["parent_generator"] + child_gen["script"] = "cat $in/parent_generator/my_value > $out/my_value" + flake = generate_flake( + temporary_home, + flake_template=CLAN_CORE / "templates" / "minimal", + machine_configs=dict(my_machine=config), + ) + monkeypatch.chdir(flake.path) + cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) + parent_file_path = ( + flake.path / "machines" / "my_machine" / "vars" / "child_generator" / "my_value" + ) + assert parent_file_path.is_file() + assert parent_file_path.read_text() == "hello\n" + child_file_path = ( + flake.path / "machines" / "my_machine" / "vars" / "child_generator" / "my_value" + ) + assert child_file_path.is_file() + assert child_file_path.read_text() == "hello\n"