Merge pull request 'vars: implement dependencies' (#1771) from DavHau/clan-core:DavHau-vars into main

nixosModules/clanCore/vars/default.nix

@@ -39,7 +39,7 @@ in
     vars = {
       generators = lib.flip lib.mapAttrs config.clan.core.vars.generators (
         _name: generator: {
-          inherit (generator) finalScript;
+          inherit (generator) dependencies finalScript;
           files = lib.flip lib.mapAttrs generator.files (_name: file: { inherit (file) secret; });
         }
       );

nixosModules/clanCore/vars/interface.nix

@@ -44,7 +44,7 @@ in
               description = ''
                 A list of other generators that this generator depends on.
                 The output values of these generators will be available to the generator script as files.
-                For example, the file 'file1' of a dependency named 'dep1' will be available via $dependencies/dep1/file1.
+                For example, the file 'file1' of a dependency named 'dep1' will be available via $in/dep1/file1.
               '';
               type = listOf str;
               default = [ ];
@@ -147,7 +147,7 @@ in
               description = ''
                 The script to run to generate the files.
                 The script will be run with the following environment variables:
-                  - $dependencies: The directory containing the output values of all declared dependencies
+                  - $in: The directory containing the output values of all declared dependencies
                   - $out: The output directory to put the generated files
                   - $prompts: The directory containing the prompted values as files
                 The script should produce the files specified in the 'files' attribute under $out.

pkgs/clan-cli/clan_cli/vars/generate.py

@@ -5,6 +5,7 @@ import os
 import subprocess
 import sys
 from collections.abc import Callable
+from graphlib import TopologicalSorter
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -37,7 +38,7 @@ def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str:
     return proc.stdout
 
 
-def bubblewrap_cmd(generator: str, generator_dir: Path) -> list[str]:
+def bubblewrap_cmd(generator: str, generator_dir: Path, dep_tmpdir: Path) -> list[str]:
     # fmt: off
     return run_cmd(
         [
@@ -50,6 +51,7 @@ def bubblewrap_cmd(generator: str, generator_dir: Path) -> list[str]:
             "--tmpfs",  "/usr/lib/systemd",
             "--dev", "/dev",
             "--bind", str(generator_dir), str(generator_dir),
+            "--ro-bind", str(dep_tmpdir), str(dep_tmpdir),
             "--unshare-all",
             "--unshare-user",
             "--uid", "1000",
@@ -60,16 +62,58 @@ def bubblewrap_cmd(generator: str, generator_dir: Path) -> list[str]:
     # fmt: on
 
 
+# TODO: implement caching to not decrypt the same secret multiple times
+def decrypt_dependencies(
+    machine: Machine,
+    generator_name: str,
+    secret_vars_store: SecretStoreBase,
+    public_vars_store: FactStoreBase,
+) -> dict[str, dict[str, bytes]]:
+    generator = machine.vars_generators[generator_name]
+    dependencies = set(generator["dependencies"])
+    decrypted_dependencies = {}
+    for dep_generator in dependencies:
+        decrypted_dependencies[dep_generator] = {}
+        dep_files = machine.vars_generators[dep_generator]["files"]
+        for file_name, file in dep_files.items():
+            if file["secret"]:
+                decrypted_dependencies[dep_generator][file_name] = (
+                    secret_vars_store.get(dep_generator, file_name)
+                )
+            else:
+                decrypted_dependencies[dep_generator][file_name] = (
+                    public_vars_store.get(dep_generator, file_name)
+                )
+    return decrypted_dependencies
+
+
+# decrypt dependencies and return temporary file tree
+def dependencies_as_dir(
+    decrypted_dependencies: dict[str, dict[str, bytes]],
+    tmpdir: Path,
+) -> Path:
+    for dep_generator, files in decrypted_dependencies.items():
+        dep_generator_dir = tmpdir / dep_generator
+        dep_generator_dir.mkdir()
+        dep_generator_dir.chmod(0o700)
+        for file_name, file in files.items():
+            file_path = dep_generator_dir / file_name
+            file_path.touch()
+            file_path.chmod(0o600)
+            file_path.write_bytes(file)
+    return tmpdir
+
+
 def execute_generator(
     machine: Machine,
     generator_name: str,
     regenerate: bool,
     secret_vars_store: SecretStoreBase,
     public_vars_store: FactStoreBase,
-    tmpdir: Path,
+    dep_tmpdir: Path,
     prompt: Callable[[str], str],
 ) -> bool:
-    generator_dir = tmpdir / generator_name
+    generator_dir = dep_tmpdir / generator_name
     # check if all secrets exist and generate them if at least one is missing
     needs_regeneration = not check_secrets(machine, generator_name=generator_name)
     log.debug(f"{generator_name} needs_regeneration: {needs_regeneration}")
@@ -79,22 +123,30 @@ def execute_generator(
         msg = f"flake is not a Path: {machine.flake}"
         msg += "fact/secret generation is only supported for local flakes"
 
-    env = os.environ.copy()
-    generator_dir.mkdir(parents=True)
-    env["out"] = str(generator_dir)
     # compatibility for old outputs.nix users
     generator = machine.vars_generators[generator_name]["finalScript"]
     # if machine.vars_data[generator_name]["generator"]["prompt"]:
     #     prompt_value = prompt(machine.vars_data[generator_name]["generator"]["prompt"])
     #     env["prompt_value"] = prompt_value
-    if sys.platform == "linux":
+
-        cmd = bubblewrap_cmd(generator, generator_dir)
+    # build temporary file tree of dependencies
-    else:
+    decrypted_dependencies = decrypt_dependencies(
-        cmd = ["bash", "-c", generator]
+        machine, generator_name, secret_vars_store, public_vars_store
-    run(
-        cmd,
-        env=env,
     )
+    env = os.environ.copy()
+    generator_dir.mkdir(parents=True)
+    env["out"] = str(generator_dir)
+    with TemporaryDirectory() as tmp:
+        dep_tmpdir = dependencies_as_dir(decrypted_dependencies, Path(tmp))
+        env["in"] = str(dep_tmpdir)
+        if sys.platform == "linux":
+            cmd = bubblewrap_cmd(generator, generator_dir, dep_tmpdir=dep_tmpdir)
+        else:
+            cmd = ["bash", "-c", generator]
+        run(
+            cmd,
+            env=env,
+        )
     files_to_commit = []
     # store secrets
     files = machine.vars_generators[generator_name]["files"]
@@ -129,6 +181,17 @@ def prompt_func(text: str) -> str:
     return read_multiline_input()
 
 
+def _get_subgraph(graph: dict[str, set], vertex: str) -> dict[str, set]:
+    visited = set()
+    queue = [vertex]
+    while queue:
+        vertex = queue.pop(0)
+        if vertex not in visited:
+            visited.add(vertex)
+            queue.extend(graph[vertex] - visited)
+    return {k: v for k, v in graph.items() if k in visited}
+
+
 def _generate_vars_for_machine(
     machine: Machine,
     generator_name: str | None,
@@ -152,21 +215,40 @@ def _generate_vars_for_machine(
             f"Could not find generator with name: {generator_name}. The following generators are available: {generators}"
         )
 
-    if generator_name:
+    # if generator_name:
-        machine_generator_facts = {
+    #     machine_generator_facts = {
-            generator_name: machine.vars_generators[generator_name]
+    #         generator_name: machine.vars_generators[generator_name]
-        }
+    #     }
-    else:
+    # else:
-        machine_generator_facts = machine.vars_generators
+    #     machine_generator_facts = machine.vars_generators
 
-    for generator_name in machine_generator_facts:
+    graph = {
+        gen_name: set(generator["dependencies"])
+        for gen_name, generator in machine.vars_generators.items()
+    }
+
+    # extract sub-graph if specific generator selected
+    if generator_name:
+        graph = _get_subgraph(graph, generator_name)
+
+    # check if all dependencies actually exist
+    for gen_name, dependencies in graph.items():
+        for dep in dependencies:
+            if dep not in graph:
+                raise ClanError(
+                    f"Generator {gen_name} has a dependency on {dep}, which does not exist"
+                )
+
+    # process generators in topological order
+    sorter = TopologicalSorter(graph)
+    for generator_name in sorter.static_order():
         machine_updated |= execute_generator(
             machine=machine,
             generator_name=generator_name,
             regenerate=regenerate,
             secret_vars_store=secret_vars_store,
             public_vars_store=public_vars_store,
-            tmpdir=local_temp,
+            dep_tmpdir=local_temp,
             prompt=prompt,
         )
     if machine_updated:

pkgs/clan-cli/tests/test_vars.py

@@ -2,6 +2,7 @@ import os
 from collections import defaultdict
 from collections.abc import Callable
 from pathlib import Path
+from tempfile import TemporaryDirectory
 from typing import Any
 
 import pytest
@@ -23,6 +24,50 @@ def def_value() -> defaultdict:
 nested_dict: Callable[[], dict[str, Any]] = lambda: defaultdict(def_value)
 
 
+def test_get_subgraph() -> None:
+    from clan_cli.vars.generate import _get_subgraph
+
+    graph = dict(
+        a={"b", "c"},
+        b={"c"},
+        c=set(),
+        d=set(),
+    )
+    assert _get_subgraph(graph, "a") == {
+        "a": {"b", "c"},
+        "b": {"c"},
+        "c": set(),
+    }
+    assert _get_subgraph(graph, "b") == {"b": {"c"}, "c": set()}
+
+
+def test_dependencies_as_files() -> None:
+    from clan_cli.vars.generate import dependencies_as_dir
+
+    decrypted_dependencies = dict(
+        gen_1=dict(
+            var_1a=b"var_1a",
+            var_1b=b"var_1b",
+        ),
+        gen_2=dict(
+            var_2a=b"var_2a",
+            var_2b=b"var_2b",
+        ),
+    )
+    with TemporaryDirectory() as tmpdir:
+        dep_tmpdir = dependencies_as_dir(decrypted_dependencies, Path(tmpdir))
+        assert dep_tmpdir.is_dir()
+        assert (dep_tmpdir / "gen_1" / "var_1a").read_bytes() == b"var_1a"
+        assert (dep_tmpdir / "gen_1" / "var_1b").read_bytes() == b"var_1b"
+        assert (dep_tmpdir / "gen_2" / "var_2a").read_bytes() == b"var_2a"
+        assert (dep_tmpdir / "gen_2" / "var_2b").read_bytes() == b"var_2b"
+        # ensure the files are not world readable
+        assert (dep_tmpdir / "gen_1" / "var_1a").stat().st_mode & 0o777 == 0o600
+        assert (dep_tmpdir / "gen_1" / "var_1b").stat().st_mode & 0o777 == 0o600
+        assert (dep_tmpdir / "gen_2" / "var_2a").stat().st_mode & 0o777 == 0o600
+        assert (dep_tmpdir / "gen_2" / "var_2b").stat().st_mode & 0o777 == 0o600
+
+
 @pytest.mark.impure
 def test_generate_public_var(
     monkeypatch: pytest.MonkeyPatch,
@@ -155,3 +200,35 @@ def test_generate_secret_for_multiple_machines(
     assert sops_store2.exists("my_generator", "my_secret")
     assert sops_store1.get("my_generator", "my_secret").decode() == "machine1\n"
     assert sops_store2.get("my_generator", "my_secret").decode() == "machine2\n"
+
+
+@pytest.mark.impure
+def test_dependant_generators(
+    monkeypatch: pytest.MonkeyPatch,
+    temporary_home: Path,
+) -> None:
+    config = nested_dict()
+    parent_gen = config["clan"]["core"]["vars"]["generators"]["parent_generator"]
+    parent_gen["files"]["my_value"]["secret"] = False
+    parent_gen["script"] = "echo hello > $out/my_value"
+    child_gen = config["clan"]["core"]["vars"]["generators"]["child_generator"]
+    child_gen["files"]["my_value"]["secret"] = False
+    child_gen["dependencies"] = ["parent_generator"]
+    child_gen["script"] = "cat $in/parent_generator/my_value > $out/my_value"
+    flake = generate_flake(
+        temporary_home,
+        flake_template=CLAN_CORE / "templates" / "minimal",
+        machine_configs=dict(my_machine=config),
+    )
+    monkeypatch.chdir(flake.path)
+    cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
+    parent_file_path = (
+        flake.path / "machines" / "my_machine" / "vars" / "child_generator" / "my_value"
+    )
+    assert parent_file_path.is_file()
+    assert parent_file_path.read_text() == "hello\n"
+    child_file_path = (
+        flake.path / "machines" / "my_machine" / "vars" / "child_generator" / "my_value"
+    )
+    assert child_file_path.is_file()
+    assert child_file_path.read_text() == "hello\n"