Merge pull request 'vars: always generate dependents' (#4996) from vars into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4996

pkgs/clan-cli/clan_cli/tests/test_vars.py

@@ -156,6 +156,12 @@ def test_generate_public_and_secret_vars(
     vars_text = stringify_all_vars(machine)
     flake_obj = Flake(str(flake.path))
     my_generator = Generator("my_generator", machine="my_machine", _flake=flake_obj)
+    shared_generator = Generator(
+        "my_shared_generator",
+        share=True,
+        machine="my_machine",
+        _flake=flake_obj,
+    )
     dependent_generator = Generator(
         "dependent_generator",
         machine="my_machine",
@@ -262,6 +268,36 @@ def test_generate_public_and_secret_vars(
         "my_generator value should NOT change after regenerating only my_shared_generator"
     )
 
+    # test that a dependent is generated on a clean slate even when no --regenerate is given
+    # remove all generated vars
+    in_repo_store.delete_store("my_machine")
+    sops_store.delete(shared_generator, "my_shared_value")
+    sops_store.delete_store("my_machine")
+    cli.run(
+        [
+            "vars",
+            "generate",
+            "--flake",
+            str(flake.path),
+            "my_machine",
+            "--generator",
+            "my_shared_generator",
+        ]
+    )
+    # check that both my_shared_generator and dependent_generator are generated
+    shared_value_clean = get_machine_var(
+        machine,
+        "my_shared_generator/my_shared_value",
+    ).printable_value
+    assert shared_value_clean.startswith("shared"), "Shared value should be generated"
+    assert sops_store.exists(dependent_generator, "my_secret"), (
+        "Dependent generator's secret should be generated"
+    )
+    secret_value_clean = sops_store.get(dependent_generator, "my_secret").decode()
+    assert secret_value_clean == shared_value_clean, (
+        "Dependent generator's secret should match the shared value"
+    )
+
 
 # TODO: it doesn't actually test if the group has access
 @pytest.mark.with_core

pkgs/clan-cli/clan_cli/vars/graph.py

@@ -87,31 +87,27 @@ def toposort_closure(
     return [generators[gen_key] for gen_key in result]
 
 
-# all generators in topological order
-def full_closure(generators: dict[GeneratorKey, Generator]) -> list[Generator]:
-    """From a set of generators, return all generators in topological order.
-    This includes all dependencies and dependents of the generators.
-    Returns all generators in topological order.
-    """
-    return toposort_closure(generators.keys(), generators)
-
-
 # just the missing generators including their dependents
-def all_missing_closure(generators: dict[GeneratorKey, Generator]) -> list[Generator]:
+def all_missing_closure(
+    requested_generators: Iterable[GeneratorKey],
+    generators: dict[GeneratorKey, Generator],
+) -> list[Generator]:
     """From a set of generators, return all incomplete generators in topological order.
 
     incomplete
     : A generator is missing if at least one of its files is missing.
     """
     # collect all generators that are missing from disk
-    closure = {gen_key for gen_key, gen in generators.items() if not gen.exists}
+    closure = {
+        gen_key for gen_key in requested_generators if not generators[gen_key].exists
+    }
     closure = add_dependents(closure, generators)
     return toposort_closure(closure, generators)
 
 
 # only a selected list of generators including their missing dependencies and their dependents
 def requested_closure(
-    requested_generators: list[GeneratorKey],
+    requested_generators: Iterable[GeneratorKey],
     generators: dict[GeneratorKey, Generator],
 ) -> list[Generator]:
     closure = set(requested_generators)
@@ -119,18 +115,3 @@ def requested_closure(
     closure = add_missing_dependencies(closure, generators)
     closure = add_dependents(closure, generators)
     return toposort_closure(closure, generators)
-
-
-# just enough to ensure that the list of selected generators are in a consistent state.
-# empty if nothing is missing.
-def minimal_closure(
-    requested_generators: list[GeneratorKey],
-    generators: dict[GeneratorKey, Generator],
-) -> list[Generator]:
-    closure = set(requested_generators)
-    final_closure = missing_dependency_closure(closure, generators)
-    # add requested generators if not already exist
-    for gen_key in closure:
-        if not generators[gen_key].exists:
-            final_closure.add(gen_key)
-    return toposort_closure(final_closure, generators)

pkgs/clan-cli/clan_cli/vars/graph_test.py

@@ -5,6 +5,10 @@ from clan_cli.vars.generator import (
 from clan_cli.vars.graph import all_missing_closure, requested_closure
 
 
+def generator_names(generator: list[Generator]) -> list[str]:
+    return [gen.name for gen in generator]
+
+
 def test_required_generators() -> None:
     # Create generators with proper machine context
     machine_name = "test_machine"
@@ -33,9 +37,6 @@ def test_required_generators() -> None:
         generator.key: generator for generator in [gen_1, gen_2, gen_2a, gen_2b]
     }
 
-    def generator_names(generator: list[Generator]) -> list[str]:
-        return [gen.name for gen in generator]
-
     assert generator_names(requested_closure([gen_1.key], generators)) == [
         "gen_1",
         "gen_2",
@@ -58,8 +59,50 @@ def test_required_generators() -> None:
         "gen_2b",
     ]
 
-    assert generator_names(all_missing_closure(generators)) == [
+    assert generator_names(all_missing_closure(generators.keys(), generators)) == [
         "gen_2",
         "gen_2a",
         "gen_2b",
     ]
+
+
+def test_shared_generator_invalidates_multiple_machines_dependents() -> None:
+    # Create generators with proper machine context
+    machine_1 = "machine_1"
+    machine_2 = "machine_2"
+    shared_gen = Generator(
+        name="shared_gen",
+        dependencies=[],
+        machine=None,  # Shared generator
+    )
+    gen_1 = Generator(
+        name="gen_1",
+        dependencies=[shared_gen.key],
+        machine=machine_1,
+    )
+    gen_2 = Generator(
+        name="gen_2",
+        dependencies=[shared_gen.key],
+        machine=machine_2,
+    )
+
+    shared_gen.exists = False
+    gen_1.exists = True
+    gen_2.exists = True
+    generators: dict[GeneratorKey, Generator] = {
+        generator.key: generator for generator in [shared_gen, gen_1, gen_2]
+    }
+
+    assert generator_names(all_missing_closure(generators.keys(), generators)) == [
+        "shared_gen",
+        "gen_1",
+        "gen_2",
+    ], (
+        "All generators should be included in all_missing_closure due to shared dependency"
+    )
+
+    assert generator_names(requested_closure([shared_gen.key], generators)) == [
+        "shared_gen",
+        "gen_1",
+        "gen_2",
+    ], "All generators should be included in requested_closure due to shared dependency"

pkgs/clan-cli/clan_lib/vars/generate.py

@@ -3,7 +3,7 @@ from collections.abc import Callable
 
 from clan_cli.vars import graph
 from clan_cli.vars.generator import Generator
-from clan_cli.vars.graph import minimal_closure, requested_closure
+from clan_cli.vars.graph import requested_closure
 from clan_cli.vars.migration import check_can_migrate, migrate_files
 
 from clan_lib.api import API
@@ -43,16 +43,16 @@ def get_generators(
     result_closure = []
     if generator_name is None:  # all generators selected
         if full_closure:
-            result_closure = graph.full_closure(generators)
+            result_closure = graph.requested_closure(generators.keys(), generators)
         else:
-            result_closure = graph.all_missing_closure(generators)
+            result_closure = graph.all_missing_closure(generators.keys(), generators)
     # specific generator selected
     elif full_closure:
         roots = [key for key in generators if key.name == generator_name]
         result_closure = requested_closure(roots, generators)
     else:
         roots = [key for key in generators if key.name == generator_name]
-        result_closure = minimal_closure(roots, generators)
+        result_closure = graph.all_missing_closure(roots, generators)
 
     return result_closure