vars: retrieve generators for multiple machines

This is necessary ground work for fixing regeneration behavior spanning over multiple machines

pkgs/clan-app/ui/src/hooks/queries.ts

@@ -433,12 +433,14 @@ export const useMachineGenerators = (
     ],
     queryFn: async () => {
       const call = client.fetch("get_generators", {
-        machine: {
+        machines: [
-          name: machineName,
+          {
-          flake: {
+            name: machineName,
-            identifier: clanUri,
+            flake: {
+              identifier: clanUri,
+            },
           },
-        },
+        ],
         full_closure: true, // TODO: Make this configurable
         // TODO: Make this configurable
         include_previous_values: true,

pkgs/clan-cli/clan_cli/tests/test_vars.py

@@ -891,7 +891,7 @@ def test_api_set_prompts(
 
     machine = Machine(name="my_machine", flake=Flake(str(flake.path)))
     generators = get_generators(
-        machine=machine,
+        machines=[machine],
         full_closure=True,
         include_previous_values=True,
     )

pkgs/clan-cli/clan_cli/vars/check.py

@@ -40,7 +40,7 @@ def vars_status(
     invalid_generators = []
     from clan_cli.vars.generator import Generator
 
-    generators = Generator.get_machine_generators(machine.name, machine.flake)
+    generators = Generator.get_machine_generators([machine.name], machine.flake)
     if generator_name:
         for generator in generators:
             if generator_name == generator.name:

pkgs/clan-cli/clan_cli/vars/fix.py

@@ -12,7 +12,7 @@ log = logging.getLogger(__name__)
 def fix_vars(machine: Machine, generator_name: None | str = None) -> None:
     from clan_cli.vars.generator import Generator
 
-    generators = Generator.get_machine_generators(machine.name, machine.flake)
+    generators = Generator.get_machine_generators([machine.name], machine.flake)
     if generator_name:
         for generator in generators:
             if generator_name == generator.name:

pkgs/clan-cli/clan_cli/vars/generator.py

@@ -76,90 +76,111 @@ class Generator:
     @classmethod
     def get_machine_generators(
         cls: type["Generator"],
-        machine_name: str,
+        machine_names: list[str],
         flake: "Flake",
         include_previous_values: bool = False,
     ) -> list["Generator"]:
         """Get all generators for a machine from the flake.
 
         Args:
-            machine_name (str): The name of the machine.
+            machine_names: The names of the machines.
-            flake (Flake): The flake to get the generators from.
+            flake: The flake to get the generators from.
             include_previous_values: Whether to include previous values in the generators.
 
         Returns:
             list[Generator]: A list of (unsorted) generators for the machine.
 
         """
-        # Get all generator metadata in one select (safe fields only)
+        from clan_lib.nix import nix_config
-        generators_data = flake.select_machine(
-            machine_name,
-            "config.clan.core.vars.generators.*.{share,dependencies,migrateFact,prompts}",
-        )
-        if not generators_data:
-            return []
 
-        # Get all file metadata in one select
+        config = nix_config()
-        files_data = flake.select_machine(
+        system = config["system"]
-            machine_name,
-            "config.clan.core.vars.generators.*.files.*.{secret,deploy,owner,group,mode,neededFor}",
-        )
 
-        from clan_lib.machines.machines import Machine
+        generators_selector = "config.clan.core.vars.generators.*.{share,dependencies,migrateFact,prompts}"
+        files_selector = "config.clan.core.vars.generators.*.files.*.{secret,deploy,owner,group,mode,neededFor}"
 
-        machine = Machine(name=machine_name, flake=flake)
+        # precache all machines generators and files to avoid multiple calls to nix
-        pub_store = machine.public_vars_store
+        all_selectors = []
-        sec_store = machine.secret_vars_store
+        for machine_name in machine_names:
+            all_selectors += [
+                f'clanInternals.machines."{system}"."{machine_name}".{generators_selector}',
+                f'clanInternals.machines."{system}"."{machine_name}".{files_selector}',
+            ]
+        flake.precache(all_selectors)
 
         generators = []
-        for gen_name, gen_data in generators_data.items():
-            # Build files from the files_data
-            files = []
-            gen_files = files_data.get(gen_name, {})
-            for file_name, file_data in gen_files.items():
-                var = Var(
-                    id=f"{gen_name}/{file_name}",
-                    name=file_name,
-                    secret=file_data["secret"],
-                    deploy=file_data["deploy"],
-                    owner=file_data["owner"],
-                    group=file_data["group"],
-                    mode=(
-                        file_data["mode"]
-                        if isinstance(file_data["mode"], int)
-                        else int(file_data["mode"], 8)
-                    ),
-                    needed_for=file_data["neededFor"],
-                    _store=pub_store if not file_data["secret"] else sec_store,
-                )
-                files.append(var)
 
-            # Build prompts
+        for machine_name in machine_names:
-            prompts = [Prompt.from_nix(p) for p in gen_data.get("prompts", {}).values()]
+            # Get all generator metadata in one select (safe fields only)
-
+            generators_data = flake.select_machine(
-            generator = cls(
+                machine_name,
-                name=gen_name,
+                generators_selector,
-                share=gen_data["share"],
-                files=files,
-                dependencies=[
-                    GeneratorKey(machine=machine_name, name=dep)
-                    for dep in gen_data["dependencies"]
-                ],
-                migrate_fact=gen_data.get("migrateFact"),
-                prompts=prompts,
-                machine=machine_name,
-                _flake=flake,
             )
-            generators.append(generator)
+            if not generators_data:
+                return []
 
-        # TODO: This should be done in a non-mutable way.
+            # Get all file metadata in one select
-        if include_previous_values:
+            files_data = flake.select_machine(
-            for generator in generators:
+                machine_name,
-                for prompt in generator.prompts:
+                files_selector,
-                    prompt.previous_value = generator.get_previous_value(
+            )
-                        machine,
+
-                        prompt,
+            from clan_lib.machines.machines import Machine
+
+            machine = Machine(name=machine_name, flake=flake)
+            pub_store = machine.public_vars_store
+            sec_store = machine.secret_vars_store
+
+            for gen_name, gen_data in generators_data.items():
+                # Build files from the files_data
+                files = []
+                gen_files = files_data.get(gen_name, {})
+                for file_name, file_data in gen_files.items():
+                    var = Var(
+                        id=f"{gen_name}/{file_name}",
+                        name=file_name,
+                        secret=file_data["secret"],
+                        deploy=file_data["deploy"],
+                        owner=file_data["owner"],
+                        group=file_data["group"],
+                        mode=(
+                            file_data["mode"]
+                            if isinstance(file_data["mode"], int)
+                            else int(file_data["mode"], 8)
+                        ),
+                        needed_for=file_data["neededFor"],
+                        _store=pub_store if not file_data["secret"] else sec_store,
                     )
+                    files.append(var)
+
+                # Build prompts
+                prompts = [
+                    Prompt.from_nix(p) for p in gen_data.get("prompts", {}).values()
+                ]
+
+                generator = cls(
+                    name=gen_name,
+                    share=gen_data["share"],
+                    files=files,
+                    dependencies=[
+                        GeneratorKey(machine=machine_name, name=dep)
+                        for dep in gen_data["dependencies"]
+                    ],
+                    migrate_fact=gen_data.get("migrateFact"),
+                    prompts=prompts,
+                    machine=machine_name,
+                    _flake=flake,
+                )
+                generators.append(generator)
+
+            # TODO: This should be done in a non-mutable way.
+            if include_previous_values:
+                for generator in generators:
+                    for prompt in generator.prompts:
+                        prompt.previous_value = generator.get_previous_value(
+                            machine,
+                            prompt,
+                        )
 
         return generators
 
@@ -231,7 +252,7 @@ class Generator:
         """
         from clan_lib.errors import ClanError
 
-        generators = self.get_machine_generators(machine.name, machine.flake)
+        generators = self.get_machine_generators([machine.name], machine.flake)
         result: dict[str, dict[str, bytes]] = {}
 
         for dep_key in set(self.dependencies):

pkgs/clan-cli/clan_cli/vars/list.py

@@ -19,7 +19,7 @@ def get_machine_vars(machine: Machine) -> list[Var]:
 
     all_vars = []
 
-    generators = get_generators(machine=machine, full_closure=True)
+    generators = get_generators(machines=[machine], full_closure=True)
     for generator in generators:
         for var in generator.files:
             if var.secret:

pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py

@@ -164,7 +164,7 @@ class SecretStore(StoreBase):
 
         from clan_cli.vars.generator import Generator
 
-        generators = Generator.get_machine_generators(machine, self.flake)
+        generators = Generator.get_machine_generators([machine], self.flake)
         manifest = [
             f"{generator.name}/{file.name}".encode()
             for generator in generators
@@ -197,7 +197,7 @@ class SecretStore(StoreBase):
     def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None:
         from clan_cli.vars.generator import Generator
 
-        vars_generators = Generator.get_machine_generators(machine, self.flake)
+        vars_generators = Generator.get_machine_generators([machine], self.flake)
         if "users" in phases:
             with tarfile.open(
                 output_dir / "secrets_for_users.tar.gz",

pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py

@@ -56,7 +56,7 @@ class SecretStore(StoreBase):
         # no need to generate keys if we don't manage secrets
         from clan_cli.vars.generator import Generator
 
-        vars_generators = Generator.get_machine_generators(machine, self.flake)
+        vars_generators = Generator.get_machine_generators([machine], self.flake)
         if not vars_generators:
             return
         has_secrets = False
@@ -143,7 +143,7 @@ class SecretStore(StoreBase):
         if generators is None:
             from clan_cli.vars.generator import Generator
 
-            generators = Generator.get_machine_generators(machine, self.flake)
+            generators = Generator.get_machine_generators([machine], self.flake)
         file_found = False
         outdated = []
         for generator in generators:
@@ -221,7 +221,7 @@ class SecretStore(StoreBase):
     def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None:
         from clan_cli.vars.generator import Generator
 
-        vars_generators = Generator.get_machine_generators(machine, self.flake)
+        vars_generators = Generator.get_machine_generators([machine], self.flake)
         if "users" in phases or "services" in phases:
             key_name = f"{machine}-age.key"
             if not has_secret(sops_secrets_folder(self.flake.path) / key_name):
@@ -357,7 +357,7 @@ class SecretStore(StoreBase):
         if generators is None:
             from clan_cli.vars.generator import Generator
 
-            generators = Generator.get_machine_generators(machine, self.flake)
+            generators = Generator.get_machine_generators([machine], self.flake)
         file_found = False
         for generator in generators:
             for file in generator.files:

pkgs/clan-cli/clan_lib/flash/flash.py

@@ -118,7 +118,9 @@ def run_machine_flash(
 
         from clan_cli.vars.generator import Generator
 
-        for generator in Generator.get_machine_generators(machine.name, machine.flake):
+        for generator in Generator.get_machine_generators(
+            [machine.name], machine.flake
+        ):
             for file in generator.files:
                 if file.needed_for == "partitioning":
                     msg = f"Partitioning time secrets are not supported with `clan flash write`: clan.core.vars.generators.{generator.name}.files.{file.name}"

pkgs/clan-cli/clan_lib/tests/test_create.py

@@ -228,7 +228,7 @@ def test_clan_create_api(
     # Invalidate cache because of new inventory
     clan_dir_flake.invalidate_cache()
 
-    generators = get_generators(machine=machine, full_closure=True)
+    generators = get_generators(machines=[machine], full_closure=True)
     collected_prompt_values = {}
     for generator in generators:
         prompt_values = {}

pkgs/clan-cli/clan_lib/vars/generate.py

@@ -14,7 +14,7 @@ log = logging.getLogger(__name__)
 
 @API.register
 def get_generators(
-    machine: Machine,
+    machines: list[Machine],
     full_closure: bool,
     generator_name: str | None = None,
     include_previous_values: bool = False,
@@ -22,7 +22,7 @@ def get_generators(
     """Get generators for a machine, with optional closure computation.
 
     Args:
-        machine: The machine to get generators for.
+        machines: The machines to get generators for.
         full_closure: If True, include all dependency generators. If False, only include missing ones.
         generator_name: Name of a specific generator to get, or None for all generators.
         include_previous_values: If True, populate prompts with their previous values.
@@ -33,7 +33,12 @@ def get_generators(
     """
     from clan_cli.vars import graph
 
-    vars_generators = Generator.get_machine_generators(machine.name, machine.flake)
+    machine_names = [machine.name for machine in machines]
+    vars_generators = Generator.get_machine_generators(
+        machine_names,
+        machines[0].flake,
+        include_previous_values=include_previous_values,
+    )
     generators = {generator.key: generator for generator in vars_generators}
 
     result_closure = []
@@ -44,16 +49,11 @@ def get_generators(
             result_closure = graph.all_missing_closure(generators)
     # specific generator selected
     elif full_closure:
-        gen_key = GeneratorKey(machine=machine.name, name=generator_name)
+        roots = [key for key in generators if key.name == generator_name]
-        result_closure = requested_closure([gen_key], generators)
+        result_closure = requested_closure(roots, generators)
     else:
-        gen_key = GeneratorKey(machine=machine.name, name=generator_name)
+        roots = [key for key in generators if key.name == generator_name]
-        result_closure = minimal_closure([gen_key], generators)
+        result_closure = minimal_closure(roots, generators)
-
-    if include_previous_values:
-        for generator in result_closure:
-            for prompt in generator.prompts:
-                prompt.previous_value = generator.get_previous_value(machine, prompt)
 
     return result_closure
 
@@ -66,7 +66,7 @@ def _ensure_healthy(
     Fails if any of the generators' health checks fail.
     """
     if generators is None:
-        generators = Generator.get_machine_generators(machine.name, machine.flake)
+        generators = Generator.get_machine_generators([machine.name], machine.flake)
 
     pub_healtcheck_msg = machine.public_vars_store.health_check(
         machine.name,
@@ -133,12 +133,12 @@ def run_generators(
             generator_keys = {
                 GeneratorKey(machine=machine.name, name=name) for name in generators
             }
-            all_generators = get_generators(machine, full_closure=True)
+            all_generators = get_generators([machine], full_closure=True)
             generator_objects = [g for g in all_generators if g.key in generator_keys]
         else:
             # None or single string - use get_generators with closure parameter
             generator_objects = get_generators(
-                machine,
+                [machine],
                 full_closure=full_closure,
                 generator_name=generators,
             )