diff --git a/pkgs/clan-app/ui/src/hooks/queries.ts b/pkgs/clan-app/ui/src/hooks/queries.ts index f84b0724e..5398e0e86 100644 --- a/pkgs/clan-app/ui/src/hooks/queries.ts +++ b/pkgs/clan-app/ui/src/hooks/queries.ts @@ -433,12 +433,14 @@ export const useMachineGenerators = ( ], queryFn: async () => { const call = client.fetch("get_generators", { - machine: { - name: machineName, - flake: { - identifier: clanUri, + machines: [ + { + name: machineName, + flake: { + identifier: clanUri, + }, }, - }, + ], full_closure: true, // TODO: Make this configurable // TODO: Make this configurable include_previous_values: true, diff --git a/pkgs/clan-cli/clan_cli/tests/test_vars.py b/pkgs/clan-cli/clan_cli/tests/test_vars.py index 2430f16a7..a2bf1a63e 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_vars.py +++ b/pkgs/clan-cli/clan_cli/tests/test_vars.py @@ -891,7 +891,7 @@ def test_api_set_prompts( machine = Machine(name="my_machine", flake=Flake(str(flake.path))) generators = get_generators( - machine=machine, + machines=[machine], full_closure=True, include_previous_values=True, ) diff --git a/pkgs/clan-cli/clan_cli/vars/check.py b/pkgs/clan-cli/clan_cli/vars/check.py index 2d99a0937..d79567c9b 100644 --- a/pkgs/clan-cli/clan_cli/vars/check.py +++ b/pkgs/clan-cli/clan_cli/vars/check.py @@ -40,7 +40,7 @@ def vars_status( invalid_generators = [] from clan_cli.vars.generator import Generator - generators = Generator.get_machine_generators(machine.name, machine.flake) + generators = Generator.get_machine_generators([machine.name], machine.flake) if generator_name: for generator in generators: if generator_name == generator.name: diff --git a/pkgs/clan-cli/clan_cli/vars/fix.py b/pkgs/clan-cli/clan_cli/vars/fix.py index 574937ade..4019534fb 100644 --- a/pkgs/clan-cli/clan_cli/vars/fix.py +++ b/pkgs/clan-cli/clan_cli/vars/fix.py @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) def fix_vars(machine: Machine, generator_name: None | str = None) -> None: from clan_cli.vars.generator import Generator - generators = Generator.get_machine_generators(machine.name, machine.flake) + generators = Generator.get_machine_generators([machine.name], machine.flake) if generator_name: for generator in generators: if generator_name == generator.name: diff --git a/pkgs/clan-cli/clan_cli/vars/generator.py b/pkgs/clan-cli/clan_cli/vars/generator.py index a8c95aace..8b9b285fa 100644 --- a/pkgs/clan-cli/clan_cli/vars/generator.py +++ b/pkgs/clan-cli/clan_cli/vars/generator.py @@ -76,90 +76,111 @@ class Generator: @classmethod def get_machine_generators( cls: type["Generator"], - machine_name: str, + machine_names: list[str], flake: "Flake", include_previous_values: bool = False, ) -> list["Generator"]: """Get all generators for a machine from the flake. Args: - machine_name (str): The name of the machine. - flake (Flake): The flake to get the generators from. + machine_names: The names of the machines. + flake: The flake to get the generators from. include_previous_values: Whether to include previous values in the generators. Returns: list[Generator]: A list of (unsorted) generators for the machine. """ - # Get all generator metadata in one select (safe fields only) - generators_data = flake.select_machine( - machine_name, - "config.clan.core.vars.generators.*.{share,dependencies,migrateFact,prompts}", - ) - if not generators_data: - return [] + from clan_lib.nix import nix_config - # Get all file metadata in one select - files_data = flake.select_machine( - machine_name, - "config.clan.core.vars.generators.*.files.*.{secret,deploy,owner,group,mode,neededFor}", - ) + config = nix_config() + system = config["system"] - from clan_lib.machines.machines import Machine + generators_selector = "config.clan.core.vars.generators.*.{share,dependencies,migrateFact,prompts}" + files_selector = "config.clan.core.vars.generators.*.files.*.{secret,deploy,owner,group,mode,neededFor}" - machine = Machine(name=machine_name, flake=flake) - pub_store = machine.public_vars_store - sec_store = machine.secret_vars_store + # precache all machines generators and files to avoid multiple calls to nix + all_selectors = [] + for machine_name in machine_names: + all_selectors += [ + f'clanInternals.machines."{system}"."{machine_name}".{generators_selector}', + f'clanInternals.machines."{system}"."{machine_name}".{files_selector}', + ] + flake.precache(all_selectors) generators = [] - for gen_name, gen_data in generators_data.items(): - # Build files from the files_data - files = [] - gen_files = files_data.get(gen_name, {}) - for file_name, file_data in gen_files.items(): - var = Var( - id=f"{gen_name}/{file_name}", - name=file_name, - secret=file_data["secret"], - deploy=file_data["deploy"], - owner=file_data["owner"], - group=file_data["group"], - mode=( - file_data["mode"] - if isinstance(file_data["mode"], int) - else int(file_data["mode"], 8) - ), - needed_for=file_data["neededFor"], - _store=pub_store if not file_data["secret"] else sec_store, - ) - files.append(var) - # Build prompts - prompts = [Prompt.from_nix(p) for p in gen_data.get("prompts", {}).values()] - - generator = cls( - name=gen_name, - share=gen_data["share"], - files=files, - dependencies=[ - GeneratorKey(machine=machine_name, name=dep) - for dep in gen_data["dependencies"] - ], - migrate_fact=gen_data.get("migrateFact"), - prompts=prompts, - machine=machine_name, - _flake=flake, + for machine_name in machine_names: + # Get all generator metadata in one select (safe fields only) + generators_data = flake.select_machine( + machine_name, + generators_selector, ) - generators.append(generator) + if not generators_data: + return [] - # TODO: This should be done in a non-mutable way. - if include_previous_values: - for generator in generators: - for prompt in generator.prompts: - prompt.previous_value = generator.get_previous_value( - machine, - prompt, + # Get all file metadata in one select + files_data = flake.select_machine( + machine_name, + files_selector, + ) + + from clan_lib.machines.machines import Machine + + machine = Machine(name=machine_name, flake=flake) + pub_store = machine.public_vars_store + sec_store = machine.secret_vars_store + + for gen_name, gen_data in generators_data.items(): + # Build files from the files_data + files = [] + gen_files = files_data.get(gen_name, {}) + for file_name, file_data in gen_files.items(): + var = Var( + id=f"{gen_name}/{file_name}", + name=file_name, + secret=file_data["secret"], + deploy=file_data["deploy"], + owner=file_data["owner"], + group=file_data["group"], + mode=( + file_data["mode"] + if isinstance(file_data["mode"], int) + else int(file_data["mode"], 8) + ), + needed_for=file_data["neededFor"], + _store=pub_store if not file_data["secret"] else sec_store, ) + files.append(var) + + # Build prompts + prompts = [ + Prompt.from_nix(p) for p in gen_data.get("prompts", {}).values() + ] + + generator = cls( + name=gen_name, + share=gen_data["share"], + files=files, + dependencies=[ + GeneratorKey(machine=machine_name, name=dep) + for dep in gen_data["dependencies"] + ], + migrate_fact=gen_data.get("migrateFact"), + prompts=prompts, + machine=machine_name, + _flake=flake, + ) + generators.append(generator) + + # TODO: This should be done in a non-mutable way. + if include_previous_values: + for generator in generators: + for prompt in generator.prompts: + prompt.previous_value = generator.get_previous_value( + machine, + prompt, + ) return generators @@ -231,7 +252,7 @@ class Generator: """ from clan_lib.errors import ClanError - generators = self.get_machine_generators(machine.name, machine.flake) + generators = self.get_machine_generators([machine.name], machine.flake) result: dict[str, dict[str, bytes]] = {} for dep_key in set(self.dependencies): diff --git a/pkgs/clan-cli/clan_cli/vars/list.py b/pkgs/clan-cli/clan_cli/vars/list.py index 03ecf5ca4..93c3babbe 100644 --- a/pkgs/clan-cli/clan_cli/vars/list.py +++ b/pkgs/clan-cli/clan_cli/vars/list.py @@ -19,7 +19,7 @@ def get_machine_vars(machine: Machine) -> list[Var]: all_vars = [] - generators = get_generators(machine=machine, full_closure=True) + generators = get_generators(machines=[machine], full_closure=True) for generator in generators: for var in generator.files: if var.secret: diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py index bb857d6a7..d9e94ec43 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py @@ -164,7 +164,7 @@ class SecretStore(StoreBase): from clan_cli.vars.generator import Generator - generators = Generator.get_machine_generators(machine, self.flake) + generators = Generator.get_machine_generators([machine], self.flake) manifest = [ f"{generator.name}/{file.name}".encode() for generator in generators @@ -197,7 +197,7 @@ class SecretStore(StoreBase): def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None: from clan_cli.vars.generator import Generator - vars_generators = Generator.get_machine_generators(machine, self.flake) + vars_generators = Generator.get_machine_generators([machine], self.flake) if "users" in phases: with tarfile.open( output_dir / "secrets_for_users.tar.gz", diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py index 847284bd5..ad52cbad1 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py @@ -56,7 +56,7 @@ class SecretStore(StoreBase): # no need to generate keys if we don't manage secrets from clan_cli.vars.generator import Generator - vars_generators = Generator.get_machine_generators(machine, self.flake) + vars_generators = Generator.get_machine_generators([machine], self.flake) if not vars_generators: return has_secrets = False @@ -143,7 +143,7 @@ class SecretStore(StoreBase): if generators is None: from clan_cli.vars.generator import Generator - generators = Generator.get_machine_generators(machine, self.flake) + generators = Generator.get_machine_generators([machine], self.flake) file_found = False outdated = [] for generator in generators: @@ -221,7 +221,7 @@ class SecretStore(StoreBase): def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None: from clan_cli.vars.generator import Generator - vars_generators = Generator.get_machine_generators(machine, self.flake) + vars_generators = Generator.get_machine_generators([machine], self.flake) if "users" in phases or "services" in phases: key_name = f"{machine}-age.key" if not has_secret(sops_secrets_folder(self.flake.path) / key_name): @@ -357,7 +357,7 @@ class SecretStore(StoreBase): if generators is None: from clan_cli.vars.generator import Generator - generators = Generator.get_machine_generators(machine, self.flake) + generators = Generator.get_machine_generators([machine], self.flake) file_found = False for generator in generators: for file in generator.files: diff --git a/pkgs/clan-cli/clan_lib/flash/flash.py b/pkgs/clan-cli/clan_lib/flash/flash.py index ec82fddb6..781bd094e 100644 --- a/pkgs/clan-cli/clan_lib/flash/flash.py +++ b/pkgs/clan-cli/clan_lib/flash/flash.py @@ -118,7 +118,9 @@ def run_machine_flash( from clan_cli.vars.generator import Generator - for generator in Generator.get_machine_generators(machine.name, machine.flake): + for generator in Generator.get_machine_generators( + [machine.name], machine.flake + ): for file in generator.files: if file.needed_for == "partitioning": msg = f"Partitioning time secrets are not supported with `clan flash write`: clan.core.vars.generators.{generator.name}.files.{file.name}" diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index 227ef4fe0..3edee4a10 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -228,7 +228,7 @@ def test_clan_create_api( # Invalidate cache because of new inventory clan_dir_flake.invalidate_cache() - generators = get_generators(machine=machine, full_closure=True) + generators = get_generators(machines=[machine], full_closure=True) collected_prompt_values = {} for generator in generators: prompt_values = {} diff --git a/pkgs/clan-cli/clan_lib/vars/generate.py b/pkgs/clan-cli/clan_lib/vars/generate.py index e8efe6e1c..d5b263611 100644 --- a/pkgs/clan-cli/clan_lib/vars/generate.py +++ b/pkgs/clan-cli/clan_lib/vars/generate.py @@ -14,7 +14,7 @@ log = logging.getLogger(__name__) @API.register def get_generators( - machine: Machine, + machines: list[Machine], full_closure: bool, generator_name: str | None = None, include_previous_values: bool = False, @@ -22,7 +22,7 @@ def get_generators( """Get generators for a machine, with optional closure computation. Args: - machine: The machine to get generators for. + machines: The machines to get generators for. full_closure: If True, include all dependency generators. If False, only include missing ones. generator_name: Name of a specific generator to get, or None for all generators. include_previous_values: If True, populate prompts with their previous values. @@ -33,7 +33,12 @@ def get_generators( """ from clan_cli.vars import graph - vars_generators = Generator.get_machine_generators(machine.name, machine.flake) + machine_names = [machine.name for machine in machines] + vars_generators = Generator.get_machine_generators( + machine_names, + machines[0].flake, + include_previous_values=include_previous_values, + ) generators = {generator.key: generator for generator in vars_generators} result_closure = [] @@ -44,16 +49,11 @@ def get_generators( result_closure = graph.all_missing_closure(generators) # specific generator selected elif full_closure: - gen_key = GeneratorKey(machine=machine.name, name=generator_name) - result_closure = requested_closure([gen_key], generators) + roots = [key for key in generators if key.name == generator_name] + result_closure = requested_closure(roots, generators) else: - gen_key = GeneratorKey(machine=machine.name, name=generator_name) - result_closure = minimal_closure([gen_key], generators) - - if include_previous_values: - for generator in result_closure: - for prompt in generator.prompts: - prompt.previous_value = generator.get_previous_value(machine, prompt) + roots = [key for key in generators if key.name == generator_name] + result_closure = minimal_closure(roots, generators) return result_closure @@ -66,7 +66,7 @@ def _ensure_healthy( Fails if any of the generators' health checks fail. """ if generators is None: - generators = Generator.get_machine_generators(machine.name, machine.flake) + generators = Generator.get_machine_generators([machine.name], machine.flake) pub_healtcheck_msg = machine.public_vars_store.health_check( machine.name, @@ -133,12 +133,12 @@ def run_generators( generator_keys = { GeneratorKey(machine=machine.name, name=name) for name in generators } - all_generators = get_generators(machine, full_closure=True) + all_generators = get_generators([machine], full_closure=True) generator_objects = [g for g in all_generators if g.key in generator_keys] else: # None or single string - use get_generators with closure parameter generator_objects = get_generators( - machine, + [machine], full_closure=full_closure, generator_name=generators, )