diff --git a/pkgs/clan-app/ui/src/workflows/Install/steps/installSteps.tsx b/pkgs/clan-app/ui/src/workflows/Install/steps/installSteps.tsx index c39df7dd9..fdaedd0fc 100644 --- a/pkgs/clan-app/ui/src/workflows/Install/steps/installSteps.tsx +++ b/pkgs/clan-app/ui/src/workflows/Install/steps/installSteps.tsx @@ -552,12 +552,14 @@ const InstallSummary = () => { const runGenerators = client.fetch("run_generators", { prompt_values: store.install.promptValues, - machine: { - name: store.install.machineName, - flake: { - identifier: clanUri, + machines: [ + { + name: store.install.machineName, + flake: { + identifier: clanUri, + }, }, - }, + ], }); set("install", (s) => ({ diff --git a/pkgs/clan-cli/clan_cli/tests/test_vars.py b/pkgs/clan-cli/clan_cli/tests/test_vars.py index 486317bf7..89eae7a82 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_vars.py +++ b/pkgs/clan-cli/clan_cli/tests/test_vars.py @@ -9,11 +9,13 @@ from clan_cli.tests.fixtures_flakes import ClanFlake from clan_cli.tests.helpers import cli from clan_cli.vars.check import check_vars from clan_cli.vars.generate import ( - Generator, - GeneratorKey, get_generators, run_generators, ) +from clan_cli.vars.generator import ( + Generator, + GeneratorKey, +) from clan_cli.vars.get import get_machine_var from clan_cli.vars.graph import all_missing_closure, requested_closure from clan_cli.vars.list import stringify_all_vars @@ -698,8 +700,8 @@ def test_api_set_prompts( monkeypatch.chdir(flake.path) run_generators( - machine=Machine(name="my_machine", flake=Flake(str(flake.path))), - generators=[GeneratorKey(machine="my_machine", name="my_generator")], + machines=[Machine(name="my_machine", flake=Flake(str(flake.path)))], + generators=["my_generator"], prompt_values={ "my_generator": { "prompt1": "input1", @@ -712,8 +714,8 @@ def test_api_set_prompts( assert store.exists(my_generator, "prompt1") assert store.get(my_generator, "prompt1").decode() == "input1" run_generators( - machine=Machine(name="my_machine", flake=Flake(str(flake.path))), - generators=[GeneratorKey(machine="my_machine", name="my_generator")], + machines=[Machine(name="my_machine", flake=Flake(str(flake.path)))], + generators=["my_generator"], prompt_values={ "my_generator": { "prompt1": "input2", @@ -759,8 +761,8 @@ def test_stdout_of_generate( # with capture_output as output: with caplog.at_level(logging.INFO): run_generators( - Machine(name="my_machine", flake=flake), - generators=[GeneratorKey(machine="my_machine", name="my_generator")], + machines=[Machine(name="my_machine", flake=flake)], + generators=["my_generator"], ) assert "Updated var my_generator/my_value" in caplog.text @@ -771,8 +773,8 @@ def test_stdout_of_generate( set_var("my_machine", "my_generator/my_value", b"world", flake) with caplog.at_level(logging.INFO): run_generators( - Machine(name="my_machine", flake=flake), - generators=[GeneratorKey(machine="my_machine", name="my_generator")], + machines=[Machine(name="my_machine", flake=flake)], + generators=["my_generator"], ) assert "Updated var my_generator/my_value" in caplog.text assert "old: world" in caplog.text @@ -781,16 +783,16 @@ def test_stdout_of_generate( # check the output when nothing gets regenerated with caplog.at_level(logging.INFO): run_generators( - Machine(name="my_machine", flake=flake), - generators=[GeneratorKey(machine="my_machine", name="my_generator")], + machines=[Machine(name="my_machine", flake=flake)], + generators=["my_generator"], ) assert "Updated var" not in caplog.text assert "hello" in caplog.text caplog.clear() with caplog.at_level(logging.INFO): run_generators( - Machine(name="my_machine", flake=flake), - generators=[GeneratorKey(machine="my_machine", name="my_secret_generator")], + machines=[Machine(name="my_machine", flake=flake)], + generators=["my_secret_generator"], ) assert "Updated secret var my_secret_generator/my_secret" in caplog.text assert "hello" not in caplog.text @@ -803,8 +805,8 @@ def test_stdout_of_generate( ) with caplog.at_level(logging.INFO): run_generators( - Machine(name="my_machine", flake=flake), - generators=[GeneratorKey(machine="my_machine", name="my_secret_generator")], + machines=[Machine(name="my_machine", flake=flake)], + generators=["my_secret_generator"], ) assert "Updated secret var my_secret_generator/my_secret" in caplog.text assert "world" not in caplog.text @@ -892,8 +894,8 @@ def test_fails_when_files_are_left_from_other_backend( monkeypatch.chdir(flake.path) for generator in ["my_secret_generator", "my_value_generator"]: run_generators( - Machine(name="my_machine", flake=Flake(str(flake.path))), - generators=GeneratorKey(machine="my_machine", name=generator), + machines=[Machine(name="my_machine", flake=Flake(str(flake.path)))], + generators=generator, ) # Will raise. It was secret before, but now it's not. my_secret_generator["files"]["my_secret"]["secret"] = ( @@ -908,13 +910,13 @@ def test_fails_when_files_are_left_from_other_backend( if generator == "my_secret_generator": with pytest.raises(ClanError): run_generators( - Machine(name="my_machine", flake=Flake(str(flake.path))), - generators=GeneratorKey(machine="my_machine", name=generator), + machines=[Machine(name="my_machine", flake=Flake(str(flake.path)))], + generators=generator, ) else: run_generators( - Machine(name="my_machine", flake=Flake(str(flake.path))), - generators=GeneratorKey(machine="my_machine", name=generator), + machines=[Machine(name="my_machine", flake=Flake(str(flake.path)))], + generators=generator, ) diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 6f385144c..2fa018253 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -1,7 +1,6 @@ import argparse import logging from collections.abc import Callable -from typing import Literal from clan_cli.completions import ( add_dynamic_completer, @@ -106,19 +105,17 @@ strategies (e.g., interactive CLI, GUI, or programmatic). @API.register def run_generators( - machine: Machine, - generators: GeneratorKey - | list[GeneratorKey] - | Literal["all", "minimal"] = "minimal", + machines: list[Machine], + generators: str | list[str] = "minimal", prompt_values: dict[str, dict[str, str]] | PromptFunc = lambda g: g.ask_prompts(), no_sandbox: bool = False, ) -> None: - """Run the specified generators for a machine. + """Run the specified generators for machines. Args: - machine: The machine to run generators for. + machines: The machines to run generators for. generators: Can be: - - GeneratorKey: Single generator to run (ensuring dependencies are met) - - list[GeneratorKey]: Specific generators to run exactly as provided. + - str: Single generator name to run (ensuring dependencies are met) + - list[str]: Specific generator names to run exactly as provided. Dependency generators are not added automatically in this case. The caller must ensure that all dependencies are included. - "all": Run all generators (full closure) @@ -130,77 +127,49 @@ def run_generators( ClanError: If the machine or generator is not found, or if there are issues with executing the generator. """ - - if generators == "all": - generator_objects = get_generators(machine, full_closure=True) - elif generators == "minimal": - generator_objects = get_generators(machine, full_closure=False) - elif isinstance(generators, GeneratorKey): - # Single generator - compute minimal closure for it - generator_objects = get_generators( - machine, full_closure=False, generator_name=generators.name - ) - elif isinstance(generators, list): - if len(generators) == 0: - return - generator_keys = set(generators) - all_generators = get_generators(machine, full_closure=True) - generator_objects = [g for g in all_generators if g.key in generator_keys] - else: - msg = f"Invalid generators argument: {generators}. Must be 'all', 'minimal', GeneratorKey, or a list of GeneratorKey" - raise ValueError(msg) - - # If prompt function provided, ask all prompts - # TODO: make this more lazy and ask for every generator on execution - if callable(prompt_values): - prompt_values = { - generator.name: prompt_values(generator) for generator in generator_objects - } - # execute health check - _ensure_healthy(machine=machine, generators=generator_objects) - - # execute generators - for generator in generator_objects: - if check_can_migrate(machine, generator): - migrate_files(machine, generator) - else: - generator.execute( - machine=machine, - prompt_values=prompt_values.get(generator.name, {}), - no_sandbox=no_sandbox, - ) - - -def generate_vars( - machines: list["Machine"], - generator_name: str | None = None, - regenerate: bool = False, - no_sandbox: bool = False, -) -> None: for machine in machines: - errors = [] - try: - generators: GeneratorKey | Literal["all", "minimal"] - if generator_name: - generators = GeneratorKey(machine=machine.name, name=generator_name) - else: - generators = "all" if regenerate else "minimal" - - run_generators( - machine, - generators=generators, - no_sandbox=no_sandbox, + if generators == "all": + generator_objects = get_generators(machine, full_closure=True) + elif generators == "minimal": + generator_objects = get_generators(machine, full_closure=False) + elif isinstance(generators, str) and generators not in ["all", "minimal"]: + # Single generator name - compute minimal closure for it + generator_objects = get_generators( + machine, full_closure=False, generator_name=generators ) - machine.info("All vars are up to date") - except Exception as exc: - errors += [(machine, exc)] - if len(errors) == 1: - raise errors[0][1] - if len(errors) > 1: - msg = f"Failed to generate vars for {len(errors)} hosts:" - for machine, error in errors: - msg += f"\n{machine}: {error}" - raise ClanError(msg) from errors[0][1] + elif isinstance(generators, list): + if len(generators) == 0: + return + # Create GeneratorKeys for this specific machine + generator_keys = { + GeneratorKey(machine=machine.name, name=name) for name in generators + } + all_generators = get_generators(machine, full_closure=True) + generator_objects = [g for g in all_generators if g.key in generator_keys] + else: + msg = f"Invalid generators argument: {generators}. Must be 'all', 'minimal', a generator name, or a list of generator names" + raise ValueError(msg) + + # If prompt function provided, ask all prompts + # TODO: make this more lazy and ask for every generator on execution + if callable(prompt_values): + prompt_values = { + generator.name: prompt_values(generator) + for generator in generator_objects + } + # execute health check + _ensure_healthy(machine=machine, generators=generator_objects) + + # execute generators + for generator in generator_objects: + if check_can_migrate(machine, generator): + migrate_files(machine, generator) + else: + generator.execute( + machine=machine, + prompt_values=prompt_values.get(generator.name, {}), + no_sandbox=no_sandbox, + ) def generate_command(args: argparse.Namespace) -> None: @@ -225,10 +194,14 @@ def generate_command(args: argparse.Namespace) -> None: f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.validationHash", ] ) - generate_vars( + + run_generators( machines, - args.generator, - args.regenerate, + generators=args.generator + if args.generator + else "all" + if args.regenerate + else "minimal", no_sandbox=args.no_sandbox, ) diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index 621a5bfff..955ee0699 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -22,7 +22,7 @@ from clan_cli.completions import add_dynamic_completer, complete_machines from clan_cli.facts.generate import generate_facts from clan_cli.qemu.qga import QgaSession from clan_cli.qemu.qmp import QEMUMonitorProtocol -from clan_cli.vars.generate import generate_vars +from clan_cli.vars.generate import run_generators from clan_cli.vars.upload import populate_secret_vars from .inspect import VmConfig, inspect_vm @@ -85,7 +85,7 @@ def get_secrets( secrets_dir.mkdir(parents=True, exist_ok=True) generate_facts([machine]) - generate_vars([machine]) + run_generators([machine]) machine.secret_facts_store.upload(secrets_dir) populate_secret_vars(machine, secrets_dir) diff --git a/pkgs/clan-cli/clan_lib/flash/flash.py b/pkgs/clan-cli/clan_lib/flash/flash.py index 7127e15d0..74a980ea5 100644 --- a/pkgs/clan-cli/clan_lib/flash/flash.py +++ b/pkgs/clan-cli/clan_lib/flash/flash.py @@ -7,7 +7,7 @@ from tempfile import TemporaryDirectory from typing import Any, Literal from clan_cli.facts.generate import generate_facts -from clan_cli.vars.generate import generate_vars +from clan_cli.vars.generate import run_generators from clan_cli.vars.upload import populate_secret_vars from clan_lib.api import API @@ -78,7 +78,7 @@ def run_machine_flash( system_config_nix: dict[str, Any] = {} generate_facts([machine]) - generate_vars([machine]) + run_generators([machine]) if system_config.language: if system_config.language not in list_languages(): @@ -113,7 +113,7 @@ def run_machine_flash( "users": {"root": {"openssh": {"authorizedKeys": {"keys": root_keys}}}} } - from clan_cli.vars.generate import Generator + from clan_cli.vars.generator import Generator for generator in Generator.get_machine_generators(machine.name, machine.flake): for file in generator.files: diff --git a/pkgs/clan-cli/clan_lib/machines/install.py b/pkgs/clan-cli/clan_lib/machines/install.py index 3d8fd2741..aaa6d8568 100644 --- a/pkgs/clan-cli/clan_lib/machines/install.py +++ b/pkgs/clan-cli/clan_lib/machines/install.py @@ -7,7 +7,7 @@ from typing import Literal from clan_cli.facts.generate import generate_facts from clan_cli.machines.hardware import HardwareConfig -from clan_cli.vars.generate import generate_vars +from clan_cli.vars.generate import run_generators from clan_lib.api import API, message_queue from clan_lib.cmd import Log, RunOpts, run @@ -88,7 +88,7 @@ def run_machine_install(opts: InstallOptions, target_host: Remote) -> None: # Notify the UI about what we are doing notify_install_step("generators") generate_facts([machine]) - generate_vars([machine]) + run_generators([machine]) with ( TemporaryDirectory(prefix="nixos-install-") as _base_directory, diff --git a/pkgs/clan-cli/clan_lib/machines/morph.py b/pkgs/clan-cli/clan_lib/machines/morph.py index 478899b6f..2b6fd7aaa 100644 --- a/pkgs/clan-cli/clan_lib/machines/morph.py +++ b/pkgs/clan-cli/clan_lib/machines/morph.py @@ -7,7 +7,7 @@ from pathlib import Path from tempfile import TemporaryDirectory from clan_cli.machines.create import CreateOptions, create_machine -from clan_cli.vars.generate import generate_vars +from clan_cli.vars.generate import run_generators from clan_lib.cmd import Log, RunOpts, run from clan_lib.dirs import specific_machine_dir @@ -78,7 +78,7 @@ def morph_machine( machine = Machine(name=name, flake=Flake(str(flakedir))) - generate_vars([machine], generator_name=None, regenerate=False) + run_generators([machine], generators="minimal") machine.secret_vars_store.populate_dir( machine.name, diff --git a/pkgs/clan-cli/clan_lib/machines/update.py b/pkgs/clan-cli/clan_lib/machines/update.py index ad760e76f..b42e26ae1 100644 --- a/pkgs/clan-cli/clan_lib/machines/update.py +++ b/pkgs/clan-cli/clan_lib/machines/update.py @@ -7,7 +7,7 @@ from contextlib import ExitStack from clan_cli.facts.generate import generate_facts from clan_cli.facts.upload import upload_secrets -from clan_cli.vars.generate import generate_vars +from clan_cli.vars.generate import run_generators from clan_cli.vars.upload import upload_secret_vars from clan_lib.api import API @@ -148,7 +148,7 @@ def run_machine_update( target_host_root = stack.enter_context(_target_host.become_root()) generate_facts([machine], service=None, regenerate=False) - generate_vars([machine], generator_name=None, regenerate=False) + run_generators([machine], generators="minimal") # Upload secrets to the target host using root upload_secrets(machine, target_host_root) diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index f4da2a0d9..e21dd3097 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -231,8 +231,8 @@ def test_clan_create_api( collected_prompt_values[generator.name] = prompt_values run_generators( - machine=machine, - generators=[gen.key for gen in generators], + machines=[machine], + generators=[gen.name for gen in generators], prompt_values=collected_prompt_values, ) diff --git a/pkgs/generate-test-vars/generate_test_vars/cli.py b/pkgs/generate-test-vars/generate_test_vars/cli.py index b07566aff..b96ed65ae 100755 --- a/pkgs/generate-test-vars/generate_test_vars/cli.py +++ b/pkgs/generate-test-vars/generate_test_vars/cli.py @@ -10,9 +10,9 @@ from dataclasses import dataclass from pathlib import Path from tempfile import NamedTemporaryFile from typing import Any, override -from unittest.mock import patch -from clan_cli.vars.generate import Generator, generate_vars +from clan_cli.vars.generate import run_generators +from clan_cli.vars.generator import Generator from clan_cli.vars.prompt import PromptType from clan_lib.dirs import find_toplevel from clan_lib.errors import ClanError @@ -241,16 +241,13 @@ def main() -> None: raise ClanError(msg) return prompt_values - with ( - patch("clan_cli.vars.generate._ask_prompts", new=mocked_prompts), - NamedTemporaryFile("w") as f, - ): + with NamedTemporaryFile("w") as f: f.write("# created: 2023-07-17T10:51:45+02:00\n") f.write(f"# public key: {sops_pub_key}\n") f.write(sops_priv_key) f.seek(0) os.environ["SOPS_AGE_KEY_FILE"] = f.name - generate_vars(list(machines)) + run_generators(list(machines), prompt_values=mocked_prompts) if __name__ == "__main__":