yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

vars: retrieve generators for multiple machines

Browse Source 
This is necessary ground work for fixing regeneration behavior spanning over multiple machines
This commit is contained in:
DavHau
2025-08-26 17:13:01 +07:00
parent a9bafd71e1
commit 501d020562
11 changed files with 121 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
pkgs/clan-app/ui/src/hooks/queries.ts
View File

@@ -433,12 +433,14 @@ export const useMachineGenerators = (
],
queryFn: async () => {
const call = client.fetch("get_generators", {
machine: {
name: machineName,
flake: {
identifier: clanUri,
machines: [
{
name: machineName,
flake: {
identifier: clanUri,
},
},
},
],
full_closure: true, // TODO: Make this configurable
// TODO: Make this configurable
include_previous_values: true,

2
pkgs/clan-cli/clan_cli/tests/test_vars.py
View File

@@ -891,7 +891,7 @@ def test_api_set_prompts(
machine = Machine(name="my_machine", flake=Flake(str(flake.path)))
generators = get_generators(
machine=machine,
machines=[machine],
full_closure=True,
include_previous_values=True,
)

2
pkgs/clan-cli/clan_cli/vars/check.py
View File

@@ -40,7 +40,7 @@ def vars_status(
invalid_generators = []
from clan_cli.vars.generator import Generator
generators = Generator.get_machine_generators(machine.name, machine.flake)
generators = Generator.get_machine_generators([machine.name], machine.flake)
if generator_name:
for generator in generators:
if generator_name == generator.name:

2
pkgs/clan-cli/clan_cli/vars/fix.py
View File

@@ -12,7 +12,7 @@ log = logging.getLogger(__name__)
def fix_vars(machine: Machine, generator_name: None | str = None) -> None:
from clan_cli.vars.generator import Generator
generators = Generator.get_machine_generators(machine.name, machine.flake)
generators = Generator.get_machine_generators([machine.name], machine.flake)
if generator_name:
for generator in generators:
if generator_name == generator.name:

149
pkgs/clan-cli/clan_cli/vars/generator.py
View File

@@ -76,90 +76,111 @@ class Generator:
@classmethod
def get_machine_generators(
cls: type["Generator"],
machine_name: str,
machine_names: list[str],
flake: "Flake",
include_previous_values: bool = False,
) -> list["Generator"]:
"""Get all generators for a machine from the flake.
Args:
machine_name (str): The name of the machine.
flake (Flake): The flake to get the generators from.
machine_names: The names of the machines.
flake: The flake to get the generators from.
include_previous_values: Whether to include previous values in the generators.
Returns:
list[Generator]: A list of (unsorted) generators for the machine.
"""
# Get all generator metadata in one select (safe fields only)
generators_data = flake.select_machine(
machine_name,
"config.clan.core.vars.generators.*.{share,dependencies,migrateFact,prompts}",
)
if not generators_data:
return []
from clan_lib.nix import nix_config
# Get all file metadata in one select
files_data = flake.select_machine(
machine_name,
"config.clan.core.vars.generators.*.files.*.{secret,deploy,owner,group,mode,neededFor}",
)
config = nix_config()
system = config["system"]
from clan_lib.machines.machines import Machine
generators_selector = "config.clan.core.vars.generators.*.{share,dependencies,migrateFact,prompts}"
files_selector = "config.clan.core.vars.generators.*.files.*.{secret,deploy,owner,group,mode,neededFor}"
machine = Machine(name=machine_name, flake=flake)
pub_store = machine.public_vars_store
sec_store = machine.secret_vars_store
# precache all machines generators and files to avoid multiple calls to nix
all_selectors = []
for machine_name in machine_names:
all_selectors += [
f'clanInternals.machines."{system}"."{machine_name}".{generators_selector}',
f'clanInternals.machines."{system}"."{machine_name}".{files_selector}',
]
flake.precache(all_selectors)
generators = []
for gen_name, gen_data in generators_data.items():
# Build files from the files_data
files = []
gen_files = files_data.get(gen_name, {})
for file_name, file_data in gen_files.items():
var = Var(
id=f"{gen_name}/{file_name}",
name=file_name,
secret=file_data["secret"],
deploy=file_data["deploy"],
owner=file_data["owner"],
group=file_data["group"],
mode=(
file_data["mode"]
if isinstance(file_data["mode"], int)
else int(file_data["mode"], 8)
),
needed_for=file_data["neededFor"],
_store=pub_store if not file_data["secret"] else sec_store,
)
files.append(var)
# Build prompts
prompts = [Prompt.from_nix(p) for p in gen_data.get("prompts", {}).values()]
generator = cls(
name=gen_name,
share=gen_data["share"],
files=files,
dependencies=[
GeneratorKey(machine=machine_name, name=dep)
for dep in gen_data["dependencies"]
],
migrate_fact=gen_data.get("migrateFact"),
prompts=prompts,
machine=machine_name,
_flake=flake,
for machine_name in machine_names:
# Get all generator metadata in one select (safe fields only)
generators_data = flake.select_machine(
machine_name,
generators_selector,
)
generators.append(generator)
if not generators_data:
return []
# TODO: This should be done in a non-mutable way.
if include_previous_values:
for generator in generators:
for prompt in generator.prompts:
prompt.previous_value = generator.get_previous_value(
machine,
prompt,
# Get all file metadata in one select
files_data = flake.select_machine(
machine_name,
files_selector,
)
from clan_lib.machines.machines import Machine
machine = Machine(name=machine_name, flake=flake)
pub_store = machine.public_vars_store
sec_store = machine.secret_vars_store
for gen_name, gen_data in generators_data.items():
# Build files from the files_data
files = []
gen_files = files_data.get(gen_name, {})
for file_name, file_data in gen_files.items():
var = Var(
id=f"{gen_name}/{file_name}",
name=file_name,
secret=file_data["secret"],
deploy=file_data["deploy"],
owner=file_data["owner"],
group=file_data["group"],
mode=(
file_data["mode"]
if isinstance(file_data["mode"], int)
else int(file_data["mode"], 8)
),
needed_for=file_data["neededFor"],
_store=pub_store if not file_data["secret"] else sec_store,
)
files.append(var)
# Build prompts
prompts = [
Prompt.from_nix(p) for p in gen_data.get("prompts", {}).values()
]
generator = cls(
name=gen_name,
share=gen_data["share"],
files=files,
dependencies=[
GeneratorKey(machine=machine_name, name=dep)
for dep in gen_data["dependencies"]
],
migrate_fact=gen_data.get("migrateFact"),
prompts=prompts,
machine=machine_name,
_flake=flake,
)
generators.append(generator)
# TODO: This should be done in a non-mutable way.
if include_previous_values:
for generator in generators:
for prompt in generator.prompts:
prompt.previous_value = generator.get_previous_value(
machine,
prompt,
)
return generators
@@ -231,7 +252,7 @@ class Generator:
"""
from clan_lib.errors import ClanError
generators = self.get_machine_generators(machine.name, machine.flake)
generators = self.get_machine_generators([machine.name], machine.flake)
result: dict[str, dict[str, bytes]] = {}
for dep_key in set(self.dependencies):

2
pkgs/clan-cli/clan_cli/vars/list.py
View File

@@ -19,7 +19,7 @@ def get_machine_vars(machine: Machine) -> list[Var]:
all_vars = []
generators = get_generators(machine=machine, full_closure=True)
generators = get_generators(machines=[machine], full_closure=True)
for generator in generators:
for var in generator.files:
if var.secret:

4
pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py
View File

@@ -164,7 +164,7 @@ class SecretStore(StoreBase):
from clan_cli.vars.generator import Generator
generators = Generator.get_machine_generators(machine, self.flake)
generators = Generator.get_machine_generators([machine], self.flake)
manifest = [
f"{generator.name}/{file.name}".encode()
for generator in generators
@@ -197,7 +197,7 @@ class SecretStore(StoreBase):
def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None:
from clan_cli.vars.generator import Generator
vars_generators = Generator.get_machine_generators(machine, self.flake)
vars_generators = Generator.get_machine_generators([machine], self.flake)
if "users" in phases:
with tarfile.open(
output_dir / "secrets_for_users.tar.gz",

8
pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py
View File

@@ -56,7 +56,7 @@ class SecretStore(StoreBase):
# no need to generate keys if we don't manage secrets
from clan_cli.vars.generator import Generator
vars_generators = Generator.get_machine_generators(machine, self.flake)
vars_generators = Generator.get_machine_generators([machine], self.flake)
if not vars_generators:
return
has_secrets = False
@@ -143,7 +143,7 @@ class SecretStore(StoreBase):
if generators is None:
from clan_cli.vars.generator import Generator
generators = Generator.get_machine_generators(machine, self.flake)
generators = Generator.get_machine_generators([machine], self.flake)
file_found = False
outdated = []
for generator in generators:
@@ -221,7 +221,7 @@ class SecretStore(StoreBase):
def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None:
from clan_cli.vars.generator import Generator
vars_generators = Generator.get_machine_generators(machine, self.flake)
vars_generators = Generator.get_machine_generators([machine], self.flake)
if "users" in phases or "services" in phases:
key_name = f"{machine}-age.key"
if not has_secret(sops_secrets_folder(self.flake.path) / key_name):
@@ -357,7 +357,7 @@ class SecretStore(StoreBase):
if generators is None:
from clan_cli.vars.generator import Generator
generators = Generator.get_machine_generators(machine, self.flake)
generators = Generator.get_machine_generators([machine], self.flake)
file_found = False
for generator in generators:
for file in generator.files:

4
pkgs/clan-cli/clan_lib/flash/flash.py
View File

@@ -118,7 +118,9 @@ def run_machine_flash(
from clan_cli.vars.generator import Generator
for generator in Generator.get_machine_generators(machine.name, machine.flake):
for generator in Generator.get_machine_generators(
[machine.name], machine.flake
):
for file in generator.files:
if file.needed_for == "partitioning":
msg = f"Partitioning time secrets are not supported with `clan flash write`: clan.core.vars.generators.{generator.name}.files.{file.name}"

2
pkgs/clan-cli/clan_lib/tests/test_create.py
View File

@@ -228,7 +228,7 @@ def test_clan_create_api(
# Invalidate cache because of new inventory
clan_dir_flake.invalidate_cache()
generators = get_generators(machine=machine, full_closure=True)
generators = get_generators(machines=[machine], full_closure=True)
collected_prompt_values = {}
for generator in generators:
prompt_values = {}

30
pkgs/clan-cli/clan_lib/vars/generate.py
View File

@@ -14,7 +14,7 @@ log = logging.getLogger(__name__)
@API.register
def get_generators(
machine: Machine,
machines: list[Machine],
full_closure: bool,
generator_name: str | None = None,
include_previous_values: bool = False,
@@ -22,7 +22,7 @@ def get_generators(
"""Get generators for a machine, with optional closure computation.
Args:
machine: The machine to get generators for.
machines: The machines to get generators for.
full_closure: If True, include all dependency generators. If False, only include missing ones.
generator_name: Name of a specific generator to get, or None for all generators.
include_previous_values: If True, populate prompts with their previous values.
@@ -33,7 +33,12 @@ def get_generators(
"""
from clan_cli.vars import graph
vars_generators = Generator.get_machine_generators(machine.name, machine.flake)
machine_names = [machine.name for machine in machines]
vars_generators = Generator.get_machine_generators(
machine_names,
machines[0].flake,
include_previous_values=include_previous_values,
)
generators = {generator.key: generator for generator in vars_generators}
result_closure = []
@@ -44,16 +49,11 @@ def get_generators(
result_closure = graph.all_missing_closure(generators)
# specific generator selected
elif full_closure:
gen_key = GeneratorKey(machine=machine.name, name=generator_name)
result_closure = requested_closure([gen_key], generators)
roots = [key for key in generators if key.name == generator_name]
result_closure = requested_closure(roots, generators)
else:
gen_key = GeneratorKey(machine=machine.name, name=generator_name)
result_closure = minimal_closure([gen_key], generators)
if include_previous_values:
for generator in result_closure:
for prompt in generator.prompts:
prompt.previous_value = generator.get_previous_value(machine, prompt)
roots = [key for key in generators if key.name == generator_name]
result_closure = minimal_closure(roots, generators)
return result_closure
@@ -66,7 +66,7 @@ def _ensure_healthy(
Fails if any of the generators' health checks fail.
"""
if generators is None:
generators = Generator.get_machine_generators(machine.name, machine.flake)
generators = Generator.get_machine_generators([machine.name], machine.flake)
pub_healtcheck_msg = machine.public_vars_store.health_check(
machine.name,
@@ -133,12 +133,12 @@ def run_generators(
generator_keys = {
GeneratorKey(machine=machine.name, name=name) for name in generators
}
all_generators = get_generators(machine, full_closure=True)
all_generators = get_generators([machine], full_closure=True)
generator_objects = [g for g in all_generators if g.key in generator_keys]
else:
# None or single string - use get_generators with closure parameter
generator_objects = get_generators(
machine,
[machine],
full_closure=full_closure,
generator_name=generators,
)