yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'refactor: remove _serialized field and implement efficient vars selection' (#4187) from remove_serialized into main

Browse Source 
Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4187
This commit is contained in:
lassulus
2025-07-03 17:18:27 +00:00
parent d5a1aba480 c760561dbd
commit 401799b5bc
9 changed files with 90 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
nixosModules/clanCore/vars/interface.nix
View File

@@ -34,50 +34,6 @@ let
in
{
options = {
_serialized = lib.mkOption {
readOnly = true;
internal = true;
description = ''
JSON serialization of the generators.
This is read from the python client to generate the specified resources.
'';
default = {
# TODO: We don't support per-machine choice of backends
# Configuring different backend doesn't work, this information should be made read only and configured
# Via clan.settings instead.
inherit (config.settings) secretModule publicModule;
# Serialize generators, so that we can use them in the python client
# This need to be done because we have some non-serializable values in the generators
# Like the finalScript (derivation) or pkgs.
generators = lib.flip lib.mapAttrs config.generators (
_name: generator: {
inherit (generator)
name
dependencies
validationHash
migrateFact
share
prompts
;
files = lib.flip lib.mapAttrs generator.files (
_name: file: {
inherit (file)
name
owner
group
mode
deploy
secret
neededFor
;
}
);
}
);
};
};
settings = import ./settings-opts.nix { inherit lib; };
generators = lib.mkOption {
description = ''

3
pkgs/clan-cli/clan_cli/machines/update.py
View File

@@ -106,7 +106,8 @@ def update_command(args: argparse.Namespace) -> None:
f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.secretModule",
f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.publicModule",
f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.services",
f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars._serialized.generators",
f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.{{share,dependencies,migrateFact,prompts}}",
f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.files.*.{{secret,deploy,owner,group,mode,neededFor}}",
f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.secretUploadDirectory",
f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.vars.password-store.secretLocation",
f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.passBackend",

2
pkgs/clan-cli/clan_cli/tests/test_secrets_generate.py
View File

@@ -107,7 +107,7 @@ def test_generate_secret(
)
assert store2.exists("", "age.key")
(
assert (
test_flake_with_core.path
/ "vars"
/ "per-machine"

69
pkgs/clan-cli/clan_cli/vars/generate.py
View File

@@ -35,6 +35,7 @@ from .var import Var
log = logging.getLogger(__name__)
if TYPE_CHECKING:
from clan_lib.flake import Flake
from clan_lib.machines.machines import Machine
@@ -60,15 +61,63 @@ class Generator:
return check_vars(self._machine, generator_name=self.name)
@classmethod
def from_json(cls: type["Generator"], data: dict[str, Any]) -> "Generator":
return cls(
name=data["name"],
share=data["share"],
files=[Var.from_json(data["name"], f) for f in data["files"].values()],
dependencies=data["dependencies"],
migrate_fact=data["migrateFact"],
prompts=[Prompt.from_json(p) for p in data["prompts"].values()],
def generators_from_flake(
cls: type["Generator"], machine_name: str, flake: "Flake", machine: "Machine"
) -> list["Generator"]:
config = nix_config()
system = config["system"]
# Get all generator metadata in one select (safe fields only)
generators_data = flake.select(
f'clanInternals.machines."{system}"."{machine_name}".config.clan.core.vars.generators.*.{{share,dependencies,migrateFact,prompts}}'
)
if not generators_data:
return []
# Get all file metadata in one select
files_data = flake.select(
f'clanInternals.machines."{system}"."{machine_name}".config.clan.core.vars.generators.*.files.*.{{secret,deploy,owner,group,mode,neededFor}}'
)
generators = []
for gen_name, gen_data in generators_data.items():
# Build files from the files_data
files = []
gen_files = files_data.get(gen_name, {})
for file_name, file_data in gen_files.items():
# Handle mode conversion properly
mode = file_data["mode"]
if isinstance(mode, str):
mode = int(mode, 8)
var = Var(
id=f"{gen_name}/{file_name}",
name=file_name,
secret=file_data["secret"],
deploy=file_data["deploy"],
owner=file_data["owner"],
group=file_data["group"],
mode=mode,
needed_for=file_data["neededFor"],
)
files.append(var)
# Build prompts
prompts = [Prompt.from_nix(p) for p in gen_data.get("prompts", {}).values()]
generator = cls(
name=gen_name,
share=gen_data["share"],
files=files,
dependencies=gen_data["dependencies"],
migrate_fact=gen_data.get("migrateFact"),
prompts=prompts,
)
# Set the machine immediately
generator.machine(machine)
generators.append(generator)
return generators
def final_script(self) -> Path:
assert self._machine is not None
@@ -351,10 +400,6 @@ def get_closure(
generator.name: generator for generator in vars_generators
}
# TODO: we should remove this
for generator in vars_generators:
generator.machine(machine)
result_closure = []
if generator_name is None: # all generators selected
if full_closure:

8
pkgs/clan-cli/clan_cli/vars/prompt.py
View File

@@ -32,12 +32,12 @@ class Prompt:
previous_value: str | None = None
@classmethod
def from_json(cls: type["Prompt"], data: dict[str, Any]) -> "Prompt":
def from_nix(cls: type["Prompt"], data: dict[str, Any]) -> "Prompt":
return cls(
name=data["name"],
description=data["description"],
prompt_type=PromptType(data["type"]),
persist=data.get("persist", data["persist"]),
description=data.get("description", data["name"]),
prompt_type=PromptType(data.get("type", "line")),
persist=data.get("persist", False),
)

15
pkgs/clan-cli/clan_cli/vars/var.py
View File

@@ -1,6 +1,6 @@
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from clan_cli.vars.generate import Generator
@@ -67,16 +67,3 @@ class Var:
return f"{self.id}: ********"
return f"{self.id}: {self.printable_value}"
return f"{self.id}: <not set>"
@classmethod
def from_json(cls: type["Var"], generator_name: str, data: dict[str, Any]) -> "Var":
return cls(
id=f"{generator_name}/{data['name']}",
name=data["name"],
secret=data["secret"],
deploy=data["deploy"],
owner=data.get("owner", "root"),
group=data.get("group", "root"),
mode=int(data.get("mode", "0400"), 8),
needed_for=data.get("neededFor", "services"),
)

10
pkgs/clan-cli/clan_lib/flake/flake.py
View File

@@ -168,6 +168,16 @@ def parse_selector(selector: str) -> list[Selector]:
else:
set_select_type = SetSelectorType.STR
acc_selectors.append(SetSelector(type=set_select_type, value=acc_str))
# Check for invalid multiselect patterns with outPath
for subselector in acc_selectors:
if subselector.value == "outPath":
msg = (
"Cannot use 'outPath' in multiselect {...}. "
"When nix evaluates attrsets with outPath in a multiselect, "
"it collapses the entire attrset to just the outPath string, "
"breaking further selection. Use individual selectors instead."
)
raise ValueError(msg)
selectors.append(Selector(type=SelectorType.SET, value=acc_selectors))
submode = ""

14
pkgs/clan-cli/clan_lib/flake/flake_test.py
View File

@@ -168,6 +168,20 @@ def test_out_path() -> None:
assert test_cache.select(selectors) == "/nix/store/bla"
def test_out_path_in_multiselect_raises_exception() -> None:
with pytest.raises(ValueError, match="Cannot use 'outPath' in multiselect"):
parse_selector("{outPath}")
with pytest.raises(ValueError, match="Cannot use 'outPath' in multiselect"):
parse_selector("x.{y,outPath}")
with pytest.raises(ValueError, match="Cannot use 'outPath' in multiselect"):
parse_selector("a.b.{c,d,outPath,e}")
with pytest.raises(ValueError, match="Cannot use 'outPath' in multiselect"):
parse_selector("{?outPath}")
@pytest.mark.with_core
def test_conditional_all_selector(flake: ClanFlake) -> None:
m1 = flake.machines["machine1"]

14
pkgs/clan-cli/clan_lib/machines/machines.py
View File

@@ -112,19 +112,7 @@ class Machine:
def vars_generators(self) -> list["Generator"]:
from clan_cli.vars.generate import Generator
try:
generators_data = self.select(
"config.clan.core.vars._serialized.generators"
)
if generators_data is None:
return []
_generators = [Generator.from_json(gen) for gen in generators_data.values()]
for gen in _generators:
gen.machine(self)
except Exception:
return []
else:
return _generators
return Generator.generators_from_flake(self.name, self.flake, self)
@property
def secrets_upload_directory(self) -> str: