From c760561dbd80229f012deec87d0e61e1f7c200e4 Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 2 Jul 2025 10:19:54 +0200 Subject: [PATCH] refactor: remove _serialized field and implement efficient vars selection - Remove _serialized field from vars interface to prevent serialization errors with throwing passBackend field - Implement direct selection of generator fields using multi-select syntax - Refactor vars_generators() to use new Generator.from_flake() method that selects only safe fields (avoiding non-serializable values) - Remove unused legacy methods: Generator.from_json(), Var.from_json(), Prompt.from_json() - Update precaching to match new selection approach This fixes the serialization errors that were preventing vars from working with the new password-store implementation by avoiding the problematic _serialized field entirely. --- nixosModules/clanCore/vars/interface.nix | 44 ------------ pkgs/clan-cli/clan_cli/machines/update.py | 3 +- .../clan_cli/tests/test_secrets_generate.py | 2 +- pkgs/clan-cli/clan_cli/vars/generate.py | 69 +++++++++++++++---- pkgs/clan-cli/clan_cli/vars/prompt.py | 8 +-- pkgs/clan-cli/clan_cli/vars/var.py | 15 +--- pkgs/clan-cli/clan_lib/machines/machines.py | 14 +--- 7 files changed, 66 insertions(+), 89 deletions(-) diff --git a/nixosModules/clanCore/vars/interface.nix b/nixosModules/clanCore/vars/interface.nix index 5bfa45d8e..e69298e6a 100644 --- a/nixosModules/clanCore/vars/interface.nix +++ b/nixosModules/clanCore/vars/interface.nix @@ -34,50 +34,6 @@ let in { options = { - _serialized = lib.mkOption { - readOnly = true; - internal = true; - description = '' - JSON serialization of the generators. - This is read from the python client to generate the specified resources. - ''; - default = { - # TODO: We don't support per-machine choice of backends - # Configuring different backend doesn't work, this information should be made read only and configured - # Via clan.settings instead. - inherit (config.settings) secretModule publicModule; - # Serialize generators, so that we can use them in the python client - # This need to be done because we have some non-serializable values in the generators - # Like the finalScript (derivation) or pkgs. - generators = lib.flip lib.mapAttrs config.generators ( - _name: generator: { - inherit (generator) - name - dependencies - validationHash - migrateFact - share - prompts - ; - - files = lib.flip lib.mapAttrs generator.files ( - _name: file: { - inherit (file) - name - owner - group - mode - deploy - secret - neededFor - ; - } - ); - } - ); - }; - }; - settings = import ./settings-opts.nix { inherit lib; }; generators = lib.mkOption { description = '' diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index c6ad2b119..978db7cef 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -106,7 +106,8 @@ def update_command(args: argparse.Namespace) -> None: f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.secretModule", f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.publicModule", f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.services", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars._serialized.generators", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.{{share,dependencies,migrateFact,prompts}}", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.files.*.{{secret,deploy,owner,group,mode,neededFor}}", f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.secretUploadDirectory", f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.vars.password-store.secretLocation", f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.passBackend", diff --git a/pkgs/clan-cli/clan_cli/tests/test_secrets_generate.py b/pkgs/clan-cli/clan_cli/tests/test_secrets_generate.py index 4e311384a..10829f66e 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_secrets_generate.py +++ b/pkgs/clan-cli/clan_cli/tests/test_secrets_generate.py @@ -107,7 +107,7 @@ def test_generate_secret( ) assert store2.exists("", "age.key") - ( + assert ( test_flake_with_core.path / "vars" / "per-machine" diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 40a0cb2d1..1a14434ca 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -35,6 +35,7 @@ from .var import Var log = logging.getLogger(__name__) if TYPE_CHECKING: + from clan_lib.flake import Flake from clan_lib.machines.machines import Machine @@ -60,15 +61,63 @@ class Generator: return check_vars(self._machine, generator_name=self.name) @classmethod - def from_json(cls: type["Generator"], data: dict[str, Any]) -> "Generator": - return cls( - name=data["name"], - share=data["share"], - files=[Var.from_json(data["name"], f) for f in data["files"].values()], - dependencies=data["dependencies"], - migrate_fact=data["migrateFact"], - prompts=[Prompt.from_json(p) for p in data["prompts"].values()], + def generators_from_flake( + cls: type["Generator"], machine_name: str, flake: "Flake", machine: "Machine" + ) -> list["Generator"]: + config = nix_config() + system = config["system"] + + # Get all generator metadata in one select (safe fields only) + generators_data = flake.select( + f'clanInternals.machines."{system}"."{machine_name}".config.clan.core.vars.generators.*.{{share,dependencies,migrateFact,prompts}}' ) + if not generators_data: + return [] + + # Get all file metadata in one select + files_data = flake.select( + f'clanInternals.machines."{system}"."{machine_name}".config.clan.core.vars.generators.*.files.*.{{secret,deploy,owner,group,mode,neededFor}}' + ) + + generators = [] + for gen_name, gen_data in generators_data.items(): + # Build files from the files_data + files = [] + gen_files = files_data.get(gen_name, {}) + for file_name, file_data in gen_files.items(): + # Handle mode conversion properly + mode = file_data["mode"] + if isinstance(mode, str): + mode = int(mode, 8) + + var = Var( + id=f"{gen_name}/{file_name}", + name=file_name, + secret=file_data["secret"], + deploy=file_data["deploy"], + owner=file_data["owner"], + group=file_data["group"], + mode=mode, + needed_for=file_data["neededFor"], + ) + files.append(var) + + # Build prompts + prompts = [Prompt.from_nix(p) for p in gen_data.get("prompts", {}).values()] + + generator = cls( + name=gen_name, + share=gen_data["share"], + files=files, + dependencies=gen_data["dependencies"], + migrate_fact=gen_data.get("migrateFact"), + prompts=prompts, + ) + # Set the machine immediately + generator.machine(machine) + generators.append(generator) + + return generators def final_script(self) -> Path: assert self._machine is not None @@ -351,10 +400,6 @@ def get_closure( generator.name: generator for generator in vars_generators } - # TODO: we should remove this - for generator in vars_generators: - generator.machine(machine) - result_closure = [] if generator_name is None: # all generators selected if full_closure: diff --git a/pkgs/clan-cli/clan_cli/vars/prompt.py b/pkgs/clan-cli/clan_cli/vars/prompt.py index ba48c8566..ffc2b9cea 100644 --- a/pkgs/clan-cli/clan_cli/vars/prompt.py +++ b/pkgs/clan-cli/clan_cli/vars/prompt.py @@ -32,12 +32,12 @@ class Prompt: previous_value: str | None = None @classmethod - def from_json(cls: type["Prompt"], data: dict[str, Any]) -> "Prompt": + def from_nix(cls: type["Prompt"], data: dict[str, Any]) -> "Prompt": return cls( name=data["name"], - description=data["description"], - prompt_type=PromptType(data["type"]), - persist=data.get("persist", data["persist"]), + description=data.get("description", data["name"]), + prompt_type=PromptType(data.get("type", "line")), + persist=data.get("persist", False), ) diff --git a/pkgs/clan-cli/clan_cli/vars/var.py b/pkgs/clan-cli/clan_cli/vars/var.py index 5c41fac04..8ccc568bb 100644 --- a/pkgs/clan-cli/clan_cli/vars/var.py +++ b/pkgs/clan-cli/clan_cli/vars/var.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING if TYPE_CHECKING: from clan_cli.vars.generate import Generator @@ -67,16 +67,3 @@ class Var: return f"{self.id}: ********" return f"{self.id}: {self.printable_value}" return f"{self.id}: " - - @classmethod - def from_json(cls: type["Var"], generator_name: str, data: dict[str, Any]) -> "Var": - return cls( - id=f"{generator_name}/{data['name']}", - name=data["name"], - secret=data["secret"], - deploy=data["deploy"], - owner=data.get("owner", "root"), - group=data.get("group", "root"), - mode=int(data.get("mode", "0400"), 8), - needed_for=data.get("neededFor", "services"), - ) diff --git a/pkgs/clan-cli/clan_lib/machines/machines.py b/pkgs/clan-cli/clan_lib/machines/machines.py index 5b2e4656a..4324a7ca2 100644 --- a/pkgs/clan-cli/clan_lib/machines/machines.py +++ b/pkgs/clan-cli/clan_lib/machines/machines.py @@ -112,19 +112,7 @@ class Machine: def vars_generators(self) -> list["Generator"]: from clan_cli.vars.generate import Generator - try: - generators_data = self.select( - "config.clan.core.vars._serialized.generators" - ) - if generators_data is None: - return [] - _generators = [Generator.from_json(gen) for gen in generators_data.values()] - for gen in _generators: - gen.machine(self) - except Exception: - return [] - else: - return _generators + return Generator.generators_from_flake(self.name, self.flake, self) @property def secrets_upload_directory(self) -> str: