refactor: remove _serialized field and implement efficient vars selection

- Remove _serialized field from vars interface to prevent serialization
  errors with throwing passBackend field
- Implement direct selection of generator fields using multi-select syntax
- Refactor vars_generators() to use new Generator.from_flake() method that
  selects only safe fields (avoiding non-serializable values)
- Remove unused legacy methods: Generator.from_json(), Var.from_json(),
  Prompt.from_json()
- Update precaching to match new selection approach

This fixes the serialization errors that were preventing vars from working
with the new password-store implementation by avoiding the problematic
_serialized field entirely.

nixosModules/clanCore/vars/interface.nix

@@ -34,50 +34,6 @@ let
 in
 {
   options = {
-    _serialized = lib.mkOption {
-      readOnly = true;
-      internal = true;
-      description = ''
-        JSON serialization of the generators.
-        This is read from the python client to generate the specified resources.
-      '';
-      default = {
-        # TODO: We don't support per-machine choice of backends
-        # Configuring different backend doesn't work, this information should be made read only and configured
-        # Via clan.settings instead.
-        inherit (config.settings) secretModule publicModule;
-        # Serialize generators, so that we can use them in the python client
-        # This need to be done because we have some non-serializable values in the generators
-        # Like the finalScript (derivation) or pkgs.
-        generators = lib.flip lib.mapAttrs config.generators (
-          _name: generator: {
-            inherit (generator)
-              name
-              dependencies
-              validationHash
-              migrateFact
-              share
-              prompts
-              ;
-
-            files = lib.flip lib.mapAttrs generator.files (
-              _name: file: {
-                inherit (file)
-                  name
-                  owner
-                  group
-                  mode
-                  deploy
-                  secret
-                  neededFor
-                  ;
-              }
-            );
-          }
-        );
-      };
-    };
-
     settings = import ./settings-opts.nix { inherit lib; };
     generators = lib.mkOption {
       description = ''

pkgs/clan-cli/clan_cli/machines/update.py

@@ -106,7 +106,8 @@ def update_command(args: argparse.Namespace) -> None:
                     f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.secretModule",
                     f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.publicModule",
                     f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.services",
-                    f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars._serialized.generators",
+                    f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.{{share,dependencies,migrateFact,prompts}}",
+                    f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.files.*.{{secret,deploy,owner,group,mode,neededFor}}",
                     f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.secretUploadDirectory",
                     f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.vars.password-store.secretLocation",
                     f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.passBackend",

pkgs/clan-cli/clan_cli/tests/test_secrets_generate.py

@@ -107,7 +107,7 @@ def test_generate_secret(
     )
 
     assert store2.exists("", "age.key")
-    (
+    assert (
         test_flake_with_core.path
         / "vars"
         / "per-machine"

pkgs/clan-cli/clan_cli/vars/generate.py

@@ -35,6 +35,7 @@ from .var import Var
 log = logging.getLogger(__name__)
 
 if TYPE_CHECKING:
+    from clan_lib.flake import Flake
     from clan_lib.machines.machines import Machine
 
 
@@ -60,15 +61,63 @@ class Generator:
         return check_vars(self._machine, generator_name=self.name)
 
     @classmethod
-    def from_json(cls: type["Generator"], data: dict[str, Any]) -> "Generator":
+    def generators_from_flake(
-        return cls(
+        cls: type["Generator"], machine_name: str, flake: "Flake", machine: "Machine"
-            name=data["name"],
+    ) -> list["Generator"]:
-            share=data["share"],
+        config = nix_config()
-            files=[Var.from_json(data["name"], f) for f in data["files"].values()],
+        system = config["system"]
-            dependencies=data["dependencies"],
+
-            migrate_fact=data["migrateFact"],
+        # Get all generator metadata in one select (safe fields only)
-            prompts=[Prompt.from_json(p) for p in data["prompts"].values()],
+        generators_data = flake.select(
+            f'clanInternals.machines."{system}"."{machine_name}".config.clan.core.vars.generators.*.{{share,dependencies,migrateFact,prompts}}'
         )
+        if not generators_data:
+            return []
+
+        # Get all file metadata in one select
+        files_data = flake.select(
+            f'clanInternals.machines."{system}"."{machine_name}".config.clan.core.vars.generators.*.files.*.{{secret,deploy,owner,group,mode,neededFor}}'
+        )
+
+        generators = []
+        for gen_name, gen_data in generators_data.items():
+            # Build files from the files_data
+            files = []
+            gen_files = files_data.get(gen_name, {})
+            for file_name, file_data in gen_files.items():
+                # Handle mode conversion properly
+                mode = file_data["mode"]
+                if isinstance(mode, str):
+                    mode = int(mode, 8)
+
+                var = Var(
+                    id=f"{gen_name}/{file_name}",
+                    name=file_name,
+                    secret=file_data["secret"],
+                    deploy=file_data["deploy"],
+                    owner=file_data["owner"],
+                    group=file_data["group"],
+                    mode=mode,
+                    needed_for=file_data["neededFor"],
+                )
+                files.append(var)
+
+            # Build prompts
+            prompts = [Prompt.from_nix(p) for p in gen_data.get("prompts", {}).values()]
+
+            generator = cls(
+                name=gen_name,
+                share=gen_data["share"],
+                files=files,
+                dependencies=gen_data["dependencies"],
+                migrate_fact=gen_data.get("migrateFact"),
+                prompts=prompts,
+            )
+            # Set the machine immediately
+            generator.machine(machine)
+            generators.append(generator)
+
+        return generators
 
     def final_script(self) -> Path:
         assert self._machine is not None
@@ -351,10 +400,6 @@ def get_closure(
         generator.name: generator for generator in vars_generators
     }
 
-    # TODO: we should remove this
-    for generator in vars_generators:
-        generator.machine(machine)
-
     result_closure = []
     if generator_name is None:  # all generators selected
         if full_closure:

pkgs/clan-cli/clan_cli/vars/prompt.py

@@ -32,12 +32,12 @@ class Prompt:
     previous_value: str | None = None
 
     @classmethod
-    def from_json(cls: type["Prompt"], data: dict[str, Any]) -> "Prompt":
+    def from_nix(cls: type["Prompt"], data: dict[str, Any]) -> "Prompt":
         return cls(
             name=data["name"],
-            description=data["description"],
+            description=data.get("description", data["name"]),
-            prompt_type=PromptType(data["type"]),
+            prompt_type=PromptType(data.get("type", "line")),
-            persist=data.get("persist", data["persist"]),
+            persist=data.get("persist", False),
         )
 
 

pkgs/clan-cli/clan_cli/vars/var.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from clan_cli.vars.generate import Generator
@@ -67,16 +67,3 @@ class Var:
                 return f"{self.id}: ********"
             return f"{self.id}: {self.printable_value}"
         return f"{self.id}: <not set>"
-
-    @classmethod
-    def from_json(cls: type["Var"], generator_name: str, data: dict[str, Any]) -> "Var":
-        return cls(
-            id=f"{generator_name}/{data['name']}",
-            name=data["name"],
-            secret=data["secret"],
-            deploy=data["deploy"],
-            owner=data.get("owner", "root"),
-            group=data.get("group", "root"),
-            mode=int(data.get("mode", "0400"), 8),
-            needed_for=data.get("neededFor", "services"),
-        )

pkgs/clan-cli/clan_lib/machines/machines.py

@@ -112,19 +112,7 @@ class Machine:
     def vars_generators(self) -> list["Generator"]:
         from clan_cli.vars.generate import Generator
 
-        try:
+        return Generator.generators_from_flake(self.name, self.flake, self)
-            generators_data = self.select(
-                "config.clan.core.vars._serialized.generators"
-            )
-            if generators_data is None:
-                return []
-            _generators = [Generator.from_json(gen) for gen in generators_data.values()]
-            for gen in _generators:
-                gen.machine(self)
-        except Exception:
-            return []
-        else:
-            return _generators
 
     @property
     def secrets_upload_directory(self) -> str: