vars: implement migration

Migrating generated files from the facts subsystem to the vars subsystem is now possible.

HowTo:
1. declare `clan.core.vars.generators.<generator>.migrateFact = my_service` where `my_service` refers to a service from `clan.core.facts.services`
2. run `clan vers generate your_machine` or `clan machines update your_machine`

Vars will only be migrated for a generator if:
1. The facts service specified via `migrateFact` does exist
2. None of the vars to generate exist yet
3. All public var names exist in the public facts store
4. All secret var names exist in the secret fact store

If the migration is deemed possible, the generator script will not be executed. Instead the files from the public or secret facts store are read and stored into the corresponding vars store

nixosModules/clanCore/vars/default.nix

@@ -42,6 +42,7 @@ in
           inherit (generator)
             dependencies
             finalScript
+            migrateFact
             prompts
             share
             ;

nixosModules/clanCore/vars/interface.nix

@@ -12,6 +12,7 @@ let
     either
     enum
     listOf
+    nullOr
     package
     path
     str
@@ -49,6 +50,16 @@ in
               type = listOf str;
               default = [ ];
             };
+            migrateFact = {
+              description = ''
+                The fact service name to import the files from.
+
+                Use this to migrate legacy facts to the new vars system.
+              '';
+              type = nullOr str;
+              example = "my_service";
+              default = null;
+            };
             files = {
               description = ''
                 A set of files to generate.

pkgs/clan-cli/clan_cli/facts/public_modules/__init__.py

@@ -1,12 +1,14 @@
+from __future__ import annotations
+
 from abc import ABC, abstractmethod
 from pathlib import Path
 
-from clan_cli.machines.machines import Machine
+import clan_cli.machines.machines as machines
 
 
 class FactStoreBase(ABC):
     @abstractmethod
-    def __init__(self, machine: Machine) -> None:
+    def __init__(self, machine: machines.Machine) -> None:
         pass
 
     @abstractmethod

pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py

@@ -1,12 +1,14 @@
+from __future__ import annotations
+
 from abc import ABC, abstractmethod
 from pathlib import Path
 
-from clan_cli.machines.machines import Machine
+import clan_cli.machines.machines as machines
 
 
 class SecretStoreBase(ABC):
     @abstractmethod
-    def __init__(self, machine: Machine) -> None:
+    def __init__(self, machine: machines.Machine) -> None:
         pass
 
     @abstractmethod

pkgs/clan-cli/clan_cli/machines/machines.py

@@ -10,6 +10,8 @@ from typing import Any, Literal
 from clan_cli.clan_uri import FlakeId
 from clan_cli.cmd import run_no_stdout
 from clan_cli.errors import ClanError
+from clan_cli.facts import public_modules as facts_public_modules
+from clan_cli.facts import secret_modules as facts_secret_modules
 from clan_cli.nix import nix_build, nix_config, nix_eval, nix_metadata
 from clan_cli.ssh import Host, parse_deployment_address
 from clan_cli.vars.public_modules import FactStoreBase
@@ -85,6 +87,16 @@ class Machine:
     ]:
         return self.deployment["facts"]["publicModule"]
 
+    @cached_property
+    def secret_facts_store(self) -> facts_secret_modules.SecretStoreBase:
+        module = importlib.import_module(self.secret_facts_module)
+        return module.SecretStore(machine=self)
+
+    @cached_property
+    def public_facts_store(self) -> facts_public_modules.FactStoreBase:
+        module = importlib.import_module(self.public_facts_module)
+        return module.FactStore(machine=self)
+
     # WIP: Vars module is not ready yet.
     @property
     def secret_vars_module(self) -> str:

pkgs/clan-cli/clan_cli/vars/generate.py

@@ -223,6 +223,99 @@ def get_closure(
     return minimal_closure([generator_name], generators)
 
 
+def _migration_file_exists(
+    machine: Machine,
+    service_name: str,
+    fact_name: str,
+) -> bool:
+    is_secret = machine.vars_generators[service_name]["files"][fact_name]["secret"]
+    if is_secret:
+        if machine.secret_facts_store.exists(service_name, fact_name):
+            return True
+        log.debug(
+            f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the secret fact store"
+        )
+    if not is_secret:
+        if machine.public_facts_store.exists(service_name, fact_name):
+            return True
+        log.debug(
+            f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the public fact store"
+        )
+    return False
+
+
+def _migrate_file(
+    machine: Machine,
+    generator_name: str,
+    var_name: str,
+    service_name: str,
+    fact_name: str,
+) -> None:
+    is_secret = machine.vars_generators[generator_name]["files"][var_name]["secret"]
+    if is_secret:
+        old_value = machine.secret_facts_store.get(service_name, fact_name)
+    else:
+        old_value = machine.public_facts_store.get(service_name, fact_name)
+    is_shared = machine.vars_generators[generator_name]["share"]
+    is_deployed = machine.vars_generators[generator_name]["files"][var_name]["deploy"]
+    machine.public_vars_store.set(
+        generator_name, var_name, old_value, shared=is_shared, deployed=is_deployed
+    )
+
+
+def _migrate_files(
+    machine: Machine,
+    generator_name: str,
+) -> None:
+    service_name = machine.vars_generators[generator_name]["migrateFact"]
+    not_found = []
+    for var_name, _file in machine.vars_generators[generator_name]["files"].items():
+        if _migration_file_exists(machine, generator_name, var_name):
+            _migrate_file(machine, generator_name, var_name, service_name, var_name)
+        else:
+            not_found.append(var_name)
+    if len(not_found) > 0:
+        msg = f"Could not migrate the following files for generator {generator_name}, as no fact or secret exists with the same name: {not_found}"
+        raise ClanError(msg)
+
+
+def _check_can_migrate(
+    machine: Machine,
+    generator_name: str,
+) -> bool:
+    vars_generator = machine.vars_generators[generator_name]
+    if "migrateFact" not in vars_generator:
+        return False
+    service_name = vars_generator["migrateFact"]
+    if not service_name:
+        return False
+    # ensure that none of the generated vars already exist in the store
+    for fname, file in vars_generator["files"].items():
+        if file["secret"]:
+            if machine.secret_vars_store.exists(
+                generator_name, fname, vars_generator["share"]
+            ):
+                return False
+        else:
+            if machine.public_vars_store.exists(
+                generator_name, fname, vars_generator["share"]
+            ):
+                return False
+    # ensure that the service to migrate from actually exists
+    if service_name not in machine.facts_data:
+        log.debug(
+            f"Could not migrate facts for generator {generator_name}, as the service {service_name} does not exist"
+        )
+        return False
+    # ensure that all files can be migrated (exists in the corresponding fact store)
+    return bool(
+        all(
+            _migration_file_exists(machine, generator_name, fname)
+            for fname in vars_generator["files"]
+        )
+    )
+
+
 def generate_vars_for_machine(
     machine: Machine,
     generator_name: str | None,
@@ -233,13 +326,16 @@ def generate_vars_for_machine(
         return False
     prompt_values = _ask_prompts(machine, closure)
     for gen_name in closure:
-        execute_generator(
+        if _check_can_migrate(machine, gen_name):
-            machine,
+            _migrate_files(machine, gen_name)
-            gen_name,
+        else:
-            machine.secret_vars_store,
+            execute_generator(
-            machine.public_vars_store,
+                machine,
-            prompt_values.get(gen_name, {}),
+                gen_name,
-        )
+                machine.secret_vars_store,
+                machine.public_vars_store,
+                prompt_values.get(gen_name, {}),
+            )
     # flush caches to make sure the new secrets are available in evaluation
     machine.flush_caches()
     return True

pkgs/clan-cli/tests/test_vars.py

@@ -716,3 +716,62 @@ def test_stdout_of_generate(
     assert "Updated secret var my_secret_generator/my_secret" in output.out
     assert "world" not in output.out
     assert "hello" not in output.out
+
+
+@pytest.mark.impure
+def test_migration_skip(
+    monkeypatch: pytest.MonkeyPatch,
+    temporary_home: Path,
+) -> None:
+    config = nested_dict()
+    my_service = config["clan"]["core"]["facts"]["services"]["my_service"]
+    my_service["secret"]["my_value"] = {}
+    my_service["generator"]["script"] = "echo -n hello > $secrets/my_value"
+    my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
+    # the var to migrate to is mistakenly marked as not secret (migration should fail)
+    my_generator["files"]["my_value"]["secret"] = False
+    my_generator["migrateFact"] = "my_service"
+    my_generator["script"] = "echo -n world > $out/my_value"
+    flake = generate_flake(
+        temporary_home,
+        flake_template=CLAN_CORE / "templates" / "minimal",
+        machine_configs={"my_machine": config},
+        monkeypatch=monkeypatch,
+    )
+    monkeypatch.chdir(flake.path)
+    cli.run(["facts", "generate", "--flake", str(flake.path), "my_machine"])
+    cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
+    in_repo_store = in_repo.FactStore(
+        Machine(name="my_machine", flake=FlakeId(str(flake.path)))
+    )
+    assert in_repo_store.exists("my_generator", "my_value")
+    assert in_repo_store.get("my_generator", "my_value").decode() == "world"
+
+
+@pytest.mark.impure
+def test_migration(
+    monkeypatch: pytest.MonkeyPatch,
+    temporary_home: Path,
+) -> None:
+    config = nested_dict()
+    my_service = config["clan"]["core"]["facts"]["services"]["my_service"]
+    my_service["public"]["my_value"] = {}
+    my_service["generator"]["script"] = "echo -n hello > $facts/my_value"
+    my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
+    my_generator["files"]["my_value"]["secret"] = False
+    my_generator["migrateFact"] = "my_service"
+    my_generator["script"] = "echo -n world > $out/my_value"
+    flake = generate_flake(
+        temporary_home,
+        flake_template=CLAN_CORE / "templates" / "minimal",
+        machine_configs={"my_machine": config},
+        monkeypatch=monkeypatch,
+    )
+    monkeypatch.chdir(flake.path)
+    cli.run(["facts", "generate", "--flake", str(flake.path), "my_machine"])
+    cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
+    in_repo_store = in_repo.FactStore(
+        Machine(name="my_machine", flake=FlakeId(str(flake.path)))
+    )
+    assert in_repo_store.exists("my_generator", "my_value")
+    assert in_repo_store.get("my_generator", "my_value").decode() == "hello"