Merge pull request 'vars: implement migration' (#2148) from DavHau/clan-core:DavHau-vars-migration into main

nixosModules/clanCore/vars/default.nix

@@ -42,6 +42,7 @@ in
           inherit (generator)
             dependencies
             finalScript
+            migrateFact
             prompts
             share
             ;

nixosModules/clanCore/vars/interface.nix

@@ -12,6 +12,7 @@ let
     either
     enum
     listOf
+    nullOr
     package
     path
     str
@@ -49,6 +50,16 @@ in
               type = listOf str;
               default = [ ];
             };
+            migrateFact = {
+              description = ''
+                The fact service name to import the files from.
+
+                Use this to migrate legacy facts to the new vars system.
+              '';
+              type = nullOr str;
+              example = "my_service";
+              default = null;
+            };
             files = {
               description = ''
                 A set of files to generate.

pkgs/clan-cli/clan_cli/facts/public_modules/__init__.py

@@ -1,12 +1,14 @@
+from __future__ import annotations
+
 from abc import ABC, abstractmethod
 from pathlib import Path
 
-from clan_cli.machines.machines import Machine
+import clan_cli.machines.machines as machines
 
 
 class FactStoreBase(ABC):
     @abstractmethod
-    def __init__(self, machine: Machine) -> None:
+    def __init__(self, machine: machines.Machine) -> None:
         pass
 
     @abstractmethod

pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py

@@ -1,12 +1,14 @@
+from __future__ import annotations
+
 from abc import ABC, abstractmethod
 from pathlib import Path
 
-from clan_cli.machines.machines import Machine
+import clan_cli.machines.machines as machines
 
 
 class SecretStoreBase(ABC):
     @abstractmethod
-    def __init__(self, machine: Machine) -> None:
+    def __init__(self, machine: machines.Machine) -> None:
         pass
 
     @abstractmethod

pkgs/clan-cli/clan_cli/machines/machines.py

@@ -10,6 +10,8 @@ from typing import Any, Literal
 from clan_cli.clan_uri import FlakeId
 from clan_cli.cmd import run_no_stdout
 from clan_cli.errors import ClanError
+from clan_cli.facts import public_modules as facts_public_modules
+from clan_cli.facts import secret_modules as facts_secret_modules
 from clan_cli.nix import nix_build, nix_config, nix_eval, nix_metadata
 from clan_cli.ssh import Host, parse_deployment_address
 from clan_cli.vars.public_modules import FactStoreBase
@@ -85,6 +87,16 @@ class Machine:
     ]:
         return self.deployment["facts"]["publicModule"]
 
+    @cached_property
+    def secret_facts_store(self) -> facts_secret_modules.SecretStoreBase:
+        module = importlib.import_module(self.secret_facts_module)
+        return module.SecretStore(machine=self)
+
+    @cached_property
+    def public_facts_store(self) -> facts_public_modules.FactStoreBase:
+        module = importlib.import_module(self.public_facts_module)
+        return module.FactStore(machine=self)
+
     # WIP: Vars module is not ready yet.
     @property
     def secret_vars_module(self) -> str:

pkgs/clan-cli/clan_cli/vars/generate.py

@@ -223,6 +223,99 @@ def get_closure(
     return minimal_closure([generator_name], generators)
 
 
+def _migration_file_exists(
+    machine: Machine,
+    service_name: str,
+    fact_name: str,
+) -> bool:
+    is_secret = machine.vars_generators[service_name]["files"][fact_name]["secret"]
+    if is_secret:
+        if machine.secret_facts_store.exists(service_name, fact_name):
+            return True
+        log.debug(
+            f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the secret fact store"
+        )
+    if not is_secret:
+        if machine.public_facts_store.exists(service_name, fact_name):
+            return True
+        log.debug(
+            f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the public fact store"
+        )
+    return False
+
+
+def _migrate_file(
+    machine: Machine,
+    generator_name: str,
+    var_name: str,
+    service_name: str,
+    fact_name: str,
+) -> None:
+    is_secret = machine.vars_generators[generator_name]["files"][var_name]["secret"]
+    if is_secret:
+        old_value = machine.secret_facts_store.get(service_name, fact_name)
+    else:
+        old_value = machine.public_facts_store.get(service_name, fact_name)
+    is_shared = machine.vars_generators[generator_name]["share"]
+    is_deployed = machine.vars_generators[generator_name]["files"][var_name]["deploy"]
+    machine.public_vars_store.set(
+        generator_name, var_name, old_value, shared=is_shared, deployed=is_deployed
+    )
+
+
+def _migrate_files(
+    machine: Machine,
+    generator_name: str,
+) -> None:
+    service_name = machine.vars_generators[generator_name]["migrateFact"]
+    not_found = []
+    for var_name, _file in machine.vars_generators[generator_name]["files"].items():
+        if _migration_file_exists(machine, generator_name, var_name):
+            _migrate_file(machine, generator_name, var_name, service_name, var_name)
+        else:
+            not_found.append(var_name)
+    if len(not_found) > 0:
+        msg = f"Could not migrate the following files for generator {generator_name}, as no fact or secret exists with the same name: {not_found}"
+        raise ClanError(msg)
+
+
+def _check_can_migrate(
+    machine: Machine,
+    generator_name: str,
+) -> bool:
+    vars_generator = machine.vars_generators[generator_name]
+    if "migrateFact" not in vars_generator:
+        return False
+    service_name = vars_generator["migrateFact"]
+    if not service_name:
+        return False
+    # ensure that none of the generated vars already exist in the store
+    for fname, file in vars_generator["files"].items():
+        if file["secret"]:
+            if machine.secret_vars_store.exists(
+                generator_name, fname, vars_generator["share"]
+            ):
+                return False
+        else:
+            if machine.public_vars_store.exists(
+                generator_name, fname, vars_generator["share"]
+            ):
+                return False
+    # ensure that the service to migrate from actually exists
+    if service_name not in machine.facts_data:
+        log.debug(
+            f"Could not migrate facts for generator {generator_name}, as the service {service_name} does not exist"
+        )
+        return False
+    # ensure that all files can be migrated (exists in the corresponding fact store)
+    return bool(
+        all(
+            _migration_file_exists(machine, generator_name, fname)
+            for fname in vars_generator["files"]
+        )
+    )
+
+
 def generate_vars_for_machine(
     machine: Machine,
     generator_name: str | None,
@@ -233,13 +326,16 @@ def generate_vars_for_machine(
         return False
     prompt_values = _ask_prompts(machine, closure)
     for gen_name in closure:
-        execute_generator(
+        if _check_can_migrate(machine, gen_name):
-            machine,
+            _migrate_files(machine, gen_name)
-            gen_name,
+        else:
-            machine.secret_vars_store,
+            execute_generator(
-            machine.public_vars_store,
+                machine,
-            prompt_values.get(gen_name, {}),
+                gen_name,
-        )
+                machine.secret_vars_store,
+                machine.public_vars_store,
+                prompt_values.get(gen_name, {}),
+            )
     # flush caches to make sure the new secrets are available in evaluation
     machine.flush_caches()
     return True

pkgs/clan-cli/tests/test_vars.py

@@ -716,3 +716,62 @@ def test_stdout_of_generate(
     assert "Updated secret var my_secret_generator/my_secret" in output.out
     assert "world" not in output.out
     assert "hello" not in output.out
+
+
+@pytest.mark.impure
+def test_migration_skip(
+    monkeypatch: pytest.MonkeyPatch,
+    temporary_home: Path,
+) -> None:
+    config = nested_dict()
+    my_service = config["clan"]["core"]["facts"]["services"]["my_service"]
+    my_service["secret"]["my_value"] = {}
+    my_service["generator"]["script"] = "echo -n hello > $secrets/my_value"
+    my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
+    # the var to migrate to is mistakenly marked as not secret (migration should fail)
+    my_generator["files"]["my_value"]["secret"] = False
+    my_generator["migrateFact"] = "my_service"
+    my_generator["script"] = "echo -n world > $out/my_value"
+    flake = generate_flake(
+        temporary_home,
+        flake_template=CLAN_CORE / "templates" / "minimal",
+        machine_configs={"my_machine": config},
+        monkeypatch=monkeypatch,
+    )
+    monkeypatch.chdir(flake.path)
+    cli.run(["facts", "generate", "--flake", str(flake.path), "my_machine"])
+    cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
+    in_repo_store = in_repo.FactStore(
+        Machine(name="my_machine", flake=FlakeId(str(flake.path)))
+    )
+    assert in_repo_store.exists("my_generator", "my_value")
+    assert in_repo_store.get("my_generator", "my_value").decode() == "world"
+
+
+@pytest.mark.impure
+def test_migration(
+    monkeypatch: pytest.MonkeyPatch,
+    temporary_home: Path,
+) -> None:
+    config = nested_dict()
+    my_service = config["clan"]["core"]["facts"]["services"]["my_service"]
+    my_service["public"]["my_value"] = {}
+    my_service["generator"]["script"] = "echo -n hello > $facts/my_value"
+    my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
+    my_generator["files"]["my_value"]["secret"] = False
+    my_generator["migrateFact"] = "my_service"
+    my_generator["script"] = "echo -n world > $out/my_value"
+    flake = generate_flake(
+        temporary_home,
+        flake_template=CLAN_CORE / "templates" / "minimal",
+        machine_configs={"my_machine": config},
+        monkeypatch=monkeypatch,
+    )
+    monkeypatch.chdir(flake.path)
+    cli.run(["facts", "generate", "--flake", str(flake.path), "my_machine"])
+    cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
+    in_repo_store = in_repo.FactStore(
+        Machine(name="my_machine", flake=FlakeId(str(flake.path)))
+    )
+    assert in_repo_store.exists("my_generator", "my_value")
+    assert in_repo_store.get("my_generator", "my_value").decode() == "hello"