diff --git a/nixosModules/clanCore/vars/default.nix b/nixosModules/clanCore/vars/default.nix index 95981d90b..d8a667ab4 100644 --- a/nixosModules/clanCore/vars/default.nix +++ b/nixosModules/clanCore/vars/default.nix @@ -42,6 +42,7 @@ in inherit (generator) dependencies finalScript + migrateFact prompts share ; diff --git a/nixosModules/clanCore/vars/interface.nix b/nixosModules/clanCore/vars/interface.nix index cd97723cd..25b224d12 100644 --- a/nixosModules/clanCore/vars/interface.nix +++ b/nixosModules/clanCore/vars/interface.nix @@ -12,6 +12,7 @@ let either enum listOf + nullOr package path str @@ -49,6 +50,16 @@ in type = listOf str; default = [ ]; }; + migrateFact = { + description = '' + The fact service name to import the files from. + + Use this to migrate legacy facts to the new vars system. + ''; + type = nullOr str; + example = "my_service"; + default = null; + }; files = { description = '' A set of files to generate. diff --git a/pkgs/clan-cli/clan_cli/facts/public_modules/__init__.py b/pkgs/clan-cli/clan_cli/facts/public_modules/__init__.py index a53ba10c0..64949bb88 100644 --- a/pkgs/clan-cli/clan_cli/facts/public_modules/__init__.py +++ b/pkgs/clan-cli/clan_cli/facts/public_modules/__init__.py @@ -1,12 +1,14 @@ +from __future__ import annotations + from abc import ABC, abstractmethod from pathlib import Path -from clan_cli.machines.machines import Machine +import clan_cli.machines.machines as machines class FactStoreBase(ABC): @abstractmethod - def __init__(self, machine: Machine) -> None: + def __init__(self, machine: machines.Machine) -> None: pass @abstractmethod diff --git a/pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py b/pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py index 5e26009c7..34dc54b16 100644 --- a/pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py +++ b/pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py @@ -1,12 +1,14 @@ +from __future__ import annotations + from abc import ABC, abstractmethod from pathlib import Path -from clan_cli.machines.machines import Machine +import clan_cli.machines.machines as machines class SecretStoreBase(ABC): @abstractmethod - def __init__(self, machine: Machine) -> None: + def __init__(self, machine: machines.Machine) -> None: pass @abstractmethod diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 9eb2b2849..43ca4459b 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -10,6 +10,8 @@ from typing import Any, Literal from clan_cli.clan_uri import FlakeId from clan_cli.cmd import run_no_stdout from clan_cli.errors import ClanError +from clan_cli.facts import public_modules as facts_public_modules +from clan_cli.facts import secret_modules as facts_secret_modules from clan_cli.nix import nix_build, nix_config, nix_eval, nix_metadata from clan_cli.ssh import Host, parse_deployment_address from clan_cli.vars.public_modules import FactStoreBase @@ -85,6 +87,16 @@ class Machine: ]: return self.deployment["facts"]["publicModule"] + @cached_property + def secret_facts_store(self) -> facts_secret_modules.SecretStoreBase: + module = importlib.import_module(self.secret_facts_module) + return module.SecretStore(machine=self) + + @cached_property + def public_facts_store(self) -> facts_public_modules.FactStoreBase: + module = importlib.import_module(self.public_facts_module) + return module.FactStore(machine=self) + # WIP: Vars module is not ready yet. @property def secret_vars_module(self) -> str: diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 4d5769230..8cd275c6f 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -223,6 +223,99 @@ def get_closure( return minimal_closure([generator_name], generators) +def _migration_file_exists( + machine: Machine, + service_name: str, + fact_name: str, +) -> bool: + is_secret = machine.vars_generators[service_name]["files"][fact_name]["secret"] + if is_secret: + if machine.secret_facts_store.exists(service_name, fact_name): + return True + log.debug( + f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the secret fact store" + ) + if not is_secret: + if machine.public_facts_store.exists(service_name, fact_name): + return True + log.debug( + f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the public fact store" + ) + return False + + +def _migrate_file( + machine: Machine, + generator_name: str, + var_name: str, + service_name: str, + fact_name: str, +) -> None: + is_secret = machine.vars_generators[generator_name]["files"][var_name]["secret"] + if is_secret: + old_value = machine.secret_facts_store.get(service_name, fact_name) + else: + old_value = machine.public_facts_store.get(service_name, fact_name) + is_shared = machine.vars_generators[generator_name]["share"] + is_deployed = machine.vars_generators[generator_name]["files"][var_name]["deploy"] + machine.public_vars_store.set( + generator_name, var_name, old_value, shared=is_shared, deployed=is_deployed + ) + + +def _migrate_files( + machine: Machine, + generator_name: str, +) -> None: + service_name = machine.vars_generators[generator_name]["migrateFact"] + not_found = [] + for var_name, _file in machine.vars_generators[generator_name]["files"].items(): + if _migration_file_exists(machine, generator_name, var_name): + _migrate_file(machine, generator_name, var_name, service_name, var_name) + else: + not_found.append(var_name) + if len(not_found) > 0: + msg = f"Could not migrate the following files for generator {generator_name}, as no fact or secret exists with the same name: {not_found}" + raise ClanError(msg) + + +def _check_can_migrate( + machine: Machine, + generator_name: str, +) -> bool: + vars_generator = machine.vars_generators[generator_name] + if "migrateFact" not in vars_generator: + return False + service_name = vars_generator["migrateFact"] + if not service_name: + return False + # ensure that none of the generated vars already exist in the store + for fname, file in vars_generator["files"].items(): + if file["secret"]: + if machine.secret_vars_store.exists( + generator_name, fname, vars_generator["share"] + ): + return False + else: + if machine.public_vars_store.exists( + generator_name, fname, vars_generator["share"] + ): + return False + # ensure that the service to migrate from actually exists + if service_name not in machine.facts_data: + log.debug( + f"Could not migrate facts for generator {generator_name}, as the service {service_name} does not exist" + ) + return False + # ensure that all files can be migrated (exists in the corresponding fact store) + return bool( + all( + _migration_file_exists(machine, generator_name, fname) + for fname in vars_generator["files"] + ) + ) + + def generate_vars_for_machine( machine: Machine, generator_name: str | None, @@ -233,13 +326,16 @@ def generate_vars_for_machine( return False prompt_values = _ask_prompts(machine, closure) for gen_name in closure: - execute_generator( - machine, - gen_name, - machine.secret_vars_store, - machine.public_vars_store, - prompt_values.get(gen_name, {}), - ) + if _check_can_migrate(machine, gen_name): + _migrate_files(machine, gen_name) + else: + execute_generator( + machine, + gen_name, + machine.secret_vars_store, + machine.public_vars_store, + prompt_values.get(gen_name, {}), + ) # flush caches to make sure the new secrets are available in evaluation machine.flush_caches() return True diff --git a/pkgs/clan-cli/tests/test_vars.py b/pkgs/clan-cli/tests/test_vars.py index 5fce487c1..e510c62ca 100644 --- a/pkgs/clan-cli/tests/test_vars.py +++ b/pkgs/clan-cli/tests/test_vars.py @@ -716,3 +716,62 @@ def test_stdout_of_generate( assert "Updated secret var my_secret_generator/my_secret" in output.out assert "world" not in output.out assert "hello" not in output.out + + +@pytest.mark.impure +def test_migration_skip( + monkeypatch: pytest.MonkeyPatch, + temporary_home: Path, +) -> None: + config = nested_dict() + my_service = config["clan"]["core"]["facts"]["services"]["my_service"] + my_service["secret"]["my_value"] = {} + my_service["generator"]["script"] = "echo -n hello > $secrets/my_value" + my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"] + # the var to migrate to is mistakenly marked as not secret (migration should fail) + my_generator["files"]["my_value"]["secret"] = False + my_generator["migrateFact"] = "my_service" + my_generator["script"] = "echo -n world > $out/my_value" + flake = generate_flake( + temporary_home, + flake_template=CLAN_CORE / "templates" / "minimal", + machine_configs={"my_machine": config}, + monkeypatch=monkeypatch, + ) + monkeypatch.chdir(flake.path) + cli.run(["facts", "generate", "--flake", str(flake.path), "my_machine"]) + cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) + in_repo_store = in_repo.FactStore( + Machine(name="my_machine", flake=FlakeId(str(flake.path))) + ) + assert in_repo_store.exists("my_generator", "my_value") + assert in_repo_store.get("my_generator", "my_value").decode() == "world" + + +@pytest.mark.impure +def test_migration( + monkeypatch: pytest.MonkeyPatch, + temporary_home: Path, +) -> None: + config = nested_dict() + my_service = config["clan"]["core"]["facts"]["services"]["my_service"] + my_service["public"]["my_value"] = {} + my_service["generator"]["script"] = "echo -n hello > $facts/my_value" + my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"] + my_generator["files"]["my_value"]["secret"] = False + my_generator["migrateFact"] = "my_service" + my_generator["script"] = "echo -n world > $out/my_value" + flake = generate_flake( + temporary_home, + flake_template=CLAN_CORE / "templates" / "minimal", + machine_configs={"my_machine": config}, + monkeypatch=monkeypatch, + ) + monkeypatch.chdir(flake.path) + cli.run(["facts", "generate", "--flake", str(flake.path), "my_machine"]) + cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) + in_repo_store = in_repo.FactStore( + Machine(name="my_machine", flake=FlakeId(str(flake.path))) + ) + assert in_repo_store.exists("my_generator", "my_value") + assert in_repo_store.get("my_generator", "my_value").decode() == "hello"