From 0a8b7f38a8a191eda82dfd017ef796aa8dfbb6ea Mon Sep 17 00:00:00 2001 From: DavHau Date: Thu, 19 Sep 2024 17:55:17 +0200 Subject: [PATCH] vars: implement migration Migrating generated files from the facts subsystem to the vars subsystem is now possible. HowTo: 1. declare `clan.core.vars.generators..migrateFact = my_service` where `my_service` refers to a service from `clan.core.facts.services` 2. run `clan vers generate your_machine` or `clan machines update your_machine` Vars will only be migrated for a generator if: 1. The facts service specified via `migrateFact` does exist 2. None of the vars to generate exist yet 3. All public var names exist in the public facts store 4. All secret var names exist in the secret fact store If the migration is deemed possible, the generator script will not be executed. Instead the files from the public or secret facts store are read and stored into the corresponding vars store --- nixosModules/clanCore/vars/default.nix | 1 + nixosModules/clanCore/vars/interface.nix | 11 ++ .../clan_cli/facts/public_modules/__init__.py | 6 +- .../clan_cli/facts/secret_modules/__init__.py | 6 +- pkgs/clan-cli/clan_cli/machines/machines.py | 12 ++ pkgs/clan-cli/clan_cli/vars/generate.py | 110 ++++++++++++++++-- pkgs/clan-cli/tests/test_vars.py | 59 ++++++++++ 7 files changed, 194 insertions(+), 11 deletions(-) diff --git a/nixosModules/clanCore/vars/default.nix b/nixosModules/clanCore/vars/default.nix index 95981d90b..d8a667ab4 100644 --- a/nixosModules/clanCore/vars/default.nix +++ b/nixosModules/clanCore/vars/default.nix @@ -42,6 +42,7 @@ in inherit (generator) dependencies finalScript + migrateFact prompts share ; diff --git a/nixosModules/clanCore/vars/interface.nix b/nixosModules/clanCore/vars/interface.nix index cd97723cd..25b224d12 100644 --- a/nixosModules/clanCore/vars/interface.nix +++ b/nixosModules/clanCore/vars/interface.nix @@ -12,6 +12,7 @@ let either enum listOf + nullOr package path str @@ -49,6 +50,16 @@ in type = listOf str; default = [ ]; }; + migrateFact = { + description = '' + The fact service name to import the files from. + + Use this to migrate legacy facts to the new vars system. + ''; + type = nullOr str; + example = "my_service"; + default = null; + }; files = { description = '' A set of files to generate. diff --git a/pkgs/clan-cli/clan_cli/facts/public_modules/__init__.py b/pkgs/clan-cli/clan_cli/facts/public_modules/__init__.py index a53ba10c0..64949bb88 100644 --- a/pkgs/clan-cli/clan_cli/facts/public_modules/__init__.py +++ b/pkgs/clan-cli/clan_cli/facts/public_modules/__init__.py @@ -1,12 +1,14 @@ +from __future__ import annotations + from abc import ABC, abstractmethod from pathlib import Path -from clan_cli.machines.machines import Machine +import clan_cli.machines.machines as machines class FactStoreBase(ABC): @abstractmethod - def __init__(self, machine: Machine) -> None: + def __init__(self, machine: machines.Machine) -> None: pass @abstractmethod diff --git a/pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py b/pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py index 5e26009c7..34dc54b16 100644 --- a/pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py +++ b/pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py @@ -1,12 +1,14 @@ +from __future__ import annotations + from abc import ABC, abstractmethod from pathlib import Path -from clan_cli.machines.machines import Machine +import clan_cli.machines.machines as machines class SecretStoreBase(ABC): @abstractmethod - def __init__(self, machine: Machine) -> None: + def __init__(self, machine: machines.Machine) -> None: pass @abstractmethod diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 9eb2b2849..43ca4459b 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -10,6 +10,8 @@ from typing import Any, Literal from clan_cli.clan_uri import FlakeId from clan_cli.cmd import run_no_stdout from clan_cli.errors import ClanError +from clan_cli.facts import public_modules as facts_public_modules +from clan_cli.facts import secret_modules as facts_secret_modules from clan_cli.nix import nix_build, nix_config, nix_eval, nix_metadata from clan_cli.ssh import Host, parse_deployment_address from clan_cli.vars.public_modules import FactStoreBase @@ -85,6 +87,16 @@ class Machine: ]: return self.deployment["facts"]["publicModule"] + @cached_property + def secret_facts_store(self) -> facts_secret_modules.SecretStoreBase: + module = importlib.import_module(self.secret_facts_module) + return module.SecretStore(machine=self) + + @cached_property + def public_facts_store(self) -> facts_public_modules.FactStoreBase: + module = importlib.import_module(self.public_facts_module) + return module.FactStore(machine=self) + # WIP: Vars module is not ready yet. @property def secret_vars_module(self) -> str: diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 4d5769230..8cd275c6f 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -223,6 +223,99 @@ def get_closure( return minimal_closure([generator_name], generators) +def _migration_file_exists( + machine: Machine, + service_name: str, + fact_name: str, +) -> bool: + is_secret = machine.vars_generators[service_name]["files"][fact_name]["secret"] + if is_secret: + if machine.secret_facts_store.exists(service_name, fact_name): + return True + log.debug( + f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the secret fact store" + ) + if not is_secret: + if machine.public_facts_store.exists(service_name, fact_name): + return True + log.debug( + f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the public fact store" + ) + return False + + +def _migrate_file( + machine: Machine, + generator_name: str, + var_name: str, + service_name: str, + fact_name: str, +) -> None: + is_secret = machine.vars_generators[generator_name]["files"][var_name]["secret"] + if is_secret: + old_value = machine.secret_facts_store.get(service_name, fact_name) + else: + old_value = machine.public_facts_store.get(service_name, fact_name) + is_shared = machine.vars_generators[generator_name]["share"] + is_deployed = machine.vars_generators[generator_name]["files"][var_name]["deploy"] + machine.public_vars_store.set( + generator_name, var_name, old_value, shared=is_shared, deployed=is_deployed + ) + + +def _migrate_files( + machine: Machine, + generator_name: str, +) -> None: + service_name = machine.vars_generators[generator_name]["migrateFact"] + not_found = [] + for var_name, _file in machine.vars_generators[generator_name]["files"].items(): + if _migration_file_exists(machine, generator_name, var_name): + _migrate_file(machine, generator_name, var_name, service_name, var_name) + else: + not_found.append(var_name) + if len(not_found) > 0: + msg = f"Could not migrate the following files for generator {generator_name}, as no fact or secret exists with the same name: {not_found}" + raise ClanError(msg) + + +def _check_can_migrate( + machine: Machine, + generator_name: str, +) -> bool: + vars_generator = machine.vars_generators[generator_name] + if "migrateFact" not in vars_generator: + return False + service_name = vars_generator["migrateFact"] + if not service_name: + return False + # ensure that none of the generated vars already exist in the store + for fname, file in vars_generator["files"].items(): + if file["secret"]: + if machine.secret_vars_store.exists( + generator_name, fname, vars_generator["share"] + ): + return False + else: + if machine.public_vars_store.exists( + generator_name, fname, vars_generator["share"] + ): + return False + # ensure that the service to migrate from actually exists + if service_name not in machine.facts_data: + log.debug( + f"Could not migrate facts for generator {generator_name}, as the service {service_name} does not exist" + ) + return False + # ensure that all files can be migrated (exists in the corresponding fact store) + return bool( + all( + _migration_file_exists(machine, generator_name, fname) + for fname in vars_generator["files"] + ) + ) + + def generate_vars_for_machine( machine: Machine, generator_name: str | None, @@ -233,13 +326,16 @@ def generate_vars_for_machine( return False prompt_values = _ask_prompts(machine, closure) for gen_name in closure: - execute_generator( - machine, - gen_name, - machine.secret_vars_store, - machine.public_vars_store, - prompt_values.get(gen_name, {}), - ) + if _check_can_migrate(machine, gen_name): + _migrate_files(machine, gen_name) + else: + execute_generator( + machine, + gen_name, + machine.secret_vars_store, + machine.public_vars_store, + prompt_values.get(gen_name, {}), + ) # flush caches to make sure the new secrets are available in evaluation machine.flush_caches() return True diff --git a/pkgs/clan-cli/tests/test_vars.py b/pkgs/clan-cli/tests/test_vars.py index 5fce487c1..e510c62ca 100644 --- a/pkgs/clan-cli/tests/test_vars.py +++ b/pkgs/clan-cli/tests/test_vars.py @@ -716,3 +716,62 @@ def test_stdout_of_generate( assert "Updated secret var my_secret_generator/my_secret" in output.out assert "world" not in output.out assert "hello" not in output.out + + +@pytest.mark.impure +def test_migration_skip( + monkeypatch: pytest.MonkeyPatch, + temporary_home: Path, +) -> None: + config = nested_dict() + my_service = config["clan"]["core"]["facts"]["services"]["my_service"] + my_service["secret"]["my_value"] = {} + my_service["generator"]["script"] = "echo -n hello > $secrets/my_value" + my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"] + # the var to migrate to is mistakenly marked as not secret (migration should fail) + my_generator["files"]["my_value"]["secret"] = False + my_generator["migrateFact"] = "my_service" + my_generator["script"] = "echo -n world > $out/my_value" + flake = generate_flake( + temporary_home, + flake_template=CLAN_CORE / "templates" / "minimal", + machine_configs={"my_machine": config}, + monkeypatch=monkeypatch, + ) + monkeypatch.chdir(flake.path) + cli.run(["facts", "generate", "--flake", str(flake.path), "my_machine"]) + cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) + in_repo_store = in_repo.FactStore( + Machine(name="my_machine", flake=FlakeId(str(flake.path))) + ) + assert in_repo_store.exists("my_generator", "my_value") + assert in_repo_store.get("my_generator", "my_value").decode() == "world" + + +@pytest.mark.impure +def test_migration( + monkeypatch: pytest.MonkeyPatch, + temporary_home: Path, +) -> None: + config = nested_dict() + my_service = config["clan"]["core"]["facts"]["services"]["my_service"] + my_service["public"]["my_value"] = {} + my_service["generator"]["script"] = "echo -n hello > $facts/my_value" + my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"] + my_generator["files"]["my_value"]["secret"] = False + my_generator["migrateFact"] = "my_service" + my_generator["script"] = "echo -n world > $out/my_value" + flake = generate_flake( + temporary_home, + flake_template=CLAN_CORE / "templates" / "minimal", + machine_configs={"my_machine": config}, + monkeypatch=monkeypatch, + ) + monkeypatch.chdir(flake.path) + cli.run(["facts", "generate", "--flake", str(flake.path), "my_machine"]) + cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) + in_repo_store = in_repo.FactStore( + Machine(name="my_machine", flake=FlakeId(str(flake.path))) + ) + assert in_repo_store.exists("my_generator", "my_value") + assert in_repo_store.get("my_generator", "my_value").decode() == "hello"