yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

vars: implement migration

Browse Source 
Migrating generated files from the facts subsystem to the vars subsystem is now possible.

HowTo:
1. declare `clan.core.vars.generators.<generator>.migrateFact = my_service` where `my_service` refers to a service from `clan.core.facts.services`
2. run `clan vers generate your_machine` or `clan machines update your_machine`

Vars will only be migrated for a generator if:
1. The facts service specified via `migrateFact` does exist
2. None of the vars to generate exist yet
3. All public var names exist in the public facts store
4. All secret var names exist in the secret fact store

If the migration is deemed possible, the generator script will not be executed. Instead the files from the public or secret facts store are read and stored into the corresponding vars store
This commit is contained in:
DavHau
2024-09-19 17:55:17 +02:00
parent 733c44b7b6
commit 0a8b7f38a8
7 changed files with 194 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
nixosModules/clanCore/vars/default.nix
View File

@@ -42,6 +42,7 @@ in
inherit (generator)
dependencies
finalScript
migrateFact
prompts
share
;

11
nixosModules/clanCore/vars/interface.nix
View File

@@ -12,6 +12,7 @@ let
either
enum
listOf
nullOr
package
path
str
@@ -49,6 +50,16 @@ in
type = listOf str;
default = [ ];
};
migrateFact = {
description = ''
The fact service name to import the files from.
Use this to migrate legacy facts to the new vars system.
'';
type = nullOr str;
example = "my_service";
default = null;
};
files = {
description = ''
A set of files to generate.

6
pkgs/clan-cli/clan_cli/facts/public_modules/__init__.py
View File

@@ -1,12 +1,14 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from pathlib import Path
from clan_cli.machines.machines import Machine
import clan_cli.machines.machines as machines
class FactStoreBase(ABC):
@abstractmethod
def __init__(self, machine: Machine) -> None:
def __init__(self, machine: machines.Machine) -> None:
pass
@abstractmethod

6
pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py
View File

@@ -1,12 +1,14 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from pathlib import Path
from clan_cli.machines.machines import Machine
import clan_cli.machines.machines as machines
class SecretStoreBase(ABC):
@abstractmethod
def __init__(self, machine: Machine) -> None:
def __init__(self, machine: machines.Machine) -> None:
pass
@abstractmethod

12
pkgs/clan-cli/clan_cli/machines/machines.py
View File

@@ -10,6 +10,8 @@ from typing import Any, Literal
from clan_cli.clan_uri import FlakeId
from clan_cli.cmd import run_no_stdout
from clan_cli.errors import ClanError
from clan_cli.facts import public_modules as facts_public_modules
from clan_cli.facts import secret_modules as facts_secret_modules
from clan_cli.nix import nix_build, nix_config, nix_eval, nix_metadata
from clan_cli.ssh import Host, parse_deployment_address
from clan_cli.vars.public_modules import FactStoreBase
@@ -85,6 +87,16 @@ class Machine:
]:
return self.deployment["facts"]["publicModule"]
@cached_property
def secret_facts_store(self) -> facts_secret_modules.SecretStoreBase:
module = importlib.import_module(self.secret_facts_module)
return module.SecretStore(machine=self)
@cached_property
def public_facts_store(self) -> facts_public_modules.FactStoreBase:
module = importlib.import_module(self.public_facts_module)
return module.FactStore(machine=self)
# WIP: Vars module is not ready yet.
@property
def secret_vars_module(self) -> str:

110
pkgs/clan-cli/clan_cli/vars/generate.py
View File

@@ -223,6 +223,99 @@ def get_closure(
return minimal_closure([generator_name], generators)
def _migration_file_exists(
machine: Machine,
service_name: str,
fact_name: str,
) -> bool:
is_secret = machine.vars_generators[service_name]["files"][fact_name]["secret"]
if is_secret:
if machine.secret_facts_store.exists(service_name, fact_name):
return True
log.debug(
f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the secret fact store"
)
if not is_secret:
if machine.public_facts_store.exists(service_name, fact_name):
return True
log.debug(
f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the public fact store"
)
return False
def _migrate_file(
machine: Machine,
generator_name: str,
var_name: str,
service_name: str,
fact_name: str,
) -> None:
is_secret = machine.vars_generators[generator_name]["files"][var_name]["secret"]
if is_secret:
old_value = machine.secret_facts_store.get(service_name, fact_name)
else:
old_value = machine.public_facts_store.get(service_name, fact_name)
is_shared = machine.vars_generators[generator_name]["share"]
is_deployed = machine.vars_generators[generator_name]["files"][var_name]["deploy"]
machine.public_vars_store.set(
generator_name, var_name, old_value, shared=is_shared, deployed=is_deployed
)
def _migrate_files(
machine: Machine,
generator_name: str,
) -> None:
service_name = machine.vars_generators[generator_name]["migrateFact"]
not_found = []
for var_name, _file in machine.vars_generators[generator_name]["files"].items():
if _migration_file_exists(machine, generator_name, var_name):
_migrate_file(machine, generator_name, var_name, service_name, var_name)
else:
not_found.append(var_name)
if len(not_found) > 0:
msg = f"Could not migrate the following files for generator {generator_name}, as no fact or secret exists with the same name: {not_found}"
raise ClanError(msg)
def _check_can_migrate(
machine: Machine,
generator_name: str,
) -> bool:
vars_generator = machine.vars_generators[generator_name]
if "migrateFact" not in vars_generator:
return False
service_name = vars_generator["migrateFact"]
if not service_name:
return False
# ensure that none of the generated vars already exist in the store
for fname, file in vars_generator["files"].items():
if file["secret"]:
if machine.secret_vars_store.exists(
generator_name, fname, vars_generator["share"]
):
return False
else:
if machine.public_vars_store.exists(
generator_name, fname, vars_generator["share"]
):
return False
# ensure that the service to migrate from actually exists
if service_name not in machine.facts_data:
log.debug(
f"Could not migrate facts for generator {generator_name}, as the service {service_name} does not exist"
)
return False
# ensure that all files can be migrated (exists in the corresponding fact store)
return bool(
all(
_migration_file_exists(machine, generator_name, fname)
for fname in vars_generator["files"]
)
)
def generate_vars_for_machine(
machine: Machine,
generator_name: str | None,
@@ -233,13 +326,16 @@ def generate_vars_for_machine(
return False
prompt_values = _ask_prompts(machine, closure)
for gen_name in closure:
execute_generator(
machine,
gen_name,
machine.secret_vars_store,
machine.public_vars_store,
prompt_values.get(gen_name, {}),
)
if _check_can_migrate(machine, gen_name):
_migrate_files(machine, gen_name)
else:
execute_generator(
machine,
gen_name,
machine.secret_vars_store,
machine.public_vars_store,
prompt_values.get(gen_name, {}),
)
# flush caches to make sure the new secrets are available in evaluation
machine.flush_caches()
return True

59
pkgs/clan-cli/tests/test_vars.py
View File

@@ -716,3 +716,62 @@ def test_stdout_of_generate(
assert "Updated secret var my_secret_generator/my_secret" in output.out
assert "world" not in output.out
assert "hello" not in output.out
@pytest.mark.impure
def test_migration_skip(
monkeypatch: pytest.MonkeyPatch,
temporary_home: Path,
) -> None:
config = nested_dict()
my_service = config["clan"]["core"]["facts"]["services"]["my_service"]
my_service["secret"]["my_value"] = {}
my_service["generator"]["script"] = "echo -n hello > $secrets/my_value"
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
# the var to migrate to is mistakenly marked as not secret (migration should fail)
my_generator["files"]["my_value"]["secret"] = False
my_generator["migrateFact"] = "my_service"
my_generator["script"] = "echo -n world > $out/my_value"
flake = generate_flake(
temporary_home,
flake_template=CLAN_CORE / "templates" / "minimal",
machine_configs={"my_machine": config},
monkeypatch=monkeypatch,
)
monkeypatch.chdir(flake.path)
cli.run(["facts", "generate", "--flake", str(flake.path), "my_machine"])
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
in_repo_store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert in_repo_store.exists("my_generator", "my_value")
assert in_repo_store.get("my_generator", "my_value").decode() == "world"
@pytest.mark.impure
def test_migration(
monkeypatch: pytest.MonkeyPatch,
temporary_home: Path,
) -> None:
config = nested_dict()
my_service = config["clan"]["core"]["facts"]["services"]["my_service"]
my_service["public"]["my_value"] = {}
my_service["generator"]["script"] = "echo -n hello > $facts/my_value"
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
my_generator["files"]["my_value"]["secret"] = False
my_generator["migrateFact"] = "my_service"
my_generator["script"] = "echo -n world > $out/my_value"
flake = generate_flake(
temporary_home,
flake_template=CLAN_CORE / "templates" / "minimal",
machine_configs={"my_machine": config},
monkeypatch=monkeypatch,
)
monkeypatch.chdir(flake.path)
cli.run(["facts", "generate", "--flake", str(flake.path), "my_machine"])
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
in_repo_store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert in_repo_store.exists("my_generator", "my_value")
assert in_repo_store.get("my_generator", "my_value").decode() == "hello"