yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'vars: implement migration' (#2148) from DavHau/clan-core:DavHau-vars-migration into main

Browse Source
This commit is contained in:
clan-bot
2024-09-19 16:04:39 +00:00
parent dcb759b0dc 501ade7de7
commit bef19592cd
7 changed files with 194 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
nixosModules/clanCore/vars/default.nix
View File

@@ -42,6 +42,7 @@ in
inherit (generator)
dependencies
finalScript
migrateFact
prompts
share
;

11
nixosModules/clanCore/vars/interface.nix
View File

@@ -12,6 +12,7 @@ let
either
enum
listOf
nullOr
package
path
str
@@ -49,6 +50,16 @@ in
type = listOf str;
default = [ ];
};
migrateFact = {
description = ''
The fact service name to import the files from.
Use this to migrate legacy facts to the new vars system.
'';
type = nullOr str;
example = "my_service";
default = null;
};
files = {
description = ''
A set of files to generate.

6
pkgs/clan-cli/clan_cli/facts/public_modules/__init__.py
View File

@@ -1,12 +1,14 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from pathlib import Path
from clan_cli.machines.machines import Machine
import clan_cli.machines.machines as machines
class FactStoreBase(ABC):
@abstractmethod
def __init__(self, machine: Machine) -> None:
def __init__(self, machine: machines.Machine) -> None:
pass
@abstractmethod

6
pkgs/clan-cli/clan_cli/facts/secret_modules/__init__.py
View File

@@ -1,12 +1,14 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from pathlib import Path
from clan_cli.machines.machines import Machine
import clan_cli.machines.machines as machines
class SecretStoreBase(ABC):
@abstractmethod
def __init__(self, machine: Machine) -> None:
def __init__(self, machine: machines.Machine) -> None:
pass
@abstractmethod

12
pkgs/clan-cli/clan_cli/machines/machines.py
View File

@@ -10,6 +10,8 @@ from typing import Any, Literal
from clan_cli.clan_uri import FlakeId
from clan_cli.cmd import run_no_stdout
from clan_cli.errors import ClanError
from clan_cli.facts import public_modules as facts_public_modules
from clan_cli.facts import secret_modules as facts_secret_modules
from clan_cli.nix import nix_build, nix_config, nix_eval, nix_metadata
from clan_cli.ssh import Host, parse_deployment_address
from clan_cli.vars.public_modules import FactStoreBase
@@ -85,6 +87,16 @@ class Machine:
]:
return self.deployment["facts"]["publicModule"]
@cached_property
def secret_facts_store(self) -> facts_secret_modules.SecretStoreBase:
module = importlib.import_module(self.secret_facts_module)
return module.SecretStore(machine=self)
@cached_property
def public_facts_store(self) -> facts_public_modules.FactStoreBase:
module = importlib.import_module(self.public_facts_module)
return module.FactStore(machine=self)
# WIP: Vars module is not ready yet.
@property
def secret_vars_module(self) -> str:

110
pkgs/clan-cli/clan_cli/vars/generate.py
View File

@@ -223,6 +223,99 @@ def get_closure(
return minimal_closure([generator_name], generators)
def _migration_file_exists(
machine: Machine,
service_name: str,
fact_name: str,
) -> bool:
is_secret = machine.vars_generators[service_name]["files"][fact_name]["secret"]
if is_secret:
if machine.secret_facts_store.exists(service_name, fact_name):
return True
log.debug(
f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the secret fact store"
)
if not is_secret:
if machine.public_facts_store.exists(service_name, fact_name):
return True
log.debug(
f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the public fact store"
)
return False
def _migrate_file(
machine: Machine,
generator_name: str,
var_name: str,
service_name: str,
fact_name: str,
) -> None:
is_secret = machine.vars_generators[generator_name]["files"][var_name]["secret"]
if is_secret:
old_value = machine.secret_facts_store.get(service_name, fact_name)
else:
old_value = machine.public_facts_store.get(service_name, fact_name)
is_shared = machine.vars_generators[generator_name]["share"]
is_deployed = machine.vars_generators[generator_name]["files"][var_name]["deploy"]
machine.public_vars_store.set(
generator_name, var_name, old_value, shared=is_shared, deployed=is_deployed
)
def _migrate_files(
machine: Machine,
generator_name: str,
) -> None:
service_name = machine.vars_generators[generator_name]["migrateFact"]
not_found = []
for var_name, _file in machine.vars_generators[generator_name]["files"].items():
if _migration_file_exists(machine, generator_name, var_name):
_migrate_file(machine, generator_name, var_name, service_name, var_name)
else:
not_found.append(var_name)
if len(not_found) > 0:
msg = f"Could not migrate the following files for generator {generator_name}, as no fact or secret exists with the same name: {not_found}"
raise ClanError(msg)
def _check_can_migrate(
machine: Machine,
generator_name: str,
) -> bool:
vars_generator = machine.vars_generators[generator_name]
if "migrateFact" not in vars_generator:
return False
service_name = vars_generator["migrateFact"]
if not service_name:
return False
# ensure that none of the generated vars already exist in the store
for fname, file in vars_generator["files"].items():
if file["secret"]:
if machine.secret_vars_store.exists(
generator_name, fname, vars_generator["share"]
):
return False
else:
if machine.public_vars_store.exists(
generator_name, fname, vars_generator["share"]
):
return False
# ensure that the service to migrate from actually exists
if service_name not in machine.facts_data:
log.debug(
f"Could not migrate facts for generator {generator_name}, as the service {service_name} does not exist"
)
return False
# ensure that all files can be migrated (exists in the corresponding fact store)
return bool(
all(
_migration_file_exists(machine, generator_name, fname)
for fname in vars_generator["files"]
)
)
def generate_vars_for_machine(
machine: Machine,
generator_name: str | None,
@@ -233,13 +326,16 @@ def generate_vars_for_machine(
return False
prompt_values = _ask_prompts(machine, closure)
for gen_name in closure:
execute_generator(
machine,
gen_name,
machine.secret_vars_store,
machine.public_vars_store,
prompt_values.get(gen_name, {}),
)
if _check_can_migrate(machine, gen_name):
_migrate_files(machine, gen_name)
else:
execute_generator(
machine,
gen_name,
machine.secret_vars_store,
machine.public_vars_store,
prompt_values.get(gen_name, {}),
)
# flush caches to make sure the new secrets are available in evaluation
machine.flush_caches()
return True

59
pkgs/clan-cli/tests/test_vars.py
View File

@@ -716,3 +716,62 @@ def test_stdout_of_generate(
assert "Updated secret var my_secret_generator/my_secret" in output.out
assert "world" not in output.out
assert "hello" not in output.out
@pytest.mark.impure
def test_migration_skip(
monkeypatch: pytest.MonkeyPatch,
temporary_home: Path,
) -> None:
config = nested_dict()
my_service = config["clan"]["core"]["facts"]["services"]["my_service"]
my_service["secret"]["my_value"] = {}
my_service["generator"]["script"] = "echo -n hello > $secrets/my_value"
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
# the var to migrate to is mistakenly marked as not secret (migration should fail)
my_generator["files"]["my_value"]["secret"] = False
my_generator["migrateFact"] = "my_service"
my_generator["script"] = "echo -n world > $out/my_value"
flake = generate_flake(
temporary_home,
flake_template=CLAN_CORE / "templates" / "minimal",
machine_configs={"my_machine": config},
monkeypatch=monkeypatch,
)
monkeypatch.chdir(flake.path)
cli.run(["facts", "generate", "--flake", str(flake.path), "my_machine"])
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
in_repo_store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert in_repo_store.exists("my_generator", "my_value")
assert in_repo_store.get("my_generator", "my_value").decode() == "world"
@pytest.mark.impure
def test_migration(
monkeypatch: pytest.MonkeyPatch,
temporary_home: Path,
) -> None:
config = nested_dict()
my_service = config["clan"]["core"]["facts"]["services"]["my_service"]
my_service["public"]["my_value"] = {}
my_service["generator"]["script"] = "echo -n hello > $facts/my_value"
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
my_generator["files"]["my_value"]["secret"] = False
my_generator["migrateFact"] = "my_service"
my_generator["script"] = "echo -n world > $out/my_value"
flake = generate_flake(
temporary_home,
flake_template=CLAN_CORE / "templates" / "minimal",
machine_configs={"my_machine": config},
monkeypatch=monkeypatch,
)
monkeypatch.chdir(flake.path)
cli.run(["facts", "generate", "--flake", str(flake.path), "my_machine"])
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
in_repo_store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert in_repo_store.exists("my_generator", "my_value")
assert in_repo_store.get("my_generator", "my_value").decode() == "hello"