yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'vars: make interface more type-safe' (#2459) from vars into main

Browse Source 
Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/2459
Reviewed-by: kenji <aks.kenji@protonmail.com>
This commit is contained in:
Mic92
2024-11-26 16:15:55 +00:00
parent 71bcf76feb 45dfbf54db
commit 90578d47f1
19 changed files with 676 additions and 522 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
nixosModules/clanCore/vars/default.nix
View File

@@ -40,6 +40,7 @@ in
generators = lib.flip lib.mapAttrs config.clan.core.vars.generators (
_name: generator: {
inherit (generator)
name
dependencies
finalScript
invalidationHash
@@ -49,7 +50,7 @@ in
;
files = lib.flip lib.mapAttrs generator.files (
_name: file: {
inherit (file) deploy secret;
inherit (file) name deploy secret;
}
);
}

18
nixosModules/clanCore/vars/interface.nix
View File

@@ -46,6 +46,16 @@ in
submodule (generator: {
imports = [ ./generator.nix ];
options = {
name = lib.mkOption {
type = lib.types.str;
description = ''
The name of the generator.
This name will be used to refer to the generator in other generators.
'';
readOnly = true;
default = generator.config._module.args.name;
};
dependencies = lib.mkOption {
description = ''
A list of other generators that this generator depends on.
@@ -212,6 +222,14 @@ in
type = attrsOf (
submodule (prompt: {
options = {
name = lib.mkOption {
description = ''
The name of the prompt.
This name will be used to refer to the prompt in the generator script.
'';
type = str;
default = prompt.config._module.args.name;
};
createFile = lib.mkOption {
description = ''
Whether the prompted value should be stored in a file with the same name as the prompt.

19
pkgs/clan-cli/clan_cli/machines/machines.py
View File

@@ -5,7 +5,7 @@ from dataclasses import dataclass, field
from functools import cached_property
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any, Literal
from typing import TYPE_CHECKING, Any, Literal
from clan_cli.clan_uri import FlakeId
from clan_cli.cmd import run_no_stdout
@@ -21,6 +21,9 @@ from clan_cli.vars.secret_modules import SecretStoreBase
log = logging.getLogger(__name__)
if TYPE_CHECKING:
from clan_cli.vars.generate import Generator
@dataclass
class Machine:
@@ -156,10 +159,16 @@ class Machine:
return {}
@property
def vars_generators(self) -> dict[str, dict[str, Any]]:
if self.deployment["vars"]["generators"]:
return self.deployment["vars"]["generators"]
return {}
def vars_generators(self) -> list["Generator"]:
from clan_cli.vars.generate import Generator
clan_vars = self.deployment.get("vars")
if clan_vars is None:
return []
generators: dict[str, Any] = clan_vars.get("generators")
if generators is None:
return []
return [Generator.from_json(gen) for gen in generators.values()]
@property
def secrets_upload_directory(self) -> str:

145
pkgs/clan-cli/clan_cli/vars/_types.py
View File

@@ -2,10 +2,14 @@ import logging
from abc import ABC, abstractmethod
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING
from clan_cli.errors import ClanError
from clan_cli.machines import machines
if TYPE_CHECKING:
from .generate import Generator, Var
log = logging.getLogger(__name__)
@@ -16,69 +20,12 @@ def string_repr(value: bytes) -> str:
return "<binary blob>"
@dataclass
class Prompt:
name: str
description: str
type: str
has_file: bool
generator: str
previous_value: str | None = None
# TODO: add flag 'pending' generator needs to be executed
@dataclass
class Generator:
name: str
share: bool
prompts: list[Prompt]
@dataclass
class GeneratorUpdate:
generator: str
prompt_values: dict[str, str]
@dataclass
class Var:
_store: "StoreBase"
generator: str
name: str
id: str
secret: bool
shared: bool
deployed: bool
owner: str
group: str
@property
def value(self) -> bytes:
if not self._store.exists(self.generator, self.name, self.shared):
msg = f"Var {self.id} has not been generated yet"
raise ValueError(msg)
# try decode the value or return <binary blob>
return self._store.get(self.generator, self.name, self.shared)
@property
def printable_value(self) -> str:
return string_repr(self.value)
def set(self, value: bytes) -> None:
self._store.set(self.generator, self.name, value, self.shared, self.deployed)
@property
def exists(self) -> bool:
return self._store.exists(self.generator, self.name, self.shared)
def __str__(self) -> str:
if self._store.exists(self.generator, self.name, self.shared):
if self.secret:
return f"{self.id}: ********"
return f"{self.id}: {self.printable_value}"
return f"{self.id}: <not set>"
class StoreBase(ABC):
def __init__(self, machine: "machines.Machine") -> None:
self.machine = machine
@@ -90,24 +37,22 @@ class StoreBase(ABC):
# get a single fact
@abstractmethod
def get(self, generator_name: str, name: str, shared: bool = False) -> bytes:
def get(self, generator: "Generator", name: str) -> bytes:
pass
@abstractmethod
def _set(
self,
generator_name: str,
name: str,
generator: "Generator",
var: "Var",
value: bytes,
shared: bool = False,
deployed: bool = True,
) -> Path | None:
"""
override this method to implement the actual creation of the file
"""
@abstractmethod
def exists(self, generator_name: str, name: str, shared: bool = False) -> bool:
def exists(self, generator: "Generator", name: str) -> bool:
pass
@property
@@ -122,106 +67,88 @@ class StoreBase(ABC):
)
raise ClanError(msg)
def rel_dir(self, generator_name: str, var_name: str, shared: bool = False) -> Path:
if shared:
return Path(f"shared/{generator_name}/{var_name}")
return Path(f"per-machine/{self.machine.name}/{generator_name}/{var_name}")
def rel_dir(self, generator: "Generator", var_name: str) -> Path:
if generator.share:
return Path(f"shared/{generator.name}/{var_name}")
return Path(f"per-machine/{self.machine.name}/{generator.name}/{var_name}")
def directory(
self, generator_name: str, var_name: str, shared: bool = False
) -> Path:
return (
Path(self.machine.flake_dir)
/ "vars"
/ self.rel_dir(generator_name, var_name, shared)
)
def directory(self, generator: "Generator", var_name: str) -> Path:
return Path(self.machine.flake_dir) / "vars" / self.rel_dir(generator, var_name)
def set(
self,
generator_name: str,
var_name: str,
generator: "Generator",
var: "Var",
value: bytes,
shared: bool = False,
deployed: bool = True,
) -> Path | None:
if self.exists(generator_name, var_name, shared):
if self.exists(generator, var.name):
if self.is_secret_store:
old_val = None
old_val_str = "********"
else:
old_val = self.get(generator_name, var_name, shared)
old_val = self.get(generator, var.name)
old_val_str = string_repr(old_val)
else:
old_val = None
old_val_str = "<not set>"
new_file = self._set(generator_name, var_name, value, shared, deployed)
new_file = self._set(generator, var, value)
if self.is_secret_store:
print(f"Updated secret var {generator_name}/{var_name}\n")
print(f"Updated secret var {generator.name}/{var.name}\n")
else:
if value != old_val:
print(
f"Updated var {generator_name}/{var_name}\n"
f"Updated var {generator.name}/{var.name}\n"
f" old: {old_val_str}\n"
f" new: {string_repr(value)}"
)
else:
print(
f"Var {generator_name}/{var_name} remains unchanged: {old_val_str}"
f"Var {generator.name}/{var.name} remains unchanged: {old_val_str}"
)
return new_file
def get_all(self) -> list[Var]:
def get_all(self) -> list["Var"]:
all_vars = []
for gen_name, generator in self.machine.vars_generators.items():
for f_name, file in generator["files"].items():
for generator in self.machine.vars_generators:
for var in generator.files:
# only handle vars compatible to this store
if self.is_secret_store != file["secret"]:
if self.is_secret_store != var.secret:
continue
all_vars.append(
Var(
_store=self,
generator=gen_name,
name=f_name,
id=f"{gen_name}/{f_name}",
secret=file["secret"],
shared=generator["share"],
deployed=file["deploy"],
owner=file.get("owner", "root"),
group=file.get("group", "root"),
)
)
var.store(self)
var.generator(generator)
all_vars.append(var)
return all_vars
def get_invalidation_hash(self, generator_name: str) -> str | None:
def get_invalidation_hash(self, generator: "Generator") -> str | None:
"""
Return the invalidation hash that indicates if a generator needs to be re-run
due to a change in its definition
"""
hash_file = (
self.machine.flake_dir / "vars" / generator_name / "invalidation_hash"
self.machine.flake_dir / "vars" / generator.name / "invalidation_hash"
)
if not hash_file.exists():
return None
return hash_file.read_text().strip()
def set_invalidation_hash(self, generator_name: str, hash_str: str) -> None:
def set_invalidation_hash(self, generator: "Generator", hash_str: str) -> None:
"""
Store the invalidation hash that indicates if a generator needs to be re-run
"""
hash_file = (
self.machine.flake_dir / "vars" / generator_name / "invalidation_hash"
self.machine.flake_dir / "vars" / generator.name / "invalidation_hash"
)
hash_file.parent.mkdir(parents=True, exist_ok=True)
hash_file.write_text(hash_str)
def hash_is_valid(self, generator_name: str) -> bool:
def hash_is_valid(self, generator: "Generator") -> bool:
"""
Check if the invalidation hash is up to date
If the hash is not set in nix and hasn't been stored before, it is considered valid
-> this provides backward and forward compatibility
"""
stored_hash = self.get_invalidation_hash(generator_name)
target_hash = self.machine.vars_generators[generator_name]["invalidationHash"]
stored_hash = self.get_invalidation_hash(generator)
target_hash = generator.invalidation_hash
# if the hash is neither set in nix nor on disk, it is considered valid (provides backwards compat)
if target_hash is None and stored_hash is None:
return True

82
pkgs/clan-cli/clan_cli/vars/check.py
View File

@@ -1,75 +1,91 @@
import argparse
import importlib
import logging
from dataclasses import dataclass
from clan_cli.completions import add_dynamic_completer, complete_machines
from clan_cli.errors import ClanError
from clan_cli.machines.machines import Machine
from clan_cli.vars.public_modules import FactStoreBase
from clan_cli.vars.secret_modules import SecretStoreBase
log = logging.getLogger(__name__)
from typing import TYPE_CHECKING
@dataclass
class Var:
generator: str
name: str
if TYPE_CHECKING:
from .generate import Var
@dataclass
class VarStatus:
missing_secret_vars: list[Var]
missing_public_vars: list[Var]
unfixed_secret_vars: list[Var]
invalid_generators: list[str]
def __init__(
self,
missing_secret_vars: list["Var"],
missing_public_vars: list["Var"],
unfixed_secret_vars: list["Var"],
invalid_generators: list[str],
) -> None:
self.missing_secret_vars = missing_secret_vars
self.missing_public_vars = missing_public_vars
self.unfixed_secret_vars = unfixed_secret_vars
self.invalid_generators = invalid_generators
def vars_status(machine: Machine, generator_name: None | str = None) -> VarStatus:
secret_vars_module = importlib.import_module(machine.secret_vars_module)
secret_vars_store = secret_vars_module.SecretStore(machine=machine)
secret_vars_store: SecretStoreBase = secret_vars_module.SecretStore(machine=machine)
public_vars_module = importlib.import_module(machine.public_vars_module)
public_vars_store = public_vars_module.FactStore(machine=machine)
public_vars_store: FactStoreBase = public_vars_module.FactStore(machine=machine)
missing_secret_vars = []
missing_public_vars = []
# signals if a var needs to be updated (eg. needs re-encryption due to new users added)
unfixed_secret_vars = []
invalid_generators = []
generators = machine.vars_generators
if generator_name:
generators = [generator_name]
for generator in generators:
if generator_name == generator.name:
generators = [generator]
break
else:
generators = list(machine.vars_generators.keys())
for generator_name in generators:
shared = machine.vars_generators[generator_name]["share"]
for name, file in machine.vars_generators[generator_name]["files"].items():
if file["secret"]:
if not secret_vars_store.exists(generator_name, name, shared=shared):
err_msg = (
f"Generator '{generator_name}' not found in machine {machine.name}"
)
raise ClanError(err_msg)
for generator in generators:
generator.machine(machine)
for file in generator.files:
file.store(secret_vars_store if file.secret else public_vars_store)
file.generator(generator)
if file.secret:
if not secret_vars_store.exists(generator, file.name):
log.info(
f"Secret var '{name}' for service '{generator_name}' in machine {machine.name} is missing."
f"Secret var '{file.name}' for service '{generator.name}' in machine {machine.name} is missing."
)
missing_secret_vars.append(Var(generator_name, name))
missing_secret_vars.append(file)
else:
needs_fix, msg = secret_vars_store.needs_fix(
generator_name, name, shared=shared
)
needs_fix, msg = secret_vars_store.needs_fix(generator, file.name)
if needs_fix:
log.info(
f"Secret var '{name}' for service '{generator_name}' in machine {machine.name} needs update: {msg}"
f"Secret var '{file.name}' for service '{generator.name}' in machine {machine.name} needs update: {msg}"
)
unfixed_secret_vars.append(Var(generator_name, name))
unfixed_secret_vars.append(file)
elif not public_vars_store.exists(generator_name, name, shared=shared):
elif not public_vars_store.exists(generator, file.name):
log.info(
f"Public var '{name}' for service '{generator_name}' in machine {machine.name} is missing."
f"Public var '{file.name}' for service '{generator.name}' in machine {machine.name} is missing."
)
missing_public_vars.append(Var(generator_name, name))
missing_public_vars.append(file)
# check if invalidation hash is up to date
if not (
secret_vars_store.hash_is_valid(generator_name)
and public_vars_store.hash_is_valid(generator_name)
secret_vars_store.hash_is_valid(generator)
and public_vars_store.hash_is_valid(generator)
):
invalid_generators.append(generator_name)
invalid_generators.append(generator.name)
log.info(
f"Generator '{generator_name}' in machine {machine.name} has outdated invalidation hash."
f"Generator '{generator.name}' in machine {machine.name} has outdated invalidation hash."
)
log.debug(f"missing_secret_vars: {missing_secret_vars}")
log.debug(f"missing_public_vars: {missing_public_vars}")

323
pkgs/clan-cli/clan_cli/vars/generate.py
View File

@@ -2,9 +2,11 @@ import argparse
import logging
import os
import sys
from dataclasses import dataclass, field
from functools import cached_property
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any
from typing import TYPE_CHECKING, Any
from clan_cli.cmd import run
from clan_cli.completions import (
@@ -15,19 +17,60 @@ from clan_cli.completions import (
from clan_cli.errors import ClanError
from clan_cli.git import commit_files
from clan_cli.machines.inventory import get_all_machines, get_selected_machines
from clan_cli.machines.machines import Machine
from clan_cli.nix import nix_shell
from .check import check_vars
from .graph import (
minimal_closure,
requested_closure,
)
from .prompt import ask
from .prompt import Prompt, ask
from .public_modules import FactStoreBase
from .secret_modules import SecretStoreBase
from .var import Var
log = logging.getLogger(__name__)
if TYPE_CHECKING:
from clan_cli.machines.machines import Machine
@dataclass
class Generator:
name: str
files: list[Var] = field(default_factory=list)
share: bool = False
invalidation_hash: str | None = None
final_script: str = ""
prompts: list[Prompt] = field(default_factory=list)
dependencies: list[str] = field(default_factory=list)
migrate_fact: str | None = None
# TODO: remove this
_machine: "Machine | None" = None
def machine(self, machine: "Machine") -> None:
self._machine = machine
@cached_property
def exists(self) -> bool:
assert self._machine is not None
return check_vars(self._machine, generator_name=self.name)
@classmethod
def from_json(cls: type["Generator"], data: dict[str, Any]) -> "Generator":
return cls(
name=data["name"],
share=data["share"],
final_script=data["finalScript"],
files=[Var.from_json(data["name"], f) for f in data["files"].values()],
invalidation_hash=data["invalidationHash"],
dependencies=data["dependencies"],
migrate_fact=data["migrateFact"],
prompts=[Prompt.from_json(p) for p in data["prompts"].values()],
)
def bubblewrap_cmd(generator: str, tmpdir: Path) -> list[str]:
# fmt: off
@@ -54,26 +97,29 @@ def bubblewrap_cmd(generator: str, tmpdir: Path) -> list[str]:
# TODO: implement caching to not decrypt the same secret multiple times
def decrypt_dependencies(
machine: Machine,
generator_name: str,
machine: "Machine",
generator: Generator,
secret_vars_store: SecretStoreBase,
public_vars_store: FactStoreBase,
) -> dict[str, dict[str, bytes]]:
generator = machine.vars_generators[generator_name]
dependencies = set(generator["dependencies"])
decrypted_dependencies: dict[str, Any] = {}
for dep_generator in dependencies:
decrypted_dependencies[dep_generator] = {}
dep_files = machine.vars_generators[dep_generator]["files"]
shared = machine.vars_generators[dep_generator]["share"]
for file_name, file in dep_files.items():
if file["secret"]:
decrypted_dependencies[dep_generator][file_name] = (
secret_vars_store.get(dep_generator, file_name, shared=shared)
for generator_name in set(generator.dependencies):
decrypted_dependencies[generator_name] = {}
for dep_generator in machine.vars_generators:
if generator_name == dep_generator.name:
break
else:
msg = f"Could not find dependent generator {generator_name} in machine {machine.name}"
raise ClanError(msg)
dep_files = dep_generator.files
for file in dep_files:
if file.secret:
decrypted_dependencies[generator_name][file.name] = (
secret_vars_store.get(dep_generator, file.name)
)
else:
decrypted_dependencies[dep_generator][file_name] = (
public_vars_store.get(dep_generator, file_name, shared=shared)
decrypted_dependencies[generator_name][file.name] = (
public_vars_store.get(dep_generator, file.name)
)
return decrypted_dependencies
@@ -95,8 +141,8 @@ def dependencies_as_dir(
def execute_generator(
machine: Machine,
generator_name: str,
machine: "Machine",
generator: Generator,
secret_vars_store: SecretStoreBase,
public_vars_store: FactStoreBase,
prompt_values: dict[str, str],
@@ -105,14 +151,10 @@ def execute_generator(
msg = f"flake is not a Path: {machine.flake}"
msg += "fact/secret generation is only supported for local flakes"
generator = machine.vars_generators[generator_name]
script = generator["finalScript"]
is_shared = generator["share"]
# build temporary file tree of dependencies
decrypted_dependencies = decrypt_dependencies(
machine,
generator_name,
generator,
secret_vars_store,
public_vars_store,
)
@@ -121,7 +163,7 @@ def execute_generator(
try:
return prompt_values[prompt_name]
except KeyError as e:
msg = f"prompt value for '{prompt_name}' in generator {generator_name} not provided"
msg = f"prompt value for '{prompt_name}' in generator {generator.name} not provided"
raise ClanError(msg) from e
env = os.environ.copy()
@@ -138,97 +180,94 @@ def execute_generator(
dependencies_as_dir(decrypted_dependencies, tmpdir_in)
# populate prompted values
# TODO: make prompts rest API friendly
if generator["prompts"]:
if generator.prompts:
tmpdir_prompts.mkdir()
env["prompts"] = str(tmpdir_prompts)
for prompt_name in generator["prompts"]:
prompt_file = tmpdir_prompts / prompt_name
value = get_prompt_value(prompt_name)
for prompt in generator.prompts:
prompt_file = tmpdir_prompts / prompt.name
value = get_prompt_value(prompt.name)
prompt_file.write_text(value)
if sys.platform == "linux":
cmd = bubblewrap_cmd(script, tmpdir)
cmd = bubblewrap_cmd(generator.final_script, tmpdir)
else:
cmd = ["bash", "-c", script]
cmd = ["bash", "-c", generator.final_script]
run(
cmd,
env=env,
)
files_to_commit = []
# store secrets
files = generator["files"]
files = generator.files
public_changed = False
secret_changed = False
for file_name, file in files.items():
is_deployed = file["deploy"]
secret_file = tmpdir_out / file_name
for file in files:
secret_file = tmpdir_out / file.name
if not secret_file.is_file():
msg = f"did not generate a file for '{file_name}' when running the following command:\n"
msg += script
msg = f"did not generate a file for '{file.name}' when running the following command:\n"
msg += generator.final_script
raise ClanError(msg)
if file["secret"]:
if file.secret:
file_path = secret_vars_store.set(
generator_name,
file_name,
generator,
file,
secret_file.read_bytes(),
shared=is_shared,
deployed=is_deployed,
)
secret_changed = True
else:
file_path = public_vars_store.set(
generator_name,
file_name,
generator,
file,
secret_file.read_bytes(),
shared=is_shared,
)
public_changed = True
if file_path:
files_to_commit.append(file_path)
if generator["invalidationHash"] is not None:
if generator.invalidation_hash is not None:
if public_changed:
public_vars_store.set_invalidation_hash(
generator_name, generator["invalidationHash"]
generator, generator.invalidation_hash
)
if secret_changed:
secret_vars_store.set_invalidation_hash(
generator_name, generator["invalidationHash"]
generator, generator.invalidation_hash
)
commit_files(
files_to_commit,
machine.flake_dir,
f"Update vars via generator {generator_name} for machine {machine.name}",
f"Update vars via generator {generator.name} for machine {machine.name}",
)
def _ask_prompts(
machine: Machine,
generator_names: list[str],
generators: list[Generator],
) -> dict[str, dict[str, str]]:
prompt_values: dict[str, dict[str, str]] = {}
for generator in generator_names:
prompts = machine.vars_generators[generator]["prompts"]
for prompt_name, _prompt in prompts.items():
if generator not in prompt_values:
prompt_values[generator] = {}
var_id = f"{generator}/{prompt_name}"
prompt_values[generator][prompt_name] = ask(var_id, _prompt["type"])
for generator in generators:
for prompt in generator.prompts:
if generator.name not in prompt_values:
prompt_values[generator.name] = {}
var_id = f"{generator.name}/{prompt.name}"
prompt_values[generator.name][prompt.name] = ask(var_id, prompt.prompt_type)
return prompt_values
def get_closure(
machine: Machine,
machine: "Machine",
generator_name: str | None,
regenerate: bool,
) -> list[str]:
from .graph import Generator, all_missing_closure, full_closure
) -> list[Generator]:
from .graph import all_missing_closure, full_closure
vars_generators = machine.vars_generators
generators: dict[str, Generator] = {
name: Generator(name, generator["dependencies"], _machine=machine)
for name, generator in vars_generators.items()
generator.name: generator for generator in vars_generators
}
# TODO: we should remove this
for generator in vars_generators:
generator.machine(machine)
if generator_name is None: # all generators selected
if regenerate:
return full_closure(generators)
@@ -240,109 +279,109 @@ def get_closure(
def _migration_file_exists(
machine: Machine,
service_name: str,
machine: "Machine",
generator: Generator,
fact_name: str,
) -> bool:
is_secret = machine.vars_generators[service_name]["files"][fact_name]["secret"]
for file in generator.files:
if file.name == fact_name:
break
else:
msg = f"Could not find file {fact_name} in generator {generator.name}"
raise ClanError(msg)
is_secret = file.secret
if is_secret:
if machine.secret_facts_store.exists(service_name, fact_name):
if machine.secret_facts_store.exists(generator.name, fact_name):
return True
log.debug(
f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the secret fact store"
f"Cannot migrate fact {fact_name} for service {generator.name}, as it does not exist in the secret fact store"
)
if not is_secret:
if machine.public_facts_store.exists(service_name, fact_name):
if machine.public_facts_store.exists(generator.name, fact_name):
return True
log.debug(
f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the public fact store"
f"Cannot migrate fact {fact_name} for service {generator.name}, as it does not exist in the public fact store"
)
return False
def _migrate_file(
machine: Machine,
generator_name: str,
machine: "Machine",
generator: Generator,
var_name: str,
service_name: str,
fact_name: str,
) -> None:
is_secret = machine.vars_generators[generator_name]["files"][var_name]["secret"]
if is_secret:
for file in generator.files:
if file.name == var_name:
break
else:
msg = f"Could not find file {fact_name} in generator {generator.name}"
raise ClanError(msg)
if file.secret:
old_value = machine.secret_facts_store.get(service_name, fact_name)
machine.secret_vars_store.set(generator, file, old_value)
else:
old_value = machine.public_facts_store.get(service_name, fact_name)
is_shared = machine.vars_generators[generator_name]["share"]
is_deployed = machine.vars_generators[generator_name]["files"][var_name]["deploy"]
if is_secret:
machine.secret_vars_store.set(
generator_name, var_name, old_value, shared=is_shared, deployed=is_deployed
)
else:
machine.public_vars_store.set(
generator_name, var_name, old_value, shared=is_shared, deployed=is_deployed
)
machine.public_vars_store.set(generator, file, old_value)
def _migrate_files(
machine: Machine,
generator_name: str,
machine: "Machine",
generator: Generator,
) -> None:
service_name = machine.vars_generators[generator_name]["migrateFact"]
not_found = []
for var_name, _file in machine.vars_generators[generator_name]["files"].items():
if _migration_file_exists(machine, generator_name, var_name):
_migrate_file(machine, generator_name, var_name, service_name, var_name)
for file in generator.files:
if _migration_file_exists(machine, generator, file.name):
assert generator.migrate_fact is not None
_migrate_file(
machine, generator, file.name, generator.migrate_fact, file.name
)
else:
not_found.append(var_name)
not_found.append(file.name)
if len(not_found) > 0:
msg = f"Could not migrate the following files for generator {generator_name}, as no fact or secret exists with the same name: {not_found}"
msg = f"Could not migrate the following files for generator {generator.name}, as no fact or secret exists with the same name: {not_found}"
raise ClanError(msg)
def _check_can_migrate(
machine: Machine,
generator_name: str,
machine: "Machine",
generator: Generator,
) -> bool:
vars_generator = machine.vars_generators[generator_name]
if "migrateFact" not in vars_generator:
return False
service_name = vars_generator["migrateFact"]
service_name = generator.migrate_fact
if not service_name:
return False
# ensure that none of the generated vars already exist in the store
for fname, file in vars_generator["files"].items():
if file["secret"]:
if machine.secret_vars_store.exists(
generator_name, fname, vars_generator["share"]
):
if vars_generator["deploy"]:
for file in generator.files:
if file.secret:
if machine.secret_vars_store.exists(generator, file.name):
if file.deploy:
machine.secret_vars_store.ensure_machine_has_access(
generator_name, fname, vars_generator["share"]
generator, file.name
)
return False
else:
if machine.public_vars_store.exists(
generator_name, fname, vars_generator["share"]
):
if machine.public_vars_store.exists(generator, file.name):
return False
# ensure that the service to migrate from actually exists
if service_name not in machine.facts_data:
log.debug(
f"Could not migrate facts for generator {generator_name}, as the service {service_name} does not exist"
f"Could not migrate facts for generator {generator.name}, as the service {service_name} does not exist"
)
return False
# ensure that all files can be migrated (exists in the corresponding fact store)
return bool(
all(
_migration_file_exists(machine, generator_name, fname)
for fname in vars_generator["files"]
_migration_file_exists(machine, generator, file.name)
for file in generator.files
)
)
def ensure_consistent_state(
machine: Machine,
machine: "Machine",
generator_name: str | None,
fix: bool,
) -> None:
@@ -352,25 +391,30 @@ def ensure_consistent_state(
"""
if generator_name is None:
generators = list(machine.vars_generators.keys())
generators = machine.vars_generators
else:
generators = [generator_name]
for generator in machine.vars_generators:
if generator_name == generator.name:
generators = [generator]
break
else:
err_msg = (
f"Could not find generator {generator_name} in machine {machine.name}"
)
raise ClanError(err_msg)
outdated = []
for generator_name in generators:
for name, file in machine.vars_generators[generator_name]["files"].items():
shared = machine.vars_generators[generator_name]["share"]
if file["secret"] and machine.secret_vars_store.exists(
generator_name, name, shared=shared
):
if file["deploy"]:
for generator in generators:
for file in generator.files:
if file.secret and machine.secret_vars_store.exists(generator, file.name):
if file.deploy:
machine.secret_vars_store.ensure_machine_has_access(
generator_name, name, shared=shared
generator, file.name
)
needs_update, msg = machine.secret_vars_store.needs_fix(
generator_name, name, shared=shared
generator, file.name
)
if needs_update:
outdated.append((generator_name, name, msg))
outdated.append((generator_name, file.name, msg))
if not fix and outdated:
msg = (
"The local state of some secret vars is inconsistent and needs to be updated.\n"
@@ -382,7 +426,7 @@ def ensure_consistent_state(
def generate_vars_for_machine(
machine: Machine,
machine: "Machine",
generator_name: str | None,
regenerate: bool,
fix: bool,
@@ -391,17 +435,17 @@ def generate_vars_for_machine(
closure = get_closure(machine, generator_name, regenerate)
if len(closure) == 0:
return False
prompt_values = _ask_prompts(machine, closure)
for gen_name in closure:
if _check_can_migrate(machine, gen_name):
_migrate_files(machine, gen_name)
prompt_values = _ask_prompts(closure)
for generator in closure:
if _check_can_migrate(machine, generator):
_migrate_files(machine, generator)
else:
execute_generator(
machine,
gen_name,
generator,
machine.secret_vars_store,
machine.public_vars_store,
prompt_values.get(gen_name, {}),
prompt_values.get(generator.name, {}),
)
# flush caches to make sure the new secrets are available in evaluation
machine.flush_caches()
@@ -409,7 +453,7 @@ def generate_vars_for_machine(
def generate_vars(
machines: list[Machine],
machines: list["Machine"],
generator_name: str | None = None,
regenerate: bool = False,
fix: bool = False,
@@ -423,11 +467,14 @@ def generate_vars(
)
machine.flush_caches()
except Exception as exc:
machine.error(f"Failed to generate facts: {exc}")
errors += [exc]
if len(errors) > 0:
msg = f"Failed to generate facts for {len(errors)} hosts. Check the logs above"
raise ClanError(msg) from errors[0]
errors += [(machine, exc)]
if len(errors) == 1:
raise errors[0][1]
if len(errors) > 1:
msg = f"Failed to generate facts for {len(errors)} hosts:"
for machine, error in errors:
msg += f"\n{machine}: {error}"
raise ClanError(msg) from errors[0][1]
if not was_regenerated and len(machines) > 0:
machine.info("All vars are already up to date")

2
pkgs/clan-cli/clan_cli/vars/get.py
View File

@@ -7,7 +7,7 @@ from clan_cli.completions import add_dynamic_completer, complete_machines
from clan_cli.errors import ClanError
from clan_cli.machines.machines import Machine
from ._types import Var
from .generate import Var
from .list import get_vars
log = logging.getLogger(__name__)

44
pkgs/clan-cli/clan_cli/vars/graph.py
View File

@@ -1,31 +1,21 @@
from __future__ import annotations
from collections.abc import Iterable
from dataclasses import dataclass
from functools import cached_property
from graphlib import TopologicalSorter
from typing import TYPE_CHECKING
from clan_cli.errors import ClanError
from clan_cli.machines.machines import Machine
from .check import check_vars
if TYPE_CHECKING:
from .generate import Generator
class GeneratorNotFoundError(ClanError):
pass
@dataclass
class Generator:
name: str
dependencies: list[str]
_machine: Machine
@cached_property
def exists(self) -> bool:
return check_vars(self._machine, generator_name=self.name)
def missing_dependency_closure(
requested_generators: Iterable[str], generators: dict
requested_generators: Iterable[str], generators: dict[str, Generator]
) -> set[str]:
closure = set(requested_generators)
# extend the graph to include all dependencies which are not on disk
@@ -52,7 +42,9 @@ def add_missing_dependencies(
return missing_dependency_closure(closure, generators) | closure
def add_dependents(requested_generators: Iterable[str], generators: dict) -> set[str]:
def add_dependents(
requested_generators: Iterable[str], generators: dict[str, Generator]
) -> set[str]:
closure = set(requested_generators)
# build reverse dependency graph (graph of dependents)
dependents_graph: dict[str, set[str]] = {}
@@ -72,7 +64,9 @@ def add_dependents(requested_generators: Iterable[str], generators: dict) -> set
return closure
def toposort_closure(_closure: Iterable[str], generators: dict) -> list[str]:
def toposort_closure(
_closure: Iterable[str], generators: dict[str, Generator]
) -> list[Generator]:
closure = set(_closure)
# return the topological sorted list of generators to execute
final_dep_graph = {}
@@ -81,16 +75,16 @@ def toposort_closure(_closure: Iterable[str], generators: dict) -> list[str]:
final_dep_graph[gen_name] = deps
sorter = TopologicalSorter(final_dep_graph)
result = list(sorter.static_order())
return result
return [generators[gen_name] for gen_name in result]
# all generators in topological order
def full_closure(generators: dict) -> list[str]:
def full_closure(generators: dict[str, Generator]) -> list[Generator]:
return toposort_closure(generators.keys(), generators)
# just the missing generators including their dependents
def all_missing_closure(generators: dict) -> list[str]:
def all_missing_closure(generators: dict[str, Generator]) -> list[Generator]:
# collect all generators that are missing from disk
closure = {gen_name for gen_name, gen in generators.items() if not gen.exists}
closure = add_dependents(closure, generators)
@@ -98,7 +92,9 @@ def all_missing_closure(generators: dict) -> list[str]:
# only a selected list of generators including their missing dependencies and their dependents
def requested_closure(requested_generators: list[str], generators: dict) -> list[str]:
def requested_closure(
requested_generators: list[str], generators: dict[str, Generator]
) -> list[Generator]:
closure = set(requested_generators)
# extend the graph to include all dependencies which are not on disk
closure = add_missing_dependencies(closure, generators)
@@ -108,7 +104,9 @@ def requested_closure(requested_generators: list[str], generators: dict) -> list
# just enough to ensure that the list of selected generators are in a consistent state.
# empty if nothing is missing.
def minimal_closure(requested_generators: list[str], generators: dict) -> list[str]:
def minimal_closure(
requested_generators: list[str], generators: dict[str, Generator]
) -> list[Generator]:
closure = set(requested_generators)
final_closure = missing_dependency_closure(closure, generators)
# add requested generators if not already exist

52
pkgs/clan-cli/clan_cli/vars/list.py
View File

@@ -4,10 +4,11 @@ import logging
from clan_cli.api import API
from clan_cli.completions import add_dynamic_completer, complete_machines
from clan_cli.errors import ClanError
from clan_cli.machines.machines import Machine
from ._types import Generator, GeneratorUpdate, Prompt, Var
from .generate import execute_generator
from ._types import GeneratorUpdate
from .generate import Generator, Prompt, Var, execute_generator
from .public_modules import FactStoreBase
from .secret_modules import SecretStoreBase
@@ -35,44 +36,25 @@ def _get_previous_value(
generator: Generator,
prompt: Prompt,
) -> str | None:
if not prompt.has_file:
if not prompt.create_file:
return None
pub_store = public_store(machine)
if pub_store.exists(generator.name, prompt.name, shared=generator.share):
return pub_store.get(
generator.name, prompt.name, shared=generator.share
).decode()
if pub_store.exists(generator, prompt.name):
return pub_store.get(generator, prompt.name).decode()
sec_store = secret_store(machine)
if sec_store.exists(generator.name, prompt.name, shared=generator.share):
return sec_store.get(
generator.name, prompt.name, shared=generator.share
).decode()
if sec_store.exists(generator, prompt.name):
return sec_store.get(generator, prompt.name).decode()
return None
@API.register
# TODO: use machine_name
def get_prompts(machine: Machine) -> list[Generator]:
generators = []
for gen_name, generator in machine.vars_generators.items():
prompts: list[Prompt] = []
gen = Generator(
name=gen_name,
prompts=prompts,
share=generator["share"],
)
for prompt_name, prompt in generator["prompts"].items():
prompt = Prompt(
name=prompt_name,
description=prompt["description"],
type=prompt["type"],
has_file=prompt["createFile"],
generator=gen_name,
)
prompt.previous_value = _get_previous_value(machine, gen, prompt)
prompts.append(prompt)
generators.append(gen)
generators: list[Generator] = machine.vars_generators
for generator in generators:
for prompt in generator.prompts:
prompt.previous_value = _get_previous_value(machine, generator, prompt)
return generators
@@ -82,9 +64,15 @@ def get_prompts(machine: Machine) -> list[Generator]:
@API.register
def set_prompts(machine: Machine, updates: list[GeneratorUpdate]) -> None:
for update in updates:
for generator in machine.vars_generators:
if generator.name == update.generator:
break
else:
msg = f"Generator '{update.generator}' not found in machine {machine.name}"
raise ClanError(msg)
execute_generator(
machine,
update.generator,
generator,
secret_vars_store=secret_store(machine),
public_vars_store=public_store(machine),
prompt_values=update.prompt_values,

43
pkgs/clan-cli/clan_cli/vars/prompt.py
View File

@@ -1,8 +1,9 @@
import enum
import logging
import sys
from dataclasses import dataclass
from getpass import getpass
from clan_cli.errors import ClanError
from typing import Any
log = logging.getLogger(__name__)
@@ -10,18 +11,42 @@ log = logging.getLogger(__name__)
MOCK_PROMPT_RESPONSE = None
def ask(description: str, input_type: str) -> str:
class PromptType(enum.Enum):
LINE = "line"
HIDDEN = "hidden"
MULTILINE = "multiline"
@dataclass
class Prompt:
name: str
description: str
prompt_type: PromptType
create_file: bool = False
previous_value: str | None = None
@classmethod
def from_json(cls: type["Prompt"], data: dict[str, Any]) -> "Prompt":
return cls(
name=data["name"],
description=data["description"],
prompt_type=PromptType(data["type"]),
create_file=data["createFile"],
previous_value=data.get("previousValue"),
)
def ask(description: str, input_type: PromptType) -> str:
if MOCK_PROMPT_RESPONSE:
return next(MOCK_PROMPT_RESPONSE)
if input_type == "line":
match input_type:
case PromptType.LINE:
result = input(f"Enter the value for {description}: ")
elif input_type == "multiline":
case PromptType.MULTILINE:
print(f"Enter the value for {description} (Finish with Ctrl-D): ")
result = sys.stdin.read()
elif input_type == "hidden":
case PromptType.HIDDEN:
result = getpass(f"Enter the value for {description} (hidden): ")
else:
msg = f"Unknown input type: {input_type} for prompt {description}"
raise ClanError(msg)
log.info("Input received. Processing...")
return result

17
pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py
View File

@@ -3,6 +3,7 @@ from pathlib import Path
from clan_cli.errors import ClanError
from clan_cli.machines.machines import Machine
from clan_cli.vars.generate import Generator, Var
from . import FactStoreBase
@@ -18,16 +19,14 @@ class FactStore(FactStoreBase):
def _set(
self,
generator_name: str,
name: str,
generator: Generator,
var: Var,
value: bytes,
shared: bool = False,
deployed: bool = True,
) -> Path | None:
if not self.machine.flake.is_local():
msg = f"in_flake fact storage is only supported for local flakes: {self.machine.flake}"
raise ClanError(msg)
folder = self.directory(generator_name, name, shared)
folder = self.directory(generator, var.name)
if folder.exists():
if not (folder / "value").exists():
# another backend has used that folder before -> error out
@@ -42,8 +41,8 @@ class FactStore(FactStoreBase):
return file_path
# get a single fact
def get(self, generator_name: str, name: str, shared: bool = False) -> bytes:
return (self.directory(generator_name, name, shared) / "value").read_bytes()
def get(self, generator: Generator, name: str) -> bytes:
return (self.directory(generator, name) / "value").read_bytes()
def exists(self, generator_name: str, name: str, shared: bool = False) -> bool:
return (self.directory(generator_name, name, shared) / "value").exists()
def exists(self, generator: Generator, name: str) -> bool:
return (self.directory(generator, name) / "value").exists()

19
pkgs/clan-cli/clan_cli/vars/public_modules/vm.py
View File

@@ -4,6 +4,7 @@ from pathlib import Path
from clan_cli.dirs import vm_state_dir
from clan_cli.errors import ClanError
from clan_cli.machines.machines import Machine
from clan_cli.vars.generate import Generator, Var
from . import FactStoreBase
@@ -21,27 +22,25 @@ class FactStore(FactStoreBase):
def store_name(self) -> str:
return "vm"
def exists(self, service: str, name: str, shared: bool = False) -> bool:
fact_path = self.dir / service / name
def exists(self, generator: Generator, name: str) -> bool:
fact_path = self.dir / generator.name / name
return fact_path.exists()
def _set(
self,
service: str,
name: str,
generator: Generator,
var: Var,
value: bytes,
shared: bool = False,
deployed: bool = True,
) -> Path | None:
fact_path = self.dir / service / name
fact_path = self.dir / generator.name / var.name
fact_path.parent.mkdir(parents=True, exist_ok=True)
fact_path.write_bytes(value)
return None
# get a single fact
def get(self, service: str, name: str, shared: bool = False) -> bytes:
fact_path = self.dir / service / name
def get(self, generator: Generator, name: str) -> bytes:
fact_path = self.dir / generator.name / name
if fact_path.exists():
return fact_path.read_bytes()
msg = f"Fact {name} for service {service} not found"
msg = f"Fact {name} for service {generator.name} not found"
raise ClanError(msg)

14
pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py
View File

@@ -1,8 +1,12 @@
from abc import abstractmethod
from pathlib import Path
from typing import TYPE_CHECKING
from clan_cli.vars._types import StoreBase
if TYPE_CHECKING:
from clan_cli.vars.generate import Generator
class SecretStoreBase(StoreBase):
@property
@@ -12,16 +16,13 @@ class SecretStoreBase(StoreBase):
def needs_upload(self) -> bool:
return True
def ensure_machine_has_access(
self, generator_name: str, name: str, shared: bool = False
) -> None:
def ensure_machine_has_access(self, generator: "Generator", name: str) -> None:
pass
def needs_fix(
self,
generator_name: str,
generator: "Generator",
name: str,
shared: bool,
) -> tuple[bool, str | None]:
"""
Check if local state needs updating, eg. secret needs to be re-encrypted with new keys
@@ -30,9 +31,8 @@ class SecretStoreBase(StoreBase):
def fix(
self,
generator_name: str,
generator: "Generator",
name: str,
shared: bool,
) -> None:
"""
Update local state, eg make sure secret is encrypted with correct keys

46
pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py
View File

@@ -9,6 +9,7 @@ from typing import override
from clan_cli.cmd import Log, run
from clan_cli.machines.machines import Machine
from clan_cli.nix import nix_shell
from clan_cli.vars.generate import Generator, Var
from . import SecretStoreBase
@@ -30,16 +31,14 @@ class SecretStore(SecretStoreBase):
"PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store"
)
def entry_dir(self, generator_name: str, name: str, shared: bool) -> Path:
return Path(self.entry_prefix) / self.rel_dir(generator_name, name, shared)
def entry_dir(self, generator: Generator, name: str) -> Path:
return Path(self.entry_prefix) / self.rel_dir(generator, name)
def _set(
self,
generator_name: str,
name: str,
generator: Generator,
var: Var,
value: bytes,
shared: bool = False,
deployed: bool = True,
) -> Path | None:
run(
nix_shell(
@@ -48,7 +47,7 @@ class SecretStore(SecretStoreBase):
"pass",
"insert",
"-m",
str(self.entry_dir(generator_name, name, shared)),
str(self.entry_dir(generator, var.name)),
],
),
input=value,
@@ -56,22 +55,21 @@ class SecretStore(SecretStoreBase):
)
return None # we manage the files outside of the git repo
def get(self, generator_name: str, name: str, shared: bool = False) -> bytes:
def get(self, generator: Generator, name: str) -> bytes:
return run(
nix_shell(
["nixpkgs#pass"],
[
"pass",
"show",
str(self.entry_dir(generator_name, name, shared)),
str(self.entry_dir(generator, name)),
],
),
).stdout.encode()
def exists(self, generator_name: str, name: str, shared: bool = False) -> bool:
def exists(self, generator: Generator, name: str) -> bool:
return (
Path(self._password_store_dir)
/ f"{self.entry_dir(generator_name, name, shared)}.gpg"
Path(self._password_store_dir) / f"{self.entry_dir(generator, name)}.gpg"
).exists()
def generate_hash(self) -> bytes:
@@ -128,9 +126,9 @@ class SecretStore(SecretStoreBase):
hashes.sort()
manifest = []
for gen_name, generator in self.machine.vars_generators.items():
for f_name in generator["files"]:
manifest.append(f"{gen_name}/{f_name}".encode())
for generator in self.machine.vars_generators:
for file in generator.files:
manifest.append(f"{generator.name}/{file.name}".encode())
manifest += hashes
return b"\n".join(manifest)
@@ -152,24 +150,24 @@ class SecretStore(SecretStoreBase):
def upload(self, output_dir: Path) -> None:
with tarfile.open(output_dir / "secrets.tar.gz", "w:gz") as tar:
for gen_name, generator in self.machine.vars_generators.items():
for generator in self.machine.vars_generators:
dir_exists = False
for f_name, file in generator["files"].items():
if not file["deploy"]:
for file in generator.files:
if not file.deploy:
continue
if not file["secret"]:
if not file.secret:
continue
if not dir_exists:
tar_dir = tarfile.TarInfo(name=gen_name)
tar_dir = tarfile.TarInfo(name=generator.name)
tar_dir.type = tarfile.DIRTYPE
tar_dir.mode = 0o511
tar.addfile(tarinfo=tar_dir)
dir_exists = True
tar_file = tarfile.TarInfo(name=f"{gen_name}/{f_name}")
content = self.get(gen_name, f_name, generator["share"])
tar_file = tarfile.TarInfo(name=f"{generator.name}/{file.name}")
content = self.get(generator, file.name)
tar_file.size = len(content)
tar_file.mode = 0o440
tar_file.uname = file.get("owner", "root")
tar_file.gname = file.get("group", "root")
tar_file.uname = file.owner
tar_file.gname = file.group
tar.addfile(tarinfo=tar_file, fileobj=io.BytesIO(content))
(output_dir / ".pass_info").write_bytes(self.generate_hash())

64
pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py
View File

@@ -18,6 +18,7 @@ from clan_cli.secrets.secrets import (
has_secret,
)
from clan_cli.secrets.sops import KeyType, generate_private_key
from clan_cli.vars.generate import Generator, Var
from . import SecretStoreBase
@@ -42,10 +43,9 @@ class SecretStore(SecretStoreBase):
if not self.machine.vars_generators:
return
has_secrets = False
for generator in self.machine.vars_generators.values():
if "files" in generator:
for file in generator["files"].values():
if file["secret"]:
for generator in self.machine.vars_generators:
for file in generator.files:
if file.secret:
has_secrets = True
if not has_secrets:
return
@@ -67,9 +67,9 @@ class SecretStore(SecretStoreBase):
return "sops"
def user_has_access(
self, user: str, generator_name: str, secret_name: str, shared: bool
self, user: str, generator: Generator, secret_name: str
) -> bool:
secret_path = self.secret_path(generator_name, secret_name, shared=shared)
secret_path = self.secret_path(generator, secret_name)
secret = json.loads((secret_path / "secret").read_text())
recipients = [r["recipient"] for r in (secret["sops"].get("age") or [])]
users_folder_path = sops_users_folder(self.machine.flake_dir)
@@ -78,10 +78,8 @@ class SecretStore(SecretStoreBase):
]
return user_pubkey in recipients
def machine_has_access(
self, generator_name: str, secret_name: str, shared: bool
) -> bool:
secret_path = self.secret_path(generator_name, secret_name, shared)
def machine_has_access(self, generator: Generator, secret_name: str) -> bool:
secret_path = self.secret_path(generator, secret_name)
secret = json.loads((secret_path / "secret").read_text())
recipients = [r["recipient"] for r in (secret["sops"].get("age") or [])]
machines_folder_path = sops_machines_folder(self.machine.flake_dir)
@@ -90,20 +88,16 @@ class SecretStore(SecretStoreBase):
)["publickey"]
return machine_pubkey in recipients
def secret_path(
self, generator_name: str, secret_name: str, shared: bool = False
) -> Path:
return self.directory(generator_name, secret_name, shared=shared)
def secret_path(self, generator: Generator, secret_name: str) -> Path:
return self.directory(generator, secret_name)
def _set(
self,
generator_name: str,
name: str,
generator: Generator,
var: Var,
value: bytes,
shared: bool = False,
deployed: bool = True,
) -> Path | None:
secret_folder = self.secret_path(generator_name, name, shared)
secret_folder = self.secret_path(generator, var.name)
# delete directory
if secret_folder.exists() and not (secret_folder / "secret").exists():
# another backend has used that folder before -> error out
@@ -115,15 +109,15 @@ class SecretStore(SecretStoreBase):
self.machine.flake_dir,
secret_folder,
value,
add_machines=[self.machine.name] if deployed else [],
add_machines=[self.machine.name] if var.deploy else [],
add_groups=self.machine.deployment["sops"]["defaultGroups"],
git_commit=False,
)
return secret_folder
def get(self, generator_name: str, name: str, shared: bool = False) -> bytes:
def get(self, generator: Generator, name: str) -> bytes:
return decrypt_secret(
self.machine.flake_dir, self.secret_path(generator_name, name, shared)
self.machine.flake_dir, self.secret_path(generator, name)
).encode("utf-8")
def upload(self, output_dir: Path) -> None:
@@ -137,16 +131,14 @@ class SecretStore(SecretStoreBase):
)
(output_dir / "key.txt").write_text(key)
def exists(self, generator_name: str, name: str, shared: bool = False) -> bool:
secret_folder = self.secret_path(generator_name, name, shared)
def exists(self, generator: Generator, name: str) -> bool:
secret_folder = self.secret_path(generator, name)
return (secret_folder / "secret").exists()
def ensure_machine_has_access(
self, generator_name: str, name: str, shared: bool = False
) -> None:
if self.machine_has_access(generator_name, name, shared):
def ensure_machine_has_access(self, generator: Generator, name: str) -> None:
if self.machine_has_access(generator, name):
return
secret_folder = self.secret_path(generator_name, name, shared)
secret_folder = self.secret_path(generator, name)
add_secret(self.machine.flake_dir, self.machine.name, secret_folder)
def collect_keys_for_secret(self, path: Path) -> set[tuple[str, KeyType]]:
@@ -170,10 +162,8 @@ class SecretStore(SecretStoreBase):
return keys
@override
def needs_fix(
self, generator_name: str, name: str, shared: bool
) -> tuple[bool, str | None]:
secret_path = self.secret_path(generator_name, name, shared)
def needs_fix(self, generator: Generator, name: str) -> tuple[bool, str | None]:
secret_path = self.secret_path(generator, name)
recipients_ = json.loads((secret_path / "secret").read_text())["sops"]["age"]
current_recipients = {r["recipient"] for r in recipients_}
wanted_recipients = {
@@ -181,19 +171,19 @@ class SecretStore(SecretStoreBase):
}
needs_update = current_recipients != wanted_recipients
recipients_to_add = wanted_recipients - current_recipients
var_id = f"{generator_name}/{name}"
var_id = f"{generator.name}/{name}"
msg = (
f"One or more recipient keys were added to secret{' shared' if shared else ''} var '{var_id}', but it was never re-encrypted. "
f"One or more recipient keys were added to secret{' shared' if generator.share else ''} var '{var_id}', but it was never re-encrypted. "
f"This could have been a malicious actor trying to add their keys, please investigate. "
f"Added keys: {', '.join(recipients_to_add)}"
)
return needs_update, msg
@override
def fix(self, generator_name: str, name: str, shared: bool) -> None:
def fix(self, generator: Generator, name: str) -> None:
from clan_cli.secrets.secrets import update_keys
secret_path = self.secret_path(generator_name, name, shared)
secret_path = self.secret_path(generator, name)
update_keys(
secret_path,
collect_keys_for_path(secret_path),

13
pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py
View File

@@ -3,6 +3,7 @@ from pathlib import Path
from clan_cli.dirs import vm_state_dir
from clan_cli.machines.machines import Machine
from clan_cli.vars.generate import Generator, Var
from . import SecretStoreBase
@@ -19,19 +20,17 @@ class SecretStore(SecretStoreBase):
def _set(
self,
service: str,
name: str,
generator: Generator,
var: Var,
value: bytes,
shared: bool = False,
deployed: bool = True,
) -> Path | None:
secret_file = self.dir / service / name
secret_file = self.dir / generator.name / var.name
secret_file.parent.mkdir(parents=True, exist_ok=True)
secret_file.write_bytes(value)
return None # we manage the files outside of the git repo
def get(self, service: str, name: str, shared: bool = False) -> bytes:
secret_file = self.dir / service / name
def get(self, generator: Generator, name: str) -> bytes:
secret_file = self.dir / generator.name / name
return secret_file.read_bytes()
def upload(self, output_dir: Path) -> None:

5
pkgs/clan-cli/clan_cli/vars/set.py
View File

@@ -6,8 +6,9 @@ from clan_cli.clan_uri import FlakeId
from clan_cli.completions import add_dynamic_completer, complete_machines
from clan_cli.machines.machines import Machine
from clan_cli.vars.get import get_var
from clan_cli.vars.prompt import PromptType
from ._types import Var
from .generate import Var
from .prompt import ask
log = logging.getLogger(__name__)
@@ -31,7 +32,7 @@ def set_via_stdin(machine: str, var_id: str, flake: FlakeId) -> None:
_machine = Machine(name=machine, flake=flake)
var = get_var(_machine, var_id)
if sys.stdin.isatty():
new_value = ask(var.id, "hidden").encode("utf-8")
new_value = ask(var.id, PromptType.HIDDEN).encode("utf-8")
else:
new_value = sys.stdin.buffer.read()
set_var(_machine, var, new_value, flake)

77
pkgs/clan-cli/clan_cli/vars/var.py Normal file
View File

@@ -0,0 +1,77 @@
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from clan_cli.vars.generate import Generator
from ._types import StoreBase
@dataclass
class Var:
id: str
name: str
secret: bool = True
deploy: bool = False
owner: str = "root"
group: str = "root"
# TODO: those shouldn't be set here
_store: "StoreBase | None" = None
_generator: "Generator | None" = None
def store(self, store: "StoreBase") -> None:
self._store = store
def generator(self, generator: "Generator") -> None:
self._generator = generator
@property
def value(self) -> bytes:
assert self._store is not None
assert self._generator is not None
if not self._store.exists(self._generator, self.name):
msg = f"Var {self.id} has not been generated yet"
raise ValueError(msg)
# try decode the value or return <binary blob>
return self._store.get(self._generator, self.name)
@property
def printable_value(self) -> str:
try:
return self.value.decode()
except UnicodeDecodeError:
return "<binary blob>"
def set(self, value: bytes) -> None:
assert self._store is not None
assert self._generator is not None
self._store.set(self._generator, self, value)
@property
def exists(self) -> bool:
assert self._store is not None
assert self._generator is not None
return self._store.exists(self._generator, self.name)
def __str__(self) -> str:
if self._store is None or self._generator is None:
return f"{self.id}: <not initialized>"
# TODO: we don't want __str__ with side effects, this should be a separate method
if self._store.exists(self._generator, self.name):
if self.secret:
return f"{self.id}: ********"
return f"{self.id}: {self.printable_value}"
return f"{self.id}: <not set>"
@classmethod
def from_json(cls: type["Var"], generator_name: str, data: dict[str, Any]) -> "Var":
return cls(
id=f"{generator_name}/{data['name']}",
name=data["name"],
secret=data["secret"],
deploy=data["deploy"],
owner=data.get("owner", "root"),
group=data.get("group", "root"),
)

200
pkgs/clan-cli/tests/test_vars.py
View File

@@ -1,6 +1,5 @@
import json
import shutil
from dataclasses import dataclass
from pathlib import Path
import pytest
@@ -10,8 +9,9 @@ from clan_cli.errors import ClanError
from clan_cli.machines.machines import Machine
from clan_cli.nix import nix_eval, run
from clan_cli.vars.check import check_vars
from clan_cli.vars.generate import generate_vars_for_machine
from clan_cli.vars.generate import Generator, generate_vars_for_machine
from clan_cli.vars.get import get_var
from clan_cli.vars.graph import all_missing_closure, requested_closure
from clan_cli.vars.list import stringify_all_vars
from clan_cli.vars.public_modules import in_repo
from clan_cli.vars.secret_modules import password_store, sops
@@ -48,31 +48,49 @@ def test_dependencies_as_files(temp_dir: Path) -> None:
def test_required_generators() -> None:
from clan_cli.vars.graph import all_missing_closure, requested_closure
@dataclass
class Generator:
dependencies: list[str]
exists: bool # result is already on disk
gen_1 = Generator(name="gen_1", dependencies=[])
gen_2 = Generator(name="gen_2", dependencies=["gen_1"])
gen_2a = Generator(name="gen_2a", dependencies=["gen_2"])
gen_2b = Generator(name="gen_2b", dependencies=["gen_2"])
gen_1.exists = True
gen_2.exists = False
gen_2a.exists = False
gen_2b.exists = True
generators = {
"gen_1": Generator([], True),
"gen_2": Generator(["gen_1"], False),
"gen_2a": Generator(["gen_2"], False),
"gen_2b": Generator(["gen_2"], True),
generator.name: generator for generator in [gen_1, gen_2, gen_2a, gen_2b]
}
assert requested_closure(["gen_1"], generators) == [
def generator_names(generator: list[Generator]) -> list[str]:
return [gen.name for gen in generator]
assert generator_names(requested_closure(["gen_1"], generators)) == [
"gen_1",
"gen_2",
"gen_2a",
"gen_2b",
]
assert requested_closure(["gen_2"], generators) == ["gen_2", "gen_2a", "gen_2b"]
assert requested_closure(["gen_2a"], generators) == ["gen_2", "gen_2a", "gen_2b"]
assert requested_closure(["gen_2b"], generators) == ["gen_2", "gen_2a", "gen_2b"]
assert generator_names(requested_closure(["gen_2"], generators)) == [
"gen_2",
"gen_2a",
"gen_2b",
]
assert generator_names(requested_closure(["gen_2a"], generators)) == [
"gen_2",
"gen_2a",
"gen_2b",
]
assert generator_names(requested_closure(["gen_2b"], generators)) == [
"gen_2",
"gen_2a",
"gen_2b",
]
assert all_missing_closure(generators) == ["gen_2", "gen_2a", "gen_2b"]
assert generator_names(all_missing_closure(generators)) == [
"gen_2",
"gen_2a",
"gen_2b",
]
@pytest.mark.impure
@@ -96,8 +114,8 @@ def test_generate_public_var(
store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert store.exists("my_generator", "my_value")
assert store.get("my_generator", "my_value").decode() == "hello\n"
assert store.exists(Generator("my_generator"), "my_value")
assert store.get(Generator("my_generator"), "my_value").decode() == "hello\n"
vars_text = stringify_all_vars(machine)
assert "my_generator/my_value: hello" in vars_text
vars_eval = run(
@@ -133,12 +151,12 @@ def test_generate_secret_var_sops(
in_repo_store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert not in_repo_store.exists("my_generator", "my_secret")
assert not in_repo_store.exists(Generator("my_generator"), "my_secret")
sops_store = sops.SecretStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert sops_store.exists("my_generator", "my_secret")
assert sops_store.get("my_generator", "my_secret").decode() == "hello\n"
assert sops_store.exists(Generator("my_generator"), "my_secret")
assert sops_store.get(Generator("my_generator"), "my_secret").decode() == "hello\n"
vars_text = stringify_all_vars(machine)
assert "my_generator/my_secret" in vars_text
# test regeneration works
@@ -168,12 +186,12 @@ def test_generate_secret_var_sops_with_default_group(
in_repo_store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert not in_repo_store.exists("my_generator", "my_secret")
assert not in_repo_store.exists(Generator("my_generator"), "my_secret")
sops_store = sops.SecretStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert sops_store.exists("my_generator", "my_secret")
assert sops_store.get("my_generator", "my_secret").decode() == "hello\n"
assert sops_store.exists(Generator("my_generator"), "my_secret")
assert sops_store.get(Generator("my_generator"), "my_secret").decode() == "hello\n"
# add another user and check if secret gets re-encrypted
from clan_cli.secrets.sops import generate_private_key
@@ -197,7 +215,7 @@ def test_generate_secret_var_sops_with_default_group(
# check if new user can access the secret
monkeypatch.setenv("USER", "uschi")
assert sops_store.user_has_access(
"uschi", "my_generator", "my_secret", shared=False
"uschi", Generator("my_generator", share=False), "my_secret"
)
@@ -232,13 +250,17 @@ def test_generated_shared_secret_sops(
assert check_vars(machine2)
m1_sops_store = sops.SecretStore(machine1)
m2_sops_store = sops.SecretStore(machine2)
assert m1_sops_store.exists("my_shared_generator", "my_shared_secret", shared=True)
assert m2_sops_store.exists("my_shared_generator", "my_shared_secret", shared=True)
assert m1_sops_store.exists(
Generator("my_shared_generator", share=True), "my_shared_secret"
)
assert m2_sops_store.exists(
Generator("my_shared_generator", share=True), "my_shared_secret"
)
assert m1_sops_store.machine_has_access(
"my_shared_generator", "my_shared_secret", shared=True
Generator("my_shared_generator", share=True), "my_shared_secret"
)
assert m2_sops_store.machine_has_access(
"my_shared_generator", "my_shared_secret", shared=True
Generator("my_shared_generator", share=True), "my_shared_secret"
)
@@ -277,11 +299,19 @@ def test_generate_secret_var_password_store(
store = password_store.SecretStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert store.exists("my_generator", "my_secret", shared=False)
assert not store.exists("my_generator", "my_secret", shared=True)
assert store.exists("my_shared_generator", "my_shared_secret", shared=True)
assert not store.exists("my_shared_generator", "my_shared_secret", shared=False)
assert store.get("my_generator", "my_secret", shared=False).decode() == "hello\n"
assert store.exists(Generator("my_generator", share=False, files=[]), "my_secret")
assert not store.exists(
Generator("my_generator", share=True, files=[]), "my_secret"
)
assert store.exists(
Generator("my_shared_generator", share=True, files=[]), "my_shared_secret"
)
assert not store.exists(
Generator("my_shared_generator", share=False, files=[]), "my_shared_secret"
)
generator = Generator(name="my_generator", share=False, files=[])
assert store.get(generator, "my_secret").decode() == "hello\n"
vars_text = stringify_all_vars(machine)
assert "my_generator/my_secret" in vars_text
@@ -321,10 +351,16 @@ def test_generate_secret_for_multiple_machines(
in_repo_store2 = in_repo.FactStore(
Machine(name="machine2", flake=FlakeId(str(flake.path)))
)
assert in_repo_store1.exists("my_generator", "my_value")
assert in_repo_store2.exists("my_generator", "my_value")
assert in_repo_store1.get("my_generator", "my_value").decode() == "machine1\n"
assert in_repo_store2.get("my_generator", "my_value").decode() == "machine2\n"
assert in_repo_store1.exists(Generator("my_generator"), "my_value")
assert in_repo_store2.exists(Generator("my_generator"), "my_value")
assert (
in_repo_store1.get(Generator("my_generator"), "my_value").decode()
== "machine1\n"
)
assert (
in_repo_store2.get(Generator("my_generator"), "my_value").decode()
== "machine2\n"
)
# check if secret vars have been created correctly
sops_store1 = sops.SecretStore(
Machine(name="machine1", flake=FlakeId(str(flake.path)))
@@ -332,10 +368,14 @@ def test_generate_secret_for_multiple_machines(
sops_store2 = sops.SecretStore(
Machine(name="machine2", flake=FlakeId(str(flake.path)))
)
assert sops_store1.exists("my_generator", "my_secret")
assert sops_store2.exists("my_generator", "my_secret")
assert sops_store1.get("my_generator", "my_secret").decode() == "machine1\n"
assert sops_store2.get("my_generator", "my_secret").decode() == "machine2\n"
assert sops_store1.exists(Generator("my_generator"), "my_secret")
assert sops_store2.exists(Generator("my_generator"), "my_secret")
assert (
sops_store1.get(Generator("my_generator"), "my_secret").decode() == "machine1\n"
)
assert (
sops_store2.get(Generator("my_generator"), "my_secret").decode() == "machine2\n"
)
@pytest.mark.impure
@@ -357,10 +397,16 @@ def test_dependant_generators(
in_repo_store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert in_repo_store.exists("parent_generator", "my_value")
assert in_repo_store.get("parent_generator", "my_value").decode() == "hello\n"
assert in_repo_store.exists("child_generator", "my_value")
assert in_repo_store.get("child_generator", "my_value").decode() == "hello\n"
assert in_repo_store.exists(Generator("parent_generator"), "my_value")
assert (
in_repo_store.get(Generator("parent_generator"), "my_value").decode()
== "hello\n"
)
assert in_repo_store.exists(Generator("child_generator"), "my_value")
assert (
in_repo_store.get(Generator("child_generator"), "my_value").decode()
== "hello\n"
)
@pytest.mark.impure
@@ -395,8 +441,10 @@ def test_prompt(
in_repo_store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert in_repo_store.exists("my_generator", "my_value")
assert in_repo_store.get("my_generator", "my_value").decode() == input_value
assert in_repo_store.exists(Generator("my_generator"), "my_value")
assert (
in_repo_store.get(Generator("my_generator"), "my_value").decode() == input_value
)
@pytest.mark.impure
@@ -437,15 +485,25 @@ def test_share_flag(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
# check secrets stored correctly
assert sops_store.exists("shared_generator", "my_secret", shared=True)
assert not sops_store.exists("shared_generator", "my_secret", shared=False)
assert sops_store.exists("unshared_generator", "my_secret", shared=False)
assert not sops_store.exists("unshared_generator", "my_secret", shared=True)
assert sops_store.exists(Generator("shared_generator", share=True), "my_secret")
assert not sops_store.exists(
Generator("shared_generator", share=False), "my_secret"
)
assert sops_store.exists(Generator("unshared_generator", share=False), "my_secret")
assert not sops_store.exists(
Generator("unshared_generator", share=True), "my_secret"
)
# check values stored correctly
assert in_repo_store.exists("shared_generator", "my_value", shared=True)
assert not in_repo_store.exists("shared_generator", "my_value", shared=False)
assert in_repo_store.exists("unshared_generator", "my_value", shared=False)
assert not in_repo_store.exists("unshared_generator", "my_value", shared=True)
assert in_repo_store.exists(Generator("shared_generator", share=True), "my_value")
assert not in_repo_store.exists(
Generator("shared_generator", share=False), "my_value"
)
assert in_repo_store.exists(
Generator("unshared_generator", share=False), "my_value"
)
assert not in_repo_store.exists(
Generator("unshared_generator", share=True), "my_value"
)
vars_eval = run(
nix_eval(
[
@@ -505,9 +563,13 @@ def test_prompt_create_file(
sops_store = sops.SecretStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert sops_store.exists("my_generator", "prompt1")
assert not sops_store.exists("my_generator", "prompt2")
assert sops_store.get("my_generator", "prompt1").decode() == "input1"
assert sops_store.exists(
Generator(name="my_generator", share=False, files=[]), "prompt1"
)
assert not sops_store.exists(Generator(name="my_generator"), "prompt2")
assert (
sops_store.get(Generator(name="my_generator"), "prompt1").decode() == "input1"
)
@pytest.mark.impure
@@ -558,8 +620,8 @@ def test_api_set_prompts(
],
)
store = in_repo.FactStore(machine)
assert store.exists("my_generator", "prompt1")
assert store.get("my_generator", "prompt1").decode() == "input1"
assert store.exists(Generator("my_generator"), "prompt1")
assert store.get(Generator("my_generator"), "prompt1").decode() == "input1"
set_prompts(
machine,
[
@@ -569,7 +631,7 @@ def test_api_set_prompts(
)
],
)
assert store.get("my_generator", "prompt1").decode() == "input2"
assert store.get(Generator("my_generator"), "prompt1").decode() == "input2"
@pytest.mark.impure
@@ -771,8 +833,8 @@ def test_migration_skip(
in_repo_store = in_repo.FactStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert in_repo_store.exists("my_generator", "my_value")
assert in_repo_store.get("my_generator", "my_value").decode() == "world"
assert in_repo_store.exists(Generator("my_generator"), "my_value")
assert in_repo_store.get(Generator("my_generator"), "my_value").decode() == "world"
@pytest.mark.impure
@@ -805,10 +867,10 @@ def test_migration(
sops_store = sops.SecretStore(
Machine(name="my_machine", flake=FlakeId(str(flake.path)))
)
assert in_repo_store.exists("my_generator", "my_value")
assert in_repo_store.get("my_generator", "my_value").decode() == "hello"
assert sops_store.exists("my_generator", "my_secret")
assert sops_store.get("my_generator", "my_secret").decode() == "hello"
assert in_repo_store.exists(Generator("my_generator"), "my_value")
assert in_repo_store.get(Generator("my_generator"), "my_value").decode() == "hello"
assert sops_store.exists(Generator("my_generator"), "my_secret")
assert sops_store.get(Generator("my_generator"), "my_secret").decode() == "hello"
@pytest.mark.impure