From c9e80f38cae840164946ceace08384afa3a3ee68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 20 Nov 2024 19:03:35 +0100 Subject: [PATCH] vars: make interface more type-safe --- pkgs/clan-cli/clan_cli/machines/machines.py | 19 +- pkgs/clan-cli/clan_cli/vars/_types.py | 145 ++------ pkgs/clan-cli/clan_cli/vars/check.py | 84 +++-- pkgs/clan-cli/clan_cli/vars/generate.py | 323 ++++++++++-------- pkgs/clan-cli/clan_cli/vars/get.py | 2 +- pkgs/clan-cli/clan_cli/vars/graph.py | 44 ++- pkgs/clan-cli/clan_cli/vars/list.py | 52 ++- pkgs/clan-cli/clan_cli/vars/prompt.py | 51 ++- .../clan_cli/vars/public_modules/in_repo.py | 17 +- .../clan_cli/vars/public_modules/vm.py | 19 +- .../clan_cli/vars/secret_modules/__init__.py | 14 +- .../vars/secret_modules/password_store.py | 46 ++- .../clan_cli/vars/secret_modules/sops.py | 66 ++-- .../clan_cli/vars/secret_modules/vm.py | 13 +- pkgs/clan-cli/clan_cli/vars/set.py | 5 +- pkgs/clan-cli/clan_cli/vars/var.py | 77 +++++ pkgs/clan-cli/tests/test_vars.py | 200 +++++++---- 17 files changed, 656 insertions(+), 521 deletions(-) create mode 100644 pkgs/clan-cli/clan_cli/vars/var.py diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 29e802d7d..be7547263 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field from functools import cached_property from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal from clan_cli.clan_uri import FlakeId from clan_cli.cmd import run_no_stdout @@ -21,6 +21,9 @@ from clan_cli.vars.secret_modules import SecretStoreBase log = logging.getLogger(__name__) +if TYPE_CHECKING: + from clan_cli.vars.generate import Generator + @dataclass class Machine: @@ -156,10 +159,16 @@ class Machine: return {} @property - def vars_generators(self) -> dict[str, dict[str, Any]]: - if self.deployment["vars"]["generators"]: - return self.deployment["vars"]["generators"] - return {} + def vars_generators(self) -> list["Generator"]: + from clan_cli.vars.generate import Generator + + clan_vars = self.deployment.get("vars") + if clan_vars is None: + return [] + generators: dict[str, Any] = clan_vars.get("generators") + if generators is None: + return [] + return [Generator.from_json(gen) for gen in generators.values()] @property def secrets_upload_directory(self) -> str: diff --git a/pkgs/clan-cli/clan_cli/vars/_types.py b/pkgs/clan-cli/clan_cli/vars/_types.py index 6e76dbe90..32b4b2fcf 100644 --- a/pkgs/clan-cli/clan_cli/vars/_types.py +++ b/pkgs/clan-cli/clan_cli/vars/_types.py @@ -2,10 +2,14 @@ import logging from abc import ABC, abstractmethod from dataclasses import dataclass from pathlib import Path +from typing import TYPE_CHECKING from clan_cli.errors import ClanError from clan_cli.machines import machines +if TYPE_CHECKING: + from .generate import Generator, Var + log = logging.getLogger(__name__) @@ -16,69 +20,12 @@ def string_repr(value: bytes) -> str: return "" -@dataclass -class Prompt: - name: str - description: str - type: str - has_file: bool - generator: str - previous_value: str | None = None - - -# TODO: add flag 'pending' generator needs to be executed -@dataclass -class Generator: - name: str - share: bool - prompts: list[Prompt] - - @dataclass class GeneratorUpdate: generator: str prompt_values: dict[str, str] -@dataclass -class Var: - _store: "StoreBase" - generator: str - name: str - id: str - secret: bool - shared: bool - deployed: bool - owner: str - group: str - - @property - def value(self) -> bytes: - if not self._store.exists(self.generator, self.name, self.shared): - msg = f"Var {self.id} has not been generated yet" - raise ValueError(msg) - # try decode the value or return - return self._store.get(self.generator, self.name, self.shared) - - @property - def printable_value(self) -> str: - return string_repr(self.value) - - def set(self, value: bytes) -> None: - self._store.set(self.generator, self.name, value, self.shared, self.deployed) - - @property - def exists(self) -> bool: - return self._store.exists(self.generator, self.name, self.shared) - - def __str__(self) -> str: - if self._store.exists(self.generator, self.name, self.shared): - if self.secret: - return f"{self.id}: ********" - return f"{self.id}: {self.printable_value}" - return f"{self.id}: " - - class StoreBase(ABC): def __init__(self, machine: "machines.Machine") -> None: self.machine = machine @@ -90,24 +37,22 @@ class StoreBase(ABC): # get a single fact @abstractmethod - def get(self, generator_name: str, name: str, shared: bool = False) -> bytes: + def get(self, generator: "Generator", name: str) -> bytes: pass @abstractmethod def _set( self, - generator_name: str, - name: str, + generator: "Generator", + var: "Var", value: bytes, - shared: bool = False, - deployed: bool = True, ) -> Path | None: """ override this method to implement the actual creation of the file """ @abstractmethod - def exists(self, generator_name: str, name: str, shared: bool = False) -> bool: + def exists(self, generator: "Generator", name: str) -> bool: pass @property @@ -122,106 +67,88 @@ class StoreBase(ABC): ) raise ClanError(msg) - def rel_dir(self, generator_name: str, var_name: str, shared: bool = False) -> Path: - if shared: - return Path(f"shared/{generator_name}/{var_name}") - return Path(f"per-machine/{self.machine.name}/{generator_name}/{var_name}") + def rel_dir(self, generator: "Generator", var_name: str) -> Path: + if generator.share: + return Path(f"shared/{generator.name}/{var_name}") + return Path(f"per-machine/{self.machine.name}/{generator.name}/{var_name}") - def directory( - self, generator_name: str, var_name: str, shared: bool = False - ) -> Path: - return ( - Path(self.machine.flake_dir) - / "vars" - / self.rel_dir(generator_name, var_name, shared) - ) + def directory(self, generator: "Generator", var_name: str) -> Path: + return Path(self.machine.flake_dir) / "vars" / self.rel_dir(generator, var_name) def set( self, - generator_name: str, - var_name: str, + generator: "Generator", + var: "Var", value: bytes, - shared: bool = False, - deployed: bool = True, ) -> Path | None: - if self.exists(generator_name, var_name, shared): + if self.exists(generator, var.name): if self.is_secret_store: old_val = None old_val_str = "********" else: - old_val = self.get(generator_name, var_name, shared) + old_val = self.get(generator, var.name) old_val_str = string_repr(old_val) else: old_val = None old_val_str = "" - new_file = self._set(generator_name, var_name, value, shared, deployed) + new_file = self._set(generator, var, value) if self.is_secret_store: - print(f"Updated secret var {generator_name}/{var_name}\n") + print(f"Updated secret var {generator.name}/{var.name}\n") else: if value != old_val: print( - f"Updated var {generator_name}/{var_name}\n" + f"Updated var {generator.name}/{var.name}\n" f" old: {old_val_str}\n" f" new: {string_repr(value)}" ) else: print( - f"Var {generator_name}/{var_name} remains unchanged: {old_val_str}" + f"Var {generator.name}/{var.name} remains unchanged: {old_val_str}" ) return new_file - def get_all(self) -> list[Var]: + def get_all(self) -> list["Var"]: all_vars = [] - for gen_name, generator in self.machine.vars_generators.items(): - for f_name, file in generator["files"].items(): + for generator in self.machine.vars_generators: + for var in generator.files: # only handle vars compatible to this store - if self.is_secret_store != file["secret"]: + if self.is_secret_store != var.secret: continue - all_vars.append( - Var( - _store=self, - generator=gen_name, - name=f_name, - id=f"{gen_name}/{f_name}", - secret=file["secret"], - shared=generator["share"], - deployed=file["deploy"], - owner=file.get("owner", "root"), - group=file.get("group", "root"), - ) - ) + var.store(self) + var.generator(generator) + all_vars.append(var) return all_vars - def get_invalidation_hash(self, generator_name: str) -> str | None: + def get_invalidation_hash(self, generator: "Generator") -> str | None: """ Return the invalidation hash that indicates if a generator needs to be re-run due to a change in its definition """ hash_file = ( - self.machine.flake_dir / "vars" / generator_name / "invalidation_hash" + self.machine.flake_dir / "vars" / generator.name / "invalidation_hash" ) if not hash_file.exists(): return None return hash_file.read_text().strip() - def set_invalidation_hash(self, generator_name: str, hash_str: str) -> None: + def set_invalidation_hash(self, generator: "Generator", hash_str: str) -> None: """ Store the invalidation hash that indicates if a generator needs to be re-run """ hash_file = ( - self.machine.flake_dir / "vars" / generator_name / "invalidation_hash" + self.machine.flake_dir / "vars" / generator.name / "invalidation_hash" ) hash_file.parent.mkdir(parents=True, exist_ok=True) hash_file.write_text(hash_str) - def hash_is_valid(self, generator_name: str) -> bool: + def hash_is_valid(self, generator: "Generator") -> bool: """ Check if the invalidation hash is up to date If the hash is not set in nix and hasn't been stored before, it is considered valid -> this provides backward and forward compatibility """ - stored_hash = self.get_invalidation_hash(generator_name) - target_hash = self.machine.vars_generators[generator_name]["invalidationHash"] + stored_hash = self.get_invalidation_hash(generator) + target_hash = generator.invalidation_hash # if the hash is neither set in nix nor on disk, it is considered valid (provides backwards compat) if target_hash is None and stored_hash is None: return True diff --git a/pkgs/clan-cli/clan_cli/vars/check.py b/pkgs/clan-cli/clan_cli/vars/check.py index e0d86fd8a..7c91f75a6 100644 --- a/pkgs/clan-cli/clan_cli/vars/check.py +++ b/pkgs/clan-cli/clan_cli/vars/check.py @@ -1,75 +1,91 @@ import argparse import importlib import logging -from dataclasses import dataclass from clan_cli.completions import add_dynamic_completer, complete_machines +from clan_cli.errors import ClanError from clan_cli.machines.machines import Machine +from clan_cli.vars.public_modules import FactStoreBase +from clan_cli.vars.secret_modules import SecretStoreBase log = logging.getLogger(__name__) +from typing import TYPE_CHECKING -@dataclass -class Var: - generator: str - name: str +if TYPE_CHECKING: + from .generate import Var -@dataclass class VarStatus: - missing_secret_vars: list[Var] - missing_public_vars: list[Var] - unfixed_secret_vars: list[Var] - invalid_generators: list[str] + def __init__( + self, + missing_secret_vars: list["Var"], + missing_public_vars: list["Var"], + unfixed_secret_vars: list["Var"], + invalid_generators: list[str], + ) -> None: + self.missing_secret_vars = missing_secret_vars + self.missing_public_vars = missing_public_vars + self.unfixed_secret_vars = unfixed_secret_vars + self.invalid_generators = invalid_generators def vars_status(machine: Machine, generator_name: None | str = None) -> VarStatus: secret_vars_module = importlib.import_module(machine.secret_vars_module) - secret_vars_store = secret_vars_module.SecretStore(machine=machine) + secret_vars_store: SecretStoreBase = secret_vars_module.SecretStore(machine=machine) public_vars_module = importlib.import_module(machine.public_vars_module) - public_vars_store = public_vars_module.FactStore(machine=machine) + public_vars_store: FactStoreBase = public_vars_module.FactStore(machine=machine) missing_secret_vars = [] missing_public_vars = [] # signals if a var needs to be updated (eg. needs re-encryption due to new users added) unfixed_secret_vars = [] invalid_generators = [] + generators = machine.vars_generators if generator_name: - generators = [generator_name] - else: - generators = list(machine.vars_generators.keys()) - for generator_name in generators: - shared = machine.vars_generators[generator_name]["share"] - for name, file in machine.vars_generators[generator_name]["files"].items(): - if file["secret"]: - if not secret_vars_store.exists(generator_name, name, shared=shared): + for generator in generators: + if generator_name == generator.name: + generators = [generator] + break + else: + err_msg = ( + f"Generator '{generator_name}' not found in machine {machine.name}" + ) + raise ClanError(err_msg) + + for generator in generators: + generator.machine(machine) + for file in generator.files: + file.store(secret_vars_store if file.secret else public_vars_store) + file.generator(generator) + + if file.secret: + if not secret_vars_store.exists(generator, file.name): log.info( - f"Secret var '{name}' for service '{generator_name}' in machine {machine.name} is missing." + f"Secret var '{file.name}' for service '{generator.name}' in machine {machine.name} is missing." ) - missing_secret_vars.append(Var(generator_name, name)) + missing_secret_vars.append(file) else: - needs_fix, msg = secret_vars_store.needs_fix( - generator_name, name, shared=shared - ) + needs_fix, msg = secret_vars_store.needs_fix(generator, file.name) if needs_fix: log.info( - f"Secret var '{name}' for service '{generator_name}' in machine {machine.name} needs update: {msg}" + f"Secret var '{file.name}' for service '{generator.name}' in machine {machine.name} needs update: {msg}" ) - unfixed_secret_vars.append(Var(generator_name, name)) + unfixed_secret_vars.append(file) - elif not public_vars_store.exists(generator_name, name, shared=shared): + elif not public_vars_store.exists(generator, file.name): log.info( - f"Public var '{name}' for service '{generator_name}' in machine {machine.name} is missing." + f"Public var '{file.name}' for service '{generator.name}' in machine {machine.name} is missing." ) - missing_public_vars.append(Var(generator_name, name)) + missing_public_vars.append(file) # check if invalidation hash is up to date if not ( - secret_vars_store.hash_is_valid(generator_name) - and public_vars_store.hash_is_valid(generator_name) + secret_vars_store.hash_is_valid(generator) + and public_vars_store.hash_is_valid(generator) ): - invalid_generators.append(generator_name) + invalid_generators.append(generator.name) log.info( - f"Generator '{generator_name}' in machine {machine.name} has outdated invalidation hash." + f"Generator '{generator.name}' in machine {machine.name} has outdated invalidation hash." ) log.debug(f"missing_secret_vars: {missing_secret_vars}") log.debug(f"missing_public_vars: {missing_public_vars}") diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 138627fc8..6ee011aba 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -2,9 +2,11 @@ import argparse import logging import os import sys +from dataclasses import dataclass, field +from functools import cached_property from pathlib import Path from tempfile import TemporaryDirectory -from typing import Any +from typing import TYPE_CHECKING, Any from clan_cli.cmd import run from clan_cli.completions import ( @@ -15,19 +17,60 @@ from clan_cli.completions import ( from clan_cli.errors import ClanError from clan_cli.git import commit_files from clan_cli.machines.inventory import get_all_machines, get_selected_machines -from clan_cli.machines.machines import Machine from clan_cli.nix import nix_shell +from .check import check_vars from .graph import ( minimal_closure, requested_closure, ) -from .prompt import ask +from .prompt import Prompt, ask from .public_modules import FactStoreBase from .secret_modules import SecretStoreBase +from .var import Var log = logging.getLogger(__name__) +if TYPE_CHECKING: + from clan_cli.machines.machines import Machine + + +@dataclass +class Generator: + name: str + files: list[Var] = field(default_factory=list) + share: bool = False + invalidation_hash: str | None = None + final_script: str = "" + prompts: list[Prompt] = field(default_factory=list) + dependencies: list[str] = field(default_factory=list) + + migrate_fact: str | None = None + + # TODO: remove this + _machine: "Machine | None" = None + + def machine(self, machine: "Machine") -> None: + self._machine = machine + + @cached_property + def exists(self) -> bool: + assert self._machine is not None + return check_vars(self._machine, generator_name=self.name) + + @classmethod + def from_json(cls: type["Generator"], data: dict[str, Any]) -> "Generator": + return cls( + name=data["name"], + share=data["share"], + final_script=data["finalScript"], + files=[Var.from_json(data["name"], f) for f in data["files"].values()], + invalidation_hash=data["invalidationHash"], + dependencies=data["dependencies"], + migrate_fact=data["migrateFact"], + prompts=[Prompt.from_json(p) for p in data["prompts"].values()], + ) + def bubblewrap_cmd(generator: str, tmpdir: Path) -> list[str]: # fmt: off @@ -54,26 +97,29 @@ def bubblewrap_cmd(generator: str, tmpdir: Path) -> list[str]: # TODO: implement caching to not decrypt the same secret multiple times def decrypt_dependencies( - machine: Machine, - generator_name: str, + machine: "Machine", + generator: Generator, secret_vars_store: SecretStoreBase, public_vars_store: FactStoreBase, ) -> dict[str, dict[str, bytes]]: - generator = machine.vars_generators[generator_name] - dependencies = set(generator["dependencies"]) decrypted_dependencies: dict[str, Any] = {} - for dep_generator in dependencies: - decrypted_dependencies[dep_generator] = {} - dep_files = machine.vars_generators[dep_generator]["files"] - shared = machine.vars_generators[dep_generator]["share"] - for file_name, file in dep_files.items(): - if file["secret"]: - decrypted_dependencies[dep_generator][file_name] = ( - secret_vars_store.get(dep_generator, file_name, shared=shared) + for generator_name in set(generator.dependencies): + decrypted_dependencies[generator_name] = {} + for dep_generator in machine.vars_generators: + if generator_name == dep_generator.name: + break + else: + msg = f"Could not find dependent generator {generator_name} in machine {machine.name}" + raise ClanError(msg) + dep_files = dep_generator.files + for file in dep_files: + if file.secret: + decrypted_dependencies[generator_name][file.name] = ( + secret_vars_store.get(dep_generator, file.name) ) else: - decrypted_dependencies[dep_generator][file_name] = ( - public_vars_store.get(dep_generator, file_name, shared=shared) + decrypted_dependencies[generator_name][file.name] = ( + public_vars_store.get(dep_generator, file.name) ) return decrypted_dependencies @@ -95,8 +141,8 @@ def dependencies_as_dir( def execute_generator( - machine: Machine, - generator_name: str, + machine: "Machine", + generator: Generator, secret_vars_store: SecretStoreBase, public_vars_store: FactStoreBase, prompt_values: dict[str, str], @@ -105,14 +151,10 @@ def execute_generator( msg = f"flake is not a Path: {machine.flake}" msg += "fact/secret generation is only supported for local flakes" - generator = machine.vars_generators[generator_name] - script = generator["finalScript"] - is_shared = generator["share"] - # build temporary file tree of dependencies decrypted_dependencies = decrypt_dependencies( machine, - generator_name, + generator, secret_vars_store, public_vars_store, ) @@ -121,7 +163,7 @@ def execute_generator( try: return prompt_values[prompt_name] except KeyError as e: - msg = f"prompt value for '{prompt_name}' in generator {generator_name} not provided" + msg = f"prompt value for '{prompt_name}' in generator {generator.name} not provided" raise ClanError(msg) from e env = os.environ.copy() @@ -138,97 +180,94 @@ def execute_generator( dependencies_as_dir(decrypted_dependencies, tmpdir_in) # populate prompted values # TODO: make prompts rest API friendly - if generator["prompts"]: + if generator.prompts: tmpdir_prompts.mkdir() env["prompts"] = str(tmpdir_prompts) - for prompt_name in generator["prompts"]: - prompt_file = tmpdir_prompts / prompt_name - value = get_prompt_value(prompt_name) + for prompt in generator.prompts: + prompt_file = tmpdir_prompts / prompt.name + value = get_prompt_value(prompt.name) prompt_file.write_text(value) if sys.platform == "linux": - cmd = bubblewrap_cmd(script, tmpdir) + cmd = bubblewrap_cmd(generator.final_script, tmpdir) else: - cmd = ["bash", "-c", script] + cmd = ["bash", "-c", generator.final_script] run( cmd, env=env, ) files_to_commit = [] # store secrets - files = generator["files"] + files = generator.files public_changed = False secret_changed = False - for file_name, file in files.items(): - is_deployed = file["deploy"] - - secret_file = tmpdir_out / file_name + for file in files: + secret_file = tmpdir_out / file.name if not secret_file.is_file(): - msg = f"did not generate a file for '{file_name}' when running the following command:\n" - msg += script + msg = f"did not generate a file for '{file.name}' when running the following command:\n" + msg += generator.final_script raise ClanError(msg) - if file["secret"]: + if file.secret: file_path = secret_vars_store.set( - generator_name, - file_name, + generator, + file, secret_file.read_bytes(), - shared=is_shared, - deployed=is_deployed, ) secret_changed = True else: file_path = public_vars_store.set( - generator_name, - file_name, + generator, + file, secret_file.read_bytes(), - shared=is_shared, ) public_changed = True if file_path: files_to_commit.append(file_path) - if generator["invalidationHash"] is not None: + if generator.invalidation_hash is not None: if public_changed: public_vars_store.set_invalidation_hash( - generator_name, generator["invalidationHash"] + generator, generator.invalidation_hash ) if secret_changed: secret_vars_store.set_invalidation_hash( - generator_name, generator["invalidationHash"] + generator, generator.invalidation_hash ) commit_files( files_to_commit, machine.flake_dir, - f"Update vars via generator {generator_name} for machine {machine.name}", + f"Update vars via generator {generator.name} for machine {machine.name}", ) def _ask_prompts( - machine: Machine, - generator_names: list[str], + generators: list[Generator], ) -> dict[str, dict[str, str]]: prompt_values: dict[str, dict[str, str]] = {} - for generator in generator_names: - prompts = machine.vars_generators[generator]["prompts"] - for prompt_name, _prompt in prompts.items(): - if generator not in prompt_values: - prompt_values[generator] = {} - var_id = f"{generator}/{prompt_name}" - prompt_values[generator][prompt_name] = ask(var_id, _prompt["type"]) + for generator in generators: + for prompt in generator.prompts: + if generator.name not in prompt_values: + prompt_values[generator.name] = {} + var_id = f"{generator.name}/{prompt.name}" + prompt_values[generator.name][prompt.name] = ask(var_id, prompt.prompt_type) return prompt_values def get_closure( - machine: Machine, + machine: "Machine", generator_name: str | None, regenerate: bool, -) -> list[str]: - from .graph import Generator, all_missing_closure, full_closure +) -> list[Generator]: + from .graph import all_missing_closure, full_closure vars_generators = machine.vars_generators generators: dict[str, Generator] = { - name: Generator(name, generator["dependencies"], _machine=machine) - for name, generator in vars_generators.items() + generator.name: generator for generator in vars_generators } + + # TODO: we should remove this + for generator in vars_generators: + generator.machine(machine) + if generator_name is None: # all generators selected if regenerate: return full_closure(generators) @@ -240,109 +279,109 @@ def get_closure( def _migration_file_exists( - machine: Machine, - service_name: str, + machine: "Machine", + generator: Generator, fact_name: str, ) -> bool: - is_secret = machine.vars_generators[service_name]["files"][fact_name]["secret"] + for file in generator.files: + if file.name == fact_name: + break + else: + msg = f"Could not find file {fact_name} in generator {generator.name}" + raise ClanError(msg) + + is_secret = file.secret if is_secret: - if machine.secret_facts_store.exists(service_name, fact_name): + if machine.secret_facts_store.exists(generator.name, fact_name): return True log.debug( - f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the secret fact store" + f"Cannot migrate fact {fact_name} for service {generator.name}, as it does not exist in the secret fact store" ) if not is_secret: - if machine.public_facts_store.exists(service_name, fact_name): + if machine.public_facts_store.exists(generator.name, fact_name): return True log.debug( - f"Cannot migrate fact {fact_name} for service {service_name}, as it does not exist in the public fact store" + f"Cannot migrate fact {fact_name} for service {generator.name}, as it does not exist in the public fact store" ) return False def _migrate_file( - machine: Machine, - generator_name: str, + machine: "Machine", + generator: Generator, var_name: str, service_name: str, fact_name: str, ) -> None: - is_secret = machine.vars_generators[generator_name]["files"][var_name]["secret"] - if is_secret: + for file in generator.files: + if file.name == var_name: + break + else: + msg = f"Could not find file {fact_name} in generator {generator.name}" + raise ClanError(msg) + + if file.secret: old_value = machine.secret_facts_store.get(service_name, fact_name) + machine.secret_vars_store.set(generator, file, old_value) else: old_value = machine.public_facts_store.get(service_name, fact_name) - is_shared = machine.vars_generators[generator_name]["share"] - is_deployed = machine.vars_generators[generator_name]["files"][var_name]["deploy"] - if is_secret: - machine.secret_vars_store.set( - generator_name, var_name, old_value, shared=is_shared, deployed=is_deployed - ) - else: - machine.public_vars_store.set( - generator_name, var_name, old_value, shared=is_shared, deployed=is_deployed - ) + machine.public_vars_store.set(generator, file, old_value) def _migrate_files( - machine: Machine, - generator_name: str, + machine: "Machine", + generator: Generator, ) -> None: - service_name = machine.vars_generators[generator_name]["migrateFact"] not_found = [] - for var_name, _file in machine.vars_generators[generator_name]["files"].items(): - if _migration_file_exists(machine, generator_name, var_name): - _migrate_file(machine, generator_name, var_name, service_name, var_name) + for file in generator.files: + if _migration_file_exists(machine, generator, file.name): + assert generator.migrate_fact is not None + _migrate_file( + machine, generator, file.name, generator.migrate_fact, file.name + ) else: - not_found.append(var_name) + not_found.append(file.name) if len(not_found) > 0: - msg = f"Could not migrate the following files for generator {generator_name}, as no fact or secret exists with the same name: {not_found}" + msg = f"Could not migrate the following files for generator {generator.name}, as no fact or secret exists with the same name: {not_found}" raise ClanError(msg) def _check_can_migrate( - machine: Machine, - generator_name: str, + machine: "Machine", + generator: Generator, ) -> bool: - vars_generator = machine.vars_generators[generator_name] - if "migrateFact" not in vars_generator: - return False - service_name = vars_generator["migrateFact"] + service_name = generator.migrate_fact if not service_name: return False # ensure that none of the generated vars already exist in the store - for fname, file in vars_generator["files"].items(): - if file["secret"]: - if machine.secret_vars_store.exists( - generator_name, fname, vars_generator["share"] - ): - if vars_generator["deploy"]: + for file in generator.files: + if file.secret: + if machine.secret_vars_store.exists(generator, file.name): + if file.deploy: machine.secret_vars_store.ensure_machine_has_access( - generator_name, fname, vars_generator["share"] + generator, file.name ) return False else: - if machine.public_vars_store.exists( - generator_name, fname, vars_generator["share"] - ): + if machine.public_vars_store.exists(generator, file.name): return False # ensure that the service to migrate from actually exists if service_name not in machine.facts_data: log.debug( - f"Could not migrate facts for generator {generator_name}, as the service {service_name} does not exist" + f"Could not migrate facts for generator {generator.name}, as the service {service_name} does not exist" ) return False # ensure that all files can be migrated (exists in the corresponding fact store) return bool( all( - _migration_file_exists(machine, generator_name, fname) - for fname in vars_generator["files"] + _migration_file_exists(machine, generator, file.name) + for file in generator.files ) ) def ensure_consistent_state( - machine: Machine, + machine: "Machine", generator_name: str | None, fix: bool, ) -> None: @@ -352,25 +391,30 @@ def ensure_consistent_state( """ if generator_name is None: - generators = list(machine.vars_generators.keys()) + generators = machine.vars_generators else: - generators = [generator_name] + for generator in machine.vars_generators: + if generator_name == generator.name: + generators = [generator] + break + else: + err_msg = ( + f"Could not find generator {generator_name} in machine {machine.name}" + ) + raise ClanError(err_msg) outdated = [] - for generator_name in generators: - for name, file in machine.vars_generators[generator_name]["files"].items(): - shared = machine.vars_generators[generator_name]["share"] - if file["secret"] and machine.secret_vars_store.exists( - generator_name, name, shared=shared - ): - if file["deploy"]: + for generator in generators: + for file in generator.files: + if file.secret and machine.secret_vars_store.exists(generator, file.name): + if file.deploy: machine.secret_vars_store.ensure_machine_has_access( - generator_name, name, shared=shared + generator, file.name ) needs_update, msg = machine.secret_vars_store.needs_fix( - generator_name, name, shared=shared + generator, file.name ) if needs_update: - outdated.append((generator_name, name, msg)) + outdated.append((generator_name, file.name, msg)) if not fix and outdated: msg = ( "The local state of some secret vars is inconsistent and needs to be updated.\n" @@ -382,7 +426,7 @@ def ensure_consistent_state( def generate_vars_for_machine( - machine: Machine, + machine: "Machine", generator_name: str | None, regenerate: bool, fix: bool, @@ -391,17 +435,17 @@ def generate_vars_for_machine( closure = get_closure(machine, generator_name, regenerate) if len(closure) == 0: return False - prompt_values = _ask_prompts(machine, closure) - for gen_name in closure: - if _check_can_migrate(machine, gen_name): - _migrate_files(machine, gen_name) + prompt_values = _ask_prompts(closure) + for generator in closure: + if _check_can_migrate(machine, generator): + _migrate_files(machine, generator) else: execute_generator( machine, - gen_name, + generator, machine.secret_vars_store, machine.public_vars_store, - prompt_values.get(gen_name, {}), + prompt_values.get(generator.name, {}), ) # flush caches to make sure the new secrets are available in evaluation machine.flush_caches() @@ -409,7 +453,7 @@ def generate_vars_for_machine( def generate_vars( - machines: list[Machine], + machines: list["Machine"], generator_name: str | None = None, regenerate: bool = False, fix: bool = False, @@ -423,11 +467,14 @@ def generate_vars( ) machine.flush_caches() except Exception as exc: - machine.error(f"Failed to generate facts: {exc}") - errors += [exc] - if len(errors) > 0: - msg = f"Failed to generate facts for {len(errors)} hosts. Check the logs above" - raise ClanError(msg) from errors[0] + errors += [(machine, exc)] + if len(errors) == 1: + raise errors[0][1] + if len(errors) > 1: + msg = f"Failed to generate facts for {len(errors)} hosts:" + for machine, error in errors: + msg += f"\n{machine}: {error}" + raise ClanError(msg) from errors[0][1] if not was_regenerated and len(machines) > 0: machine.info("All vars are already up to date") diff --git a/pkgs/clan-cli/clan_cli/vars/get.py b/pkgs/clan-cli/clan_cli/vars/get.py index 253167b0f..5e975c550 100644 --- a/pkgs/clan-cli/clan_cli/vars/get.py +++ b/pkgs/clan-cli/clan_cli/vars/get.py @@ -7,7 +7,7 @@ from clan_cli.completions import add_dynamic_completer, complete_machines from clan_cli.errors import ClanError from clan_cli.machines.machines import Machine -from ._types import Var +from .generate import Var from .list import get_vars log = logging.getLogger(__name__) diff --git a/pkgs/clan-cli/clan_cli/vars/graph.py b/pkgs/clan-cli/clan_cli/vars/graph.py index d716ab372..f333c33a0 100644 --- a/pkgs/clan-cli/clan_cli/vars/graph.py +++ b/pkgs/clan-cli/clan_cli/vars/graph.py @@ -1,31 +1,21 @@ +from __future__ import annotations + from collections.abc import Iterable -from dataclasses import dataclass -from functools import cached_property from graphlib import TopologicalSorter +from typing import TYPE_CHECKING from clan_cli.errors import ClanError -from clan_cli.machines.machines import Machine -from .check import check_vars +if TYPE_CHECKING: + from .generate import Generator class GeneratorNotFoundError(ClanError): pass -@dataclass -class Generator: - name: str - dependencies: list[str] - _machine: Machine - - @cached_property - def exists(self) -> bool: - return check_vars(self._machine, generator_name=self.name) - - def missing_dependency_closure( - requested_generators: Iterable[str], generators: dict + requested_generators: Iterable[str], generators: dict[str, Generator] ) -> set[str]: closure = set(requested_generators) # extend the graph to include all dependencies which are not on disk @@ -52,7 +42,9 @@ def add_missing_dependencies( return missing_dependency_closure(closure, generators) | closure -def add_dependents(requested_generators: Iterable[str], generators: dict) -> set[str]: +def add_dependents( + requested_generators: Iterable[str], generators: dict[str, Generator] +) -> set[str]: closure = set(requested_generators) # build reverse dependency graph (graph of dependents) dependents_graph: dict[str, set[str]] = {} @@ -72,7 +64,9 @@ def add_dependents(requested_generators: Iterable[str], generators: dict) -> set return closure -def toposort_closure(_closure: Iterable[str], generators: dict) -> list[str]: +def toposort_closure( + _closure: Iterable[str], generators: dict[str, Generator] +) -> list[Generator]: closure = set(_closure) # return the topological sorted list of generators to execute final_dep_graph = {} @@ -81,16 +75,16 @@ def toposort_closure(_closure: Iterable[str], generators: dict) -> list[str]: final_dep_graph[gen_name] = deps sorter = TopologicalSorter(final_dep_graph) result = list(sorter.static_order()) - return result + return [generators[gen_name] for gen_name in result] # all generators in topological order -def full_closure(generators: dict) -> list[str]: +def full_closure(generators: dict[str, Generator]) -> list[Generator]: return toposort_closure(generators.keys(), generators) # just the missing generators including their dependents -def all_missing_closure(generators: dict) -> list[str]: +def all_missing_closure(generators: dict[str, Generator]) -> list[Generator]: # collect all generators that are missing from disk closure = {gen_name for gen_name, gen in generators.items() if not gen.exists} closure = add_dependents(closure, generators) @@ -98,7 +92,9 @@ def all_missing_closure(generators: dict) -> list[str]: # only a selected list of generators including their missing dependencies and their dependents -def requested_closure(requested_generators: list[str], generators: dict) -> list[str]: +def requested_closure( + requested_generators: list[str], generators: dict[str, Generator] +) -> list[Generator]: closure = set(requested_generators) # extend the graph to include all dependencies which are not on disk closure = add_missing_dependencies(closure, generators) @@ -108,7 +104,9 @@ def requested_closure(requested_generators: list[str], generators: dict) -> list # just enough to ensure that the list of selected generators are in a consistent state. # empty if nothing is missing. -def minimal_closure(requested_generators: list[str], generators: dict) -> list[str]: +def minimal_closure( + requested_generators: list[str], generators: dict[str, Generator] +) -> list[Generator]: closure = set(requested_generators) final_closure = missing_dependency_closure(closure, generators) # add requested generators if not already exist diff --git a/pkgs/clan-cli/clan_cli/vars/list.py b/pkgs/clan-cli/clan_cli/vars/list.py index 699c42f7f..f89e229f5 100644 --- a/pkgs/clan-cli/clan_cli/vars/list.py +++ b/pkgs/clan-cli/clan_cli/vars/list.py @@ -4,10 +4,11 @@ import logging from clan_cli.api import API from clan_cli.completions import add_dynamic_completer, complete_machines +from clan_cli.errors import ClanError from clan_cli.machines.machines import Machine -from ._types import Generator, GeneratorUpdate, Prompt, Var -from .generate import execute_generator +from ._types import GeneratorUpdate +from .generate import Generator, Prompt, Var, execute_generator from .public_modules import FactStoreBase from .secret_modules import SecretStoreBase @@ -35,44 +36,25 @@ def _get_previous_value( generator: Generator, prompt: Prompt, ) -> str | None: - if not prompt.has_file: + if not prompt.create_file: return None + pub_store = public_store(machine) - if pub_store.exists(generator.name, prompt.name, shared=generator.share): - return pub_store.get( - generator.name, prompt.name, shared=generator.share - ).decode() + if pub_store.exists(generator, prompt.name): + return pub_store.get(generator, prompt.name).decode() sec_store = secret_store(machine) - if sec_store.exists(generator.name, prompt.name, shared=generator.share): - return sec_store.get( - generator.name, prompt.name, shared=generator.share - ).decode() + if sec_store.exists(generator, prompt.name): + return sec_store.get(generator, prompt.name).decode() return None @API.register # TODO: use machine_name def get_prompts(machine: Machine) -> list[Generator]: - generators = [] - for gen_name, generator in machine.vars_generators.items(): - prompts: list[Prompt] = [] - gen = Generator( - name=gen_name, - prompts=prompts, - share=generator["share"], - ) - for prompt_name, prompt in generator["prompts"].items(): - prompt = Prompt( - name=prompt_name, - description=prompt["description"], - type=prompt["type"], - has_file=prompt["createFile"], - generator=gen_name, - ) - prompt.previous_value = _get_previous_value(machine, gen, prompt) - prompts.append(prompt) - - generators.append(gen) + generators: list[Generator] = machine.vars_generators + for generator in generators: + for prompt in generator.prompts: + prompt.previous_value = _get_previous_value(machine, generator, prompt) return generators @@ -82,9 +64,15 @@ def get_prompts(machine: Machine) -> list[Generator]: @API.register def set_prompts(machine: Machine, updates: list[GeneratorUpdate]) -> None: for update in updates: + for generator in machine.vars_generators: + if generator.name == update.generator: + break + else: + msg = f"Generator '{update.generator}' not found in machine {machine.name}" + raise ClanError(msg) execute_generator( machine, - update.generator, + generator, secret_vars_store=secret_store(machine), public_vars_store=public_store(machine), prompt_values=update.prompt_values, diff --git a/pkgs/clan-cli/clan_cli/vars/prompt.py b/pkgs/clan-cli/clan_cli/vars/prompt.py index 0362ced22..629dba747 100644 --- a/pkgs/clan-cli/clan_cli/vars/prompt.py +++ b/pkgs/clan-cli/clan_cli/vars/prompt.py @@ -1,8 +1,9 @@ +import enum import logging import sys +from dataclasses import dataclass from getpass import getpass - -from clan_cli.errors import ClanError +from typing import Any log = logging.getLogger(__name__) @@ -10,18 +11,42 @@ log = logging.getLogger(__name__) MOCK_PROMPT_RESPONSE = None -def ask(description: str, input_type: str) -> str: +class PromptType(enum.Enum): + LINE = "line" + HIDDEN = "hidden" + MULTILINE = "multiline" + + +@dataclass +class Prompt: + name: str + description: str + prompt_type: PromptType + + create_file: bool = False + previous_value: str | None = None + + @classmethod + def from_json(cls: type["Prompt"], data: dict[str, Any]) -> "Prompt": + return cls( + name=data["name"], + description=data["description"], + prompt_type=PromptType(data["type"]), + create_file=data["createFile"], + previous_value=data.get("previousValue"), + ) + + +def ask(description: str, input_type: PromptType) -> str: if MOCK_PROMPT_RESPONSE: return next(MOCK_PROMPT_RESPONSE) - if input_type == "line": - result = input(f"Enter the value for {description}: ") - elif input_type == "multiline": - print(f"Enter the value for {description} (Finish with Ctrl-D): ") - result = sys.stdin.read() - elif input_type == "hidden": - result = getpass(f"Enter the value for {description} (hidden): ") - else: - msg = f"Unknown input type: {input_type} for prompt {description}" - raise ClanError(msg) + match input_type: + case PromptType.LINE: + result = input(f"Enter the value for {description}: ") + case PromptType.MULTILINE: + print(f"Enter the value for {description} (Finish with Ctrl-D): ") + result = sys.stdin.read() + case PromptType.HIDDEN: + result = getpass(f"Enter the value for {description} (hidden): ") log.info("Input received. Processing...") return result diff --git a/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py b/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py index ce5c102d1..0c8a63a96 100644 --- a/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py +++ b/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py @@ -3,6 +3,7 @@ from pathlib import Path from clan_cli.errors import ClanError from clan_cli.machines.machines import Machine +from clan_cli.vars.generate import Generator, Var from . import FactStoreBase @@ -18,16 +19,14 @@ class FactStore(FactStoreBase): def _set( self, - generator_name: str, - name: str, + generator: Generator, + var: Var, value: bytes, - shared: bool = False, - deployed: bool = True, ) -> Path | None: if not self.machine.flake.is_local(): msg = f"in_flake fact storage is only supported for local flakes: {self.machine.flake}" raise ClanError(msg) - folder = self.directory(generator_name, name, shared) + folder = self.directory(generator, var.name) if folder.exists(): if not (folder / "value").exists(): # another backend has used that folder before -> error out @@ -42,8 +41,8 @@ class FactStore(FactStoreBase): return file_path # get a single fact - def get(self, generator_name: str, name: str, shared: bool = False) -> bytes: - return (self.directory(generator_name, name, shared) / "value").read_bytes() + def get(self, generator: Generator, name: str) -> bytes: + return (self.directory(generator, name) / "value").read_bytes() - def exists(self, generator_name: str, name: str, shared: bool = False) -> bool: - return (self.directory(generator_name, name, shared) / "value").exists() + def exists(self, generator: Generator, name: str) -> bool: + return (self.directory(generator, name) / "value").exists() diff --git a/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py b/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py index e54bb1633..def8173aa 100644 --- a/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py +++ b/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py @@ -4,6 +4,7 @@ from pathlib import Path from clan_cli.dirs import vm_state_dir from clan_cli.errors import ClanError from clan_cli.machines.machines import Machine +from clan_cli.vars.generate import Generator, Var from . import FactStoreBase @@ -21,27 +22,25 @@ class FactStore(FactStoreBase): def store_name(self) -> str: return "vm" - def exists(self, service: str, name: str, shared: bool = False) -> bool: - fact_path = self.dir / service / name + def exists(self, generator: Generator, name: str) -> bool: + fact_path = self.dir / generator.name / name return fact_path.exists() def _set( self, - service: str, - name: str, + generator: Generator, + var: Var, value: bytes, - shared: bool = False, - deployed: bool = True, ) -> Path | None: - fact_path = self.dir / service / name + fact_path = self.dir / generator.name / var.name fact_path.parent.mkdir(parents=True, exist_ok=True) fact_path.write_bytes(value) return None # get a single fact - def get(self, service: str, name: str, shared: bool = False) -> bytes: - fact_path = self.dir / service / name + def get(self, generator: Generator, name: str) -> bytes: + fact_path = self.dir / generator.name / name if fact_path.exists(): return fact_path.read_bytes() - msg = f"Fact {name} for service {service} not found" + msg = f"Fact {name} for service {generator.name} not found" raise ClanError(msg) diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py index 31845eb76..697a8ff9b 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py @@ -1,8 +1,12 @@ from abc import abstractmethod from pathlib import Path +from typing import TYPE_CHECKING from clan_cli.vars._types import StoreBase +if TYPE_CHECKING: + from clan_cli.vars.generate import Generator + class SecretStoreBase(StoreBase): @property @@ -12,16 +16,13 @@ class SecretStoreBase(StoreBase): def needs_upload(self) -> bool: return True - def ensure_machine_has_access( - self, generator_name: str, name: str, shared: bool = False - ) -> None: + def ensure_machine_has_access(self, generator: "Generator", name: str) -> None: pass def needs_fix( self, - generator_name: str, + generator: "Generator", name: str, - shared: bool, ) -> tuple[bool, str | None]: """ Check if local state needs updating, eg. secret needs to be re-encrypted with new keys @@ -30,9 +31,8 @@ class SecretStoreBase(StoreBase): def fix( self, - generator_name: str, + generator: "Generator", name: str, - shared: bool, ) -> None: """ Update local state, eg make sure secret is encrypted with correct keys diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py index 4af78b6ad..80db0c624 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py @@ -9,6 +9,7 @@ from typing import override from clan_cli.cmd import Log, run from clan_cli.machines.machines import Machine from clan_cli.nix import nix_shell +from clan_cli.vars.generate import Generator, Var from . import SecretStoreBase @@ -30,16 +31,14 @@ class SecretStore(SecretStoreBase): "PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store" ) - def entry_dir(self, generator_name: str, name: str, shared: bool) -> Path: - return Path(self.entry_prefix) / self.rel_dir(generator_name, name, shared) + def entry_dir(self, generator: Generator, name: str) -> Path: + return Path(self.entry_prefix) / self.rel_dir(generator, name) def _set( self, - generator_name: str, - name: str, + generator: Generator, + var: Var, value: bytes, - shared: bool = False, - deployed: bool = True, ) -> Path | None: run( nix_shell( @@ -48,7 +47,7 @@ class SecretStore(SecretStoreBase): "pass", "insert", "-m", - str(self.entry_dir(generator_name, name, shared)), + str(self.entry_dir(generator, var.name)), ], ), input=value, @@ -56,22 +55,21 @@ class SecretStore(SecretStoreBase): ) return None # we manage the files outside of the git repo - def get(self, generator_name: str, name: str, shared: bool = False) -> bytes: + def get(self, generator: Generator, name: str) -> bytes: return run( nix_shell( ["nixpkgs#pass"], [ "pass", "show", - str(self.entry_dir(generator_name, name, shared)), + str(self.entry_dir(generator, name)), ], ), ).stdout.encode() - def exists(self, generator_name: str, name: str, shared: bool = False) -> bool: + def exists(self, generator: Generator, name: str) -> bool: return ( - Path(self._password_store_dir) - / f"{self.entry_dir(generator_name, name, shared)}.gpg" + Path(self._password_store_dir) / f"{self.entry_dir(generator, name)}.gpg" ).exists() def generate_hash(self) -> bytes: @@ -128,9 +126,9 @@ class SecretStore(SecretStoreBase): hashes.sort() manifest = [] - for gen_name, generator in self.machine.vars_generators.items(): - for f_name in generator["files"]: - manifest.append(f"{gen_name}/{f_name}".encode()) + for generator in self.machine.vars_generators: + for file in generator.files: + manifest.append(f"{generator.name}/{file.name}".encode()) manifest += hashes return b"\n".join(manifest) @@ -152,24 +150,24 @@ class SecretStore(SecretStoreBase): def upload(self, output_dir: Path) -> None: with tarfile.open(output_dir / "secrets.tar.gz", "w:gz") as tar: - for gen_name, generator in self.machine.vars_generators.items(): + for generator in self.machine.vars_generators: dir_exists = False - for f_name, file in generator["files"].items(): - if not file["deploy"]: + for file in generator.files: + if not file.deploy: continue - if not file["secret"]: + if not file.secret: continue if not dir_exists: - tar_dir = tarfile.TarInfo(name=gen_name) + tar_dir = tarfile.TarInfo(name=generator.name) tar_dir.type = tarfile.DIRTYPE tar_dir.mode = 0o511 tar.addfile(tarinfo=tar_dir) dir_exists = True - tar_file = tarfile.TarInfo(name=f"{gen_name}/{f_name}") - content = self.get(gen_name, f_name, generator["share"]) + tar_file = tarfile.TarInfo(name=f"{generator.name}/{file.name}") + content = self.get(generator, file.name) tar_file.size = len(content) tar_file.mode = 0o440 - tar_file.uname = file.get("owner", "root") - tar_file.gname = file.get("group", "root") + tar_file.uname = file.owner + tar_file.gname = file.group tar.addfile(tarinfo=tar_file, fileobj=io.BytesIO(content)) (output_dir / ".pass_info").write_bytes(self.generate_hash()) diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py index ca4899cae..5b02d55ac 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py @@ -18,6 +18,7 @@ from clan_cli.secrets.secrets import ( has_secret, ) from clan_cli.secrets.sops import KeyType, generate_private_key +from clan_cli.vars.generate import Generator, Var from . import SecretStoreBase @@ -42,11 +43,10 @@ class SecretStore(SecretStoreBase): if not self.machine.vars_generators: return has_secrets = False - for generator in self.machine.vars_generators.values(): - if "files" in generator: - for file in generator["files"].values(): - if file["secret"]: - has_secrets = True + for generator in self.machine.vars_generators: + for file in generator.files: + if file.secret: + has_secrets = True if not has_secrets: return @@ -67,9 +67,9 @@ class SecretStore(SecretStoreBase): return "sops" def user_has_access( - self, user: str, generator_name: str, secret_name: str, shared: bool + self, user: str, generator: Generator, secret_name: str ) -> bool: - secret_path = self.secret_path(generator_name, secret_name, shared=shared) + secret_path = self.secret_path(generator, secret_name) secret = json.loads((secret_path / "secret").read_text()) recipients = [r["recipient"] for r in (secret["sops"].get("age") or [])] users_folder_path = sops_users_folder(self.machine.flake_dir) @@ -78,10 +78,8 @@ class SecretStore(SecretStoreBase): ] return user_pubkey in recipients - def machine_has_access( - self, generator_name: str, secret_name: str, shared: bool - ) -> bool: - secret_path = self.secret_path(generator_name, secret_name, shared) + def machine_has_access(self, generator: Generator, secret_name: str) -> bool: + secret_path = self.secret_path(generator, secret_name) secret = json.loads((secret_path / "secret").read_text()) recipients = [r["recipient"] for r in (secret["sops"].get("age") or [])] machines_folder_path = sops_machines_folder(self.machine.flake_dir) @@ -90,20 +88,16 @@ class SecretStore(SecretStoreBase): )["publickey"] return machine_pubkey in recipients - def secret_path( - self, generator_name: str, secret_name: str, shared: bool = False - ) -> Path: - return self.directory(generator_name, secret_name, shared=shared) + def secret_path(self, generator: Generator, secret_name: str) -> Path: + return self.directory(generator, secret_name) def _set( self, - generator_name: str, - name: str, + generator: Generator, + var: Var, value: bytes, - shared: bool = False, - deployed: bool = True, ) -> Path | None: - secret_folder = self.secret_path(generator_name, name, shared) + secret_folder = self.secret_path(generator, var.name) # delete directory if secret_folder.exists() and not (secret_folder / "secret").exists(): # another backend has used that folder before -> error out @@ -115,15 +109,15 @@ class SecretStore(SecretStoreBase): self.machine.flake_dir, secret_folder, value, - add_machines=[self.machine.name] if deployed else [], + add_machines=[self.machine.name] if var.deploy else [], add_groups=self.machine.deployment["sops"]["defaultGroups"], git_commit=False, ) return secret_folder - def get(self, generator_name: str, name: str, shared: bool = False) -> bytes: + def get(self, generator: Generator, name: str) -> bytes: return decrypt_secret( - self.machine.flake_dir, self.secret_path(generator_name, name, shared) + self.machine.flake_dir, self.secret_path(generator, name) ).encode("utf-8") def upload(self, output_dir: Path) -> None: @@ -137,16 +131,14 @@ class SecretStore(SecretStoreBase): ) (output_dir / "key.txt").write_text(key) - def exists(self, generator_name: str, name: str, shared: bool = False) -> bool: - secret_folder = self.secret_path(generator_name, name, shared) + def exists(self, generator: Generator, name: str) -> bool: + secret_folder = self.secret_path(generator, name) return (secret_folder / "secret").exists() - def ensure_machine_has_access( - self, generator_name: str, name: str, shared: bool = False - ) -> None: - if self.machine_has_access(generator_name, name, shared): + def ensure_machine_has_access(self, generator: Generator, name: str) -> None: + if self.machine_has_access(generator, name): return - secret_folder = self.secret_path(generator_name, name, shared) + secret_folder = self.secret_path(generator, name) add_secret(self.machine.flake_dir, self.machine.name, secret_folder) def collect_keys_for_secret(self, path: Path) -> set[tuple[str, KeyType]]: @@ -170,10 +162,8 @@ class SecretStore(SecretStoreBase): return keys @override - def needs_fix( - self, generator_name: str, name: str, shared: bool - ) -> tuple[bool, str | None]: - secret_path = self.secret_path(generator_name, name, shared) + def needs_fix(self, generator: Generator, name: str) -> tuple[bool, str | None]: + secret_path = self.secret_path(generator, name) recipients_ = json.loads((secret_path / "secret").read_text())["sops"]["age"] current_recipients = {r["recipient"] for r in recipients_} wanted_recipients = { @@ -181,19 +171,19 @@ class SecretStore(SecretStoreBase): } needs_update = current_recipients != wanted_recipients recipients_to_add = wanted_recipients - current_recipients - var_id = f"{generator_name}/{name}" + var_id = f"{generator.name}/{name}" msg = ( - f"One or more recipient keys were added to secret{' shared' if shared else ''} var '{var_id}', but it was never re-encrypted. " + f"One or more recipient keys were added to secret{' shared' if generator.share else ''} var '{var_id}', but it was never re-encrypted. " f"This could have been a malicious actor trying to add their keys, please investigate. " f"Added keys: {', '.join(recipients_to_add)}" ) return needs_update, msg @override - def fix(self, generator_name: str, name: str, shared: bool) -> None: + def fix(self, generator: Generator, name: str) -> None: from clan_cli.secrets.secrets import update_keys - secret_path = self.secret_path(generator_name, name, shared) + secret_path = self.secret_path(generator, name) update_keys( secret_path, collect_keys_for_path(secret_path), diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py index 2aef6770b..7699345dd 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py @@ -3,6 +3,7 @@ from pathlib import Path from clan_cli.dirs import vm_state_dir from clan_cli.machines.machines import Machine +from clan_cli.vars.generate import Generator, Var from . import SecretStoreBase @@ -19,19 +20,17 @@ class SecretStore(SecretStoreBase): def _set( self, - service: str, - name: str, + generator: Generator, + var: Var, value: bytes, - shared: bool = False, - deployed: bool = True, ) -> Path | None: - secret_file = self.dir / service / name + secret_file = self.dir / generator.name / var.name secret_file.parent.mkdir(parents=True, exist_ok=True) secret_file.write_bytes(value) return None # we manage the files outside of the git repo - def get(self, service: str, name: str, shared: bool = False) -> bytes: - secret_file = self.dir / service / name + def get(self, generator: Generator, name: str) -> bytes: + secret_file = self.dir / generator.name / name return secret_file.read_bytes() def upload(self, output_dir: Path) -> None: diff --git a/pkgs/clan-cli/clan_cli/vars/set.py b/pkgs/clan-cli/clan_cli/vars/set.py index 51445848a..89681f0b7 100644 --- a/pkgs/clan-cli/clan_cli/vars/set.py +++ b/pkgs/clan-cli/clan_cli/vars/set.py @@ -6,8 +6,9 @@ from clan_cli.clan_uri import FlakeId from clan_cli.completions import add_dynamic_completer, complete_machines from clan_cli.machines.machines import Machine from clan_cli.vars.get import get_var +from clan_cli.vars.prompt import PromptType -from ._types import Var +from .generate import Var from .prompt import ask log = logging.getLogger(__name__) @@ -31,7 +32,7 @@ def set_via_stdin(machine: str, var_id: str, flake: FlakeId) -> None: _machine = Machine(name=machine, flake=flake) var = get_var(_machine, var_id) if sys.stdin.isatty(): - new_value = ask(var.id, "hidden").encode("utf-8") + new_value = ask(var.id, PromptType.HIDDEN).encode("utf-8") else: new_value = sys.stdin.buffer.read() set_var(_machine, var, new_value, flake) diff --git a/pkgs/clan-cli/clan_cli/vars/var.py b/pkgs/clan-cli/clan_cli/vars/var.py new file mode 100644 index 000000000..1b584c70f --- /dev/null +++ b/pkgs/clan-cli/clan_cli/vars/var.py @@ -0,0 +1,77 @@ +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from clan_cli.vars.generate import Generator + + from ._types import StoreBase + + +@dataclass +class Var: + id: str + name: str + secret: bool = True + deploy: bool = False + owner: str = "root" + group: str = "root" + + # TODO: those shouldn't be set here + _store: "StoreBase | None" = None + _generator: "Generator | None" = None + + def store(self, store: "StoreBase") -> None: + self._store = store + + def generator(self, generator: "Generator") -> None: + self._generator = generator + + @property + def value(self) -> bytes: + assert self._store is not None + assert self._generator is not None + if not self._store.exists(self._generator, self.name): + msg = f"Var {self.id} has not been generated yet" + raise ValueError(msg) + # try decode the value or return + return self._store.get(self._generator, self.name) + + @property + def printable_value(self) -> str: + try: + return self.value.decode() + except UnicodeDecodeError: + return "" + + def set(self, value: bytes) -> None: + assert self._store is not None + assert self._generator is not None + self._store.set(self._generator, self, value) + + @property + def exists(self) -> bool: + assert self._store is not None + assert self._generator is not None + return self._store.exists(self._generator, self.name) + + def __str__(self) -> str: + if self._store is None or self._generator is None: + return f"{self.id}: " + + # TODO: we don't want __str__ with side effects, this should be a separate method + if self._store.exists(self._generator, self.name): + if self.secret: + return f"{self.id}: ********" + return f"{self.id}: {self.printable_value}" + return f"{self.id}: " + + @classmethod + def from_json(cls: type["Var"], generator_name: str, data: dict[str, Any]) -> "Var": + return cls( + id=f"{generator_name}/{data['name']}", + name=data["name"], + secret=data["secret"], + deploy=data["deploy"], + owner=data.get("owner", "root"), + group=data.get("group", "root"), + ) diff --git a/pkgs/clan-cli/tests/test_vars.py b/pkgs/clan-cli/tests/test_vars.py index 3f1b52256..16a8bbee3 100644 --- a/pkgs/clan-cli/tests/test_vars.py +++ b/pkgs/clan-cli/tests/test_vars.py @@ -1,6 +1,5 @@ import json import shutil -from dataclasses import dataclass from pathlib import Path import pytest @@ -10,8 +9,9 @@ from clan_cli.errors import ClanError from clan_cli.machines.machines import Machine from clan_cli.nix import nix_eval, run from clan_cli.vars.check import check_vars -from clan_cli.vars.generate import generate_vars_for_machine +from clan_cli.vars.generate import Generator, generate_vars_for_machine from clan_cli.vars.get import get_var +from clan_cli.vars.graph import all_missing_closure, requested_closure from clan_cli.vars.list import stringify_all_vars from clan_cli.vars.public_modules import in_repo from clan_cli.vars.secret_modules import password_store, sops @@ -48,31 +48,49 @@ def test_dependencies_as_files(temp_dir: Path) -> None: def test_required_generators() -> None: - from clan_cli.vars.graph import all_missing_closure, requested_closure - - @dataclass - class Generator: - dependencies: list[str] - exists: bool # result is already on disk + gen_1 = Generator(name="gen_1", dependencies=[]) + gen_2 = Generator(name="gen_2", dependencies=["gen_1"]) + gen_2a = Generator(name="gen_2a", dependencies=["gen_2"]) + gen_2b = Generator(name="gen_2b", dependencies=["gen_2"]) + gen_1.exists = True + gen_2.exists = False + gen_2a.exists = False + gen_2b.exists = True generators = { - "gen_1": Generator([], True), - "gen_2": Generator(["gen_1"], False), - "gen_2a": Generator(["gen_2"], False), - "gen_2b": Generator(["gen_2"], True), + generator.name: generator for generator in [gen_1, gen_2, gen_2a, gen_2b] } - assert requested_closure(["gen_1"], generators) == [ + def generator_names(generator: list[Generator]) -> list[str]: + return [gen.name for gen in generator] + + assert generator_names(requested_closure(["gen_1"], generators)) == [ "gen_1", "gen_2", "gen_2a", "gen_2b", ] - assert requested_closure(["gen_2"], generators) == ["gen_2", "gen_2a", "gen_2b"] - assert requested_closure(["gen_2a"], generators) == ["gen_2", "gen_2a", "gen_2b"] - assert requested_closure(["gen_2b"], generators) == ["gen_2", "gen_2a", "gen_2b"] + assert generator_names(requested_closure(["gen_2"], generators)) == [ + "gen_2", + "gen_2a", + "gen_2b", + ] + assert generator_names(requested_closure(["gen_2a"], generators)) == [ + "gen_2", + "gen_2a", + "gen_2b", + ] + assert generator_names(requested_closure(["gen_2b"], generators)) == [ + "gen_2", + "gen_2a", + "gen_2b", + ] - assert all_missing_closure(generators) == ["gen_2", "gen_2a", "gen_2b"] + assert generator_names(all_missing_closure(generators)) == [ + "gen_2", + "gen_2a", + "gen_2b", + ] @pytest.mark.impure @@ -96,8 +114,8 @@ def test_generate_public_var( store = in_repo.FactStore( Machine(name="my_machine", flake=FlakeId(str(flake.path))) ) - assert store.exists("my_generator", "my_value") - assert store.get("my_generator", "my_value").decode() == "hello\n" + assert store.exists(Generator("my_generator"), "my_value") + assert store.get(Generator("my_generator"), "my_value").decode() == "hello\n" vars_text = stringify_all_vars(machine) assert "my_generator/my_value: hello" in vars_text vars_eval = run( @@ -133,12 +151,12 @@ def test_generate_secret_var_sops( in_repo_store = in_repo.FactStore( Machine(name="my_machine", flake=FlakeId(str(flake.path))) ) - assert not in_repo_store.exists("my_generator", "my_secret") + assert not in_repo_store.exists(Generator("my_generator"), "my_secret") sops_store = sops.SecretStore( Machine(name="my_machine", flake=FlakeId(str(flake.path))) ) - assert sops_store.exists("my_generator", "my_secret") - assert sops_store.get("my_generator", "my_secret").decode() == "hello\n" + assert sops_store.exists(Generator("my_generator"), "my_secret") + assert sops_store.get(Generator("my_generator"), "my_secret").decode() == "hello\n" vars_text = stringify_all_vars(machine) assert "my_generator/my_secret" in vars_text # test regeneration works @@ -168,12 +186,12 @@ def test_generate_secret_var_sops_with_default_group( in_repo_store = in_repo.FactStore( Machine(name="my_machine", flake=FlakeId(str(flake.path))) ) - assert not in_repo_store.exists("my_generator", "my_secret") + assert not in_repo_store.exists(Generator("my_generator"), "my_secret") sops_store = sops.SecretStore( Machine(name="my_machine", flake=FlakeId(str(flake.path))) ) - assert sops_store.exists("my_generator", "my_secret") - assert sops_store.get("my_generator", "my_secret").decode() == "hello\n" + assert sops_store.exists(Generator("my_generator"), "my_secret") + assert sops_store.get(Generator("my_generator"), "my_secret").decode() == "hello\n" # add another user and check if secret gets re-encrypted from clan_cli.secrets.sops import generate_private_key @@ -197,7 +215,7 @@ def test_generate_secret_var_sops_with_default_group( # check if new user can access the secret monkeypatch.setenv("USER", "uschi") assert sops_store.user_has_access( - "uschi", "my_generator", "my_secret", shared=False + "uschi", Generator("my_generator", share=False), "my_secret" ) @@ -232,13 +250,17 @@ def test_generated_shared_secret_sops( assert check_vars(machine2) m1_sops_store = sops.SecretStore(machine1) m2_sops_store = sops.SecretStore(machine2) - assert m1_sops_store.exists("my_shared_generator", "my_shared_secret", shared=True) - assert m2_sops_store.exists("my_shared_generator", "my_shared_secret", shared=True) + assert m1_sops_store.exists( + Generator("my_shared_generator", share=True), "my_shared_secret" + ) + assert m2_sops_store.exists( + Generator("my_shared_generator", share=True), "my_shared_secret" + ) assert m1_sops_store.machine_has_access( - "my_shared_generator", "my_shared_secret", shared=True + Generator("my_shared_generator", share=True), "my_shared_secret" ) assert m2_sops_store.machine_has_access( - "my_shared_generator", "my_shared_secret", shared=True + Generator("my_shared_generator", share=True), "my_shared_secret" ) @@ -277,11 +299,19 @@ def test_generate_secret_var_password_store( store = password_store.SecretStore( Machine(name="my_machine", flake=FlakeId(str(flake.path))) ) - assert store.exists("my_generator", "my_secret", shared=False) - assert not store.exists("my_generator", "my_secret", shared=True) - assert store.exists("my_shared_generator", "my_shared_secret", shared=True) - assert not store.exists("my_shared_generator", "my_shared_secret", shared=False) - assert store.get("my_generator", "my_secret", shared=False).decode() == "hello\n" + assert store.exists(Generator("my_generator", share=False, files=[]), "my_secret") + assert not store.exists( + Generator("my_generator", share=True, files=[]), "my_secret" + ) + assert store.exists( + Generator("my_shared_generator", share=True, files=[]), "my_shared_secret" + ) + assert not store.exists( + Generator("my_shared_generator", share=False, files=[]), "my_shared_secret" + ) + + generator = Generator(name="my_generator", share=False, files=[]) + assert store.get(generator, "my_secret").decode() == "hello\n" vars_text = stringify_all_vars(machine) assert "my_generator/my_secret" in vars_text @@ -321,10 +351,16 @@ def test_generate_secret_for_multiple_machines( in_repo_store2 = in_repo.FactStore( Machine(name="machine2", flake=FlakeId(str(flake.path))) ) - assert in_repo_store1.exists("my_generator", "my_value") - assert in_repo_store2.exists("my_generator", "my_value") - assert in_repo_store1.get("my_generator", "my_value").decode() == "machine1\n" - assert in_repo_store2.get("my_generator", "my_value").decode() == "machine2\n" + assert in_repo_store1.exists(Generator("my_generator"), "my_value") + assert in_repo_store2.exists(Generator("my_generator"), "my_value") + assert ( + in_repo_store1.get(Generator("my_generator"), "my_value").decode() + == "machine1\n" + ) + assert ( + in_repo_store2.get(Generator("my_generator"), "my_value").decode() + == "machine2\n" + ) # check if secret vars have been created correctly sops_store1 = sops.SecretStore( Machine(name="machine1", flake=FlakeId(str(flake.path))) @@ -332,10 +368,14 @@ def test_generate_secret_for_multiple_machines( sops_store2 = sops.SecretStore( Machine(name="machine2", flake=FlakeId(str(flake.path))) ) - assert sops_store1.exists("my_generator", "my_secret") - assert sops_store2.exists("my_generator", "my_secret") - assert sops_store1.get("my_generator", "my_secret").decode() == "machine1\n" - assert sops_store2.get("my_generator", "my_secret").decode() == "machine2\n" + assert sops_store1.exists(Generator("my_generator"), "my_secret") + assert sops_store2.exists(Generator("my_generator"), "my_secret") + assert ( + sops_store1.get(Generator("my_generator"), "my_secret").decode() == "machine1\n" + ) + assert ( + sops_store2.get(Generator("my_generator"), "my_secret").decode() == "machine2\n" + ) @pytest.mark.impure @@ -357,10 +397,16 @@ def test_dependant_generators( in_repo_store = in_repo.FactStore( Machine(name="my_machine", flake=FlakeId(str(flake.path))) ) - assert in_repo_store.exists("parent_generator", "my_value") - assert in_repo_store.get("parent_generator", "my_value").decode() == "hello\n" - assert in_repo_store.exists("child_generator", "my_value") - assert in_repo_store.get("child_generator", "my_value").decode() == "hello\n" + assert in_repo_store.exists(Generator("parent_generator"), "my_value") + assert ( + in_repo_store.get(Generator("parent_generator"), "my_value").decode() + == "hello\n" + ) + assert in_repo_store.exists(Generator("child_generator"), "my_value") + assert ( + in_repo_store.get(Generator("child_generator"), "my_value").decode() + == "hello\n" + ) @pytest.mark.impure @@ -395,8 +441,10 @@ def test_prompt( in_repo_store = in_repo.FactStore( Machine(name="my_machine", flake=FlakeId(str(flake.path))) ) - assert in_repo_store.exists("my_generator", "my_value") - assert in_repo_store.get("my_generator", "my_value").decode() == input_value + assert in_repo_store.exists(Generator("my_generator"), "my_value") + assert ( + in_repo_store.get(Generator("my_generator"), "my_value").decode() == input_value + ) @pytest.mark.impure @@ -437,15 +485,25 @@ def test_share_flag( Machine(name="my_machine", flake=FlakeId(str(flake.path))) ) # check secrets stored correctly - assert sops_store.exists("shared_generator", "my_secret", shared=True) - assert not sops_store.exists("shared_generator", "my_secret", shared=False) - assert sops_store.exists("unshared_generator", "my_secret", shared=False) - assert not sops_store.exists("unshared_generator", "my_secret", shared=True) + assert sops_store.exists(Generator("shared_generator", share=True), "my_secret") + assert not sops_store.exists( + Generator("shared_generator", share=False), "my_secret" + ) + assert sops_store.exists(Generator("unshared_generator", share=False), "my_secret") + assert not sops_store.exists( + Generator("unshared_generator", share=True), "my_secret" + ) # check values stored correctly - assert in_repo_store.exists("shared_generator", "my_value", shared=True) - assert not in_repo_store.exists("shared_generator", "my_value", shared=False) - assert in_repo_store.exists("unshared_generator", "my_value", shared=False) - assert not in_repo_store.exists("unshared_generator", "my_value", shared=True) + assert in_repo_store.exists(Generator("shared_generator", share=True), "my_value") + assert not in_repo_store.exists( + Generator("shared_generator", share=False), "my_value" + ) + assert in_repo_store.exists( + Generator("unshared_generator", share=False), "my_value" + ) + assert not in_repo_store.exists( + Generator("unshared_generator", share=True), "my_value" + ) vars_eval = run( nix_eval( [ @@ -505,9 +563,13 @@ def test_prompt_create_file( sops_store = sops.SecretStore( Machine(name="my_machine", flake=FlakeId(str(flake.path))) ) - assert sops_store.exists("my_generator", "prompt1") - assert not sops_store.exists("my_generator", "prompt2") - assert sops_store.get("my_generator", "prompt1").decode() == "input1" + assert sops_store.exists( + Generator(name="my_generator", share=False, files=[]), "prompt1" + ) + assert not sops_store.exists(Generator(name="my_generator"), "prompt2") + assert ( + sops_store.get(Generator(name="my_generator"), "prompt1").decode() == "input1" + ) @pytest.mark.impure @@ -558,8 +620,8 @@ def test_api_set_prompts( ], ) store = in_repo.FactStore(machine) - assert store.exists("my_generator", "prompt1") - assert store.get("my_generator", "prompt1").decode() == "input1" + assert store.exists(Generator("my_generator"), "prompt1") + assert store.get(Generator("my_generator"), "prompt1").decode() == "input1" set_prompts( machine, [ @@ -569,7 +631,7 @@ def test_api_set_prompts( ) ], ) - assert store.get("my_generator", "prompt1").decode() == "input2" + assert store.get(Generator("my_generator"), "prompt1").decode() == "input2" @pytest.mark.impure @@ -771,8 +833,8 @@ def test_migration_skip( in_repo_store = in_repo.FactStore( Machine(name="my_machine", flake=FlakeId(str(flake.path))) ) - assert in_repo_store.exists("my_generator", "my_value") - assert in_repo_store.get("my_generator", "my_value").decode() == "world" + assert in_repo_store.exists(Generator("my_generator"), "my_value") + assert in_repo_store.get(Generator("my_generator"), "my_value").decode() == "world" @pytest.mark.impure @@ -805,10 +867,10 @@ def test_migration( sops_store = sops.SecretStore( Machine(name="my_machine", flake=FlakeId(str(flake.path))) ) - assert in_repo_store.exists("my_generator", "my_value") - assert in_repo_store.get("my_generator", "my_value").decode() == "hello" - assert sops_store.exists("my_generator", "my_secret") - assert sops_store.get("my_generator", "my_secret").decode() == "hello" + assert in_repo_store.exists(Generator("my_generator"), "my_value") + assert in_repo_store.get(Generator("my_generator"), "my_value").decode() == "hello" + assert sops_store.exists(Generator("my_generator"), "my_secret") + assert sops_store.get(Generator("my_generator"), "my_secret").decode() == "hello" @pytest.mark.impure