From 98139ac48d4744ff8d6e7a7e6d06d87c61ad5eda Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 12 Feb 2024 13:31:12 +0100 Subject: [PATCH] add factsStore modules --- nixosModules/clanCore/outputs.nix | 9 +++- pkgs/clan-cli/clan_cli/__init__.py | 5 ++- pkgs/clan-cli/clan_cli/facts/__init__.py | 21 ++++++++++ pkgs/clan-cli/clan_cli/facts/check.py | 37 ++++++++++++++++ pkgs/clan-cli/clan_cli/facts/list.py | 36 ++++++++++++++++ .../clan_cli/facts/modules/__init__.py | 0 .../clan_cli/facts/modules/in_repo.py | 42 +++++++++++++++++++ pkgs/clan-cli/clan_cli/machines/machines.py | 4 ++ pkgs/clan-cli/clan_cli/secrets/check.py | 4 +- pkgs/clan-cli/clan_cli/secrets/generate.py | 12 +++--- 10 files changed, 162 insertions(+), 8 deletions(-) create mode 100644 pkgs/clan-cli/clan_cli/facts/__init__.py create mode 100644 pkgs/clan-cli/clan_cli/facts/check.py create mode 100644 pkgs/clan-cli/clan_cli/facts/list.py create mode 100644 pkgs/clan-cli/clan_cli/facts/modules/__init__.py create mode 100644 pkgs/clan-cli/clan_cli/facts/modules/in_repo.py diff --git a/nixosModules/clanCore/outputs.nix b/nixosModules/clanCore/outputs.nix index d3411599b..19efb9b11 100644 --- a/nixosModules/clanCore/outputs.nix +++ b/nixosModules/clanCore/outputs.nix @@ -44,6 +44,13 @@ the directory on the deployment server where secrets are uploaded ''; }; + factsModule = lib.mkOption { + type = lib.types.str; + description = '' + the python import path to the facts module + ''; + default = "clan_cli.facts.modules.in_repo"; + }; secretsModule = lib.mkOption { type = lib.types.str; description = '' @@ -84,7 +91,7 @@ # optimization for faster secret generate/upload and machines update config = { system.clan.deployment.data = { - inherit (config.system.clan) secretsModule secretsData; + inherit (config.system.clan) factsModule secretsModule secretsData; inherit (config.clan.networking) targetHost buildHost; inherit (config.clan.deployment) requireExplicitUpdate; inherit (config.clanCore) secretsUploadDirectory; diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index 88201103d..b3acf473c 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -6,7 +6,7 @@ from pathlib import Path from types import ModuleType from typing import Any -from . import backups, config, flakes, flash, history, machines, secrets, vms +from . import backups, config, flakes, flash, history, machines, secrets, vms, facts from .custom_logger import setup_logging from .dirs import get_clan_flake_toplevel from .errors import ClanCmdError, ClanError @@ -91,6 +91,9 @@ def create_parser(prog: str | None = None) -> argparse.ArgumentParser: parser_secrets = subparsers.add_parser("secrets", help="manage secrets") secrets.register_parser(parser_secrets) + parser_facts = subparsers.add_parser("facts", help="manage facts") + facts.register_parser(parser_facts) + parser_machine = subparsers.add_parser( "machines", help="Manage machines and their configuration" ) diff --git a/pkgs/clan-cli/clan_cli/facts/__init__.py b/pkgs/clan-cli/clan_cli/facts/__init__.py new file mode 100644 index 000000000..66fe82d1a --- /dev/null +++ b/pkgs/clan-cli/clan_cli/facts/__init__.py @@ -0,0 +1,21 @@ +# !/usr/bin/env python3 +import argparse + +from .check import register_check_parser +from .list import register_list_parser + + +# takes a (sub)parser and configures it +def register_parser(parser: argparse.ArgumentParser) -> None: + subparser = parser.add_subparsers( + title="command", + description="the command to run", + help="the command to run", + required=True, + ) + + check_parser = subparser.add_parser("check", help="check if facts are up to date") + register_check_parser(check_parser) + + list_parser = subparser.add_parser("list", help="list all facts") + register_list_parser(list_parser) diff --git a/pkgs/clan-cli/clan_cli/facts/check.py b/pkgs/clan-cli/clan_cli/facts/check.py new file mode 100644 index 000000000..11a9cc3a1 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/facts/check.py @@ -0,0 +1,37 @@ +import argparse +import importlib +import logging + +from ..machines.machines import Machine + +log = logging.getLogger(__name__) + + +def check_facts(machine: Machine) -> bool: + facts_module = importlib.import_module(machine.facts_module) + fact_store = facts_module.FactStore(machine=machine) + + missing_facts = [] + for service in machine.secrets_data: + for fact in machine.secrets_data[service]["facts"]: + if not fact_store.get(service, fact): + log.info(f"Fact {fact} for service {service} is missing") + missing_facts.append((service, fact)) + + if missing_facts: + return False + return True + + +def check_command(args: argparse.Namespace) -> None: + machine = Machine(name=args.machine, flake=args.flake) + if check_facts(machine): + print("All facts are present") + + +def register_check_parser(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "machine", + help="The machine to check facts for", + ) + parser.set_defaults(func=check_command) diff --git a/pkgs/clan-cli/clan_cli/facts/list.py b/pkgs/clan-cli/clan_cli/facts/list.py new file mode 100644 index 000000000..59eae3fc5 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/facts/list.py @@ -0,0 +1,36 @@ +import json +import argparse +import importlib +import logging + +from ..machines.machines import Machine + +log = logging.getLogger(__name__) + + +def get_all_facts(machine: Machine) -> dict: + facts_module = importlib.import_module(machine.facts_module) + fact_store = facts_module.FactStore(machine=machine) + + # for service in machine.secrets_data: + # facts[service] = {} + # for fact in machine.secrets_data[service]["facts"]: + # fact_content = fact_store.get(service, fact) + # if fact_content: + # facts[service][fact] = fact_content.decode() + # else: + # log.error(f"Fact {fact} for service {service} is missing") + return fact_store.get_all() + + +def get_command(args: argparse.Namespace) -> None: + machine = Machine(name=args.machine, flake=args.flake) + print(json.dumps(get_all_facts(machine), indent=4)) + + +def register_list_parser(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "machine", + help="The machine to print facts for", + ) + parser.set_defaults(func=get_command) diff --git a/pkgs/clan-cli/clan_cli/facts/modules/__init__.py b/pkgs/clan-cli/clan_cli/facts/modules/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/clan-cli/clan_cli/facts/modules/in_repo.py b/pkgs/clan-cli/clan_cli/facts/modules/in_repo.py new file mode 100644 index 000000000..225d608e5 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/facts/modules/in_repo.py @@ -0,0 +1,42 @@ +from pathlib import Path + +from clan_cli.errors import ClanError +from clan_cli.machines.machines import Machine + + +class FactStore: + def __init__(self, machine: Machine) -> None: + self.machine = machine + + def set(self, _service: str, name: str, value: bytes) -> Path | None: + if isinstance(self.machine.flake, Path): + fact_path = ( + self.machine.flake / "machines" / self.machine.name / "facts" / name + ) + fact_path.parent.mkdir(parents=True, exist_ok=True) + fact_path.touch() + fact_path.write_bytes(value) + return fact_path + else: + raise ClanError( + f"in_flake fact storage is only supported for local flakes: {self.machine.flake}" + ) + + def exists(self, _service: str, name: str) -> bool: + fact_path = self.machine.flake_dir / "machines" / self.machine.name / "facts" / name + return fact_path.exists() + + # get a single fact + def get(self, _service: str, name: str) -> bytes: + fact_path = self.machine.flake_dir / "machines" / self.machine.name / "facts" / name + return fact_path.read_bytes() + + # get all facts + def get_all(self) -> dict[str, dict[str, bytes]]: + facts_folder = self.machine.flake_dir / "machines" / self.machine.name / "facts" + facts: dict[str, dict[str, bytes]] = {} + facts["TODO"] = {} + if facts_folder.exists(): + for fact_path in facts_folder.iterdir(): + facts["TODO"][fact_path.name] = fact_path.read_bytes() + return facts diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index fdb91b510..17cfc581d 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -96,6 +96,10 @@ class Machine: def secrets_module(self) -> str: return self.deployment_info["secretsModule"] + @property + def facts_module(self) -> str: + return self.deployment_info["factsModule"] + @property def secrets_data(self) -> dict: if self.deployment_info["secretsData"]: diff --git a/pkgs/clan-cli/clan_cli/secrets/check.py b/pkgs/clan-cli/clan_cli/secrets/check.py index facf13c6e..a79452db0 100644 --- a/pkgs/clan-cli/clan_cli/secrets/check.py +++ b/pkgs/clan-cli/clan_cli/secrets/check.py @@ -10,6 +10,8 @@ log = logging.getLogger(__name__) def check_secrets(machine: Machine) -> bool: secrets_module = importlib.import_module(machine.secrets_module) secret_store = secrets_module.SecretStore(machine=machine) + facts_module = importlib.import_module(machine.facts_module) + fact_store = facts_module.FactsStore(machine=machine) missing_secrets = [] missing_facts = [] @@ -20,7 +22,7 @@ def check_secrets(machine: Machine) -> bool: missing_secrets.append((service, secret)) for fact in machine.secrets_data[service]["facts"].values(): - if not (machine.flake / fact).exists(): + if not fact_store.exists(service, fact): log.info(f"Fact {fact} for service {service} is missing") missing_facts.append((service, fact)) diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index 633ff2647..b38901f8a 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -2,7 +2,6 @@ import argparse import importlib import logging import os -import shutil from pathlib import Path from tempfile import TemporaryDirectory @@ -21,6 +20,9 @@ def generate_secrets(machine: Machine) -> None: secrets_module = importlib.import_module(machine.secrets_module) secret_store = secrets_module.SecretStore(machine=machine) + facts_module = importlib.import_module(machine.facts_module) + fact_store = facts_module.FactStore(machine=machine) + with TemporaryDirectory() as d: for service in machine.secrets_data: tmpdir = Path(d) / service @@ -84,10 +86,10 @@ def generate_secrets(machine: Machine) -> None: msg = f"did not generate a file for '{name}' when running the following command:\n" msg += machine.secrets_data[service]["generator"] raise ClanError(msg) - fact_path = machine.flake / fact_path - fact_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(fact_file, fact_path) - files_to_commit.append(fact_path) + fact_file = fact_store.set( + service, fact_path, fact_file.read_bytes() + ) + files_to_commit.append(fact_file) commit_files( files_to_commit, machine.flake_dir,