Merge pull request 'add factsStore modules' (#839) from fact_store into main

lib/build-clan/default.nix

@@ -37,9 +37,9 @@ let
       in
       (machineImports settings)
       ++ [
-        (nixpkgs.lib.mkOverride 51 extraConfig)
         settings
         clan-core.nixosModules.clanCore
+        extraConfig
         (machines.${name} or { })
         ({
           clanCore.clanName = clanName;

nixosModules/clanCore/outputs.nix

@@ -44,6 +44,13 @@
             the directory on the deployment server where secrets are uploaded
           '';
         };
+        factsModule = lib.mkOption {
+          type = lib.types.str;
+          description = ''
+            the python import path to the facts module
+          '';
+          default = "clan_cli.facts.modules.in_repo";
+        };
         secretsModule = lib.mkOption {
           type = lib.types.str;
           description = ''
@@ -84,7 +91,7 @@
   # optimization for faster secret generate/upload and machines update
   config = {
     system.clan.deployment.data = {
-      inherit (config.system.clan) secretsModule secretsData;
+      inherit (config.system.clan) factsModule secretsModule secretsData;
       inherit (config.clan.networking) targetHost buildHost;
       inherit (config.clan.deployment) requireExplicitUpdate;
       inherit (config.clanCore) secretsUploadDirectory;

nixosModules/clanCore/secrets/default.nix

@@ -1,7 +1,7 @@
 { config, lib, pkgs, ... }:
 {
   options.clanCore.secretStore = lib.mkOption {
-    type = lib.types.enum [ "sops" "password-store" "custom" ];
+    type = lib.types.enum [ "sops" "password-store" "vm" "custom" ];
     default = "sops";
     description = ''
       method to store secrets
@@ -150,5 +150,6 @@
   imports = [
     ./sops.nix
     ./password-store.nix
+    ./vm.nix
   ];
 }

nixosModules/clanCore/secrets/vm.nix

@@ -0,0 +1,10 @@
+{ config, lib, ... }:
+{
+  config = lib.mkIf (config.clanCore.secretStore == "vm") {
+    clanCore.secretsDirectory = "/etc/secrets";
+    clanCore.secretsUploadDirectory = "/etc/secrets";
+    system.clan.secretsModule = "clan_cli.secrets.modules.vm";
+    system.clan.factsModule = "clan_cli.facts.modules.vm";
+  };
+}
+

nixosModules/clanCore/zerotier/default.nix

@@ -190,7 +190,7 @@ in
 
       environment.systemPackages = [ config.clanCore.clanPkgs.zerotier-members ];
     })
-    (lib.mkIf (config.clanCore.secretsUploadDirectory != null && !cfg.controller.enable && cfg.networkId != null) {
+    (lib.mkIf (!cfg.controller.enable && cfg.networkId != null) {
       clanCore.secrets.zerotier = {
         facts.zerotier-ip = { };
         facts.zerotier-meshname = { };

pkgs/clan-cli/clan_cli/__init__.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from types import ModuleType
 from typing import Any
 
-from . import backups, config, flakes, flash, history, machines, secrets, vms
+from . import backups, config, facts, flakes, flash, history, machines, secrets, vms
 from .custom_logger import setup_logging
 from .dirs import get_clan_flake_toplevel
 from .errors import ClanCmdError, ClanError
@@ -91,6 +91,9 @@ def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
     parser_secrets = subparsers.add_parser("secrets", help="manage secrets")
     secrets.register_parser(parser_secrets)
 
+    parser_facts = subparsers.add_parser("facts", help="manage facts")
+    facts.register_parser(parser_facts)
+
     parser_machine = subparsers.add_parser(
         "machines", help="Manage machines and their configuration"
     )

pkgs/clan-cli/clan_cli/facts/__init__.py

@@ -0,0 +1,21 @@
+# !/usr/bin/env python3
+import argparse
+
+from .check import register_check_parser
+from .list import register_list_parser
+
+
+# takes a (sub)parser and configures it
+def register_parser(parser: argparse.ArgumentParser) -> None:
+    subparser = parser.add_subparsers(
+        title="command",
+        description="the command to run",
+        help="the command to run",
+        required=True,
+    )
+
+    check_parser = subparser.add_parser("check", help="check if facts are up to date")
+    register_check_parser(check_parser)
+
+    list_parser = subparser.add_parser("list", help="list all facts")
+    register_list_parser(list_parser)

pkgs/clan-cli/clan_cli/facts/check.py

@@ -0,0 +1,38 @@
+import argparse
+import importlib
+import logging
+
+from ..machines.machines import Machine
+
+log = logging.getLogger(__name__)
+
+
+def check_facts(machine: Machine) -> bool:
+    facts_module = importlib.import_module(machine.facts_module)
+    fact_store = facts_module.FactStore(machine=machine)
+
+    existing_facts = fact_store.get_all()
+    missing_facts = []
+    for service in machine.secrets_data:
+        for fact in machine.secrets_data[service]["facts"]:
+            if fact not in existing_facts.get(service, {}):
+                log.info(f"Fact {fact} for service {service} is missing")
+                missing_facts.append((service, fact))
+
+    if missing_facts:
+        return False
+    return True
+
+
+def check_command(args: argparse.Namespace) -> None:
+    machine = Machine(name=args.machine, flake=args.flake)
+    if check_facts(machine):
+        print("All facts are present")
+
+
+def register_check_parser(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "machine",
+        help="The machine to check facts for",
+    )
+    parser.set_defaults(func=check_command)

pkgs/clan-cli/clan_cli/facts/list.py

@@ -0,0 +1,36 @@
+import argparse
+import importlib
+import json
+import logging
+
+from ..machines.machines import Machine
+
+log = logging.getLogger(__name__)
+
+
+def get_all_facts(machine: Machine) -> dict:
+    facts_module = importlib.import_module(machine.facts_module)
+    fact_store = facts_module.FactStore(machine=machine)
+
+    # for service in machine.secrets_data:
+    #     facts[service] = {}
+    #     for fact in machine.secrets_data[service]["facts"]:
+    #         fact_content = fact_store.get(service, fact)
+    #         if fact_content:
+    #             facts[service][fact] = fact_content.decode()
+    #         else:
+    #             log.error(f"Fact {fact} for service {service} is missing")
+    return fact_store.get_all()
+
+
+def get_command(args: argparse.Namespace) -> None:
+    machine = Machine(name=args.machine, flake=args.flake)
+    print(json.dumps(get_all_facts(machine), indent=4))
+
+
+def register_list_parser(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "machine",
+        help="The machine to print facts for",
+    )
+    parser.set_defaults(func=get_command)

pkgs/clan-cli/clan_cli/facts/modules/in_repo.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+
+from clan_cli.errors import ClanError
+from clan_cli.machines.machines import Machine
+
+
+class FactStore:
+    def __init__(self, machine: Machine) -> None:
+        self.machine = machine
+        self.works_remotely = False
+
+    def set(self, _service: str, name: str, value: bytes) -> Path | None:
+        if isinstance(self.machine.flake, Path):
+            fact_path = (
+                self.machine.flake / "machines" / self.machine.name / "facts" / name
+            )
+            fact_path.parent.mkdir(parents=True, exist_ok=True)
+            fact_path.touch()
+            fact_path.write_bytes(value)
+            return fact_path
+        else:
+            raise ClanError(
+                f"in_flake fact storage is only supported for local flakes: {self.machine.flake}"
+            )
+
+    def exists(self, _service: str, name: str) -> bool:
+        fact_path = (
+            self.machine.flake_dir / "machines" / self.machine.name / "facts" / name
+        )
+        return fact_path.exists()
+
+    # get a single fact
+    def get(self, _service: str, name: str) -> bytes:
+        fact_path = (
+            self.machine.flake_dir / "machines" / self.machine.name / "facts" / name
+        )
+        return fact_path.read_bytes()
+
+    # get all facts
+    def get_all(self) -> dict[str, dict[str, bytes]]:
+        facts_folder = self.machine.flake_dir / "machines" / self.machine.name / "facts"
+        facts: dict[str, dict[str, bytes]] = {}
+        facts["TODO"] = {}
+        if facts_folder.exists():
+            for fact_path in facts_folder.iterdir():
+                facts["TODO"][fact_path.name] = fact_path.read_bytes()
+        return facts

pkgs/clan-cli/clan_cli/facts/modules/vm.py

@@ -0,0 +1,44 @@
+import logging
+from pathlib import Path
+
+from clan_cli.dirs import vm_state_dir
+from clan_cli.errors import ClanError
+from clan_cli.machines.machines import Machine
+
+log = logging.getLogger(__name__)
+
+
+class FactStore:
+    def __init__(self, machine: Machine) -> None:
+        self.machine = machine
+        self.works_remotely = False
+        self.dir = vm_state_dir(str(machine.flake), machine.name) / "facts"
+        log.debug(f"FactStore initialized with dir {self.dir}")
+
+    def exists(self, service: str, name: str) -> bool:
+        fact_path = self.dir / service / name
+        return fact_path.exists()
+
+    def set(self, service: str, name: str, value: bytes) -> Path | None:
+        fact_path = self.dir / service / name
+        fact_path.parent.mkdir(parents=True, exist_ok=True)
+        fact_path.write_bytes(value)
+        return None
+
+    # get a single fact
+    def get(self, service: str, name: str) -> bytes:
+        fact_path = self.dir / service / name
+        if fact_path.exists():
+            return fact_path.read_bytes()
+        raise ClanError(f"Fact {name} for service {service} not found")
+
+    # get all facts
+    def get_all(self) -> dict[str, dict[str, bytes]]:
+        facts: dict[str, dict[str, bytes]] = {}
+        if self.dir.exists():
+            for service in self.dir.iterdir():
+                facts[service.name] = {}
+                for fact in service.iterdir():
+                    facts[service.name][fact.name] = fact.read_bytes()
+
+        return facts

pkgs/clan-cli/clan_cli/machines/machines.py

@@ -96,6 +96,10 @@ class Machine:
     def secrets_module(self) -> str:
         return self.deployment_info["secretsModule"]
 
+    @property
+    def facts_module(self) -> str:
+        return self.deployment_info["factsModule"]
+
     @property
     def secrets_data(self) -> dict:
         if self.deployment_info["secretsData"]:
@@ -151,6 +155,7 @@ class Machine:
         attr: str,
         extra_config: None | dict = None,
         impure: bool = False,
+        nix_options: list[str] = [],
     ) -> str | Path:
         """
         Build the machine and return the path to the result
@@ -184,17 +189,15 @@ class Machine:
             if extra_config is not None:
                 metadata = nix_metadata(self.flake_dir)
                 url = metadata["url"]
-                if "dirtyRev" in metadata:
+                if "dirtyRevision" in metadata:
-                    if not impure:
+                    # if not impure:
-                        raise ClanError(
+                    #     raise ClanError(
-                            "The machine has a dirty revision, and impure mode is not allowed"
+                    #         "The machine has a dirty revision, and impure mode is not allowed"
-                        )
+                    #     )
-                    else:
+                    # else:
+                    #     args += ["--impure"]
                     args += ["--impure"]
 
-                if "dirtyRev" in nix_metadata(self.flake_dir):
-                    dirty_rev = nix_metadata(self.flake_dir)["dirtyRevision"]
-                    url = f"{url}?rev={dirty_rev}"
                 args += [
                     "--expr",
                     f"""
@@ -216,7 +219,8 @@ class Machine:
                 else:
                     flake = self.flake
                 args += [
-                    f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}'
+                    f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}',
+                    *nix_options,
                 ]
 
             if method == "eval":
@@ -234,6 +238,7 @@ class Machine:
         refresh: bool = False,
         extra_config: None | dict = None,
         impure: bool = False,
+        nix_options: list[str] = [],
     ) -> str:
         """
         eval a nix attribute of the machine
@@ -242,7 +247,7 @@ class Machine:
         if attr in self.eval_cache and not refresh and extra_config is None:
             return self.eval_cache[attr]
 
-        output = self.nix("eval", attr, extra_config, impure)
+        output = self.nix("eval", attr, extra_config, impure, nix_options)
         if isinstance(output, str):
             self.eval_cache[attr] = output
             return output
@@ -255,6 +260,7 @@ class Machine:
         refresh: bool = False,
         extra_config: None | dict = None,
         impure: bool = False,
+        nix_options: list[str] = [],
     ) -> Path:
         """
         build a nix attribute of the machine
@@ -264,7 +270,7 @@ class Machine:
         if attr in self.build_cache and not refresh and extra_config is None:
             return self.build_cache[attr]
 
-        output = self.nix("build", attr, extra_config, impure)
+        output = self.nix("build", attr, extra_config, impure, nix_options)
         if isinstance(output, Path):
             self.build_cache[attr] = output
             return output

pkgs/clan-cli/clan_cli/secrets/check.py

@@ -10,6 +10,8 @@ log = logging.getLogger(__name__)
 def check_secrets(machine: Machine) -> bool:
     secrets_module = importlib.import_module(machine.secrets_module)
     secret_store = secrets_module.SecretStore(machine=machine)
+    facts_module = importlib.import_module(machine.facts_module)
+    fact_store = facts_module.FactStore(machine=machine)
 
     missing_secrets = []
     missing_facts = []
@@ -19,11 +21,13 @@ def check_secrets(machine: Machine) -> bool:
                 log.info(f"Secret {secret} for service {service} is missing")
                 missing_secrets.append((service, secret))
 
-        for fact in machine.secrets_data[service]["facts"].values():
+        for fact in machine.secrets_data[service]["facts"]:
-            if not (machine.flake / fact).exists():
+            if not fact_store.exists(service, fact):
                 log.info(f"Fact {fact} for service {service} is missing")
                 missing_facts.append((service, fact))
 
+    log.debug(f"missing_secrets: {missing_secrets}")
+    log.debug(f"missing_facts: {missing_facts}")
     if missing_secrets or missing_facts:
         return False
     return True

pkgs/clan-cli/clan_cli/secrets/generate.py

@@ -2,7 +2,6 @@ import argparse
 import importlib
 import logging
 import os
-import shutil
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -21,11 +20,15 @@ def generate_secrets(machine: Machine) -> None:
     secrets_module = importlib.import_module(machine.secrets_module)
     secret_store = secrets_module.SecretStore(machine=machine)
 
+    facts_module = importlib.import_module(machine.facts_module)
+    fact_store = facts_module.FactStore(machine=machine)
+
     with TemporaryDirectory() as d:
         for service in machine.secrets_data:
             tmpdir = Path(d) / service
             # check if all secrets exist and generate them if at least one is missing
             needs_regeneration = not check_secrets(machine)
+            log.debug(f"{service} needs_regeneration: {needs_regeneration}")
             if needs_regeneration:
                 if not isinstance(machine.flake, Path):
                     msg = f"flake is not a Path: {machine.flake}"
@@ -78,16 +81,15 @@ def generate_secrets(machine: Machine) -> None:
                         files_to_commit.append(secret_path)
 
                 # store facts
-                for name, fact_path in machine.secrets_data[service]["facts"].items():
+                for name in machine.secrets_data[service]["facts"]:
                     fact_file = facts_dir / name
                     if not fact_file.is_file():
                         msg = f"did not generate a file for '{name}' when running the following command:\n"
                         msg += machine.secrets_data[service]["generator"]
                         raise ClanError(msg)
-                    fact_path = machine.flake / fact_path
+                    fact_file = fact_store.set(service, name, fact_file.read_bytes())
-                    fact_path.parent.mkdir(parents=True, exist_ok=True)
+                    if fact_file:
-                    shutil.copyfile(fact_file, fact_path)
+                        files_to_commit.append(fact_file)
-                    files_to_commit.append(fact_path)
                 commit_files(
                     files_to_commit,
                     machine.flake_dir,

pkgs/clan-cli/clan_cli/secrets/modules/vm.py

@@ -0,0 +1,31 @@
+import os
+import shutil
+from pathlib import Path
+
+from clan_cli.dirs import vm_state_dir
+from clan_cli.machines.machines import Machine
+
+
+class SecretStore:
+    def __init__(self, machine: Machine) -> None:
+        self.machine = machine
+        self.dir = vm_state_dir(str(machine.flake), machine.name) / "secrets"
+        self.dir.mkdir(parents=True, exist_ok=True)
+
+    def set(self, service: str, name: str, value: bytes) -> Path | None:
+        secret_file = self.dir / service / name
+        secret_file.parent.mkdir(parents=True, exist_ok=True)
+        secret_file.write_bytes(value)
+        return None  # we manage the files outside of the git repo
+
+    def get(self, service: str, name: str) -> bytes:
+        secret_file = self.dir / service / name
+        return secret_file.read_bytes()
+
+    def exists(self, service: str, name: str) -> bool:
+        return (self.dir / service / name).exists()
+
+    def upload(self, output_dir: Path) -> None:
+        if os.path.exists(output_dir):
+            shutil.rmtree(output_dir)
+        shutil.copytree(self.dir, output_dir)

pkgs/clan-cli/clan_cli/vms/run.py

@@ -15,10 +15,10 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 
 from ..cmd import Log, run
-from ..dirs import machine_gcroot, module_root, user_cache_dir, vm_state_dir
+from ..dirs import module_root, user_cache_dir, vm_state_dir
 from ..errors import ClanError
 from ..machines.machines import Machine
-from ..nix import nix_build, nix_config, nix_shell
+from ..nix import nix_shell
 from ..secrets.generate import generate_secrets
 from .inspect import VmConfig, inspect_vm
 
@@ -153,26 +153,39 @@ def qemu_command(
     return QemuCommand(command, vsock_cid=vsock_cid)
 
 
+def facts_to_nixos_config(facts: dict[str, dict[str, bytes]]) -> dict:
+    nixos_config: dict = {}
+    nixos_config["clanCore"] = {}
+    nixos_config["clanCore"]["secrets"] = {}
+    for service, service_facts in facts.items():
+        nixos_config["clanCore"]["secrets"][service] = {}
+        nixos_config["clanCore"]["secrets"][service]["facts"] = {}
+        for fact, value in service_facts.items():
+            nixos_config["clanCore"]["secrets"][service]["facts"][fact] = {
+                "value": value.decode()
+            }
+    return nixos_config
+
+
 # TODO move this to the Machines class
 def build_vm(
-    machine: Machine, vm: VmConfig, nix_options: list[str] = []
+    machine: Machine, vm: VmConfig, tmpdir: Path, nix_options: list[str]
 ) -> dict[str, str]:
-    config = nix_config()
+    secrets_dir = get_secrets(machine, tmpdir)
-    system = config["system"]
 
-    clan_dir = machine.flake
+    facts_module = importlib.import_module(machine.facts_module)
-    cmd = nix_build(
+    fact_store = facts_module.FactStore(machine=machine)
-        [
+    facts = fact_store.get_all()
-            f'{clan_dir}#clanInternals.machines."{system}"."{machine.name}".config.system.clan.vm.create',
+
-            *nix_options,
+    nixos_config_file = machine.build_nix(
-        ],
+        "config.system.clan.vm.create",
-        machine_gcroot(flake_url=str(vm.flake_url)) / f"vm-{machine.name}",
+        extra_config=facts_to_nixos_config(facts),
-    )
+        nix_options=nix_options,
-    proc = run(
-        cmd, log=Log.BOTH, error_msg=f"Could not build vm config for {machine.name}"
     )
     try:
-        return json.loads(Path(proc.stdout.strip()).read_text())
+        vm_data = json.loads(Path(nixos_config_file).read_text())
+        vm_data["secrets_dir"] = str(secrets_dir)
+        return vm_data
     except json.JSONDecodeError as e:
         raise ClanError(f"Failed to parse vm config: {e}")
 
@@ -182,16 +195,13 @@ def get_secrets(
     tmpdir: Path,
 ) -> Path:
     secrets_dir = tmpdir / "secrets"
-    secrets_dir.mkdir(exist_ok=True)
+    secrets_dir.mkdir(parents=True, exist_ok=True)
 
     secrets_module = importlib.import_module(machine.secrets_module)
     secret_store = secrets_module.SecretStore(machine=machine)
 
-    # Only generate secrets for local clans
+    # TODO Only generate secrets for local clans
-    if isinstance(machine.flake, Path) and machine.flake.is_dir():
     generate_secrets(machine)
-    else:
-        log.warning("won't generate secrets for non local clan")
 
     secret_store.upload(secrets_dir)
     return secrets_dir
@@ -302,20 +312,19 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None:
     machine = Machine(vm.machine_name, vm.flake_url)
     log.debug(f"Creating VM for {machine}")
 
-    # TODO: We should get this from the vm argument
-    nixos_config = build_vm(machine, vm, nix_options)
-
     # store the temporary rootfs inside XDG_CACHE_HOME on the host
     # otherwise, when using /tmp, we risk running out of memory
     cache = user_cache_dir() / "clan"
     cache.mkdir(exist_ok=True)
     with TemporaryDirectory(dir=cache) as cachedir, TemporaryDirectory() as sockets:
         tmpdir = Path(cachedir)
+
+        # TODO: We should get this from the vm argument
+        nixos_config = build_vm(machine, vm, tmpdir, nix_options)
+
         xchg_dir = tmpdir / "xchg"
         xchg_dir.mkdir(exist_ok=True)
 
-        secrets_dir = get_secrets(machine, tmpdir)
-
         state_dir = vm_state_dir(str(vm.flake_url), machine.name)
         state_dir.mkdir(parents=True, exist_ok=True)
 
@@ -350,7 +359,7 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None:
             vm,
             nixos_config,
             xchg_dir=xchg_dir,
-            secrets_dir=secrets_dir,
+            secrets_dir=Path(nixos_config["secrets_dir"]),
             rootfs_img=rootfs_img,
             state_img=state_img,
             virtiofsd_socket=virtiofsd_socket,