From a055b4d1ebdf10bb5ab02e05b0c5481631538421 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sun, 29 Jun 2025 19:00:38 +0200 Subject: [PATCH] refactor: remove deployment.json and use direct selectors - Remove deployment.json file generation from outputs.nix - Add throw for deprecated deployment.file usage with upgrade instructions - Remove vars data from deployment.data - Update Machine class to use direct select() calls instead of deployment property - Update all deployment property accesses to use direct selectors - Add precaching for frequently accessed values in update.py: - Module paths for facts and vars - Deployment settings (requireExplicitUpdate, nixosMobileWorkaround) - Services and generators data - Secret upload locations - This removes unnecessary JSON serialization and makes the code more composable --- checks/backups/flake-module.nix | 1 - checks/flash/flake-module.nix | 1 - checks/installation/flake-module.nix | 1 - checks/morph/flake-module.nix | 1 - nixosModules/clanCore/outputs.nix | 12 +++-- nixosModules/clanCore/vars/default.nix | 5 -- .../clanCore/vars/secret/password-store.nix | 2 - .../clan_cli/facts/secret_modules/sops.py | 2 +- pkgs/clan-cli/clan_cli/machines/update.py | 19 +++++-- pkgs/clan-cli/clan_cli/vars/generate.py | 1 - .../vars/secret_modules/password_store.py | 4 +- .../clan_cli/vars/secret_modules/sops.py | 8 +-- pkgs/clan-cli/clan_lib/machines/machines.py | 52 +++++++++---------- pkgs/clan-cli/clan_lib/machines/update.py | 7 ++- .../generate_test_vars/cli.py | 1 - 15 files changed, 62 insertions(+), 55 deletions(-) diff --git a/checks/backups/flake-module.nix b/checks/backups/flake-module.nix index fe6ad4e27..a735f07ff 100644 --- a/checks/backups/flake-module.nix +++ b/checks/backups/flake-module.nix @@ -22,7 +22,6 @@ dependencies = [ self pkgs.stdenv.drvPath - self.clan.clanInternals.machines.${pkgs.hostPlatform.system}.test-backup.config.system.clan.deployment.file ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs); closureInfo = pkgs.closureInfo { rootPaths = dependencies; }; in diff --git a/checks/flash/flake-module.nix b/checks/flash/flake-module.nix index cb6ef6280..2123388b6 100644 --- a/checks/flash/flake-module.nix +++ b/checks/flash/flake-module.nix @@ -50,7 +50,6 @@ self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.build.toplevel self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.build.diskoScript self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.build.diskoScript.drvPath - self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.clan.deployment.file ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs); closureInfo = pkgs.closureInfo { rootPaths = dependencies; }; in diff --git a/checks/installation/flake-module.nix b/checks/installation/flake-module.nix index 7bef4b82f..527890f53 100644 --- a/checks/installation/flake-module.nix +++ b/checks/installation/flake-module.nix @@ -10,7 +10,6 @@ let dependencies = [ self.clan.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.toplevel self.clan.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.diskoScript - self.clan.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.clan.deployment.file pkgs.stdenv.drvPath pkgs.bash.drvPath pkgs.nixos-anywhere diff --git a/checks/morph/flake-module.nix b/checks/morph/flake-module.nix index 244bb62c7..4af4671d4 100644 --- a/checks/morph/flake-module.nix +++ b/checks/morph/flake-module.nix @@ -35,7 +35,6 @@ pkgs.stdenv.drvPath pkgs.stdenvNoCC self.nixosConfigurations.test-morph-machine.config.system.build.toplevel - self.nixosConfigurations.test-morph-machine.config.system.clan.deployment.file ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs); closureInfo = pkgs.closureInfo { rootPaths = dependencies; }; in diff --git a/nixosModules/clanCore/outputs.nix b/nixosModules/clanCore/outputs.nix index 6acc4fc28..289a8ddc9 100644 --- a/nixosModules/clanCore/outputs.nix +++ b/nixosModules/clanCore/outputs.nix @@ -1,7 +1,6 @@ { config, lib, - pkgs, ... }: { @@ -24,6 +23,14 @@ description = '' the location of the deployment.json file ''; + default = throw '' + deployment.json file generation has been removed in favor of direct selectors. + + Please upgrade your clan-cli to the latest version. + + The deployment data is now accessed directly from the configuration + instead of being written to a separate JSON file. + ''; }; deployment.buildHost = lib.mkOption { type = lib.types.nullOr lib.types.str; @@ -83,8 +90,5 @@ inherit (config.clan.core.deployment) requireExplicitUpdate; inherit (config.system.clan.deployment) nixosMobileWorkaround; }; - system.clan.deployment.file = pkgs.writeText "deployment.json" ( - builtins.toJSON config.system.clan.deployment.data - ); }; } diff --git a/nixosModules/clanCore/vars/default.nix b/nixosModules/clanCore/vars/default.nix index 7d935f46f..0fa5c6165 100644 --- a/nixosModules/clanCore/vars/default.nix +++ b/nixosModules/clanCore/vars/default.nix @@ -73,10 +73,5 @@ in ) [ ] (lib.attrValues generator.files) ) [ ] (lib.attrValues config.clan.core.vars.generators); - system.clan.deployment.data = { - vars = config.clan.core.vars._serialized; - inherit (config.clan.core.networking) targetHost buildHost; - inherit (config.clan.core.deployment) requireExplicitUpdate; - }; }; } diff --git a/nixosModules/clanCore/vars/secret/password-store.nix b/nixosModules/clanCore/vars/secret/password-store.nix index 50c4f9582..8b02be38d 100644 --- a/nixosModules/clanCore/vars/secret/password-store.nix +++ b/nixosModules/clanCore/vars/secret/password-store.nix @@ -64,8 +64,6 @@ in }; }; config = { - system.clan.deployment.data.password-store.secretLocation = - config.clan.vars.password-store.secretLocation; clan.core.vars.settings = lib.mkIf (config.clan.core.vars.settings.secretStore == "password-store") { diff --git a/pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py b/pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py index 4d45da35d..a25ed1d30 100644 --- a/pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py +++ b/pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py @@ -31,7 +31,7 @@ class SecretStore(SecretStoreBase): sops_secrets_folder(self.machine.flake_dir) / f"{self.machine.name}-age.key", priv_key, - add_groups=self.machine.deployment["sops"]["defaultGroups"], + add_groups=self.machine.select("config.clan.core.sops.defaultGroups"), age_plugins=load_age_plugins(self.machine.flake), ) add_machine(self.machine.flake_dir, self.machine.name, pub_key, False) diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 28680516b..c6ad2b119 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -58,8 +58,11 @@ def update_command(args: argparse.Namespace) -> None: raise ClanError(msg) def filter_machine(m: Machine) -> bool: - if m.deployment.get("requireExplicitUpdate", False): - return False + try: + if m.select("config.clan.deployment.requireExplicitUpdate"): + return False + except Exception: + pass try: # check if the machine has a target host set @@ -96,7 +99,17 @@ def update_command(args: argparse.Namespace) -> None: args.flake.precache( [ f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.validationHash", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.system.clan.deployment.file", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.deployment.requireExplicitUpdate", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.system.clan.deployment.nixosMobileWorkaround", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.secretModule", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.publicModule", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.secretModule", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.publicModule", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.services", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars._serialized.generators", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.secretUploadDirectory", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.vars.password-store.secretLocation", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.passBackend", ] ) diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index fc519a2f4..40a0cb2d1 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -535,7 +535,6 @@ def generate_command(args: argparse.Namespace) -> None: args.flake.precache( [ f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.validationHash", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.system.clan.deployment.file", ] ) has_changed = generate_vars( diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py index 48781f394..c04d5ce9c 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py @@ -153,7 +153,7 @@ class SecretStore(StoreBase): # TODO get the path to the secrets from the machine [ "cat", - f"{self.machine.deployment['password-store']['secretLocation']}/.{self._store_backend}_info", + f"{self.machine.select('config.clan.vars.password-store.secretLocation')}/.{self._store_backend}_info", ], RunOpts(log=Log.STDERR, check=False), ).stdout.strip() @@ -237,6 +237,6 @@ class SecretStore(StoreBase): pass_dir = Path(_tempdir).resolve() self.populate_dir(pass_dir, phases) upload_dir = Path( - self.machine.deployment["password-store"]["secretLocation"] + self.machine.select("config.clan.vars.password-store.secretLocation") ) upload(host, pass_dir, upload_dir) diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py index d71cf3c6f..b2ba2e6c7 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py @@ -71,7 +71,7 @@ class SecretStore(StoreBase): sops_secrets_folder(self.machine.flake_dir) / f"{self.machine.name}-age.key", priv_key, - add_groups=self.machine.deployment["sops"]["defaultGroups"], + add_groups=self.machine.select("config.clan.core.sops.defaultGroups"), age_plugins=load_age_plugins(self.machine.flake), ) add_machine(self.machine.flake_dir, self.machine.name, pub_key, False) @@ -158,7 +158,7 @@ class SecretStore(StoreBase): secret_folder, value, add_machines=[self.machine.name] if var.deploy else [], - add_groups=self.machine.deployment["sops"]["defaultGroups"], + add_groups=self.machine.select("config.clan.core.sops.defaultGroups"), git_commit=False, age_plugins=load_age_plugins(self.machine.flake), ) @@ -259,7 +259,7 @@ class SecretStore(StoreBase): ) keys = collect_keys_for_path(path) - for group in self.machine.deployment["sops"]["defaultGroups"]: + for group in self.machine.select("config.clan.core.sops.defaultGroups"): keys.update( collect_keys_for_type( self.machine.flake_dir / "sops" / "groups" / group / "machines" @@ -314,7 +314,7 @@ class SecretStore(StoreBase): age_plugins = load_age_plugins(self.machine.flake) - for group in self.machine.deployment["sops"]["defaultGroups"]: + for group in self.machine.select("config.clan.core.sops.defaultGroups"): allow_member( groups_folder(secret_path), sops_groups_folder(self.machine.flake_dir), diff --git a/pkgs/clan-cli/clan_lib/machines/machines.py b/pkgs/clan-cli/clan_lib/machines/machines.py index ad72f7de8..5b2e4656a 100644 --- a/pkgs/clan-cli/clan_lib/machines/machines.py +++ b/pkgs/clan-cli/clan_lib/machines/machines.py @@ -1,5 +1,4 @@ import importlib -import json import logging import re from dataclasses import dataclass @@ -15,7 +14,7 @@ from clan_lib.api import API from clan_lib.errors import ClanCmdError, ClanError from clan_lib.flake import Flake from clan_lib.machines.actions import get_machine -from clan_lib.nix import nix_config, nix_test_store +from clan_lib.nix import nix_config from clan_lib.nix_models.clan import InventoryMachine from clan_lib.ssh.remote import Remote @@ -79,58 +78,57 @@ class Machine: f'{self._class_}Configurations."{self.name}".pkgs.hostPlatform.system' ) - @property - def deployment(self) -> dict: - output = Path(self.select("config.system.clan.deployment.file")) - if tmp_store := nix_test_store(): - output = tmp_store.joinpath(*output.parts[1:]) - deployment = json.loads(output.read_text()) - return deployment - @cached_property def secret_facts_store(self) -> facts_secret_modules.SecretStoreBase: - module = importlib.import_module(self.deployment["facts"]["secretModule"]) + secret_module = self.select("config.clan.core.facts.secretModule") + module = importlib.import_module(secret_module) return module.SecretStore(machine=self) @cached_property def public_facts_store(self) -> facts_public_modules.FactStoreBase: - module = importlib.import_module(self.deployment["facts"]["publicModule"]) + public_module = self.select("config.clan.core.facts.publicModule") + module = importlib.import_module(public_module) return module.FactStore(machine=self) @cached_property def secret_vars_store(self) -> StoreBase: - module = importlib.import_module(self.deployment["vars"]["secretModule"]) + secret_module = self.select("config.clan.core.vars.settings.secretModule") + module = importlib.import_module(secret_module) return module.SecretStore(machine=self) @cached_property def public_vars_store(self) -> StoreBase: - module = importlib.import_module(self.deployment["vars"]["publicModule"]) + public_module = self.select("config.clan.core.vars.settings.publicModule") + module = importlib.import_module(public_module) return module.FactStore(machine=self) @property def facts_data(self) -> dict[str, dict[str, Any]]: - if self.deployment["facts"]["services"]: - return self.deployment["facts"]["services"] + services = self.select("config.clan.core.facts.services") + if services: + return services return {} def vars_generators(self) -> list["Generator"]: from clan_cli.vars.generate import Generator - clan_vars = self.deployment.get("vars") - if clan_vars is None: + try: + generators_data = self.select( + "config.clan.core.vars._serialized.generators" + ) + if generators_data is None: + return [] + _generators = [Generator.from_json(gen) for gen in generators_data.values()] + for gen in _generators: + gen.machine(self) + except Exception: return [] - generators: dict[str, Any] = clan_vars.get("generators") - if generators is None: - return [] - _generators = [Generator.from_json(gen) for gen in generators.values()] - for gen in _generators: - gen.machine(self) - - return _generators + else: + return _generators @property def secrets_upload_directory(self) -> str: - return self.deployment["facts"]["secretUploadDirectory"] + return self.select("config.clan.core.facts.secretUploadDirectory") @property def flake_dir(self) -> Path: diff --git a/pkgs/clan-cli/clan_lib/machines/update.py b/pkgs/clan-cli/clan_lib/machines/update.py index 964410532..9e477562c 100644 --- a/pkgs/clan-cli/clan_lib/machines/update.py +++ b/pkgs/clan-cli/clan_lib/machines/update.py @@ -184,7 +184,12 @@ def deploy_machine( # retry nixos-rebuild switch if the first attempt failed if ret.returncode != 0: - is_mobile = machine.deployment.get("nixosMobileWorkaround", False) + try: + is_mobile = machine.select( + "config.system.clan.deployment.nixosMobileWorkaround" + ) + except Exception: + is_mobile = False # if the machine is mobile, we retry to deploy with the mobile workaround method if is_mobile: machine.info( diff --git a/pkgs/generate-test-vars/generate_test_vars/cli.py b/pkgs/generate-test-vars/generate_test_vars/cli.py index b22596ca2..489fd830e 100755 --- a/pkgs/generate-test-vars/generate_test_vars/cli.py +++ b/pkgs/generate-test-vars/generate_test_vars/cli.py @@ -161,7 +161,6 @@ def main() -> None: flake.precache( [ f"checks.{test_system}.{opts.check_attr}.machinesCross.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.validationHash", - f"checks.{test_system}.{opts.check_attr}.machinesCross.{system}.{{{','.join(machine_names)}}}.config.system.clan.deployment.file", ] )