From fbbfcc0aa518d11df5702cf06d950729445fa0fd Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 29 Nov 2024 16:51:56 +0700 Subject: [PATCH 1/3] vars: generate docs for cli and module --- docs/mkdocs.yml | 2 ++ nixosModules/clanCore/vars/default.nix | 1 - nixosModules/clanCore/vars/interface.nix | 13 +++++++++++-- pkgs/clan-cli/clan_cli/__init__.py | 6 +++--- pkgs/clan-cli/clan_cli/machines/machines.py | 1 - pkgs/clan-cli/clan_cli/vars/generate.py | 8 +++++--- 6 files changed, 21 insertions(+), 10 deletions(-) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 3505f9172..461aee185 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -120,6 +120,7 @@ nav: - reference/cli/show.md - reference/cli/ssh.md - reference/cli/state.md + - reference/cli/vars.md - reference/cli/vms.md - Clan Core: - reference/clan-core/index.md @@ -130,6 +131,7 @@ nav: - reference/clan-core/state.md - reference/clan-core/deployment.md - reference/clan-core/networking.md + - reference/clan-core/vars.md - Nix API: - buildClan: reference/nix-api/buildclan.md - Inventory: reference/nix-api/inventory.md diff --git a/nixosModules/clanCore/vars/default.nix b/nixosModules/clanCore/vars/default.nix index 0bfd072ca..e1f7477c2 100644 --- a/nixosModules/clanCore/vars/default.nix +++ b/nixosModules/clanCore/vars/default.nix @@ -22,7 +22,6 @@ in # ./secret/vm.nix ]; options.clan.core.vars = lib.mkOption { - visible = false; description = '' Generated Variables diff --git a/nixosModules/clanCore/vars/interface.nix b/nixosModules/clanCore/vars/interface.nix index 4d75abf00..30407f702 100644 --- a/nixosModules/clanCore/vars/interface.nix +++ b/nixosModules/clanCore/vars/interface.nix @@ -54,6 +54,7 @@ in ''; readOnly = true; default = generator.config._module.args.name; + defaultText = "Name of the generator"; }; dependencies = lib.mkOption { @@ -91,7 +92,10 @@ in # lists are not allowed as of now due to potential ordering issues ]); in - data; + data + // { + description = "JSON compatible data structure"; + }; }; # the invalidationHash is the validation interface to the outside world invalidationHash = lib.mkOption { @@ -108,6 +112,7 @@ in null else hashString "sha256" (toJSON generator.config.invalidationData); + defaultText = "Hash of the invalidation data"; }; files = lib.mkOption { description = '' @@ -145,6 +150,7 @@ in ''; readOnly = true; default = file.config._module.args.name; + defaultText = "Name of the file"; }; generatorName = lib.mkOption { type = lib.types.str; @@ -153,6 +159,7 @@ in ''; readOnly = true; default = generator.config._module.args.name; + defaultText = "Name of the generator that generates this file"; }; share = lib.mkOption { type = lib.types.bool; @@ -164,6 +171,7 @@ in readOnly = true; internal = true; default = generator.config.share; + defaultText = "Mirror of the share flag of the generator"; }; deploy = lib.mkOption { description = '' @@ -229,6 +237,7 @@ in ''; type = str; default = prompt.config._module.args.name; + defaultText = "Name of the prompt"; }; createFile = lib.mkOption { description = '' @@ -252,6 +261,7 @@ in type = str; example = "SSH private key"; default = prompt.config._module.args.name; + defaultText = "Name of the prompt"; }; type = lib.mkOption { description = '' @@ -301,7 +311,6 @@ in type = lib.types.str; readOnly = true; internal = true; - visible = false; }; share = lib.mkOption { description = '' diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index a1c6c150a..c967a1e67 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -293,11 +293,11 @@ For more detailed information, visit: {help_hyperlink("secrets", "https://docs.c # like facts but with vars instead of facts parser_vars = subparsers.add_parser( "vars", - help="WIP: Manage vars", - description="WIP: Manage vars", + help="Manage vars", + description="Manage vars", epilog=( f""" -This subcommand provides an interface to vars of clan machines. +This subcommand provides an interface to `vars` of clan machines. Vars are variables that a service can generate. There are public and secret vars. Public vars can be referenced by other machines directly. diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 2b14072bd..ea986c428 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -137,7 +137,6 @@ class Machine: module = importlib.import_module(self.public_facts_module) return module.FactStore(machine=self) - # WIP: Vars module is not ready yet. @property def secret_vars_module(self) -> str: return self.deployment["vars"]["secretModule"] diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 175ca3ff9..6d2e4abcb 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -487,7 +487,7 @@ def generate_command(args: argparse.Namespace) -> None: machines = get_all_machines(args.flake, args.option) else: machines = get_selected_machines(args.flake, args.option, args.machines) - generate_vars(machines, args.service, args.regenerate, args.fix) + generate_vars(machines, args.generator, args.regenerate, args.fix) def register_generate_parser(parser: argparse.ArgumentParser) -> None: @@ -501,15 +501,17 @@ def register_generate_parser(parser: argparse.ArgumentParser) -> None: add_dynamic_completer(machines_parser, complete_machines) service_parser = parser.add_argument( - "--service", + "--generator", + "-g", type=str, - help="service to generate facts for, if empty, generate facts for every service", + help="execute only the specified generator. If unset, execute all generators", default=None, ) add_dynamic_completer(service_parser, complete_services_for_machine) parser.add_argument( "--regenerate", + "-r", action=argparse.BooleanOptionalAction, help="whether to regenerate facts for the specified machine", default=None, From 5c5a87d41669fceb92efff767d5ed4be51446223 Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 29 Nov 2024 17:13:23 +0700 Subject: [PATCH 2/3] vars: rename: invalidation -> validation --- nixosModules/clanCore/vars/default.nix | 2 +- nixosModules/clanCore/vars/interface.nix | 10 +++++----- pkgs/clan-cli/clan_cli/vars/_types.py | 16 ++++++---------- pkgs/clan-cli/clan_cli/vars/check.py | 7 ++++--- pkgs/clan-cli/clan_cli/vars/cli.py | 4 ++-- pkgs/clan-cli/clan_cli/vars/generate.py | 14 +++++--------- pkgs/clan-cli/tests/test_vars.py | 6 +++--- 7 files changed, 26 insertions(+), 33 deletions(-) diff --git a/nixosModules/clanCore/vars/default.nix b/nixosModules/clanCore/vars/default.nix index e1f7477c2..264b85ea3 100644 --- a/nixosModules/clanCore/vars/default.nix +++ b/nixosModules/clanCore/vars/default.nix @@ -42,7 +42,7 @@ in name dependencies finalScript - invalidationHash + validationHash migrateFact prompts share diff --git a/nixosModules/clanCore/vars/interface.nix b/nixosModules/clanCore/vars/interface.nix index 30407f702..d5f4f62ad 100644 --- a/nixosModules/clanCore/vars/interface.nix +++ b/nixosModules/clanCore/vars/interface.nix @@ -76,7 +76,7 @@ in example = "my_service"; default = null; }; - invalidationData = lib.mkOption { + validation = lib.mkOption { description = '' A set of values that invalidate the generated values. If any of these values change, the generated values will be re-generated. @@ -97,8 +97,8 @@ in description = "JSON compatible data structure"; }; }; - # the invalidationHash is the validation interface to the outside world - invalidationHash = lib.mkOption { + # the validationHash is the validation interface to the outside world + validationHash = lib.mkOption { internal = true; description = '' A hash of the invalidation data. @@ -108,10 +108,10 @@ in # TODO: recursively traverse the structure and sort all lists in order to support lists default = # For backwards compat, the hash is null by default in which case the check is omitted - if generator.config.invalidationData == null then + if generator.config.validation == null then null else - hashString "sha256" (toJSON generator.config.invalidationData); + hashString "sha256" (toJSON generator.config.validation); defaultText = "Hash of the invalidation data"; }; files = lib.mkOption { diff --git a/pkgs/clan-cli/clan_cli/vars/_types.py b/pkgs/clan-cli/clan_cli/vars/_types.py index 32b4b2fcf..ed7ef6009 100644 --- a/pkgs/clan-cli/clan_cli/vars/_types.py +++ b/pkgs/clan-cli/clan_cli/vars/_types.py @@ -119,25 +119,21 @@ class StoreBase(ABC): all_vars.append(var) return all_vars - def get_invalidation_hash(self, generator: "Generator") -> str | None: + def get_validation(self, generator: "Generator") -> str | None: """ Return the invalidation hash that indicates if a generator needs to be re-run due to a change in its definition """ - hash_file = ( - self.machine.flake_dir / "vars" / generator.name / "invalidation_hash" - ) + hash_file = self.machine.flake_dir / "vars" / generator.name / "validation" if not hash_file.exists(): return None return hash_file.read_text().strip() - def set_invalidation_hash(self, generator: "Generator", hash_str: str) -> None: + def set_validation(self, generator: "Generator", hash_str: str) -> None: """ Store the invalidation hash that indicates if a generator needs to be re-run """ - hash_file = ( - self.machine.flake_dir / "vars" / generator.name / "invalidation_hash" - ) + hash_file = self.machine.flake_dir / "vars" / generator.name / "validation" hash_file.parent.mkdir(parents=True, exist_ok=True) hash_file.write_text(hash_str) @@ -147,8 +143,8 @@ class StoreBase(ABC): If the hash is not set in nix and hasn't been stored before, it is considered valid -> this provides backward and forward compatibility """ - stored_hash = self.get_invalidation_hash(generator) - target_hash = generator.invalidation_hash + stored_hash = self.get_validation(generator) + target_hash = generator.validation # if the hash is neither set in nix nor on disk, it is considered valid (provides backwards compat) if target_hash is None and stored_hash is None: return True diff --git a/pkgs/clan-cli/clan_cli/vars/check.py b/pkgs/clan-cli/clan_cli/vars/check.py index 7c91f75a6..2e93128a4 100644 --- a/pkgs/clan-cli/clan_cli/vars/check.py +++ b/pkgs/clan-cli/clan_cli/vars/check.py @@ -114,7 +114,7 @@ def check_command(args: argparse.Namespace) -> None: name=args.machine, flake=args.flake, ) - ok = check_vars(machine, generator_name=args.service) + ok = check_vars(machine, generator_name=args.generator) if not ok: raise SystemExit(1) @@ -127,7 +127,8 @@ def register_check_parser(parser: argparse.ArgumentParser) -> None: add_dynamic_completer(machines_parser, complete_machines) parser.add_argument( - "--service", - help="the service to check", + "--generator", + "-g", + help="the generator to check", ) parser.set_defaults(func=check_command) diff --git a/pkgs/clan-cli/clan_cli/vars/cli.py b/pkgs/clan-cli/clan_cli/vars/cli.py index e94c19297..7177be71f 100644 --- a/pkgs/clan-cli/clan_cli/vars/cli.py +++ b/pkgs/clan-cli/clan_cli/vars/cli.py @@ -145,10 +145,10 @@ Examples: $ clan vars generate [MACHINE] Will generate vars for the specified machine. - $ clan vars generate [MACHINE] --service [SERVICE] + $ clan vars generate [MACHINE] --generator [SERVICE] Will generate vars for the specified machine for the specified service. - $ clan vars generate --service [SERVICE] --regenerate + $ clan vars generate --generator [SERVICE] --regenerate Will regenerate vars, if they are already generated for a specific service. This is especially useful for resetting certain passwords while leaving the rest of the vars for a machine in place. diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 6d2e4abcb..58b2cd46b 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -40,7 +40,7 @@ class Generator: name: str files: list[Var] = field(default_factory=list) share: bool = False - invalidation_hash: str | None = None + validation: str | None = None final_script: str = "" prompts: list[Prompt] = field(default_factory=list) dependencies: list[str] = field(default_factory=list) @@ -65,7 +65,7 @@ class Generator: share=data["share"], final_script=data["finalScript"], files=[Var.from_json(data["name"], f) for f in data["files"].values()], - invalidation_hash=data["invalidationHash"], + validation=data["validationHash"], dependencies=data["dependencies"], migrate_fact=data["migrateFact"], prompts=[Prompt.from_json(p) for p in data["prompts"].values()], @@ -220,15 +220,11 @@ def execute_generator( public_changed = True if file_path: files_to_commit.append(file_path) - if generator.invalidation_hash is not None: + if generator.validation is not None: if public_changed: - public_vars_store.set_invalidation_hash( - generator, generator.invalidation_hash - ) + public_vars_store.set_validation(generator, generator.validation) if secret_changed: - secret_vars_store.set_invalidation_hash( - generator, generator.invalidation_hash - ) + secret_vars_store.set_validation(generator, generator.validation) commit_files( files_to_commit, machine.flake_dir, diff --git a/pkgs/clan-cli/tests/test_vars.py b/pkgs/clan-cli/tests/test_vars.py index 6335fedf1..3c006af91 100644 --- a/pkgs/clan-cli/tests/test_vars.py +++ b/pkgs/clan-cli/tests/test_vars.py @@ -659,7 +659,7 @@ def test_commit_message( "--flake", str(flake.path), "my_machine", - "--service", + "--generator", "my_generator", ] ) @@ -678,7 +678,7 @@ def test_commit_message( "--flake", str(flake.path), "my_machine", - "--service", + "--generator", "my_secret_generator", ] ) @@ -981,7 +981,7 @@ def test_invalidation( value1_new = get_var(machine, "my_generator/my_value").printable_value assert value1 == value1_new # set the invalidation data of the generator - my_generator["invalidationData"] = 1 + my_generator["validation"] = 1 flake.refresh() # generate again and make sure the value changes cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) From 8d007867b370a8886cea742e30269f4a3e4d356e Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 29 Nov 2024 17:21:37 +0700 Subject: [PATCH 3/3] vars/migration: remove useless check --- pkgs/clan-cli/clan_cli/vars/generate.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 58b2cd46b..9b6d6f55a 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -358,12 +358,6 @@ def _check_can_migrate( else: if machine.public_vars_store.exists(generator, file.name): return False - # ensure that the service to migrate from actually exists - if service_name not in machine.facts_data: - log.debug( - f"Could not migrate facts for generator {generator.name}, as the service {service_name} does not exist" - ) - return False # ensure that all files can be migrated (exists in the corresponding fact store) return bool( all(