From 3447a98bee2f3bcf71c1bf6fb2c0e398e4bf0502 Mon Sep 17 00:00:00 2001 From: DavHau Date: Tue, 9 Jul 2024 12:42:15 +0700 Subject: [PATCH] vars: implement generating public variables via in_repo --- nixosModules/clanCore/vars/default.nix | 39 ++++- .../clanCore/vars/eval-tests/default.nix | 15 -- nixosModules/clanCore/vars/interface.nix | 158 ++++++++++-------- nixosModules/clanCore/vars/public/in_repo.nix | 12 ++ nixosModules/clanCore/vars/secret/sops.nix | 61 +++++++ nixosModules/clanCore/vars/settings-opts.nix | 71 ++++++++ nixosModules/clanCore/vars/settings.nix | 72 -------- pkgs/clan-cli/clan_cli/__init__.py | 7 +- pkgs/clan-cli/clan_cli/machines/machines.py | 14 ++ pkgs/clan-cli/clan_cli/vars/check.py | 52 +++--- pkgs/clan-cli/clan_cli/vars/generate.py | 129 +++++++------- .../clan_cli/vars/public_modules/in_repo.py | 33 +++- .../clan_cli/vars/secret_modules/sops.py | 15 +- pkgs/clan-cli/docs.py | 3 + pkgs/clan-cli/flake-module.nix | 1 + pkgs/clan-cli/tests/test_vars.py | 49 ++++++ 16 files changed, 452 insertions(+), 279 deletions(-) create mode 100644 nixosModules/clanCore/vars/public/in_repo.nix create mode 100644 nixosModules/clanCore/vars/secret/sops.nix create mode 100644 nixosModules/clanCore/vars/settings-opts.nix delete mode 100644 nixosModules/clanCore/vars/settings.nix create mode 100644 pkgs/clan-cli/tests/test_vars.py diff --git a/nixosModules/clanCore/vars/default.nix b/nixosModules/clanCore/vars/default.nix index 38209e872..34ee569b9 100644 --- a/nixosModules/clanCore/vars/default.nix +++ b/nixosModules/clanCore/vars/default.nix @@ -1,5 +1,26 @@ -{ lib, ... }: { + lib, + config, + pkgs, + ... +}: +let + inherit (lib.types) submoduleWith; + submodule = + module: + submoduleWith { + specialArgs.pkgs = pkgs; + modules = [ module ]; + }; +in +{ + imports = [ + ./public/in_repo.nix + # ./public/vm.nix + # ./secret/password-store.nix + ./secret/sops.nix + # ./secret/vm.nix + ]; options.clan.core.vars = lib.mkOption { visible = false; description = '' @@ -11,6 +32,20 @@ - generate secrets like private keys automatically when they are needed - output multiple values like private and public keys simultaneously ''; - type = lib.types.submoduleWith { modules = [ ./interface.nix ]; }; + type = submodule { imports = [ ./interface.nix ]; }; + }; + + config.system.clan.deployment.data = { + vars = { + generators = lib.flip lib.mapAttrs config.clan.core.vars.generators ( + _name: generator: { + inherit (generator) finalScript; + files = lib.flip lib.mapAttrs generator.files (_name: file: { inherit (file) secret; }); + } + ); + inherit (config.clan.core.vars.settings) secretUploadDirectory secretModule publicModule; + }; + inherit (config.clan.networking) targetHost buildHost; + inherit (config.clan.deployment) requireExplicitUpdate; }; } diff --git a/nixosModules/clanCore/vars/eval-tests/default.nix b/nixosModules/clanCore/vars/eval-tests/default.nix index b97139154..40bd31f53 100644 --- a/nixosModules/clanCore/vars/eval-tests/default.nix +++ b/nixosModules/clanCore/vars/eval-tests/default.nix @@ -54,21 +54,6 @@ in }; }; - # Ensure that generators.imports works - # This allows importing generators from third party projects without providing - # them access to other settings. - test_generator_modules = - let - generator_module = { - my-generator.files.password = { }; - }; - config = eval { generators.imports = [ generator_module ]; }; - in - { - expr = config.generators ? my-generator; - expected = true; - }; - # script can be text test_script_text = let diff --git a/nixosModules/clanCore/vars/interface.nix b/nixosModules/clanCore/vars/interface.nix index 6e551fafe..39f0df0bb 100644 --- a/nixosModules/clanCore/vars/interface.nix +++ b/nixosModules/clanCore/vars/interface.nix @@ -1,8 +1,12 @@ -{ lib, ... }: +{ + lib, + config, + pkgs, + ... +}: let inherit (lib) mkOption; inherit (lib.types) - anything attrsOf bool either @@ -14,30 +18,27 @@ let submoduleWith ; # the original types.submodule has strange behavior - submodule = module: submoduleWith { modules = [ module ]; }; + submodule = + module: + submoduleWith { + specialArgs.pkgs = pkgs; + modules = [ module ]; + }; options = lib.mapAttrs (_: mkOption); - subOptions = opts: submodule { options = options opts; }; in { - options = options { - settings = { + options = { + settings = import ./settings-opts.nix { inherit lib; }; + generators = lib.mkOption { description = '' - Settings for the generated variables. + A set of generators that can be used to generate files. + Generators are scripts that produce files based on the values of other generators and user input. + Each generator is expected to produce a set of files under a directory. ''; - type = submodule { - freeformType = anything; - imports = [ ./settings.nix ]; - }; - }; - generators = { - default = { - imports = [ - # implementation of the generator - ./generator.nix - ]; - }; - type = submodule { - freeformType = attrsOf (subOptions { + default = { }; + type = attrsOf (submodule { + imports = [ ./generator.nix ]; + options = options { dependencies = { description = '' A list of other generators that this generator depends on. @@ -52,32 +53,45 @@ in A set of files to generate. The generator 'script' is expected to produce exactly these files under $out. ''; - type = attrsOf (subOptions { - secret = { - description = '' - Whether the file should be treated as a secret. - ''; - type = bool; - default = true; - }; - path = { - description = '' - The path to the file containing the content of the generated value. - This will be set automatically - ''; - type = str; - readOnly = true; - }; - value = { - description = '' - The content of the generated value. - Only available if the file is not secret. - ''; - type = str; - default = throw "Cannot access value of secret file"; - defaultText = "Throws error because the value of a secret file is not accessible"; - }; - }); + type = attrsOf ( + submodule (file: { + imports = [ config.settings.fileModule ]; + options = options { + name = { + type = lib.types.str; + description = '' + name of the public fact + ''; + readOnly = true; + default = file.config._module.args.name; + }; + secret = { + description = '' + Whether the file should be treated as a secret. + ''; + type = bool; + default = true; + }; + path = { + description = '' + The path to the file containing the content of the generated value. + This will be set automatically + ''; + type = str; + readOnly = true; + }; + value = { + description = '' + The content of the generated value. + Only available if the file is not secret. + ''; + type = str; + default = throw "Cannot access value of secret file"; + defaultText = "Throws error because the value of a secret file is not accessible"; + }; + }; + }) + ); }; prompts = { description = '' @@ -85,28 +99,30 @@ in Prompts are available to the generator script as files. For example, a prompt named 'prompt1' will be available via $prompts/prompt1 ''; - type = attrsOf (subOptions { - description = { - description = '' - The description of the prompted value - ''; - type = str; - example = "SSH private key"; - }; - type = { - description = '' - The input type of the prompt. - The following types are available: - - hidden: A hidden text (e.g. password) - - line: A single line of text - - multiline: A multiline text - ''; - type = enum [ - "hidden" - "line" - "multiline" - ]; - default = "line"; + type = attrsOf (submodule { + options = { + description = { + description = '' + The description of the prompted value + ''; + type = str; + example = "SSH private key"; + }; + type = { + description = '' + The input type of the prompt. + The following types are available: + - hidden: A hidden text (e.g. password) + - line: A single line of text + - multiline: A multiline text + ''; + type = enum [ + "hidden" + "line" + "multiline" + ]; + default = "line"; + }; }; }); }; @@ -140,8 +156,8 @@ in internal = true; visible = false; }; - }); - }; + }; + }); }; }; } diff --git a/nixosModules/clanCore/vars/public/in_repo.nix b/nixosModules/clanCore/vars/public/in_repo.nix new file mode 100644 index 000000000..ba7a6aa41 --- /dev/null +++ b/nixosModules/clanCore/vars/public/in_repo.nix @@ -0,0 +1,12 @@ +{ config, lib, ... }: +{ + config.clan.core.vars.settings = + lib.mkIf (config.clan.core.vars.settings.publicStore == "in_repo") + { + publicModule = "clan_cli.vars.public_modules.in_repo"; + fileModule = file: { + path = + config.clan.core.clanDir + "/machines/${config.clan.core.machineName}/vars/${file.config.name}"; + }; + }; +} diff --git a/nixosModules/clanCore/vars/secret/sops.nix b/nixosModules/clanCore/vars/secret/sops.nix new file mode 100644 index 000000000..bbf9cbc50 --- /dev/null +++ b/nixosModules/clanCore/vars/secret/sops.nix @@ -0,0 +1,61 @@ +{ + config, + lib, + pkgs, + ... +}: +let + secretsDir = config.clan.core.clanDir + "/sops/secrets"; + groupsDir = config.clan.core.clanDir + "/sops/groups"; + + # My symlink is in the nixos module detected as a directory also it works in the repl. Is this because of pure evaluation? + containsSymlink = + path: + builtins.pathExists path + && (builtins.readFileType path == "directory" || builtins.readFileType path == "symlink"); + + containsMachine = + parent: name: type: + type == "directory" && containsSymlink "${parent}/${name}/machines/${config.clan.core.machineName}"; + + containsMachineOrGroups = + name: type: + (containsMachine secretsDir name type) + || lib.any ( + group: type == "directory" && containsSymlink "${secretsDir}/${name}/groups/${group}" + ) groups; + + filterDir = + filter: dir: + lib.optionalAttrs (builtins.pathExists dir) (lib.filterAttrs filter (builtins.readDir dir)); + + groups = builtins.attrNames (filterDir (containsMachine groupsDir) groupsDir); + secrets = filterDir containsMachineOrGroups secretsDir; +in +{ + config.clan.core.vars.settings = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") { + # Before we generate a secret we cannot know the path yet, so we need to set it to an empty string + fileModule = file: { + path = + lib.mkIf file.secret + config.sops.secrets.${"${config.clan.core.machineName}-${file.config.name}"}.path + or "/no-such-path"; + }; + secretModule = "clan_cli.vars.secret_modules.sops"; + secretUploadDirectory = lib.mkDefault "/var/lib/sops-nix"; + }; + + config.sops = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") { + secrets = builtins.mapAttrs (name: _: { + sopsFile = config.clan.core.clanDir + "/sops/secrets/${name}/secret"; + format = "binary"; + }) secrets; + # To get proper error messages about missing secrets we need a dummy secret file that is always present + defaultSopsFile = lib.mkIf config.sops.validateSopsFiles ( + lib.mkDefault (builtins.toString (pkgs.writeText "dummy.yaml" "")) + ); + age.keyFile = lib.mkIf (builtins.pathExists ( + config.clan.core.clanDir + "/sops/secrets/${config.clan.core.machineName}-age.key/secret" + )) (lib.mkDefault "/var/lib/sops-nix/key.txt"); + }; +} diff --git a/nixosModules/clanCore/vars/settings-opts.nix b/nixosModules/clanCore/vars/settings-opts.nix new file mode 100644 index 000000000..cd31adf98 --- /dev/null +++ b/nixosModules/clanCore/vars/settings-opts.nix @@ -0,0 +1,71 @@ +{ lib, ... }: +{ + secretStore = lib.mkOption { + type = lib.types.enum [ + "sops" + "password-store" + "vm" + "custom" + ]; + default = "sops"; + description = '' + method to store secret facts + custom can be used to define a custom secret fact store. + ''; + }; + + secretModule = lib.mkOption { + type = lib.types.str; + internal = true; + description = '' + the python import path to the secret module + ''; + }; + + secretUploadDirectory = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + The directory where secrets are uploaded into, This is backend specific. + ''; + }; + + fileModule = lib.mkOption { + type = lib.types.deferredModule; + internal = true; + description = '' + A module to be imported in every vars.files. submodule. + Used by backends to define the `path` attribute. + ''; + default = { }; + }; + + publicStore = lib.mkOption { + type = lib.types.enum [ + "in_repo" + "vm" + "custom" + ]; + default = "in_repo"; + description = '' + method to store public facts. + custom can be used to define a custom public fact store. + ''; + }; + + publicModule = lib.mkOption { + type = lib.types.str; + internal = true; + description = '' + the python import path to the public module + ''; + }; + + publicDirectory = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + The directory where public facts are stored. + ''; + }; +} diff --git a/nixosModules/clanCore/vars/settings.nix b/nixosModules/clanCore/vars/settings.nix deleted file mode 100644 index dd3e8f7ce..000000000 --- a/nixosModules/clanCore/vars/settings.nix +++ /dev/null @@ -1,72 +0,0 @@ -{ lib, ... }: -{ - options = { - secretStore = lib.mkOption { - type = lib.types.enum [ - "sops" - "password-store" - "vm" - "custom" - ]; - default = "sops"; - description = '' - method to store secret facts - custom can be used to define a custom secret fact store. - ''; - }; - - secretModule = lib.mkOption { - type = lib.types.str; - internal = true; - description = '' - the python import path to the secret module - ''; - }; - - secretUploadDirectory = lib.mkOption { - type = lib.types.nullOr lib.types.path; - default = null; - description = '' - The directory where secrets are uploaded into, This is backend specific. - ''; - }; - - secretPathFunction = lib.mkOption { - type = lib.types.raw; - description = '' - The function to use to generate the path for a secret. - The default function will use the path attribute of the secret. - The function will be called with the secret submodule as an argument. - ''; - }; - - publicStore = lib.mkOption { - type = lib.types.enum [ - "in_repo" - "vm" - "custom" - ]; - default = "in_repo"; - description = '' - method to store public facts. - custom can be used to define a custom public fact store. - ''; - }; - - publicModule = lib.mkOption { - type = lib.types.str; - internal = true; - description = '' - the python import path to the public module - ''; - }; - - publicDirectory = lib.mkOption { - type = lib.types.nullOr lib.types.path; - default = null; - description = '' - The directory where public facts are stored. - ''; - }; - }; -} diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index dc481140c..29acb1e59 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -23,6 +23,7 @@ from . import ( machines, secrets, state, + vars, vms, ) from .clan_uri import FlakeId @@ -275,8 +276,8 @@ For more detailed information, visit: {help_hyperlink("secrets", "https://docs.c # like facts but with vars instead of facts parser_vars = subparsers.add_parser( "vars", - help="manage vars", - description="manage vars", + help="WIP: manage vars", + description="WIP: manage vars", epilog=( f""" This subcommand provides an interface to vars of clan machines. @@ -307,7 +308,7 @@ For more detailed information, visit: {help_hyperlink("secrets", "https://docs.c ), formatter_class=argparse.RawTextHelpFormatter, ) - facts.register_parser(parser_vars) + vars.register_parser(parser_vars) parser_machine = subparsers.add_parser( "machines", diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 6e1974312..5d790333d 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -69,12 +69,26 @@ class Machine: def public_facts_module(self) -> str: return self.deployment["facts"]["publicModule"] + @property + def secret_vars_module(self) -> str: + return self.deployment["vars"]["secretModule"] + + @property + def public_vars_module(self) -> str: + return self.deployment["vars"]["publicModule"] + @property def facts_data(self) -> dict[str, dict[str, Any]]: if self.deployment["facts"]["services"]: return self.deployment["facts"]["services"] return {} + @property + def vars_generators(self) -> dict[str, dict[str, Any]]: + if self.deployment["vars"]["generators"]: + return self.deployment["vars"]["generators"] + return {} + @property def secrets_upload_directory(self) -> str: return self.deployment["facts"]["secretUploadDirectory"] diff --git a/pkgs/clan-cli/clan_cli/vars/check.py b/pkgs/clan-cli/clan_cli/vars/check.py index d879b2d70..2f2c7fa34 100644 --- a/pkgs/clan-cli/clan_cli/vars/check.py +++ b/pkgs/clan-cli/clan_cli/vars/check.py @@ -8,40 +8,36 @@ from ..machines.machines import Machine log = logging.getLogger(__name__) -def check_secrets(machine: Machine, service: None | str = None) -> bool: - secret_facts_module = importlib.import_module(machine.secret_facts_module) - secret_facts_store = secret_facts_module.SecretStore(machine=machine) - public_facts_module = importlib.import_module(machine.public_facts_module) - public_facts_store = public_facts_module.FactStore(machine=machine) +def check_secrets(machine: Machine, generator_name: None | str = None) -> bool: + secret_vars_module = importlib.import_module(machine.secret_vars_module) + secret_vars_store = secret_vars_module.SecretStore(machine=machine) + public_vars_module = importlib.import_module(machine.public_vars_module) + public_vars_store = public_vars_module.FactStore(machine=machine) - missing_secret_facts = [] - missing_public_facts = [] - if service: - services = [service] + missing_secret_vars = [] + missing_public_vars = [] + if generator_name: + services = [generator_name] else: - services = list(machine.facts_data.keys()) - for service in services: - for secret_fact in machine.facts_data[service]["secret"]: - if isinstance(secret_fact, str): - secret_name = secret_fact - else: - secret_name = secret_fact["name"] - if not secret_facts_store.exists(service, secret_name): + services = list(machine.vars_generators.keys()) + for generator_name in services: + for name, file in machine.vars_generators[generator_name]["files"].items(): + if file["secret"] and not secret_vars_store.exists(generator_name, name): log.info( - f"Secret fact '{secret_fact}' for service '{service}' in machine {machine.name} is missing." + f"Secret fact '{name}' for service '{generator_name}' in machine {machine.name} is missing." ) - missing_secret_facts.append((service, secret_name)) - - for public_fact in machine.facts_data[service]["public"]: - if not public_facts_store.exists(service, public_fact): + missing_secret_vars.append((generator_name, name)) + if not file["secret"] and not public_vars_store.exists( + generator_name, name + ): log.info( - f"Public fact '{public_fact}' for service '{service}' in machine {machine.name} is missing." + f"Public fact '{name}' for service '{generator_name}' in machine {machine.name} is missing." ) - missing_public_facts.append((service, public_fact)) + missing_public_vars.append((generator_name, name)) - log.debug(f"missing_secret_facts: {missing_secret_facts}") - log.debug(f"missing_public_facts: {missing_public_facts}") - if missing_secret_facts or missing_public_facts: + log.debug(f"missing_secret_vars: {missing_secret_vars}") + log.debug(f"missing_public_vars: {missing_public_vars}") + if missing_secret_vars or missing_public_vars: return False return True @@ -51,7 +47,7 @@ def check_command(args: argparse.Namespace) -> None: name=args.machine, flake=args.flake, ) - check_secrets(machine, service=args.service) + check_secrets(machine, generator_name=args.service) def register_check_parser(parser: argparse.ArgumentParser) -> None: diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index ea5fb947c..9a54abb6a 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -37,7 +37,7 @@ def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str: return proc.stdout -def bubblewrap_cmd(generator: str, facts_dir: Path, secrets_dir: Path) -> list[str]: +def bubblewrap_cmd(generator: str, generator_dir: Path) -> list[str]: # fmt: off return nix_shell( [ @@ -49,8 +49,7 @@ def bubblewrap_cmd(generator: str, facts_dir: Path, secrets_dir: Path) -> list[s "--ro-bind", "/nix/store", "/nix/store", "--tmpfs", "/usr/lib/systemd", "--dev", "/dev", - "--bind", str(facts_dir), str(facts_dir), - "--bind", str(secrets_dir), str(secrets_dir), + "--bind", str(generator_dir), str(generator_dir), "--unshare-all", "--unshare-user", "--uid", "1000", @@ -61,19 +60,19 @@ def bubblewrap_cmd(generator: str, facts_dir: Path, secrets_dir: Path) -> list[s # fmt: on -def generate_service_facts( +def execute_generator( machine: Machine, - service: str, + generator_name: str, regenerate: bool, - secret_facts_store: SecretStoreBase, - public_facts_store: FactStoreBase, + secret_vars_store: SecretStoreBase, + public_vars_store: FactStoreBase, tmpdir: Path, prompt: Callable[[str], str], ) -> bool: - service_dir = tmpdir / service + generator_dir = tmpdir / generator_name # check if all secrets exist and generate them if at least one is missing - needs_regeneration = not check_secrets(machine, service=service) - log.debug(f"{service} needs_regeneration: {needs_regeneration}") + needs_regeneration = not check_secrets(machine, generator_name=generator_name) + log.debug(f"{generator_name} needs_regeneration: {needs_regeneration}") if not (needs_regeneration or regenerate): return False if not isinstance(machine.flake, Path): @@ -81,22 +80,15 @@ def generate_service_facts( msg += "fact/secret generation is only supported for local flakes" env = os.environ.copy() - facts_dir = service_dir / "facts" - facts_dir.mkdir(parents=True) - env["facts"] = str(facts_dir) - secrets_dir = service_dir / "secrets" - secrets_dir.mkdir(parents=True) - env["secrets"] = str(secrets_dir) + generator_dir.mkdir(parents=True) + env["out"] = str(generator_dir) # compatibility for old outputs.nix users - if isinstance(machine.facts_data[service]["generator"], str): - generator = machine.facts_data[service]["generator"] - else: - generator = machine.facts_data[service]["generator"]["finalScript"] - if machine.facts_data[service]["generator"]["prompt"]: - prompt_value = prompt(machine.facts_data[service]["generator"]["prompt"]) - env["prompt_value"] = prompt_value + generator = machine.vars_generators[generator_name]["finalScript"] + # if machine.vars_data[generator_name]["generator"]["prompt"]: + # prompt_value = prompt(machine.vars_data[generator_name]["generator"]["prompt"]) + # env["prompt_value"] = prompt_value if sys.platform == "linux": - cmd = bubblewrap_cmd(generator, facts_dir, secrets_dir) + cmd = bubblewrap_cmd(generator, generator_dir) else: cmd = ["bash", "-c", generator] run( @@ -105,40 +97,29 @@ def generate_service_facts( ) files_to_commit = [] # store secrets - for secret in machine.facts_data[service]["secret"]: - if isinstance(secret, str): - # TODO: This is the old NixOS module, can be dropped everyone has updated. - secret_name = secret - groups = [] - else: - secret_name = secret["name"] - groups = secret.get("groups", []) + files = machine.vars_generators[generator_name]["files"] + for file_name, file in files.items(): + groups = file.get("groups", []) - secret_file = secrets_dir / secret_name + secret_file = generator_dir / file_name if not secret_file.is_file(): - msg = f"did not generate a file for '{secret_name}' when running the following command:\n" + msg = f"did not generate a file for '{file_name}' when running the following command:\n" msg += generator raise ClanError(msg) - secret_path = secret_facts_store.set( - service, secret_name, secret_file.read_bytes(), groups - ) - if secret_path: - files_to_commit.append(secret_path) - - # store facts - for name in machine.facts_data[service]["public"]: - fact_file = facts_dir / name - if not fact_file.is_file(): - msg = f"did not generate a file for '{name}' when running the following command:\n" - msg += machine.facts_data[service]["generator"] - raise ClanError(msg) - fact_file = public_facts_store.set(service, name, fact_file.read_bytes()) - if fact_file: - files_to_commit.append(fact_file) + if file["secret"]: + file_path = secret_vars_store.set( + generator_name, file_name, secret_file.read_bytes(), groups + ) + else: + file_path = public_vars_store.set( + generator_name, file_name, secret_file.read_bytes() + ) + if file_path: + files_to_commit.append(file_path) commit_files( files_to_commit, machine.flake_dir, - f"Update facts/secrets for service {service} in machine {machine.name}", + f"Update facts/secrets for service {generator_name} in machine {machine.name}", ) return True @@ -148,41 +129,43 @@ def prompt_func(text: str) -> str: return read_multiline_input() -def _generate_facts_for_machine( +def _generate_vars_for_machine( machine: Machine, - service: str | None, + generator_name: str | None, regenerate: bool, tmpdir: Path, prompt: Callable[[str], str] = prompt_func, ) -> bool: local_temp = tmpdir / machine.name local_temp.mkdir() - secret_facts_module = importlib.import_module(machine.secret_facts_module) - secret_facts_store = secret_facts_module.SecretStore(machine=machine) + secret_vars_module = importlib.import_module(machine.secret_vars_module) + secret_vars_store = secret_vars_module.SecretStore(machine=machine) - public_facts_module = importlib.import_module(machine.public_facts_module) - public_facts_store = public_facts_module.FactStore(machine=machine) + public_vars_module = importlib.import_module(machine.public_vars_module) + public_vars_store = public_vars_module.FactStore(machine=machine) machine_updated = False - if service and service not in machine.facts_data: - services = list(machine.facts_data.keys()) + if generator_name and generator_name not in machine.vars_generators: + generators = list(machine.vars_generators.keys()) raise ClanError( - f"Could not find service with name: {service}. The following services are available: {services}" + f"Could not find generator with name: {generator_name}. The following generators are available: {generators}" ) - if service: - machine_service_facts = {service: machine.facts_data[service]} + if generator_name: + machine_generator_facts = { + generator_name: machine.vars_generators[generator_name] + } else: - machine_service_facts = machine.facts_data + machine_generator_facts = machine.vars_generators - for service in machine_service_facts: - machine_updated |= generate_service_facts( + for generator_name in machine_generator_facts: + machine_updated |= execute_generator( machine=machine, - service=service, + generator_name=generator_name, regenerate=regenerate, - secret_facts_store=secret_facts_store, - public_facts_store=public_facts_store, + secret_vars_store=secret_vars_store, + public_vars_store=public_vars_store, tmpdir=local_temp, prompt=prompt, ) @@ -192,9 +175,9 @@ def _generate_facts_for_machine( return machine_updated -def generate_facts( +def generate_vars( machines: list[Machine], - service: str | None, + generator_name: str | None, regenerate: bool, prompt: Callable[[str], str] = prompt_func, ) -> bool: @@ -205,8 +188,8 @@ def generate_facts( for machine in machines: errors = 0 try: - was_regenerated |= _generate_facts_for_machine( - machine, service, regenerate, tmpdir, prompt + was_regenerated |= _generate_vars_for_machine( + machine, generator_name, regenerate, tmpdir, prompt ) except Exception as exc: log.error(f"Failed to generate facts for {machine.name}: {exc}") @@ -226,7 +209,7 @@ def generate_command(args: argparse.Namespace) -> None: machines = get_all_machines(args.flake, args.option) else: machines = get_selected_machines(args.flake, args.option, args.machines) - generate_facts(machines, args.service, args.regenerate) + generate_vars(machines, args.service, args.regenerate) def register_generate_parser(parser: argparse.ArgumentParser) -> None: diff --git a/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py b/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py index f6aad79b8..c8c2c892c 100644 --- a/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py +++ b/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py @@ -11,10 +11,15 @@ class FactStore(FactStoreBase): self.machine = machine self.works_remotely = False - def set(self, service: str, name: str, value: bytes) -> Path | None: - if isinstance(self.machine.flake, Path): + def set(self, generator_name: str, name: str, value: bytes) -> Path | None: + if self.machine.flake.is_local(): fact_path = ( - self.machine.flake / "machines" / self.machine.name / "facts" / name + self.machine.flake.path + / "machines" + / self.machine.name + / "vars" + / generator_name + / name ) fact_path.parent.mkdir(parents=True, exist_ok=True) fact_path.touch() @@ -25,22 +30,32 @@ class FactStore(FactStoreBase): f"in_flake fact storage is only supported for local flakes: {self.machine.flake}" ) - def exists(self, service: str, name: str) -> bool: + def exists(self, generator_name: str, name: str) -> bool: fact_path = ( - self.machine.flake_dir / "machines" / self.machine.name / "facts" / name + self.machine.flake_dir + / "machines" + / self.machine.name + / "vars" + / generator_name + / name ) return fact_path.exists() # get a single fact - def get(self, service: str, name: str) -> bytes: + def get(self, generator_name: str, name: str) -> bytes: fact_path = ( - self.machine.flake_dir / "machines" / self.machine.name / "facts" / name + self.machine.flake_dir + / "machines" + / self.machine.name + / "vars" + / generator_name + / name ) return fact_path.read_bytes() - # get all facts + # get all public vars def get_all(self) -> dict[str, dict[str, bytes]]: - facts_folder = self.machine.flake_dir / "machines" / self.machine.name / "facts" + facts_folder = self.machine.flake_dir / "machines" / self.machine.name / "vars" facts: dict[str, dict[str, bytes]] = {} facts["TODO"] = {} if facts_folder.exists(): diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py index 38ce96c7f..9bc6bcbc1 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py @@ -14,11 +14,13 @@ class SecretStore(SecretStoreBase): self.machine = machine # no need to generate keys if we don't manage secrets - if not hasattr(self.machine, "facts_data"): - return - - if not self.machine.facts_data: + if not hasattr(self.machine, "vars_data") or not self.machine.vars_generators: return + for generator in self.machine.vars_generators.values(): + if "files" in generator: + for file in generator["files"].values(): + if file["secret"]: + return if has_machine(self.machine.flake_dir, self.machine.name): return @@ -32,10 +34,11 @@ class SecretStore(SecretStoreBase): add_machine(self.machine.flake_dir, self.machine.name, pub_key, False) def set( - self, service: str, name: str, value: bytes, groups: list[str] + self, generator_name: str, name: str, value: bytes, groups: list[str] ) -> Path | None: path = ( - sops_secrets_folder(self.machine.flake_dir) / f"{self.machine.name}-{name}" + sops_secrets_folder(self.machine.flake_dir) + / f"{self.machine.name}-{generator_name}-{name}" ) encrypt_secret( self.machine.flake_dir, diff --git a/pkgs/clan-cli/docs.py b/pkgs/clan-cli/docs.py index f2a01dd55..e1bcf0ba7 100644 --- a/pkgs/clan-cli/docs.py +++ b/pkgs/clan-cli/docs.py @@ -254,6 +254,9 @@ def collect_commands() -> list[Category]: if isinstance(action, argparse._SubParsersAction): subparsers: dict[str, argparse.ArgumentParser] = action.choices for name, subparser in subparsers.items(): + if str(subparser.description).startswith("WIP"): + print(f"Excluded {name} from documentation as it is marked as WIP") + continue (_options, _positionals, _subcommands) = get_subcommands( subparser, to=result, level=2, prefix=[name] ) diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index adc586ab3..c41143a34 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -58,6 +58,7 @@ python docs.py reference mkdir -p $out cp -r out/* $out + ls -lah $out ''; }; clan-ts-api = pkgs.stdenv.mkDerivation { diff --git a/pkgs/clan-cli/tests/test_vars.py b/pkgs/clan-cli/tests/test_vars.py new file mode 100644 index 000000000..04a141b92 --- /dev/null +++ b/pkgs/clan-cli/tests/test_vars.py @@ -0,0 +1,49 @@ +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +from fixtures_flakes import generate_flake +from helpers.cli import Cli +from root import CLAN_CORE + +if TYPE_CHECKING: + pass + + +@pytest.mark.impure +def test_generate_secret( + monkeypatch: pytest.MonkeyPatch, + temporary_home: Path, + # age_keys: list["KeyPair"], +) -> None: + flake = generate_flake( + temporary_home, + flake_template=CLAN_CORE / "templates" / "minimal", + machine_configs=dict( + my_machine=dict( + clan=dict( + core=dict( + vars=dict( + generators=dict( + my_generator=dict( + files=dict( + my_secret=dict( + secret=False, + ) + ), + script="echo hello > $out/my_secret", + ) + ) + ) + ) + ) + ) + ), + ) + monkeypatch.chdir(flake.path) + cli = Cli() + cmd = ["vars", "generate", "--flake", str(flake.path), "my_machine"] + cli.run(cmd) + assert ( + flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret" + ).is_file()