yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

vars: implement generating public variables via in_repo

Browse Source
This commit is contained in:
DavHau
2024-07-09 12:42:15 +07:00
parent 26ff2beea9
commit 3447a98bee
16 changed files with 452 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
nixosModules/clanCore/vars/default.nix
View File

@@ -1,5 +1,26 @@
{ lib, ... }:
{ {
lib,
config,
pkgs,
...
}:
let
inherit (lib.types) submoduleWith;
submodule =
module:
submoduleWith {
specialArgs.pkgs = pkgs;
modules = [ module ];
};
in
{
imports = [
./public/in_repo.nix
# ./public/vm.nix
# ./secret/password-store.nix
./secret/sops.nix
# ./secret/vm.nix
];
options.clan.core.vars = lib.mkOption { options.clan.core.vars = lib.mkOption {
visible = false; visible = false;
description = '' description = ''
@@ -11,6 +32,20 @@
- generate secrets like private keys automatically when they are needed - generate secrets like private keys automatically when they are needed
- output multiple values like private and public keys simultaneously - output multiple values like private and public keys simultaneously
''; '';
type = lib.types.submoduleWith { modules = [ ./interface.nix ]; }; type = submodule { imports = [ ./interface.nix ]; };
};
config.system.clan.deployment.data = {
vars = {
generators = lib.flip lib.mapAttrs config.clan.core.vars.generators (
_name: generator: {
inherit (generator) finalScript;
files = lib.flip lib.mapAttrs generator.files (_name: file: { inherit (file) secret; });
}
);
inherit (config.clan.core.vars.settings) secretUploadDirectory secretModule publicModule;
};
inherit (config.clan.networking) targetHost buildHost;
inherit (config.clan.deployment) requireExplicitUpdate;
}; };
} }

15
nixosModules/clanCore/vars/eval-tests/default.nix
View File

@@ -54,21 +54,6 @@ in
}; };
}; };
# Ensure that generators.imports works
# This allows importing generators from third party projects without providing
# them access to other settings.
test_generator_modules =
let
generator_module = {
my-generator.files.password = { };
};
config = eval { generators.imports = [ generator_module ]; };
in
{
expr = config.generators ? my-generator;
expected = true;
};
# script can be text # script can be text
test_script_text = test_script_text =
let let

158
nixosModules/clanCore/vars/interface.nix
View File

@@ -1,8 +1,12 @@
{ lib, ... }: {
lib,
config,
pkgs,
...
}:
let let
inherit (lib) mkOption; inherit (lib) mkOption;
inherit (lib.types) inherit (lib.types)
anything
attrsOf attrsOf
bool bool
either either
@@ -14,30 +18,27 @@ let
submoduleWith submoduleWith
; ;
# the original types.submodule has strange behavior # the original types.submodule has strange behavior
submodule = module: submoduleWith { modules = [ module ]; }; submodule =
module:
submoduleWith {
specialArgs.pkgs = pkgs;
modules = [ module ];
};
options = lib.mapAttrs (_: mkOption); options = lib.mapAttrs (_: mkOption);
subOptions = opts: submodule { options = options opts; };
in in
{ {
options = options { options = {
settings = { settings = import ./settings-opts.nix { inherit lib; };
generators = lib.mkOption {
description = '' description = ''
Settings for the generated variables. A set of generators that can be used to generate files.
Generators are scripts that produce files based on the values of other generators and user input.
Each generator is expected to produce a set of files under a directory.
''; '';
type = submodule { default = { };
freeformType = anything; type = attrsOf (submodule {
imports = [ ./settings.nix ]; imports = [ ./generator.nix ];
}; options = options {
};
generators = {
default = {
imports = [
# implementation of the generator
./generator.nix
];
};
type = submodule {
freeformType = attrsOf (subOptions {
dependencies = { dependencies = {
description = '' description = ''
A list of other generators that this generator depends on. A list of other generators that this generator depends on.
@@ -52,32 +53,45 @@ in
A set of files to generate. A set of files to generate.
The generator 'script' is expected to produce exactly these files under $out. The generator 'script' is expected to produce exactly these files under $out.
''; '';
type = attrsOf (subOptions { type = attrsOf (
secret = { submodule (file: {
description = '' imports = [ config.settings.fileModule ];
Whether the file should be treated as a secret. options = options {
''; name = {
type = bool; type = lib.types.str;
default = true; description = ''
}; name of the public fact
path = { '';
description = '' readOnly = true;
The path to the file containing the content of the generated value. default = file.config._module.args.name;
This will be set automatically };
''; secret = {
type = str; description = ''
readOnly = true; Whether the file should be treated as a secret.
}; '';
value = { type = bool;
description = '' default = true;
The content of the generated value. };
Only available if the file is not secret. path = {
''; description = ''
type = str; The path to the file containing the content of the generated value.
default = throw "Cannot access value of secret file"; This will be set automatically
defaultText = "Throws error because the value of a secret file is not accessible"; '';
}; type = str;
}); readOnly = true;
};
value = {
description = ''
The content of the generated value.
Only available if the file is not secret.
'';
type = str;
default = throw "Cannot access value of secret file";
defaultText = "Throws error because the value of a secret file is not accessible";
};
};
})
);
}; };
prompts = { prompts = {
description = '' description = ''
@@ -85,28 +99,30 @@ in
Prompts are available to the generator script as files. Prompts are available to the generator script as files.
For example, a prompt named 'prompt1' will be available via $prompts/prompt1 For example, a prompt named 'prompt1' will be available via $prompts/prompt1
''; '';
type = attrsOf (subOptions { type = attrsOf (submodule {
description = { options = {
description = '' description = {
The description of the prompted value description = ''
''; The description of the prompted value
type = str; '';
example = "SSH private key"; type = str;
}; example = "SSH private key";
type = { };
description = '' type = {
The input type of the prompt. description = ''
The following types are available: The input type of the prompt.
- hidden: A hidden text (e.g. password) The following types are available:
- line: A single line of text - hidden: A hidden text (e.g. password)
- multiline: A multiline text - line: A single line of text
''; - multiline: A multiline text
type = enum [ '';
"hidden" type = enum [
"line" "hidden"
"multiline" "line"
]; "multiline"
default = "line"; ];
default = "line";
};
}; };
}); });
}; };
@@ -140,8 +156,8 @@ in
internal = true; internal = true;
visible = false; visible = false;
}; };
}); };
}; });
}; };
}; };
} }

12
nixosModules/clanCore/vars/public/in_repo.nix Normal file
View File

@@ -0,0 +1,12 @@
{ config, lib, ... }:
{
config.clan.core.vars.settings =
lib.mkIf (config.clan.core.vars.settings.publicStore == "in_repo")
{
publicModule = "clan_cli.vars.public_modules.in_repo";
fileModule = file: {
path =
config.clan.core.clanDir + "/machines/${config.clan.core.machineName}/vars/${file.config.name}";
};
};
}

61
nixosModules/clanCore/vars/secret/sops.nix Normal file
View File

@@ -0,0 +1,61 @@
{
config,
lib,
pkgs,
...
}:
let
secretsDir = config.clan.core.clanDir + "/sops/secrets";
groupsDir = config.clan.core.clanDir + "/sops/groups";
# My symlink is in the nixos module detected as a directory also it works in the repl. Is this because of pure evaluation?
containsSymlink =
path:
builtins.pathExists path
&& (builtins.readFileType path == "directory" || builtins.readFileType path == "symlink");
containsMachine =
parent: name: type:
type == "directory" && containsSymlink "${parent}/${name}/machines/${config.clan.core.machineName}";
containsMachineOrGroups =
name: type:
(containsMachine secretsDir name type)
|| lib.any (
group: type == "directory" && containsSymlink "${secretsDir}/${name}/groups/${group}"
) groups;
filterDir =
filter: dir:
lib.optionalAttrs (builtins.pathExists dir) (lib.filterAttrs filter (builtins.readDir dir));
groups = builtins.attrNames (filterDir (containsMachine groupsDir) groupsDir);
secrets = filterDir containsMachineOrGroups secretsDir;
in
{
config.clan.core.vars.settings = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") {
# Before we generate a secret we cannot know the path yet, so we need to set it to an empty string
fileModule = file: {
path =
lib.mkIf file.secret
config.sops.secrets.${"${config.clan.core.machineName}-${file.config.name}"}.path
or "/no-such-path";
};
secretModule = "clan_cli.vars.secret_modules.sops";
secretUploadDirectory = lib.mkDefault "/var/lib/sops-nix";
};
config.sops = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") {
secrets = builtins.mapAttrs (name: _: {
sopsFile = config.clan.core.clanDir + "/sops/secrets/${name}/secret";
format = "binary";
}) secrets;
# To get proper error messages about missing secrets we need a dummy secret file that is always present
defaultSopsFile = lib.mkIf config.sops.validateSopsFiles (
lib.mkDefault (builtins.toString (pkgs.writeText "dummy.yaml" ""))
);
age.keyFile = lib.mkIf (builtins.pathExists (
config.clan.core.clanDir + "/sops/secrets/${config.clan.core.machineName}-age.key/secret"
)) (lib.mkDefault "/var/lib/sops-nix/key.txt");
};
}

71
nixosModules/clanCore/vars/settings-opts.nix Normal file
View File

@@ -0,0 +1,71 @@
{ lib, ... }:
{
secretStore = lib.mkOption {
type = lib.types.enum [
"sops"
"password-store"
"vm"
"custom"
];
default = "sops";
description = ''
method to store secret facts
custom can be used to define a custom secret fact store.
'';
};
secretModule = lib.mkOption {
type = lib.types.str;
internal = true;
description = ''
the python import path to the secret module
'';
};
secretUploadDirectory = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The directory where secrets are uploaded into, This is backend specific.
'';
};
fileModule = lib.mkOption {
type = lib.types.deferredModule;
internal = true;
description = ''
A module to be imported in every vars.files.<name> submodule.
Used by backends to define the `path` attribute.
'';
default = { };
};
publicStore = lib.mkOption {
type = lib.types.enum [
"in_repo"
"vm"
"custom"
];
default = "in_repo";
description = ''
method to store public facts.
custom can be used to define a custom public fact store.
'';
};
publicModule = lib.mkOption {
type = lib.types.str;
internal = true;
description = ''
the python import path to the public module
'';
};
publicDirectory = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The directory where public facts are stored.
'';
};
}

72
nixosModules/clanCore/vars/settings.nix
View File

@@ -1,72 +0,0 @@
{ lib, ... }:
{
options = {
secretStore = lib.mkOption {
type = lib.types.enum [
"sops"
"password-store"
"vm"
"custom"
];
default = "sops";
description = ''
method to store secret facts
custom can be used to define a custom secret fact store.
'';
};
secretModule = lib.mkOption {
type = lib.types.str;
internal = true;
description = ''
the python import path to the secret module
'';
};
secretUploadDirectory = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The directory where secrets are uploaded into, This is backend specific.
'';
};
secretPathFunction = lib.mkOption {
type = lib.types.raw;
description = ''
The function to use to generate the path for a secret.
The default function will use the path attribute of the secret.
The function will be called with the secret submodule as an argument.
'';
};
publicStore = lib.mkOption {
type = lib.types.enum [
"in_repo"
"vm"
"custom"
];
default = "in_repo";
description = ''
method to store public facts.
custom can be used to define a custom public fact store.
'';
};
publicModule = lib.mkOption {
type = lib.types.str;
internal = true;
description = ''
the python import path to the public module
'';
};
publicDirectory = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The directory where public facts are stored.
'';
};
};
}

7
pkgs/clan-cli/clan_cli/__init__.py
View File

@@ -23,6 +23,7 @@ from . import (
machines, machines,
secrets, secrets,
state, state,
vars,
vms, vms,
) )
from .clan_uri import FlakeId from .clan_uri import FlakeId
@@ -275,8 +276,8 @@ For more detailed information, visit: {help_hyperlink("secrets", "https://docs.c
# like facts but with vars instead of facts # like facts but with vars instead of facts
parser_vars = subparsers.add_parser( parser_vars = subparsers.add_parser(
"vars", "vars",
help="manage vars", help="WIP: manage vars",
description="manage vars", description="WIP: manage vars",
epilog=( epilog=(
f""" f"""
This subcommand provides an interface to vars of clan machines. This subcommand provides an interface to vars of clan machines.
@@ -307,7 +308,7 @@ For more detailed information, visit: {help_hyperlink("secrets", "https://docs.c
), ),
formatter_class=argparse.RawTextHelpFormatter, formatter_class=argparse.RawTextHelpFormatter,
) )
facts.register_parser(parser_vars) vars.register_parser(parser_vars)
parser_machine = subparsers.add_parser( parser_machine = subparsers.add_parser(
"machines", "machines",

14
pkgs/clan-cli/clan_cli/machines/machines.py
View File

@@ -69,12 +69,26 @@ class Machine:
def public_facts_module(self) -> str: def public_facts_module(self) -> str:
return self.deployment["facts"]["publicModule"] return self.deployment["facts"]["publicModule"]
@property
def secret_vars_module(self) -> str:
return self.deployment["vars"]["secretModule"]
@property
def public_vars_module(self) -> str:
return self.deployment["vars"]["publicModule"]
@property @property
def facts_data(self) -> dict[str, dict[str, Any]]: def facts_data(self) -> dict[str, dict[str, Any]]:
if self.deployment["facts"]["services"]: if self.deployment["facts"]["services"]:
return self.deployment["facts"]["services"] return self.deployment["facts"]["services"]
return {} return {}
@property
def vars_generators(self) -> dict[str, dict[str, Any]]:
if self.deployment["vars"]["generators"]:
return self.deployment["vars"]["generators"]
return {}
@property @property
def secrets_upload_directory(self) -> str: def secrets_upload_directory(self) -> str:
return self.deployment["facts"]["secretUploadDirectory"] return self.deployment["facts"]["secretUploadDirectory"]

52
pkgs/clan-cli/clan_cli/vars/check.py
View File

@@ -8,40 +8,36 @@ from ..machines.machines import Machine
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
def check_secrets(machine: Machine, service: None | str = None) -> bool: def check_secrets(machine: Machine, generator_name: None | str = None) -> bool:
secret_facts_module = importlib.import_module(machine.secret_facts_module) secret_vars_module = importlib.import_module(machine.secret_vars_module)
secret_facts_store = secret_facts_module.SecretStore(machine=machine) secret_vars_store = secret_vars_module.SecretStore(machine=machine)
public_facts_module = importlib.import_module(machine.public_facts_module) public_vars_module = importlib.import_module(machine.public_vars_module)
public_facts_store = public_facts_module.FactStore(machine=machine) public_vars_store = public_vars_module.FactStore(machine=machine)
missing_secret_facts = [] missing_secret_vars = []
missing_public_facts = [] missing_public_vars = []
if service: if generator_name:
services = [service] services = [generator_name]
else: else:
services = list(machine.facts_data.keys()) services = list(machine.vars_generators.keys())
for service in services: for generator_name in services:
for secret_fact in machine.facts_data[service]["secret"]: for name, file in machine.vars_generators[generator_name]["files"].items():
if isinstance(secret_fact, str): if file["secret"] and not secret_vars_store.exists(generator_name, name):
secret_name = secret_fact
else:
secret_name = secret_fact["name"]
if not secret_facts_store.exists(service, secret_name):
log.info( log.info(
f"Secret fact '{secret_fact}' for service '{service}' in machine {machine.name} is missing." f"Secret fact '{name}' for service '{generator_name}' in machine {machine.name} is missing."
) )
missing_secret_facts.append((service, secret_name)) missing_secret_vars.append((generator_name, name))
if not file["secret"] and not public_vars_store.exists(
for public_fact in machine.facts_data[service]["public"]: generator_name, name
if not public_facts_store.exists(service, public_fact): ):
log.info( log.info(
f"Public fact '{public_fact}' for service '{service}' in machine {machine.name} is missing." f"Public fact '{name}' for service '{generator_name}' in machine {machine.name} is missing."
) )
missing_public_facts.append((service, public_fact)) missing_public_vars.append((generator_name, name))
log.debug(f"missing_secret_facts: {missing_secret_facts}") log.debug(f"missing_secret_vars: {missing_secret_vars}")
log.debug(f"missing_public_facts: {missing_public_facts}") log.debug(f"missing_public_vars: {missing_public_vars}")
if missing_secret_facts or missing_public_facts: if missing_secret_vars or missing_public_vars:
return False return False
return True return True
@@ -51,7 +47,7 @@ def check_command(args: argparse.Namespace) -> None:
name=args.machine, name=args.machine,
flake=args.flake, flake=args.flake,
) )
check_secrets(machine, service=args.service) check_secrets(machine, generator_name=args.service)
def register_check_parser(parser: argparse.ArgumentParser) -> None: def register_check_parser(parser: argparse.ArgumentParser) -> None:

129
pkgs/clan-cli/clan_cli/vars/generate.py
View File

@@ -37,7 +37,7 @@ def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str:
return proc.stdout return proc.stdout
def bubblewrap_cmd(generator: str, facts_dir: Path, secrets_dir: Path) -> list[str]: def bubblewrap_cmd(generator: str, generator_dir: Path) -> list[str]:
# fmt: off # fmt: off
return nix_shell( return nix_shell(
[ [
@@ -49,8 +49,7 @@ def bubblewrap_cmd(generator: str, facts_dir: Path, secrets_dir: Path) -> list[s
"--ro-bind", "/nix/store", "/nix/store", "--ro-bind", "/nix/store", "/nix/store",
"--tmpfs", "/usr/lib/systemd", "--tmpfs", "/usr/lib/systemd",
"--dev", "/dev", "--dev", "/dev",
"--bind", str(facts_dir), str(facts_dir), "--bind", str(generator_dir), str(generator_dir),
"--bind", str(secrets_dir), str(secrets_dir),
"--unshare-all", "--unshare-all",
"--unshare-user", "--unshare-user",
"--uid", "1000", "--uid", "1000",
@@ -61,19 +60,19 @@ def bubblewrap_cmd(generator: str, facts_dir: Path, secrets_dir: Path) -> list[s
# fmt: on # fmt: on
def generate_service_facts( def execute_generator(
machine: Machine, machine: Machine,
service: str, generator_name: str,
regenerate: bool, regenerate: bool,
secret_facts_store: SecretStoreBase, secret_vars_store: SecretStoreBase,
public_facts_store: FactStoreBase, public_vars_store: FactStoreBase,
tmpdir: Path, tmpdir: Path,
prompt: Callable[[str], str], prompt: Callable[[str], str],
) -> bool: ) -> bool:
service_dir = tmpdir / service generator_dir = tmpdir / generator_name
# check if all secrets exist and generate them if at least one is missing # check if all secrets exist and generate them if at least one is missing
needs_regeneration = not check_secrets(machine, service=service) needs_regeneration = not check_secrets(machine, generator_name=generator_name)
log.debug(f"{service} needs_regeneration: {needs_regeneration}") log.debug(f"{generator_name} needs_regeneration: {needs_regeneration}")
if not (needs_regeneration or regenerate): if not (needs_regeneration or regenerate):
return False return False
if not isinstance(machine.flake, Path): if not isinstance(machine.flake, Path):
@@ -81,22 +80,15 @@ def generate_service_facts(
msg += "fact/secret generation is only supported for local flakes" msg += "fact/secret generation is only supported for local flakes"
env = os.environ.copy() env = os.environ.copy()
facts_dir = service_dir / "facts" generator_dir.mkdir(parents=True)
facts_dir.mkdir(parents=True) env["out"] = str(generator_dir)
env["facts"] = str(facts_dir)
secrets_dir = service_dir / "secrets"
secrets_dir.mkdir(parents=True)
env["secrets"] = str(secrets_dir)
# compatibility for old outputs.nix users # compatibility for old outputs.nix users
if isinstance(machine.facts_data[service]["generator"], str): generator = machine.vars_generators[generator_name]["finalScript"]
generator = machine.facts_data[service]["generator"] # if machine.vars_data[generator_name]["generator"]["prompt"]:
else: # prompt_value = prompt(machine.vars_data[generator_name]["generator"]["prompt"])
generator = machine.facts_data[service]["generator"]["finalScript"] # env["prompt_value"] = prompt_value
if machine.facts_data[service]["generator"]["prompt"]:
prompt_value = prompt(machine.facts_data[service]["generator"]["prompt"])
env["prompt_value"] = prompt_value
if sys.platform == "linux": if sys.platform == "linux":
cmd = bubblewrap_cmd(generator, facts_dir, secrets_dir) cmd = bubblewrap_cmd(generator, generator_dir)
else: else:
cmd = ["bash", "-c", generator] cmd = ["bash", "-c", generator]
run( run(
@@ -105,40 +97,29 @@ def generate_service_facts(
) )
files_to_commit = [] files_to_commit = []
# store secrets # store secrets
for secret in machine.facts_data[service]["secret"]: files = machine.vars_generators[generator_name]["files"]
if isinstance(secret, str): for file_name, file in files.items():
# TODO: This is the old NixOS module, can be dropped everyone has updated. groups = file.get("groups", [])
secret_name = secret
groups = []
else:
secret_name = secret["name"]
groups = secret.get("groups", [])
secret_file = secrets_dir / secret_name secret_file = generator_dir / file_name
if not secret_file.is_file(): if not secret_file.is_file():
msg = f"did not generate a file for '{secret_name}' when running the following command:\n" msg = f"did not generate a file for '{file_name}' when running the following command:\n"
msg += generator msg += generator
raise ClanError(msg) raise ClanError(msg)
secret_path = secret_facts_store.set( if file["secret"]:
service, secret_name, secret_file.read_bytes(), groups file_path = secret_vars_store.set(
) generator_name, file_name, secret_file.read_bytes(), groups
if secret_path: )
files_to_commit.append(secret_path) else:
file_path = public_vars_store.set(
# store facts generator_name, file_name, secret_file.read_bytes()
for name in machine.facts_data[service]["public"]: )
fact_file = facts_dir / name if file_path:
if not fact_file.is_file(): files_to_commit.append(file_path)
msg = f"did not generate a file for '{name}' when running the following command:\n"
msg += machine.facts_data[service]["generator"]
raise ClanError(msg)
fact_file = public_facts_store.set(service, name, fact_file.read_bytes())
if fact_file:
files_to_commit.append(fact_file)
commit_files( commit_files(
files_to_commit, files_to_commit,
machine.flake_dir, machine.flake_dir,
f"Update facts/secrets for service {service} in machine {machine.name}", f"Update facts/secrets for service {generator_name} in machine {machine.name}",
) )
return True return True
@@ -148,41 +129,43 @@ def prompt_func(text: str) -> str:
return read_multiline_input() return read_multiline_input()
def _generate_facts_for_machine( def _generate_vars_for_machine(
machine: Machine, machine: Machine,
service: str | None, generator_name: str | None,
regenerate: bool, regenerate: bool,
tmpdir: Path, tmpdir: Path,
prompt: Callable[[str], str] = prompt_func, prompt: Callable[[str], str] = prompt_func,
) -> bool: ) -> bool:
local_temp = tmpdir / machine.name local_temp = tmpdir / machine.name
local_temp.mkdir() local_temp.mkdir()
secret_facts_module = importlib.import_module(machine.secret_facts_module) secret_vars_module = importlib.import_module(machine.secret_vars_module)
secret_facts_store = secret_facts_module.SecretStore(machine=machine) secret_vars_store = secret_vars_module.SecretStore(machine=machine)
public_facts_module = importlib.import_module(machine.public_facts_module) public_vars_module = importlib.import_module(machine.public_vars_module)
public_facts_store = public_facts_module.FactStore(machine=machine) public_vars_store = public_vars_module.FactStore(machine=machine)
machine_updated = False machine_updated = False
if service and service not in machine.facts_data: if generator_name and generator_name not in machine.vars_generators:
services = list(machine.facts_data.keys()) generators = list(machine.vars_generators.keys())
raise ClanError( raise ClanError(
f"Could not find service with name: {service}. The following services are available: {services}" f"Could not find generator with name: {generator_name}. The following generators are available: {generators}"
) )
if service: if generator_name:
machine_service_facts = {service: machine.facts_data[service]} machine_generator_facts = {
generator_name: machine.vars_generators[generator_name]
}
else: else:
machine_service_facts = machine.facts_data machine_generator_facts = machine.vars_generators
for service in machine_service_facts: for generator_name in machine_generator_facts:
machine_updated |= generate_service_facts( machine_updated |= execute_generator(
machine=machine, machine=machine,
service=service, generator_name=generator_name,
regenerate=regenerate, regenerate=regenerate,
secret_facts_store=secret_facts_store, secret_vars_store=secret_vars_store,
public_facts_store=public_facts_store, public_vars_store=public_vars_store,
tmpdir=local_temp, tmpdir=local_temp,
prompt=prompt, prompt=prompt,
) )
@@ -192,9 +175,9 @@ def _generate_facts_for_machine(
return machine_updated return machine_updated
def generate_facts( def generate_vars(
machines: list[Machine], machines: list[Machine],
service: str | None, generator_name: str | None,
regenerate: bool, regenerate: bool,
prompt: Callable[[str], str] = prompt_func, prompt: Callable[[str], str] = prompt_func,
) -> bool: ) -> bool:
@@ -205,8 +188,8 @@ def generate_facts(
for machine in machines: for machine in machines:
errors = 0 errors = 0
try: try:
was_regenerated |= _generate_facts_for_machine( was_regenerated |= _generate_vars_for_machine(
machine, service, regenerate, tmpdir, prompt machine, generator_name, regenerate, tmpdir, prompt
) )
except Exception as exc: except Exception as exc:
log.error(f"Failed to generate facts for {machine.name}: {exc}") log.error(f"Failed to generate facts for {machine.name}: {exc}")
@@ -226,7 +209,7 @@ def generate_command(args: argparse.Namespace) -> None:
machines = get_all_machines(args.flake, args.option) machines = get_all_machines(args.flake, args.option)
else: else:
machines = get_selected_machines(args.flake, args.option, args.machines) machines = get_selected_machines(args.flake, args.option, args.machines)
generate_facts(machines, args.service, args.regenerate) generate_vars(machines, args.service, args.regenerate)
def register_generate_parser(parser: argparse.ArgumentParser) -> None: def register_generate_parser(parser: argparse.ArgumentParser) -> None:

33
pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py
View File

@@ -11,10 +11,15 @@ class FactStore(FactStoreBase):
self.machine = machine self.machine = machine
self.works_remotely = False self.works_remotely = False
def set(self, service: str, name: str, value: bytes) -> Path | None: def set(self, generator_name: str, name: str, value: bytes) -> Path | None:
if isinstance(self.machine.flake, Path): if self.machine.flake.is_local():
fact_path = ( fact_path = (
self.machine.flake / "machines" / self.machine.name / "facts" / name self.machine.flake.path
/ "machines"
/ self.machine.name
/ "vars"
/ generator_name
/ name
) )
fact_path.parent.mkdir(parents=True, exist_ok=True) fact_path.parent.mkdir(parents=True, exist_ok=True)
fact_path.touch() fact_path.touch()
@@ -25,22 +30,32 @@ class FactStore(FactStoreBase):
f"in_flake fact storage is only supported for local flakes: {self.machine.flake}" f"in_flake fact storage is only supported for local flakes: {self.machine.flake}"
) )
def exists(self, service: str, name: str) -> bool: def exists(self, generator_name: str, name: str) -> bool:
fact_path = ( fact_path = (
self.machine.flake_dir / "machines" / self.machine.name / "facts" / name self.machine.flake_dir
/ "machines"
/ self.machine.name
/ "vars"
/ generator_name
/ name
) )
return fact_path.exists() return fact_path.exists()
# get a single fact # get a single fact
def get(self, service: str, name: str) -> bytes: def get(self, generator_name: str, name: str) -> bytes:
fact_path = ( fact_path = (
self.machine.flake_dir / "machines" / self.machine.name / "facts" / name self.machine.flake_dir
/ "machines"
/ self.machine.name
/ "vars"
/ generator_name
/ name
) )
return fact_path.read_bytes() return fact_path.read_bytes()
# get all facts # get all public vars
def get_all(self) -> dict[str, dict[str, bytes]]: def get_all(self) -> dict[str, dict[str, bytes]]:
facts_folder = self.machine.flake_dir / "machines" / self.machine.name / "facts" facts_folder = self.machine.flake_dir / "machines" / self.machine.name / "vars"
facts: dict[str, dict[str, bytes]] = {} facts: dict[str, dict[str, bytes]] = {}
facts["TODO"] = {} facts["TODO"] = {}
if facts_folder.exists(): if facts_folder.exists():

15
pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py
View File

@@ -14,11 +14,13 @@ class SecretStore(SecretStoreBase):
self.machine = machine self.machine = machine
# no need to generate keys if we don't manage secrets # no need to generate keys if we don't manage secrets
if not hasattr(self.machine, "facts_data"): if not hasattr(self.machine, "vars_data") or not self.machine.vars_generators:
return
if not self.machine.facts_data:
return return
for generator in self.machine.vars_generators.values():
if "files" in generator:
for file in generator["files"].values():
if file["secret"]:
return
if has_machine(self.machine.flake_dir, self.machine.name): if has_machine(self.machine.flake_dir, self.machine.name):
return return
@@ -32,10 +34,11 @@ class SecretStore(SecretStoreBase):
add_machine(self.machine.flake_dir, self.machine.name, pub_key, False) add_machine(self.machine.flake_dir, self.machine.name, pub_key, False)
def set( def set(
self, service: str, name: str, value: bytes, groups: list[str] self, generator_name: str, name: str, value: bytes, groups: list[str]
) -> Path | None: ) -> Path | None:
path = ( path = (
sops_secrets_folder(self.machine.flake_dir) / f"{self.machine.name}-{name}" sops_secrets_folder(self.machine.flake_dir)
/ f"{self.machine.name}-{generator_name}-{name}"
) )
encrypt_secret( encrypt_secret(
self.machine.flake_dir, self.machine.flake_dir,

3
pkgs/clan-cli/docs.py
View File

@@ -254,6 +254,9 @@ def collect_commands() -> list[Category]:
if isinstance(action, argparse._SubParsersAction): if isinstance(action, argparse._SubParsersAction):
subparsers: dict[str, argparse.ArgumentParser] = action.choices subparsers: dict[str, argparse.ArgumentParser] = action.choices
for name, subparser in subparsers.items(): for name, subparser in subparsers.items():
if str(subparser.description).startswith("WIP"):
print(f"Excluded {name} from documentation as it is marked as WIP")
continue
(_options, _positionals, _subcommands) = get_subcommands( (_options, _positionals, _subcommands) = get_subcommands(
subparser, to=result, level=2, prefix=[name] subparser, to=result, level=2, prefix=[name]
) )

1
pkgs/clan-cli/flake-module.nix
View File

@@ -58,6 +58,7 @@
python docs.py reference python docs.py reference
mkdir -p $out mkdir -p $out
cp -r out/* $out cp -r out/* $out
ls -lah $out
''; '';
}; };
clan-ts-api = pkgs.stdenv.mkDerivation { clan-ts-api = pkgs.stdenv.mkDerivation {

49
pkgs/clan-cli/tests/test_vars.py Normal file
View File

@@ -0,0 +1,49 @@
from pathlib import Path
from typing import TYPE_CHECKING
import pytest
from fixtures_flakes import generate_flake
from helpers.cli import Cli
from root import CLAN_CORE
if TYPE_CHECKING:
pass
@pytest.mark.impure
def test_generate_secret(
monkeypatch: pytest.MonkeyPatch,
temporary_home: Path,
# age_keys: list["KeyPair"],
) -> None:
flake = generate_flake(
temporary_home,
flake_template=CLAN_CORE / "templates" / "minimal",
machine_configs=dict(
my_machine=dict(
clan=dict(
core=dict(
vars=dict(
generators=dict(
my_generator=dict(
files=dict(
my_secret=dict(
secret=False,
)
),
script="echo hello > $out/my_secret",
)
)
)
)
)
)
),
)
monkeypatch.chdir(flake.path)
cli = Cli()
cmd = ["vars", "generate", "--flake", str(flake.path), "my_machine"]
cli.run(cmd)
assert (
flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret"
).is_file()