From 79be9078dcfcb48a4558cbf10a77acc24c250d20 Mon Sep 17 00:00:00 2001 From: lassulus Date: Fri, 29 Sep 2023 11:56:02 +0200 Subject: [PATCH] restore clanInternals with valid nixos config --- lib/build-clan/default.nix | 18 +----------- nixosModules/clanCore/flake-module.nix | 10 ++++++- nixosModules/clanCore/secrets/sops.nix | 6 ++-- pkgs/clan-cli/clan_cli/machines/update.py | 13 +++++++-- pkgs/clan-cli/clan_cli/secrets/generate.py | 2 +- pkgs/clan-cli/clan_cli/secrets/upload.py | 32 ++++++++++++++++++---- 6 files changed, 52 insertions(+), 29 deletions(-) diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index e50299165..48465b07d 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -46,27 +46,11 @@ let (system: lib.nameValuePair system (lib.mapAttrs (name: _: nixosConfiguration { inherit name system; }) allMachines)) supportedSystems); - - getMachine = machine: { - inherit (machine.config.system.clan) uploadSecrets generateSecrets; - inherit (machine.config.clan.networking) deploymentAddress; - }; - - machinesPerSystem = lib.mapAttrs (_: machine: getMachine machine); - - machinesPerSystemWithJson = lib.mapAttrs (_: machine: - let - m = getMachine machine; - in - m // { - json = machine.pkgs.writers.writeJSON "machine.json" m; - }); in { inherit nixosConfigurations; clanInternals = { - machines = lib.mapAttrs (_: configs: machinesPerSystemWithJson configs) configsPerSystem; - machines-json = lib.mapAttrs (system: configs: nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" (machinesPerSystem configs)) configsPerSystem; + machines = configsPerSystem; }; } diff --git a/nixosModules/clanCore/flake-module.nix b/nixosModules/clanCore/flake-module.nix index 9e2bd5470..1ead4f2d9 100644 --- a/nixosModules/clanCore/flake-module.nix +++ b/nixosModules/clanCore/flake-module.nix @@ -1,5 +1,5 @@ { self, inputs, lib, ... }: { - flake.nixosModules.clanCore = { pkgs, options, ... }: { + flake.nixosModules.clanCore = { config, pkgs, options, ... }: { imports = [ ./secrets ./zerotier @@ -40,5 +40,13 @@ utility outputs for clan management of this machine ''; }; + # optimization for faster secret generate/upload and machines update + config = { + system.clan.deployment.text = builtins.toJSON { + inherit (config.system.clan) uploadSecrets generateSecrets; + inherit (config.clan.networking) deploymentAddress; + }; + system.clan.deployment.file = pkgs.writeText "deployment.json" config.system.clan.deployment.text; + }; }; } diff --git a/nixosModules/clanCore/secrets/sops.nix b/nixosModules/clanCore/secrets/sops.nix index c174450eb..ae561994e 100644 --- a/nixosModules/clanCore/secrets/sops.nix +++ b/nixosModules/clanCore/secrets/sops.nix @@ -37,10 +37,12 @@ in uploadSecrets = pkgs.writeScript "upload-secrets" '' #!${pkgs.python3}/bin/python import json + import sys from clan_cli.secrets.sops_generate import upload_age_key_from_nix # the second toJSON is needed to escape the string for the python - args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; deployment_address = config.clan.networking.deploymentAddress; age_key_file = config.sops.age.keyFile; })}) - upload_age_key_from_nix(**args) + deployment_address = sys.argv[1] + args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; age_key_file = config.sops.age.keyFile; })}) + upload_age_key_from_nix(**args, deployment_address=deployment_address) ''; }; sops.secrets = builtins.mapAttrs diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 95991578d..96ba12a7b 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -41,7 +41,12 @@ def deploy_nixos(hosts: HostGroup, clan_dir: Path) -> None: flake_attr = h.meta.get("flake_attr", "") run_generate_secrets(h.meta["generateSecrets"], clan_dir) - run_upload_secrets(h.meta["uploadSecrets"], clan_dir) + run_upload_secrets( + h.meta["uploadSecrets"], + clan_dir, + target=target, + target_directory=h.meta["targetDirectory"], + ) target_host = h.meta.get("target_host") if target_host: @@ -92,7 +97,7 @@ def build_json(targets: list[str]) -> list[dict[str, Any]]: def get_all_machines(clan_dir: Path) -> HostGroup: config = nix_config() system = config["system"] - what = f'{clan_dir}#clanInternals.machines-json."{system}"' + what = f'{clan_dir}#clanInternals.all-machines-json."{system}"' machines = build_json([what])[0] hosts = [] @@ -109,7 +114,9 @@ def get_selected_machines(machine_names: list[str], clan_dir: Path) -> HostGroup system = config["system"] what = [] for name in machine_names: - what.append(f'{clan_dir}#clanInternals.machines."{system}"."{name}".json') + what.append( + f'{clan_dir}#clanInternals.machines."{system}"."{name}".config.system.clan.deployment.file' + ) machines = build_json(what) hosts = [] for i, machine in enumerate(machines): diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index 56c1f7810..9e47c93cf 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -16,7 +16,7 @@ def build_generate_script(machine: str, clan_dir: Path) -> str: cmd = nix_build( [ - f'path:{clan_dir}#clanInternals.machines."{system}"."{machine}".generateSecrets' + f'path:{clan_dir}#clanInternals.machines."{system}"."{machine}".config.system.clan.generateSecrets' ] ) proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True) diff --git a/pkgs/clan-cli/clan_cli/secrets/upload.py b/pkgs/clan-cli/clan_cli/secrets/upload.py index 719ab2f28..2d628766b 100644 --- a/pkgs/clan-cli/clan_cli/secrets/upload.py +++ b/pkgs/clan-cli/clan_cli/secrets/upload.py @@ -1,4 +1,5 @@ import argparse +import json import os import shlex import subprocess @@ -6,7 +7,7 @@ from pathlib import Path from ..dirs import get_clan_flake_toplevel, module_root from ..errors import ClanError -from ..nix import nix_build, nix_config +from ..nix import nix_build, nix_config, nix_eval def build_upload_script(machine: str, clan_dir: Path) -> str: @@ -14,7 +15,9 @@ def build_upload_script(machine: str, clan_dir: Path) -> str: system = config["system"] cmd = nix_build( - [f'{clan_dir}#clanInternals.machines."{system}"."{machine}".uploadSecrets'] + [ + f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.system.clan.uploadSecrets' + ] ) proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True) if proc.returncode != 0: @@ -25,13 +28,31 @@ def build_upload_script(machine: str, clan_dir: Path) -> str: return proc.stdout.strip() -def run_upload_secrets(flake_attr: str, clan_dir: Path) -> None: +def get_deployment_address(machine: str, clan_dir: Path) -> str: + config = nix_config() + system = config["system"] + + cmd = nix_eval( + [ + f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.clan.networking.deploymentAddress' + ] + ) + proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True) + if proc.returncode != 0: + raise ClanError( + f"failed to get deploymentAddress:\n{shlex.join(cmd)}\nexited with {proc.returncode}" + ) + + return json.loads(proc.stdout.strip()) + + +def run_upload_secrets(flake_attr: str, clan_dir: Path, target: str) -> None: env = os.environ.copy() env["CLAN_DIR"] = str(clan_dir) env["PYTHONPATH"] = str(module_root().parent) # TODO do this in the clanCore module print(f"uploading secrets... {flake_attr}") proc = subprocess.run( - [flake_attr], + [flake_attr, target], env=env, ) @@ -43,7 +64,8 @@ def run_upload_secrets(flake_attr: str, clan_dir: Path) -> None: def upload_secrets(machine: str) -> None: clan_dir = get_clan_flake_toplevel() - run_upload_secrets(build_upload_script(machine, clan_dir), clan_dir) + target = get_deployment_address(machine, clan_dir) + run_upload_secrets(build_upload_script(machine, clan_dir), clan_dir, target) def upload_command(args: argparse.Namespace) -> None: