restore clanInternals with valid nixos config

lib/build-clan/default.nix

@@ -46,27 +46,11 @@ let
       (system: lib.nameValuePair system
         (lib.mapAttrs (name: _: nixosConfiguration { inherit name system; }) allMachines))
       supportedSystems);
-
-  getMachine = machine: {
-    inherit (machine.config.system.clan) uploadSecrets generateSecrets;
-    inherit (machine.config.clan.networking) deploymentAddress;
-  };
-
-  machinesPerSystem = lib.mapAttrs (_: machine: getMachine machine);
-
-  machinesPerSystemWithJson = lib.mapAttrs (_: machine:
-    let
-      m = getMachine machine;
-    in
-    m // {
-      json = machine.pkgs.writers.writeJSON "machine.json" m;
-    });
 in
 {
   inherit nixosConfigurations;
 
   clanInternals = {
-    machines = lib.mapAttrs (_: configs: machinesPerSystemWithJson configs) configsPerSystem;
+    machines = configsPerSystem;
-    machines-json = lib.mapAttrs (system: configs: nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" (machinesPerSystem configs)) configsPerSystem;
   };
 }

nixosModules/clanCore/flake-module.nix

@@ -1,5 +1,5 @@
 { self, inputs, lib, ... }: {
-  flake.nixosModules.clanCore = { pkgs, options, ... }: {
+  flake.nixosModules.clanCore = { config, pkgs, options, ... }: {
     imports = [
       ./secrets
       ./zerotier
@@ -40,5 +40,13 @@
         utility outputs for clan management of this machine
       '';
     };
+    # optimization for faster secret generate/upload and machines update
+    config = {
+      system.clan.deployment.text = builtins.toJSON {
+        inherit (config.system.clan) uploadSecrets generateSecrets;
+        inherit (config.clan.networking) deploymentAddress;
+      };
+      system.clan.deployment.file = pkgs.writeText "deployment.json" config.system.clan.deployment.text;
+    };
   };
 }

nixosModules/clanCore/secrets/sops.nix

@@ -37,10 +37,12 @@ in
       uploadSecrets = pkgs.writeScript "upload-secrets" ''
         #!${pkgs.python3}/bin/python
         import json
+        import sys
         from clan_cli.secrets.sops_generate import upload_age_key_from_nix
         # the second toJSON is needed to escape the string for the python
-        args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; deployment_address = config.clan.networking.deploymentAddress; age_key_file = config.sops.age.keyFile; })})
+        deployment_address = sys.argv[1]
-        upload_age_key_from_nix(**args)
+        args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; age_key_file = config.sops.age.keyFile; })})
+        upload_age_key_from_nix(**args, deployment_address=deployment_address)
       '';
     };
     sops.secrets = builtins.mapAttrs

pkgs/clan-cli/clan_cli/machines/update.py

@@ -41,7 +41,12 @@ def deploy_nixos(hosts: HostGroup, clan_dir: Path) -> None:
         flake_attr = h.meta.get("flake_attr", "")
 
         run_generate_secrets(h.meta["generateSecrets"], clan_dir)
-        run_upload_secrets(h.meta["uploadSecrets"], clan_dir)
+        run_upload_secrets(
+            h.meta["uploadSecrets"],
+            clan_dir,
+            target=target,
+            target_directory=h.meta["targetDirectory"],
+        )
 
         target_host = h.meta.get("target_host")
         if target_host:
@@ -92,7 +97,7 @@ def build_json(targets: list[str]) -> list[dict[str, Any]]:
 def get_all_machines(clan_dir: Path) -> HostGroup:
     config = nix_config()
     system = config["system"]
-    what = f'{clan_dir}#clanInternals.machines-json."{system}"'
+    what = f'{clan_dir}#clanInternals.all-machines-json."{system}"'
     machines = build_json([what])[0]
 
     hosts = []
@@ -109,7 +114,9 @@ def get_selected_machines(machine_names: list[str], clan_dir: Path) -> HostGroup
     system = config["system"]
     what = []
     for name in machine_names:
-        what.append(f'{clan_dir}#clanInternals.machines."{system}"."{name}".json')
+        what.append(
+            f'{clan_dir}#clanInternals.machines."{system}"."{name}".config.system.clan.deployment.file'
+        )
     machines = build_json(what)
     hosts = []
     for i, machine in enumerate(machines):

pkgs/clan-cli/clan_cli/secrets/generate.py

@@ -16,7 +16,7 @@ def build_generate_script(machine: str, clan_dir: Path) -> str:
 
     cmd = nix_build(
         [
-            f'path:{clan_dir}#clanInternals.machines."{system}"."{machine}".generateSecrets'
+            f'path:{clan_dir}#clanInternals.machines."{system}"."{machine}".config.system.clan.generateSecrets'
         ]
     )
     proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)

pkgs/clan-cli/clan_cli/secrets/upload.py

@@ -1,4 +1,5 @@
 import argparse
+import json
 import os
 import shlex
 import subprocess
@@ -6,7 +7,7 @@ from pathlib import Path
 
 from ..dirs import get_clan_flake_toplevel, module_root
 from ..errors import ClanError
-from ..nix import nix_build, nix_config
+from ..nix import nix_build, nix_config, nix_eval
 
 
 def build_upload_script(machine: str, clan_dir: Path) -> str:
@@ -14,7 +15,9 @@ def build_upload_script(machine: str, clan_dir: Path) -> str:
     system = config["system"]
 
     cmd = nix_build(
-        [f'{clan_dir}#clanInternals.machines."{system}"."{machine}".uploadSecrets']
+        [
+            f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.system.clan.uploadSecrets'
+        ]
     )
     proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
     if proc.returncode != 0:
@@ -25,13 +28,31 @@ def build_upload_script(machine: str, clan_dir: Path) -> str:
     return proc.stdout.strip()
 
 
-def run_upload_secrets(flake_attr: str, clan_dir: Path) -> None:
+def get_deployment_address(machine: str, clan_dir: Path) -> str:
+    config = nix_config()
+    system = config["system"]
+
+    cmd = nix_eval(
+        [
+            f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.clan.networking.deploymentAddress'
+        ]
+    )
+    proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
+    if proc.returncode != 0:
+        raise ClanError(
+            f"failed to get deploymentAddress:\n{shlex.join(cmd)}\nexited with {proc.returncode}"
+        )
+
+    return json.loads(proc.stdout.strip())
+
+
+def run_upload_secrets(flake_attr: str, clan_dir: Path, target: str) -> None:
     env = os.environ.copy()
     env["CLAN_DIR"] = str(clan_dir)
     env["PYTHONPATH"] = str(module_root().parent)  # TODO do this in the clanCore module
     print(f"uploading secrets... {flake_attr}")
     proc = subprocess.run(
-        [flake_attr],
+        [flake_attr, target],
         env=env,
     )
 
@@ -43,7 +64,8 @@ def run_upload_secrets(flake_attr: str, clan_dir: Path) -> None:
 
 def upload_secrets(machine: str) -> None:
     clan_dir = get_clan_flake_toplevel()
-    run_upload_secrets(build_upload_script(machine, clan_dir), clan_dir)
+    target = get_deployment_address(machine, clan_dir)
+    run_upload_secrets(build_upload_script(machine, clan_dir), clan_dir, target)
 
 
 def upload_command(args: argparse.Namespace) -> None: