Merge pull request 'fix pname of clan-cli for nix run' (#368) from Mic92-main into main

lib/build-clan/default.nix

@@ -11,6 +11,7 @@ let
       (builtins.fromJSON
         (builtins.readFile (directory + /machines/${machineName}/settings.json)));
 
+  # TODO: remove default system once we have a hardware-config mechanism
   nixosConfiguration = { system ? "x86_64-linux", name }: nixpkgs.lib.nixosSystem {
     modules = [
       self.nixosModules.clanCore
@@ -19,8 +20,7 @@ let
       {
         clanCore.machineName = name;
         clanCore.clanDir = directory;
-        # TODO: remove this once we have a hardware-config mechanism
+        nixpkgs.hostPlatform = lib.mkForce system;
-        nixpkgs.hostPlatform = lib.mkDefault system;
       }
     ];
     inherit specialArgs;
@@ -41,27 +41,32 @@ let
   # This instantiates nixos for each system that we support:
   # configPerSystem = <system>.<machine>.nixosConfiguration
   # We need this to build nixos secret generators for each system
-  configPerSystem = builtins.listToAttrs
+  configsPerSystem = builtins.listToAttrs
     (builtins.map
       (system: lib.nameValuePair system
         (lib.mapAttrs (name: _: nixosConfiguration { inherit name system; }) allMachines))
       supportedSystems);
 
-  machinesPerSystem = lib.mapAttrs (_: machine:
+  getMachine = machine: {
-    let
-      config = {
     inherit (machine.config.system.clan) uploadSecrets generateSecrets;
     inherit (machine.config.clan.networking) deploymentAddress;
   };
+
+  machinesPerSystem = lib.mapAttrs (_: machine: getMachine machine);
+
+  machinesPerSystemWithJson = lib.mapAttrs (_: machine:
+    let
+      m = getMachine machine;
     in
-    config // {
+    m // {
-      json = machine.pkgs.writeText "config.json" (builtins.toJSON config);
+      json = machine.pkgs.writers.writeJSON "machine.json" m;
     });
 in
 {
   inherit nixosConfigurations;
 
   clanInternals = {
-    machines = lib.mapAttrs (_: machinesPerSystem) configPerSystem;
+    machines = lib.mapAttrs (_: configs: machinesPerSystemWithJson configs) configsPerSystem;
+    machines-json = lib.mapAttrs (system: configs: nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" (machinesPerSystem configs)) configsPerSystem;
   };
 }

lib/default.nix

@@ -1,17 +1,5 @@
 { lib, self, nixpkgs, ... }:
 {
-  findNixFiles = folder:
-    lib.mapAttrs'
-      (name: type:
-        if
-          type == "directory"
-        then
-          lib.nameValuePair name "${folder}/${name}"
-        else
-          lib.nameValuePair (lib.removeSuffix ".nix" name) "${folder}/${name}"
-      )
-      (builtins.readDir folder);
-
   jsonschema = import ./jsonschema { inherit lib; };
 
   buildClan = import ./build-clan { inherit lib self nixpkgs; };

pkgs/clan-cli/clan_cli/machines/update.py

@@ -2,15 +2,17 @@ import argparse
 import json
 import os
 import subprocess
+from pathlib import Path
+from typing import Any
 
 from ..dirs import get_clan_flake_toplevel
-from ..nix import nix_command, nix_config, nix_eval
+from ..nix import nix_build, nix_command, nix_config
-from ..secrets.generate import generate_secrets
+from ..secrets.generate import run_generate_secrets
-from ..secrets.upload import upload_secrets
+from ..secrets.upload import run_upload_secrets
 from ..ssh import Host, HostGroup, HostKeyCheck, parse_deployment_address
 
 
-def deploy_nixos(hosts: HostGroup) -> None:
+def deploy_nixos(hosts: HostGroup, clan_dir: Path) -> None:
     """
     Deploy to all hosts in parallel
     """
@@ -38,8 +40,11 @@ def deploy_nixos(hosts: HostGroup) -> None:
 
         flake_attr = h.meta.get("flake_attr", "")
 
-        generate_secrets(flake_attr)
+        if generate_secrets_script := h.meta.get("generate_secrets"):
-        upload_secrets(flake_attr)
+            run_generate_secrets(generate_secrets_script, clan_dir)
+
+        if upload_secrets_script := h.meta.get("upload_secrets"):
+            run_upload_secrets(upload_secrets_script, clan_dir)
 
         target_host = h.meta.get("target_host")
         if target_host:
@@ -74,31 +79,65 @@ def deploy_nixos(hosts: HostGroup) -> None:
     hosts.run_function(deploy)
 
 
-# FIXME: we want some kind of inventory here.
+def build_json(targets: list[str]) -> list[dict[str, Any]]:
-def update(args: argparse.Namespace) -> None:
+    outpaths = subprocess.run(
-    clan_dir = get_clan_flake_toplevel().as_posix()
+        nix_build(targets),
-    machine = args.machine
-
-    config = nix_config()
-    system = config["system"]
-
-    address = json.loads(
-        subprocess.run(
-            nix_eval(
-                [
-                    f'{clan_dir}#clanInternals.machines."{system}"."{machine}".deploymentAddress'
-                ]
-            ),
         stdout=subprocess.PIPE,
         check=True,
         text=True,
     ).stdout
+    parsed = []
+    for outpath in outpaths.splitlines():
+        parsed.append(json.loads(Path(outpath).read_text()))
+    return parsed
+
+
+def get_all_machines(clan_dir: Path) -> HostGroup:
+    config = nix_config()
+    system = config["system"]
+    what = f'{clan_dir}#clanInternals.machines-json."{system}"'
+    machines = build_json([what])[0]
+
+    hosts = []
+    for name, machine in machines.items():
+        host = parse_deployment_address(
+            name, machine["deploymentAddress"], meta=machine
         )
-    host = parse_deployment_address(machine, address)
+        hosts.append(host)
-    print(f"deploying {machine}")
+    return HostGroup(hosts)
-    deploy_nixos(HostGroup([host]))
+
+
+def get_selected_machines(machine_names: list[str], clan_dir: Path) -> HostGroup:
+    config = nix_config()
+    system = config["system"]
+    what = []
+    for name in machine_names:
+        what.append(f'{clan_dir}#clanInternals.machines."{system}"."{name}".json')
+    machines = build_json(what)
+    hosts = []
+    for i, machine in enumerate(machines):
+        host = parse_deployment_address(machine_names[i], machine["deploymentAddress"])
+        hosts.append(host)
+    return HostGroup(hosts)
+
+
+# FIXME: we want some kind of inventory here.
+def update(args: argparse.Namespace) -> None:
+    clan_dir = get_clan_flake_toplevel()
+    if len(args.machines) == 0:
+        machines = get_all_machines(clan_dir)
+    else:
+        machines = get_selected_machines(args.machines, clan_dir)
+
+    deploy_nixos(machines, clan_dir)
 
 
 def register_update_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument("machine", type=str)
+    parser.add_argument(
+        "machines",
+        type=str,
+        help="machine to update. if empty, update all machines",
+        nargs="*",
+        default=[],
+    )
     parser.set_defaults(func=update)

pkgs/clan-cli/clan_cli/secrets/generate.py

@@ -2,6 +2,7 @@ import argparse
 import os
 import shlex
 import subprocess
+from pathlib import Path
 
 from clan_cli.errors import ClanError
 
@@ -9,11 +10,7 @@ from ..dirs import get_clan_flake_toplevel, module_root
 from ..nix import nix_build, nix_config
 
 
-def generate_secrets(machine: str) -> None:
+def build_generate_script(machine: str, clan_dir: Path) -> str:
-    clan_dir = get_clan_flake_toplevel().as_posix().strip()
-    env = os.environ.copy()
-    env["CLAN_DIR"] = clan_dir
-    env["PYTHONPATH"] = str(module_root().parent)  # TODO do this in the clanCore module
     config = nix_config()
     system = config["system"]
 
@@ -28,21 +25,32 @@ def generate_secrets(machine: str) -> None:
             f"failed to generate secrets:\n{shlex.join(cmd)}\nexited with {proc.returncode}"
         )
 
-    secret_generator_script = proc.stdout.strip()
+    return proc.stdout.strip()
-    print(secret_generator_script)
+
-    secret_generator = subprocess.run(
+
+def run_generate_secrets(secret_generator_script: str, clan_dir: Path) -> None:
+    env = os.environ.copy()
+    env["CLAN_DIR"] = str(clan_dir)
+    env["PYTHONPATH"] = str(module_root().parent)  # TODO do this in the clanCore module
+    print(f"generating secrets... {secret_generator_script}")
+    proc = subprocess.run(
         [secret_generator_script],
         env=env,
     )
 
-    if secret_generator.returncode != 0:
+    if proc.returncode != 0:
         raise ClanError("failed to generate secrets")
     else:
         print("successfully generated secrets")
 
 
+def generate(machine: str) -> None:
+    clan_dir = get_clan_flake_toplevel()
+    run_generate_secrets(build_generate_script(machine, clan_dir), clan_dir)
+
+
 def generate_command(args: argparse.Namespace) -> None:
-    generate_secrets(args.machine)
+    generate(args.machine)
 
 
 def register_generate_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/secrets/sops_generate.py

@@ -44,8 +44,8 @@ def generate_secrets_group(
 
         text = f"""\
 set -euo pipefail
-facts={shlex.quote(str(facts_dir))}
+export facts={shlex.quote(str(facts_dir))}
-secrets={shlex.quote(str(secrets_dir))}
+export secrets={shlex.quote(str(secrets_dir))}
 {generator}
         """
         try:

pkgs/clan-cli/clan_cli/secrets/upload.py

@@ -1,57 +1,51 @@
 import argparse
-import json
 import os
+import shlex
 import subprocess
+from pathlib import Path
 
 from ..dirs import get_clan_flake_toplevel, module_root
 from ..errors import ClanError
-from ..nix import nix_build, nix_config, nix_eval
+from ..nix import nix_build, nix_config
 
 
-def upload_secrets(machine: str) -> None:
+def build_upload_script(machine: str, clan_dir: Path) -> str:
-    clan_dir = get_clan_flake_toplevel().as_posix()
     config = nix_config()
     system = config["system"]
 
-    proc = subprocess.run(
+    cmd = nix_build(
-        nix_build(
         [f'{clan_dir}#clanInternals.machines."{system}"."{machine}".uploadSecrets']
-        ),
+    )
-        stdout=subprocess.PIPE,
+    proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
-        text=True,
+    if proc.returncode != 0:
-        check=True,
+        raise ClanError(
+            f"failed to upload secrets:\n{shlex.join(cmd)}\nexited with {proc.returncode}"
         )
 
+    return proc.stdout.strip()
+
+
+def run_upload_secrets(flake_attr: str, clan_dir: Path) -> None:
     env = os.environ.copy()
+    env["CLAN_DIR"] = str(clan_dir)
     env["PYTHONPATH"] = str(module_root().parent)  # TODO do this in the clanCore module
-    host = json.loads(
+    print(f"uploading secrets... {flake_attr}")
-        subprocess.run(
+    proc = subprocess.run(
-            nix_eval(
+        [flake_attr],
-                [
-                    f'{clan_dir}#clanInternals.machines."{system}"."{machine}".deploymentAddress'
-                ]
-            ),
-            stdout=subprocess.PIPE,
-            text=True,
-            check=True,
-        ).stdout
-    )
-
-    secret_upload_script = proc.stdout.strip()
-    secret_upload = subprocess.run(
-        [
-            secret_upload_script,
-            host,
-        ],
         env=env,
     )
 
-    if secret_upload.returncode != 0:
+    if proc.returncode != 0:
         raise ClanError("failed to upload secrets")
     else:
         print("successfully uploaded secrets")
 
 
+def upload_secrets(machine: str) -> None:
+    clan_dir = get_clan_flake_toplevel()
+    run_upload_secrets(build_upload_script(machine, clan_dir), clan_dir)
+
+
 def upload_command(args: argparse.Namespace) -> None:
     upload_secrets(args.machine)
 

pkgs/clan-cli/clan_cli/ssh/__init__.py

@@ -756,7 +756,9 @@ class HostGroup:
         return HostGroup(list(filter(pred, self.hosts)))
 
 
-def parse_deployment_address(machine_name: str, host: str) -> Host:
+def parse_deployment_address(
+    machine_name: str, host: str, meta: dict[str, str] = {}
+) -> Host:
     parts = host.split("@")
     user: Optional[str] = None
     if len(parts) > 1:
@@ -776,12 +778,14 @@ def parse_deployment_address(machine_name: str, host: str) -> Host:
     if len(maybe_port) > 1:
         hostname = maybe_port[0]
         port = int(maybe_port[1])
+    meta = meta.copy()
+    meta["flake_attr"] = machine_name
     return Host(
         hostname,
         user=user,
         port=port,
         command_prefix=machine_name,
-        meta=dict(flake_attr=machine_name),
+        meta=meta,
         ssh_options=options,
     )
 

pkgs/clan-cli/flake-module.nix

@@ -11,7 +11,7 @@
       };
       # Don't leak python packages into a devshell.
       # It can be very confusing if you `nix run` than than load the cli from the devshell instead.
-      clan-cli = pkgs.runCommand "clan-cli" { } ''
+      clan-cli = pkgs.runCommand "clan" { } ''
         mkdir $out
         ln -s ${self'.packages.clan-cli-unwrapped}/bin $out
       '';

pkgs/clan-cli/pyproject.toml

@@ -15,7 +15,7 @@ exclude = ["clan_cli.nixpkgs*"]
 clan_cli = [ "config/jsonschema/*", "webui/assets/**/*"]
 
 [tool.pytest.ini_options]
-faulthandler_timeout = 30
+faulthandler_timeout = 60
 addopts = "--cov . --cov-report term --cov-report html:.reports/html --no-cov-on-fail --workers auto --durations 5"
 norecursedirs = "tests/helpers"
 markers = [ "impure" ]