Merge pull request 'clan-cli: add install command' (#383) from lassulus-install into main

nixosModules/clanCore/secrets/password-store.nix

@@ -72,7 +72,7 @@ in
         )
         remote_pass_info=$(ssh ${config.clan.networking.deploymentAddress} -- ${lib.escapeShellArg ''
           cat ${config.clan.password-store.targetDirectory}/.pass_info || :
-        ''})
+        ''} || :)
 
         if test "$local_pass_info" = "$remote_pass_info"; then
           echo secrets already match

pkgs/clan-cli/clan_cli/machines/__init__.py

@@ -3,6 +3,7 @@ import argparse
 
 from .create import register_create_parser
 from .delete import register_delete_parser
+from .install import register_install_parser
 from .list import register_list_parser
 from .update import register_update_parser
 
@@ -27,3 +28,6 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
 
     list_parser = subparser.add_parser("list", help="List machines")
     register_list_parser(list_parser)
+
+    install_parser = subparser.add_parser("install", help="Install a machine")
+    register_install_parser(install_parser)

pkgs/clan-cli/clan_cli/machines/install.py

@@ -0,0 +1,60 @@
+import argparse
+import subprocess
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from ..machines.machines import Machine
+from ..nix import nix_shell
+from ..secrets.generate import generate_secrets
+
+
+def install_nixos(machine: Machine) -> None:
+    h = machine.host
+    target_host = f"{h.user or 'root'}@{h.host}"
+
+    flake_attr = h.meta.get("flake_attr", "")
+
+    generate_secrets(machine)
+
+    with TemporaryDirectory() as tmpdir_:
+        tmpdir = Path(tmpdir_)
+        machine.upload_secrets(tmpdir / machine.secrets_upload_directory)
+
+        subprocess.run(
+            nix_shell(
+                ["nixos-anywhere"],
+                [
+                    "nixos-anywhere",
+                    "-f",
+                    f"{machine.clan_dir}#{flake_attr}",
+                    "-t",
+                    "--no-reboot",
+                    "--extra-files",
+                    str(tmpdir),
+                    target_host,
+                ],
+            ),
+            check=True,
+        )
+
+
+def install_command(args: argparse.Namespace) -> None:
+    machine = Machine(args.machine)
+    machine.deployment_address = args.target_host
+
+    install_nixos(machine)
+
+
+def register_install_parser(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "machine",
+        type=str,
+        help="machine to install",
+    )
+    parser.add_argument(
+        "target_host",
+        type=str,
+        help="ssh address to install to in the form of user@host:2222",
+    )
+
+    parser.set_defaults(func=install_command)

pkgs/clan-cli/clan_cli/machines/machines.py

@@ -0,0 +1,108 @@
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional
+
+from ..dirs import get_clan_flake_toplevel
+from ..nix import nix_build, nix_config, nix_eval
+from ..ssh import Host, parse_deployment_address
+
+
+def build_machine_data(machine_name: str, clan_dir: Path) -> dict:
+    config = nix_config()
+    system = config["system"]
+
+    outpath = subprocess.run(
+        nix_build(
+            [
+                f'path:{clan_dir}#clanInternals.machines."{system}"."{machine_name}".config.system.clan.deployment.file'
+            ]
+        ),
+        stdout=subprocess.PIPE,
+        check=True,
+        text=True,
+    ).stdout.strip()
+    return json.loads(Path(outpath).read_text())
+
+
+class Machine:
+    def __init__(
+        self,
+        name: str,
+        clan_dir: Optional[Path] = None,
+        machine_data: Optional[dict] = None,
+    ) -> None:
+        """
+        Creates a Machine
+        @name: the name of the machine
+        @clan_dir: the directory of the clan, optional, if not set it will be determined from the current working directory
+        @machine_json: can be optionally used to skip evaluation of the machine, location of the json file with machine data
+        """
+        self.name = name
+        if clan_dir is None:
+            self.clan_dir = get_clan_flake_toplevel()
+        else:
+            self.clan_dir = clan_dir
+
+        if machine_data is None:
+            self.machine_data = build_machine_data(name, self.clan_dir)
+        else:
+            self.machine_data = machine_data
+
+        self.deployment_address = self.machine_data["deploymentAddress"]
+        self.upload_secrets = self.machine_data["uploadSecrets"]
+        self.generate_secrets = self.machine_data["generateSecrets"]
+        self.secrets_upload_directory = self.machine_data["secretsUploadDirectory"]
+
+    @property
+    def host(self) -> Host:
+        return parse_deployment_address(
+            self.name, self.deployment_address, meta={"machine": self}
+        )
+
+    def run_upload_secrets(self, secrets_dir: Path) -> None:
+        """
+        Upload the secrets to the provided directory
+        @secrets_dir: the directory to store the secrets in
+        """
+        env = os.environ.copy()
+        env["CLAN_DIR"] = str(self.clan_dir)
+        env["PYTHONPATH"] = str(
+            ":".join(sys.path)
+        )  # TODO do this in the clanCore module
+        env["SECRETS_DIR"] = str(secrets_dir)
+        subprocess.run(
+            [self.upload_secrets],
+            env=env,
+            check=True,
+            stdout=subprocess.PIPE,
+            text=True,
+        )
+
+    def eval_nix(self, attr: str) -> str:
+        """
+        eval a nix attribute of the machine
+        @attr: the attribute to get
+        """
+        output = subprocess.run(
+            nix_eval([f"path:{self.clan_dir}#{attr}"]),
+            stdout=subprocess.PIPE,
+            check=True,
+            text=True,
+        ).stdout.strip()
+        return output
+
+    def build_nix(self, attr: str) -> Path:
+        """
+        build a nix attribute of the machine
+        @attr: the attribute to get
+        """
+        outpath = subprocess.run(
+            nix_build([f"path:{self.clan_dir}#{attr}"]),
+            stdout=subprocess.PIPE,
+            check=True,
+            text=True,
+        ).stdout.strip()
+        return Path(outpath)

pkgs/clan-cli/clan_cli/machines/update.py

@@ -3,12 +3,12 @@ import json
 import os
 import subprocess
 from pathlib import Path
-from typing import Any
 
 from ..dirs import get_clan_flake_toplevel
+from ..machines.machines import Machine
 from ..nix import nix_build, nix_command, nix_config
-from ..secrets.generate import run_generate_secrets
+from ..secrets.generate import generate_secrets
-from ..secrets.upload import run_upload_secrets
+from ..secrets.upload import upload_secrets
 from ..ssh import Host, HostGroup, HostKeyCheck, parse_deployment_address
 
 
@@ -40,13 +40,8 @@ def deploy_nixos(hosts: HostGroup, clan_dir: Path) -> None:
 
         flake_attr = h.meta.get("flake_attr", "")
 
-        run_generate_secrets(h.meta["generateSecrets"], clan_dir)
+        generate_secrets(h.meta["machine"])
-        run_upload_secrets(
+        upload_secrets(h.meta["machine"])
-            h.meta["uploadSecrets"],
-            clan_dir,
-            target=target,
-            target_directory=h.meta["secretsUploadDirectory"],
-        )
 
         target_host = h.meta.get("target_host")
         if target_host:
@@ -81,49 +76,36 @@ def deploy_nixos(hosts: HostGroup, clan_dir: Path) -> None:
     hosts.run_function(deploy)
 
 
-def build_json(targets: list[str]) -> list[dict[str, Any]]:
+# function to speedup eval if we want to evauluate all machines
-    outpaths = subprocess.run(
+def get_all_machines(clan_dir: Path) -> HostGroup:
-        nix_build(targets),
+    config = nix_config()
+    system = config["system"]
+    machines_json = subprocess.run(
+        nix_build([f'{clan_dir}#clanInternals.all-machines-json."{system}"']),
         stdout=subprocess.PIPE,
         check=True,
         text=True,
     ).stdout
-    parsed = []
-    for outpath in outpaths.splitlines():
-        parsed.append(json.loads(Path(outpath).read_text()))
-    return parsed
 
-
+    machines = json.loads(Path(machines_json).read_text())
-def get_all_machines(clan_dir: Path) -> HostGroup:
-    config = nix_config()
-    system = config["system"]
-    what = f'{clan_dir}#clanInternals.all-machines-json."{system}"'
-    machines = build_json([what])[0]
 
     hosts = []
-    for name, machine in machines.items():
+    for name, machine_data in machines.items():
+        # very hacky. would be better to do a MachinesGroup instead
         host = parse_deployment_address(
-            name, machine["deploymentAddress"], meta=machine
+            name,
+            machine_data["deploymentAddress"],
+            meta={"machine": Machine(name=name, machine_data=machine_data)},
         )
         hosts.append(host)
     return HostGroup(hosts)
 
 
 def get_selected_machines(machine_names: list[str], clan_dir: Path) -> HostGroup:
-    config = nix_config()
-    system = config["system"]
-    what = []
-    for name in machine_names:
-        what.append(
-            f'{clan_dir}#clanInternals.machines."{system}"."{name}".config.system.clan.deployment.file'
-        )
-    machines = build_json(what)
     hosts = []
-    for i, machine in enumerate(machines):
+    for name in machine_names:
-        host = parse_deployment_address(
+        machine = Machine(name=name, clan_dir=clan_dir)
-            machine_names[i], machine["deploymentAddress"], machine
+        hosts.append(machine.host)
-        )
-        hosts.append(host)
     return HostGroup(hosts)
 
 

pkgs/clan-cli/clan_cli/secrets/generate.py

@@ -1,45 +1,24 @@
 import argparse
 import logging
 import os
-import shlex
 import subprocess
 import sys
-from pathlib import Path
 
 from clan_cli.errors import ClanError
 
-from ..dirs import get_clan_flake_toplevel
+from ..machines.machines import Machine
-from ..nix import nix_build, nix_config
 
 log = logging.getLogger(__name__)
 
 
-def build_generate_script(machine: str, clan_dir: Path) -> str:
+def generate_secrets(machine: Machine) -> None:
-    config = nix_config()
-    system = config["system"]
-
-    cmd = nix_build(
-        [
-            f'path:{clan_dir}#clanInternals.machines."{system}"."{machine}".config.system.clan.generateSecrets'
-        ]
-    )
-    proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
-    if proc.returncode != 0:
-        raise ClanError(
-            f"failed to generate secrets:\n{shlex.join(cmd)}\nexited with {proc.returncode}"
-        )
-
-    return proc.stdout.strip()
-
-
-def run_generate_secrets(secret_generator_script: str, clan_dir: Path) -> None:
     env = os.environ.copy()
-    env["CLAN_DIR"] = str(clan_dir)
+    env["CLAN_DIR"] = str(machine.clan_dir)
     env["PYTHONPATH"] = ":".join(sys.path)  # TODO do this in the clanCore module
 
-    print(f"generating secrets... {secret_generator_script}")
+    print(f"generating secrets... {machine.generate_secrets}")
     proc = subprocess.run(
-        [secret_generator_script],
+        [machine.generate_secrets],
         env=env,
     )
 
@@ -51,13 +30,9 @@ def run_generate_secrets(secret_generator_script: str, clan_dir: Path) -> None:
         print("successfully generated secrets")
 
 
-def generate(machine: str) -> None:
-    clan_dir = get_clan_flake_toplevel()
-    run_generate_secrets(build_generate_script(machine, clan_dir), clan_dir)
-
-
 def generate_command(args: argparse.Namespace) -> None:
-    generate(args.machine)
+    machine = Machine(args.machine)
+    generate_secrets(machine)
 
 
 def register_generate_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/secrets/upload.py

@@ -1,17 +1,14 @@
 import argparse
 import json
 import logging
-import os
 import shlex
 import subprocess
-import sys
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
-from ..dirs import get_clan_flake_toplevel
 from ..errors import ClanError
+from ..machines.machines import Machine
 from ..nix import nix_build, nix_config, nix_shell
-from ..ssh import parse_deployment_address
 
 log = logging.getLogger(__name__)
 
@@ -52,31 +49,13 @@ def get_deployment_info(machine: str, clan_dir: Path) -> dict:
     return json.load(open(proc.stdout.strip()))
 
 
-def run_upload_secrets(
+def upload_secrets(machine: Machine) -> None:
-    flake_attr: str, clan_dir: Path, target: str, target_directory: str
-) -> None:
-    env = os.environ.copy()
-    env["CLAN_DIR"] = str(clan_dir)
-    env["PYTHONPATH"] = ":".join(sys.path)  # TODO do this in the clanCore module
-    print(f"uploading secrets... {flake_attr}")
     with TemporaryDirectory() as tempdir_:
         tempdir = Path(tempdir_)
-        env["SECRETS_DIR"] = str(tempdir)
+        machine.run_upload_secrets(tempdir)
-        proc = subprocess.run(
+        host = machine.host
-            [flake_attr],
-            env=env,
-            check=True,
-            stdout=subprocess.PIPE,
-            text=True,
-        )
 
-        if proc.returncode != 0:
+        ssh_cmd = host.ssh_cmd()
-            log.error("Stdout: %s", proc.stdout)
-            log.error("Stderr: %s", proc.stderr)
-            raise ClanError("failed to upload secrets")
-
-        h = parse_deployment_address(flake_attr, target)
-        ssh_cmd = h.ssh_cmd()
         subprocess.run(
             nix_shell(
                 ["rsync"],
@@ -87,28 +66,16 @@ def run_upload_secrets(
                     "-az",
                     "--delete",
                     f"{str(tempdir)}/",
-                    f"{h.user}@{h.host}:{target_directory}/",
+                    f"{host.user}@{host.host}:{machine.secrets_upload_directory}/",
                 ],
             ),
             check=True,
         )
 
 
-def upload_secrets(machine: str) -> None:
-    clan_dir = get_clan_flake_toplevel()
-    deployment_info = get_deployment_info(machine, clan_dir)
-    address = deployment_info.get("deploymentAddress", "")
-    secrets_upload_directory = deployment_info.get("secretsUploadDirectory", "")
-    run_upload_secrets(
-        build_upload_script(machine, clan_dir),
-        clan_dir,
-        address,
-        secrets_upload_directory,
-    )
-
-
 def upload_command(args: argparse.Namespace) -> None:
-    upload_secrets(args.machine)
+    machine = Machine(args.machine)
+    upload_secrets(machine)
 
 
 def register_upload_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/ssh/__init__.py

@@ -759,7 +759,7 @@ class HostGroup:
 
 
 def parse_deployment_address(
-    machine_name: str, host: str, meta: dict[str, str] = {}
+    machine_name: str, host: str, meta: dict[str, Any] = {}
 ) -> Host:
     parts = host.split("@")
     user: Optional[str] = None

pkgs/clan-cli/clan_cli/vms/create.py

@@ -45,7 +45,8 @@ class BuildVmTask(BaseTask):
         vm_config = self.get_vm_create_info(cmds)
 
         with tempfile.TemporaryDirectory() as tmpdir_:
-            xchg_dir = Path(tmpdir_) / "xchg"
+            tmpdir = Path(tmpdir_)
+            xchg_dir = tmpdir / "xchg"
             xchg_dir.mkdir()
             disk_img = f"{tmpdir_}/disk.img"