Merge pull request 'clan-cli: init vm command' (#373) from lassulus-cli-vm into main

checks/impure/flake-module.nix

@@ -9,6 +9,7 @@
         export PATH="${lib.makeBinPath [
           pkgs.gitMinimal
           pkgs.nix
+          pkgs.rsync # needed to have rsync installed on the dummy ssh server
         ]}"
         ROOT=$(git rev-parse --show-toplevel)
         cd "$ROOT/pkgs/clan-cli"

lib/build-clan/default.nix

@@ -46,27 +46,11 @@ let
       (system: lib.nameValuePair system
         (lib.mapAttrs (name: _: nixosConfiguration { inherit name system; }) allMachines))
       supportedSystems);
-
-  getMachine = machine: {
-    inherit (machine.config.system.clan) uploadSecrets generateSecrets;
-    inherit (machine.config.clan.networking) deploymentAddress;
-  };
-
-  machinesPerSystem = lib.mapAttrs (_: machine: getMachine machine);
-
-  machinesPerSystemWithJson = lib.mapAttrs (_: machine:
-    let
-      m = getMachine machine;
-    in
-    m // {
-      json = machine.pkgs.writers.writeJSON "machine.json" m;
-    });
 in
 {
   inherit nixosConfigurations;
 
   clanInternals = {
-    machines = lib.mapAttrs (_: configs: machinesPerSystemWithJson configs) configsPerSystem;
+    machines = configsPerSystem;
-    machines-json = lib.mapAttrs (system: configs: nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" (machinesPerSystem configs)) configsPerSystem;
   };
 }

nixosModules/clanCore/flake-module.nix

@@ -1,5 +1,5 @@
 { self, inputs, lib, ... }: {
-  flake.nixosModules.clanCore = { pkgs, options, ... }: {
+  flake.nixosModules.clanCore = { config, pkgs, options, ... }: {
     imports = [
       ./secrets
       ./zerotier
@@ -40,5 +40,14 @@
         utility outputs for clan management of this machine
       '';
     };
+    # optimization for faster secret generate/upload and machines update
+    config = {
+      system.clan.deployment.text = builtins.toJSON {
+        inherit (config.system.clan) uploadSecrets generateSecrets;
+        inherit (config.clan.networking) deploymentAddress;
+        inherit (config.clanCore) secretsUploadDirectory;
+      };
+      system.clan.deployment.file = pkgs.writeText "deployment.json" config.system.clan.deployment.text;
+    };
   };
 }

nixosModules/clanCore/secrets/default.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, ... }:
 {
   options.clanCore.secretStore = lib.mkOption {
     type = lib.types.enum [ "sops" "password-store" "custom" ];
@@ -17,6 +17,13 @@
     '';
   };
 
+  options.clanCore.secretsUploadDirectory = lib.mkOption {
+    type = lib.types.path;
+    description = ''
+      The directory where secrets are uploaded into, This is backend specific.
+    '';
+  };
+
   options.clanCore.secretsPrefix = lib.mkOption {
     type = lib.types.str;
     default = "";
@@ -106,10 +113,6 @@
         };
       }));
   };
-  config.system.build.generateUploadSecrets = pkgs.writeScript "generate_upload_secrets" ''
-    ${config.system.clan.generateSecrets}
-    ${config.system.clan.uploadSecrets}
-  '';
   imports = [
     ./sops.nix
     ./password-store.nix

nixosModules/clanCore/secrets/password-store.nix

@@ -12,6 +12,7 @@ in
   };
   config = lib.mkIf (config.clanCore.secretStore == "password-store") {
     clanCore.secretsDirectory = config.clan.password-store.targetDirectory;
+    clanCore.secretsUploadDirectory = config.clan.password-store.targetDirectory;
     system.clan.generateSecrets = pkgs.writeScript "generate-secrets" ''
       #!/bin/sh
       set -efu
@@ -33,7 +34,7 @@ in
           trap "rm -rf $facts" EXIT
           secrets=$(mktemp -d)
           trap "rm -rf $secrets" EXIT
-          ${v.generator}
+          ( ${v.generator} )
 
           ${lib.concatMapStrings (fact: ''
             mkdir -p "$(dirname ${fact.path})"
@@ -50,8 +51,6 @@ in
       #!/bin/sh
       set -efu
 
-      target=$1
-
       umask 0077
 
       PATH=${lib.makeBinPath [
@@ -71,7 +70,7 @@ in
             sort |
             xargs -r -n 1 git -C ${passwordstoreDir} log -1 --format=%H
         )
-        remote_pass_info=$(ssh "$target" -- ${lib.escapeShellArg ''
+        remote_pass_info=$(ssh ${config.clan.networking.deploymentAddress} -- ${lib.escapeShellArg ''
           cat ${config.clan.password-store.targetDirectory}/.pass_info || :
         ''})
 
@@ -81,12 +80,6 @@ in
         fi
       fi
 
-      tmp_dir=$(mktemp -dt populate-pass.XXXXXXXX)
-      trap cleanup EXIT
-      cleanup() {
-        rm -fR "$tmp_dir"
-      }
-
       find ${passwordstoreDir}/machines/${config.clanCore.machineName} -type f -follow ! -name .gpg-id |
       while read -r gpg_path; do
 
@@ -99,7 +92,7 @@ in
           fi
         )
         pass_name=$rel_name
-        tmp_path=$tmp_dir/$(basename $rel_name)
+        tmp_path="$SECRETS_DIR"/$(basename $rel_name)
 
         mkdir -p "$(dirname "$tmp_path")"
         pass show "$pass_name" > "$tmp_path"
@@ -109,10 +102,8 @@ in
       done
 
       if test -n "''${local_pass_info-}"; then
-        echo "$local_pass_info" > "$tmp_dir"/.pass_info
+        echo "$local_pass_info" > "$SECRETS_DIR"/.pass_info
       fi
-
-      rsync --mkpath --delete -a "$tmp_dir"/ "$target":${config.clan.password-store.targetDirectory}/
     '';
   };
 }

nixosModules/clanCore/secrets/sops.nix

@@ -39,7 +39,7 @@ in
         import json
         from clan_cli.secrets.sops_generate import upload_age_key_from_nix
         # the second toJSON is needed to escape the string for the python
-        args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; deployment_address = config.clan.networking.deploymentAddress; age_key_file = config.sops.age.keyFile; })})
+        args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; })})
         upload_age_key_from_nix(**args)
       '';
     };
@@ -54,5 +54,6 @@ in
 
     sops.age.keyFile = lib.mkIf (builtins.pathExists (config.clanCore.clanDir + "/sops/secrets/${config.clanCore.machineName}-age.key/secret"))
       (lib.mkDefault "/var/lib/sops-nix/key.txt");
+    clanCore.secretsUploadDirectory = lib.mkDefault "/var/lib/sops-nix";
   };
 }

nixosModules/clanCore/vm.nix

@@ -1,4 +1,11 @@
-{ lib, config, options, ... }:
+{ lib, config, pkgs, options, extendModules, modulesPath, ... }:
+let
+  vmConfig = extendModules {
+    modules = [
+      (modulesPath + "/virtualisation/qemu-vm.nix")
+    ];
+  };
+in
 {
   options = {
     clan.virtualisation = {
@@ -33,10 +40,20 @@
   };
 
   config = {
-    system.clan.vm.config = {
+    system.clan.vm = {
+      # for clan vm inspect
+      config = {
         inherit (config.clan.virtualisation) cores graphics;
         memory_size = config.clan.virtualisation.memorySize;
       };
+      # for clan vm create
+      create = pkgs.writeText "vm.json" (builtins.toJSON {
+        initrd = "${vmConfig.config.system.build.initialRamdisk}/${vmConfig.config.system.boot.loader.initrdFile}";
+        toplevel = vmConfig.config.system.build.toplevel;
+        regInfo = (pkgs.closureInfo { rootPaths = vmConfig.config.virtualisation.additionalPaths; });
+        inherit (config.clan.virtualisation) memorySize cores graphics;
+      });
+    };
 
     virtualisation = lib.optionalAttrs (options.virtualisation ? cores) {
       memorySize = lib.mkDefault config.clan.virtualisation.memorySize;

nixosModules/clanCore/zerotier/default.nix

@@ -99,11 +99,11 @@ in
           ${pkgs.python3.interpreter} ${./generate-network.py} "$facts/zerotier-network-id" "$secrets/zerotier-identity-secret"
         '';
       };
-      environment.etc."zerotier/network-id".text = facts.zerotier-network-id.value;
       environment.systemPackages = [ config.clanCore.clanPkgs.zerotier-members ];
     })
     (lib.mkIf ((config.clanCore.secrets ? zerotier) && (facts.zerotier-network-id.value != null)) {
       clan.networking.zerotier.networkId = facts.zerotier-network-id.value;
+      environment.etc."zerotier/network-id".text = facts.zerotier-network-id.value;
 
       systemd.services.zerotierone.serviceConfig.ExecStartPre = [
         "+${pkgs.writeShellScript "init-zerotier" ''

pkgs/clan-cli/clan_cli/__init__.py

@@ -3,7 +3,7 @@ import sys
 from types import ModuleType
 from typing import Optional
 
-from . import config, create, machines, secrets, webui
+from . import config, create, machines, secrets, vms, webui
 from .errors import ClanError
 from .ssh import cli as ssh_cli
 
@@ -47,6 +47,9 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser:
     parser_webui = subparsers.add_parser("webui", help="start webui")
     webui.register_parser(parser_webui)
 
+    parser_vms = subparsers.add_parser("vms", help="manage virtual machines")
+    vms.register_parser(parser_vms)
+
     if argcomplete:
         argcomplete.autocomplete(parser)
 

pkgs/clan-cli/clan_cli/machines/update.py

@@ -41,7 +41,12 @@ def deploy_nixos(hosts: HostGroup, clan_dir: Path) -> None:
         flake_attr = h.meta.get("flake_attr", "")
 
         run_generate_secrets(h.meta["generateSecrets"], clan_dir)
-        run_upload_secrets(h.meta["uploadSecrets"], clan_dir)
+        run_upload_secrets(
+            h.meta["uploadSecrets"],
+            clan_dir,
+            target=target,
+            target_directory=h.meta["secretsUploadDirectory"],
+        )
 
         target_host = h.meta.get("target_host")
         if target_host:
@@ -92,7 +97,7 @@ def build_json(targets: list[str]) -> list[dict[str, Any]]:
 def get_all_machines(clan_dir: Path) -> HostGroup:
     config = nix_config()
     system = config["system"]
-    what = f'{clan_dir}#clanInternals.machines-json."{system}"'
+    what = f'{clan_dir}#clanInternals.all-machines-json."{system}"'
     machines = build_json([what])[0]
 
     hosts = []
@@ -109,7 +114,9 @@ def get_selected_machines(machine_names: list[str], clan_dir: Path) -> HostGroup
     system = config["system"]
     what = []
     for name in machine_names:
-        what.append(f'{clan_dir}#clanInternals.machines."{system}"."{name}".json')
+        what.append(
+            f'{clan_dir}#clanInternals.machines."{system}"."{name}".config.system.clan.deployment.file'
+        )
     machines = build_json(what)
     hosts = []
     for i, machine in enumerate(machines):

pkgs/clan-cli/clan_cli/secrets/generate.py

@@ -16,7 +16,7 @@ def build_generate_script(machine: str, clan_dir: Path) -> str:
 
     cmd = nix_build(
         [
-            f'path:{clan_dir}#clanInternals.machines."{system}"."{machine}".generateSecrets'
+            f'path:{clan_dir}#clanInternals.machines."{system}"."{machine}".config.system.clan.generateSecrets'
         ]
     )
     proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)

pkgs/clan-cli/clan_cli/secrets/sops_generate.py

@@ -1,3 +1,4 @@
+import os
 import shlex
 import shutil
 import subprocess
@@ -10,7 +11,6 @@ from clan_cli.nix import nix_shell
 
 from ..dirs import get_clan_flake_toplevel
 from ..errors import ClanError
-from ..ssh import parse_deployment_address
 from .folders import sops_secrets_folder
 from .machines import add_machine, has_machine
 from .secrets import decrypt_secret, encrypt_secret, has_secret
@@ -102,27 +102,12 @@ def generate_secrets_from_nix(
 
 # this is called by the sops.nix clan core module
 def upload_age_key_from_nix(
-    machine_name: str, deployment_address: str, age_key_file: str
+    machine_name: str,
 ) -> None:
     secret_name = f"{machine_name}-age.key"
     if not has_secret(secret_name):  # skip uploading the secret, not managed by us
         return
     secret = decrypt_secret(secret_name)
 
-    h = parse_deployment_address(machine_name, deployment_address)
+    secrets_dir = Path(os.environ["SECRETS_DIR"])
-    path = Path(age_key_file)
+    (secrets_dir / "key.txt").write_text(secret)
-
-    proc = h.run(
-        [
-            "bash",
-            "-c",
-            'mkdir -p "$0" && echo -n "$1" > "$2"',
-            str(path.parent),
-            secret,
-            age_key_file,
-        ],
-        check=False,
-    )
-    if proc.returncode != 0:
-        print(f"failed to upload age key to {deployment_address}")
-        sys.exit(1)

pkgs/clan-cli/clan_cli/secrets/upload.py

@@ -1,12 +1,15 @@
 import argparse
+import json
 import os
 import shlex
 import subprocess
 from pathlib import Path
+from tempfile import TemporaryDirectory
 
 from ..dirs import get_clan_flake_toplevel, module_root
 from ..errors import ClanError
-from ..nix import nix_build, nix_config
+from ..nix import nix_build, nix_config, nix_shell
+from ..ssh import parse_deployment_address
 
 
 def build_upload_script(machine: str, clan_dir: Path) -> str:
@@ -14,7 +17,9 @@ def build_upload_script(machine: str, clan_dir: Path) -> str:
     system = config["system"]
 
     cmd = nix_build(
-        [f'{clan_dir}#clanInternals.machines."{system}"."{machine}".uploadSecrets']
+        [
+            f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.system.clan.uploadSecrets'
+        ]
     )
     proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
     if proc.returncode != 0:
@@ -25,25 +30,75 @@ def build_upload_script(machine: str, clan_dir: Path) -> str:
     return proc.stdout.strip()
 
 
-def run_upload_secrets(flake_attr: str, clan_dir: Path) -> None:
+def get_deployment_info(machine: str, clan_dir: Path) -> dict:
+    config = nix_config()
+    system = config["system"]
+
+    cmd = nix_build(
+        [
+            f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.system.clan.deployment.file'
+        ]
+    )
+    proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
+    if proc.returncode != 0:
+        raise ClanError(
+            f"failed to get deploymentAddress:\n{shlex.join(cmd)}\nexited with {proc.returncode}"
+        )
+
+    return json.load(open(proc.stdout.strip()))
+
+
+def run_upload_secrets(
+    flake_attr: str, clan_dir: Path, target: str, target_directory: str
+) -> None:
     env = os.environ.copy()
     env["CLAN_DIR"] = str(clan_dir)
     env["PYTHONPATH"] = str(module_root().parent)  # TODO do this in the clanCore module
     print(f"uploading secrets... {flake_attr}")
+    with TemporaryDirectory() as tempdir_:
+        tempdir = Path(tempdir_)
+        env["SECRETS_DIR"] = str(tempdir)
         proc = subprocess.run(
             [flake_attr],
             env=env,
+            check=True,
+            stdout=subprocess.PIPE,
+            text=True,
         )
 
         if proc.returncode != 0:
             raise ClanError("failed to upload secrets")
-    else:
+
-        print("successfully uploaded secrets")
+        h = parse_deployment_address(flake_attr, target)
+        ssh_cmd = h.ssh_cmd()
+        subprocess.run(
+            nix_shell(
+                ["rsync"],
+                [
+                    "rsync",
+                    "-e",
+                    " ".join(["ssh"] + ssh_cmd[2:]),
+                    "-az",
+                    "--delete",
+                    f"{str(tempdir)}/",
+                    f"{h.user}@{h.host}:{target_directory}/",
+                ],
+            ),
+            check=True,
+        )
 
 
 def upload_secrets(machine: str) -> None:
     clan_dir = get_clan_flake_toplevel()
-    run_upload_secrets(build_upload_script(machine, clan_dir), clan_dir)
+    deployment_info = get_deployment_info(machine, clan_dir)
+    address = deployment_info.get("deploymentAddress", "")
+    secrets_upload_directory = deployment_info.get("secretsUploadDirectory", "")
+    run_upload_secrets(
+        build_upload_script(machine, clan_dir),
+        clan_dir,
+        address,
+        secrets_upload_directory,
+    )
 
 
 def upload_command(args: argparse.Namespace) -> None:

pkgs/clan-cli/clan_cli/ssh/__init__.py

@@ -373,7 +373,7 @@ class Host:
         Command to run locally for the host
 
         @cmd the commmand to run
-        @stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocss.PIPE
+        @stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocess.PIPE
         @stderr if not None stderr of the command will be redirected to this file i.e. stderr=subprocess.PIPE
         @extra_env environment variables to override whe running the command
         @cwd current working directory to run the process in
@@ -447,6 +447,33 @@ class Host:
             f"$ {displayed_cmd}", extra=dict(command_prefix=self.command_prefix)
         )
 
+        bash_cmd = export_cmd
+        bash_args = []
+        if isinstance(cmd, list):
+            bash_cmd += 'exec "$@"'
+            bash_args += cmd
+        else:
+            bash_cmd += cmd
+        # FIXME we assume bash to be present here? Should be documented...
+        ssh_cmd = self.ssh_cmd(verbose_ssh=verbose_ssh) + [
+            "--",
+            f"{sudo}bash -c {quote(bash_cmd)} -- {' '.join(map(quote, bash_args))}",
+        ]
+        return self._run(
+            ssh_cmd,
+            displayed_cmd,
+            shell=False,
+            stdout=stdout,
+            stderr=stderr,
+            cwd=cwd,
+            check=check,
+            timeout=timeout,
+        )
+
+    def ssh_cmd(
+        self,
+        verbose_ssh: bool = False,
+    ) -> List:
         if self.user is not None:
             ssh_target = f"{self.user}@{self.host}"
         else:
@@ -469,32 +496,7 @@ class Host:
         if verbose_ssh or self.verbose_ssh:
             ssh_opts.extend(["-v"])
 
-        bash_cmd = export_cmd
+        return ["ssh", ssh_target] + ssh_opts
-        bash_args = []
-        if isinstance(cmd, list):
-            bash_cmd += 'exec "$@"'
-            bash_args += cmd
-        else:
-            bash_cmd += cmd
-        # FIXME we assume bash to be present here? Should be documented...
-        ssh_cmd = (
-            ["ssh", ssh_target]
-            + ssh_opts
-            + [
-                "--",
-                f"{sudo}bash -c {quote(bash_cmd)} -- {' '.join(map(quote, bash_args))}",
-            ]
-        )
-        return self._run(
-            ssh_cmd,
-            displayed_cmd,
-            shell=False,
-            stdout=stdout,
-            stderr=stderr,
-            cwd=cwd,
-            check=check,
-            timeout=timeout,
-        )
 
 
 T = TypeVar("T")

pkgs/clan-cli/clan_cli/vms/__init__.py

@@ -0,0 +1,21 @@
+import argparse
+
+from .create import register_create_parser
+from .inspect import register_inspect_parser
+
+
+def register_parser(parser: argparse.ArgumentParser) -> None:
+    subparser = parser.add_subparsers(
+        title="command",
+        description="command to execute",
+        help="the command to execute",
+        required=True,
+    )
+
+    inspect_parser = subparser.add_parser(
+        "inspect", help="inspect the vm configuration"
+    )
+    register_inspect_parser(inspect_parser)
+
+    create_parser = subparser.add_parser("create", help="create a VM from a machine")
+    register_create_parser(create_parser)

pkgs/clan-cli/clan_cli/vms/create.py

@@ -0,0 +1,101 @@
+import argparse
+import json
+import subprocess
+import tempfile
+from pathlib import Path
+
+from ..dirs import get_clan_flake_toplevel
+from ..nix import nix_build, nix_shell
+
+
+def get_vm_create_info(machine: str) -> dict:
+    clan_dir = get_clan_flake_toplevel().as_posix()
+
+    # config = nix_config()
+    # system = config["system"]
+
+    vm_json = subprocess.run(
+        nix_build(
+            [
+                # f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.clan.virtualisation.createJSON' # TODO use this
+                f'{clan_dir}#nixosConfigurations."{machine}".config.system.clan.vm.create'
+            ]
+        ),
+        stdout=subprocess.PIPE,
+        check=True,
+        text=True,
+    ).stdout.strip()
+    with open(vm_json) as f:
+        return json.load(f)
+
+
+def create(args: argparse.Namespace) -> None:
+    print(f"Creating VM for {args.machine}")
+    machine = args.machine
+    vm_config = get_vm_create_info(machine)
+    with tempfile.TemporaryDirectory() as tmpdir_:
+        xchg_dir = Path(tmpdir_) / "xchg"
+        xchg_dir.mkdir()
+        disk_img = f"{tmpdir_}/disk.img"
+        subprocess.run(
+            nix_shell(
+                ["qemu"],
+                [
+                    "qemu-img",
+                    "create",
+                    "-f",
+                    "raw",
+                    disk_img,
+                    "1024M",
+                ],
+            ),
+            stdout=subprocess.PIPE,
+            check=True,
+            text=True,
+        )
+        subprocess.run(
+            [
+                "mkfs.ext4",
+                "-L",
+                "nixos",
+                disk_img,
+            ],
+            stdout=subprocess.PIPE,
+            check=True,
+            text=True,
+        )
+
+        subprocess.run(
+            nix_shell(
+                ["qemu"],
+                [
+                    # fmt: off
+                    "qemu-kvm",
+                    "-name", machine,
+                    "-m", f'{vm_config["memorySize"]}M',
+                    "-smp", str(vm_config["cores"]),
+                    "-device", "virtio-rng-pci",
+                    "-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0",
+                    "-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store",
+                    "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
+                    "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
+                    "-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report',
+                    "-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
+                    "-device", "virtio-keyboard",
+                    "-usb",
+                    "-device", "usb-tablet,bus=usb-bus.0",
+                    "-kernel", f'{vm_config["toplevel"]}/kernel',
+                    "-initrd", vm_config["initrd"],
+                    "-append", f'{(Path(vm_config["toplevel"]) / "kernel-params").read_text()} init={vm_config["toplevel"]}/init regInfo={vm_config["regInfo"]}/registration console=ttyS0,115200n8 console=tty0',
+                    # fmt: on
+                ],
+            ),
+            stdout=subprocess.PIPE,
+            check=True,
+            text=True,
+        )
+
+
+def register_create_parser(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("machine", type=str)
+    parser.set_defaults(func=create)

pkgs/clan-cli/clan_cli/vms/inspect.py

@@ -0,0 +1,38 @@
+import argparse
+import json
+import subprocess
+
+from ..dirs import get_clan_flake_toplevel
+from ..nix import nix_eval
+
+
+def get_vm_inspect_info(machine: str) -> dict:
+    clan_dir = get_clan_flake_toplevel().as_posix()
+
+    # config = nix_config()
+    # system = config["system"]
+
+    return json.loads(
+        subprocess.run(
+            nix_eval(
+                [
+                    # f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.clan.virtualisation' # TODO use this
+                    f'{clan_dir}#nixosConfigurations."{machine}".config.system.clan.vm.config'
+                ]
+            ),
+            stdout=subprocess.PIPE,
+            check=True,
+            text=True,
+        ).stdout
+    )
+
+
+def inspect(args: argparse.Namespace) -> None:
+    print(f"Creating VM for {args.machine}")
+    machine = args.machine
+    print(get_vm_inspect_info(machine))
+
+
+def register_inspect_parser(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("machine", type=str)
+    parser.set_defaults(func=inspect)

pkgs/clan-cli/default.nix

@@ -27,6 +27,8 @@
 , nixpkgs
 , makeDesktopItem
 , copyDesktopItems
+, qemu
+, gnupg
 }:
 let
 
@@ -43,6 +45,7 @@ let
     pytest-parallel
     openssh
     git
+    gnupg
     stdenv.cc
   ];
 
@@ -59,6 +62,7 @@ let
     rsync
     sops
     git
+    qemu
   ];
 
   runtimeDependenciesAsSet = builtins.listToAttrs (builtins.map (p: lib.nameValuePair (lib.getName p.name) p) runtimeDependencies);

pkgs/clan-cli/tests/test_flake.py

@@ -29,6 +29,7 @@ def create_flake(
             if clan_core_flake:
                 line = line.replace("__CLAN_CORE__", str(clan_core_flake))
             line = line.replace("__CLAN_SOPS_KEY_PATH__", sops_key)
+            line = line.replace("__CLAN_SOPS_KEY_DIR__", str(flake))
             print(line, end="")
         monkeypatch.chdir(flake)
         monkeypatch.setenv("HOME", str(home))
@@ -47,3 +48,12 @@ def test_flake_with_core(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
             "clan-core flake not found. This test requires the clan-core flake to be present"
         )
     yield from create_flake(monkeypatch, "test_flake_with_core", CLAN_CORE)
+
+
+@pytest.fixture
+def test_flake_with_core_and_pass(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
+    if not (CLAN_CORE / "flake.nix").exists():
+        raise Exception(
+            "clan-core flake not found. This test requires the clan-core flake to be present"
+        )
+    yield from create_flake(monkeypatch, "test_flake_with_core_and_pass", CLAN_CORE)

pkgs/clan-cli/tests/test_flake_with_core/flake.nix

@@ -14,6 +14,7 @@
             clan.networking.deploymentAddress = "__CLAN_DEPLOYMENT_ADDRESS__";
             system.stateVersion = lib.version;
             sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__";
+            clanCore.secretsUploadDirectory = "__CLAN_SOPS_KEY_DIR__";
 
             clan.networking.zerotier.controller.enable = true;
 

pkgs/clan-cli/tests/test_flake_with_core_and_pass/flake.nix

@@ -0,0 +1,37 @@
+{
+  # Use this path to our repo root e.g. for UI test
+  # inputs.clan-core.url = "../../../../.";
+
+  # this placeholder is replaced by the path to nixpkgs
+  inputs.clan-core.url = "__CLAN_CORE__";
+
+  outputs = { self, clan-core }:
+    let
+      clan = clan-core.lib.buildClan {
+        directory = self;
+        machines = {
+          vm1 = { lib, ... }: {
+            clan.networking.deploymentAddress = "__CLAN_DEPLOYMENT_ADDRESS__";
+            system.stateVersion = lib.version;
+            clanCore.secretStore = "password-store";
+            clanCore.secretsUploadDirectory = lib.mkForce "__CLAN_SOPS_KEY_DIR__/secrets";
+
+            clan.networking.zerotier.controller.enable = true;
+
+            systemd.services.shutdown-after-boot = {
+              enable = true;
+              wantedBy = [ "multi-user.target" ];
+              after = [ "multi-user.target" ];
+              script = ''
+                #!/usr/bin/env bash
+                shutdown -h now
+              '';
+            };
+          };
+        };
+      };
+    in
+    {
+      inherit (clan) nixosConfigurations clanInternals;
+    };
+}

pkgs/clan-cli/tests/test_secrets_generate.py

@@ -13,7 +13,7 @@ if TYPE_CHECKING:
 
 
 @pytest.mark.impure
-def test_upload_secret(
+def test_generate_secret(
     monkeypatch: pytest.MonkeyPatch,
     test_flake_with_core: Path,
     age_keys: list["KeyPair"],

pkgs/clan-cli/tests/test_secrets_password_store.py

@@ -0,0 +1,62 @@
+import subprocess
+from pathlib import Path
+
+import pytest
+from cli import Cli
+
+from clan_cli.machines.facts import machine_get_fact
+from clan_cli.nix import nix_shell
+from clan_cli.ssh import HostGroup
+
+
+@pytest.mark.impure
+def test_upload_secret(
+    monkeypatch: pytest.MonkeyPatch,
+    test_flake_with_core_and_pass: Path,
+    temporary_dir: Path,
+    host_group: HostGroup,
+) -> None:
+    monkeypatch.chdir(test_flake_with_core_and_pass)
+    gnupghome = temporary_dir / "gpg"
+    gnupghome.mkdir()
+    monkeypatch.setenv("GNUPGHOME", str(gnupghome))
+    monkeypatch.setenv("PASSWORD_STORE_DIR", str(temporary_dir / "pass"))
+    gpg_key_spec = temporary_dir / "gpg_key_spec"
+    gpg_key_spec.write_text(
+        """
+        Key-Type: 1
+        Key-Length: 1024
+        Name-Real: Root Superuser
+        Name-Email: test@local
+        Expire-Date: 0
+        %no-protection
+    """
+    )
+    cli = Cli()
+    subprocess.run(
+        nix_shell(["gnupg"], ["gpg", "--batch", "--gen-key", str(gpg_key_spec)]),
+        check=True,
+    )
+    subprocess.run(nix_shell(["pass"], ["pass", "init", "test@local"]), check=True)
+    cli.run(["secrets", "generate", "vm1"])
+    network_id = machine_get_fact("vm1", "zerotier-network-id")
+    assert len(network_id) == 16
+    identity_secret = (
+        temporary_dir / "pass" / "machines" / "vm1" / "zerotier-identity-secret.gpg"
+    )
+    secret1_mtime = identity_secret.lstat().st_mtime_ns
+
+    # test idempotency
+    cli.run(["secrets", "generate", "vm1"])
+    assert identity_secret.lstat().st_mtime_ns == secret1_mtime
+
+    flake = test_flake_with_core_and_pass.joinpath("flake.nix")
+    host = host_group.hosts[0]
+    addr = f"{host.user}@{host.host}:{host.port}?StrictHostKeyChecking=no&UserKnownHostsFile=/dev/null&IdentityFile={host.key}"
+    new_text = flake.read_text().replace("__CLAN_DEPLOYMENT_ADDRESS__", addr)
+    flake.write_text(new_text)
+    cli.run(["secrets", "upload", "vm1"])
+    zerotier_identity_secret = (
+        test_flake_with_core_and_pass / "secrets" / "zerotier-identity-secret"
+    )
+    assert zerotier_identity_secret.exists()

pkgs/clan-cli/tests/test_secrets_upload.py

@@ -36,6 +36,6 @@ def test_secrets_upload(
     cli.run(["secrets", "upload", "vm1"])
 
     # the flake defines this path as the location where the sops key should be installed
-    sops_key = test_flake_with_core.joinpath("sops.key")
+    sops_key = test_flake_with_core.joinpath("key.txt")
     assert sops_key.exists()
     assert sops_key.read_text() == age_keys[0].privkey