Merge pull request 'switch to sops-nix experimental branch' (#832) from Mic92-target_host into main

nixosModules/clanCore/vm.nix

@@ -7,19 +7,6 @@ let
       config.clanCore.state
   );
 
-  # Ensure sane mount order by topo-sorting
-  sortedStateFolders =
-    let
-      sorted = lib.toposort lib.hasPrefix stateFolders;
-    in
-      sorted.result or (
-        throw ''
-          The state folders have a cyclic dependency.
-          This is not allowed.
-          The cyclic dependencies are:
-            - ${lib.concatStringsSep "\n  - " sorted.loops}
-        ''
-      );
 
   vmModule = {
     imports = [
@@ -43,49 +30,45 @@ let
     boot.kernelPackages = pkgs.linuxPackages_latest;
 
     boot.initrd.systemd.storePaths = [ pkgs.util-linux pkgs.e2fsprogs ];
-    # Ensures, that all state paths will be persisted across reboots
+    boot.initrd.systemd.emergencyAccess = true;
-    # - Mounts the state.qcow2 disk to /vmstate.
-    # - Binds directories from /vmstate/{some-path} to /{some-path}.
-    boot.initrd.systemd.services.rw-etc-pre = {
-      unitConfig = {
-        DefaultDependencies = false;
-        RequiresMountsFor = "/sysroot /dev";
-      };
-      wantedBy = [ "initrd.target" ];
-      requiredBy = [ "rw-etc.service" ];
-      before = [ "rw-etc.service" ];
-      serviceConfig = {
-        Type = "oneshot";
-      };
-      script = ''
-        set -x
-        mkdir -p -m 0755 \
-          /sysroot/vmstate \
-          /sysroot/.rw-etc \
-          /sysroot/var/lib/nixos
 
-        ${pkgs.util-linux}/bin/blkid /dev/vdb || ${pkgs.e2fsprogs}/bin/mkfs.ext4 /dev/vdb
+    boot.initrd.kernelModules = [ "virtiofs" ];
-        sync
+    virtualisation.writableStore = false;
-        mount /dev/vdb /sysroot/vmstate
+    virtualisation.fileSystems = lib.mkForce ({
-
+      "/nix/store" = {
-        mkdir -p -m 0755 /sysroot/vmstate/{.rw-etc,var/lib/nixos}
+        device = "nix-store";
-        mount --bind /sysroot/vmstate/.rw-etc /sysroot/.rw-etc
+        options = [ "x-systemd.requires=systemd-modules-load.service" "ro" ];
-        mount --bind /sysroot/vmstate/var/lib/nixos /sysroot/var/lib/nixos
+        fsType = "virtiofs";
-
-        for folder in "${lib.concatStringsSep ''" "'' sortedStateFolders}"; do
-          mkdir -p -m 0755 "/sysroot/vmstate/$folder" "/sysroot/$folder"
-          mount --bind "/sysroot/vmstate/$folder" "/sysroot/$folder"
-        done
-      '';
       };
-    virtualisation.fileSystems = {
+
-      ${config.clanCore.secretsUploadDirectory} = lib.mkForce {
+      "/" = {
+        device = "/dev/vda";
+        fsType = "ext4";
+        options = [ "defaults" "x-systemd.makefs" ];
+      };
+
+      "/vmstate" = {
+        device = "/dev/vdb";
+        options = [ "x-systemd.makefs" ];
+        noCheck = true;
+        fsType = "ext4";
+      };
+
+      ${config.clanCore.secretsUploadDirectory} = {
         device = "secrets";
         fsType = "9p";
         neededForBoot = true;
         options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
       };
-    };
+
+    } // lib.listToAttrs (map
+      (folder:
+        lib.nameValuePair folder {
+          device = "/vmstate${folder}";
+          fsType = "none";
+          options = [ "bind" ];
+        })
+      stateFolders));
   };
 
   # We cannot simply merge the VM config into the current system config, because

pkgs/clan-cli/clan_cli/vms/run.py

@@ -5,6 +5,7 @@ import json
 import logging
 import os
 import random
+import shutil
 import socket
 import subprocess
 import time
@@ -91,6 +92,7 @@ def qemu_command(
     secrets_dir: Path,
     rootfs_img: Path,
     state_img: Path,
+    virtiofsd_socket: Path,
     qmp_socket_file: Path,
     qga_socket_file: Path,
 ) -> QemuCommand:
@@ -98,7 +100,7 @@ def qemu_command(
         (Path(nixos_config["toplevel"]) / "kernel-params").read_text(),
         f'init={nixos_config["toplevel"]}/init',
         f'regInfo={nixos_config["regInfo"]}/registration',
-        "console=ttyS0,115200n8",
+        "console=hvc0",
     ]
     if not vm.waypipe:
         kernel_cmdline.append("console=tty0")
@@ -112,14 +114,15 @@ def qemu_command(
         "-smp", str(nixos_config["cores"]),
         "-cpu", "max",
         "-enable-kvm",
+        # speed-up boot by not waiting for the boot menu
+        "-boot", "menu=off,strict=on",
         "-device", "virtio-rng-pci",
-        "-net", "nic,netdev=user.0,model=virtio",
         "-netdev", "user,id=user.0",
-        "-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store",
+        "-device", "virtio-net-pci,netdev=user.0,romfile=",
-        "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
+        "-chardev", f"socket,id=char1,path={virtiofsd_socket}",
-        "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
+        "-device", "vhost-user-fs-pci,chardev=char1,tag=nix-store",
         "-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
-        "-drive", f"cache=writeback,file={rootfs_img},format=raw,id=drive1,if=none,index=1,werror=report",
+        "-drive", f"cache=writeback,file={rootfs_img},format=qcow2,id=drive1,if=none,index=1,werror=report",
         "-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
         "-drive", f"cache=writeback,file={state_img},format=qcow2,id=state,if=none,index=2,werror=report",
         "-device", "virtio-blk-pci,drive=state",
@@ -133,6 +136,11 @@ def qemu_command(
         "-chardev", f"socket,path={qga_socket_file},server=on,wait=off,id=qga0",
         "-device", "virtio-serial",
         "-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0",
+
+        "-serial", "null",
+        "-chardev", "stdio,mux=on,id=char0,signal=off",
+        "-mon", "chardev=char0,mode=readline",
+        "-device", "virtconsole,chardev=char0,nr=0",
     ]  # fmt: on
 
     vsock_cid = None
@@ -189,9 +197,7 @@ def get_secrets(
 
 def prepare_disk(
     directory: Path,
-    disk_format: str = "raw",
     size: str = "1024M",
-    label: str = "nixos",
     file_name: str = "disk.img",
 ) -> Path:
     disk_img = directory / file_name
@@ -201,7 +207,7 @@ def prepare_disk(
             "qemu-img",
             "create",
             "-f",
-            disk_format,
+            "qcow2",
             str(disk_img),
             size,
         ],
@@ -212,21 +218,6 @@ def prepare_disk(
         error_msg=f"Could not create disk image at {disk_img}",
     )
 
-    if disk_format == "raw":
-        cmd = nix_shell(
-            ["nixpkgs#e2fsprogs"],
-            [
-                "mkfs.ext4",
-                "-L",
-                label,
-                str(disk_img),
-            ],
-        )
-        run(
-            cmd,
-            log=Log.BOTH,
-            error_msg=f"Could not create ext4 filesystem at {disk_img}",
-        )
     return disk_img
 
 
@@ -263,6 +254,42 @@ def start_waypipe(cid: int | None, title_prefix: str) -> Iterator[None]:
     with subprocess.Popen(waypipe) as proc:
         try:
             while not test_vsock_port(3049):
+                rc = proc.poll()
+                if rc is not None:
+                    msg = f"waypipe exited unexpectedly with code {rc}"
+                    raise ClanError(msg)
+                time.sleep(0.1)
+            yield
+        finally:
+            proc.kill()
+
+
+@contextlib.contextmanager
+def start_virtiofsd(socket_path: Path) -> Iterator[None]:
+    sandbox = "namespace"
+    if shutil.which("newuidmap") is None:
+        sandbox = "none"
+    virtiofsd = nix_shell(
+        ["nixpkgs#virtiofsd"],
+        [
+            "virtiofsd",
+            "--socket-path",
+            str(socket_path),
+            "--cache",
+            "always",
+            "--sandbox",
+            sandbox,
+            "--shared-dir",
+            "/nix/store",
+        ],
+    )
+    with subprocess.Popen(virtiofsd) as proc:
+        try:
+            while not socket_path.exists():
+                rc = proc.poll()
+                if rc is not None:
+                    msg = f"virtiofsd exited unexpectedly with code {rc}"
+                    raise ClanError(msg)
                 time.sleep(0.1)
             yield
         finally:
@@ -314,10 +341,9 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None:
             state_img = prepare_disk(
                 directory=state_dir,
                 file_name="state.qcow2",
-                disk_format="qcow2",
                 size="50G",
-                label="state",
             )
+        virtiofsd_socket = Path(sockets) / "virtiofsd.sock"
         qemu_cmd = qemu_command(
             vm,
             nixos_config,
@@ -325,6 +351,7 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None:
             secrets_dir=secrets_dir,
             rootfs_img=rootfs_img,
             state_img=state_img,
+            virtiofsd_socket=virtiofsd_socket,
             qmp_socket_file=qmp_socket_file,
             qga_socket_file=qga_socket_file,
         )
@@ -339,7 +366,9 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None:
                 "XDG_DATA_DIRS"
             ] = f"{remote_viewer_mimetypes}:{env.get('XDG_DATA_DIRS', '')}"
 
-        with start_waypipe(qemu_cmd.vsock_cid, f"[{vm.machine_name}] "):
+        with start_waypipe(
+            qemu_cmd.vsock_cid, f"[{vm.machine_name}] "
+        ), start_virtiofsd(virtiofsd_socket):
             run(
                 nix_shell(packages, qemu_cmd.args),
                 env=env,

pkgs/clan-cli/tests/test_vms_cli.py

@@ -266,13 +266,6 @@ def test_vm_persistence(
     # connect second time
     qga = qga_connect(state_dir)
 
-    # ensure that either /var/lib/nixos or /etc gets persisted
-    # (depending on if system.etc.overlay.enable is set or not)
-    exitcode, out, err = qga.run(
-        "ls /vmstate/var/lib/nixos/gid-map || ls /vmstate/.rw-etc/upper"
-    )
-    assert exitcode == 0, err
-
     # ensure that the file created by the service is still there and has the expected content
     exitcode, out, err = qga.run("cat /var/my-state/test")
     assert exitcode == 0, err

pkgs/clan-vm-manager/demo.sh

@@ -1,21 +1,29 @@
 #!/usr/bin/env bash
 
+set -eux -o pipefail
+
 rm -r ~/.config/clan
 
-clan history add "clan://~/Projects/democlan#syncthing-peer1"
+if [ -z "$1" ]; then
-clan history add "clan://~/Projects/democlan#syncthing-peer2"
+  echo "Usage: $0 <democlan>"
+  exit 1
+fi
 
-clan history add "clan://~/Projects/democlan#moonlight-peer1"
+democlan="$1"
-clan history add "clan://~/Projects/democlan#moonlight-peer2"
+
+clan history add "clan://$democlan#syncthing-peer1"
+clan history add "clan://$democlan#syncthing-peer2"
+
+clan history add "clan://$democlan#moonlight-peer1"
+clan history add "clan://$democlan#moonlight-peer2"
 
 clear
 cat << EOF
 Open up this link in a browser:
-"clan://~/Projects/democlan#syncthing-introducer"
+"clan://$democlan#syncthing-introducer"
 EOF
 
-
 cat << EOF
 Execute this command to show waypipe windows:
-$ clan --flake ~/Projects/democlan/ vms run --wayland wayland
+$ clan --flake $democlan vms run --wayland wayland
 EOF