diff --git a/nixosModules/clanCore/vm.nix b/nixosModules/clanCore/vm.nix index 11e827082..08f6779c8 100644 --- a/nixosModules/clanCore/vm.nix +++ b/nixosModules/clanCore/vm.nix @@ -7,19 +7,6 @@ let config.clanCore.state ); - # Ensure sane mount order by topo-sorting - sortedStateFolders = - let - sorted = lib.toposort lib.hasPrefix stateFolders; - in - sorted.result or ( - throw '' - The state folders have a cyclic dependency. - This is not allowed. - The cyclic dependencies are: - - ${lib.concatStringsSep "\n - " sorted.loops} - '' - ); vmModule = { imports = [ @@ -43,49 +30,45 @@ let boot.kernelPackages = pkgs.linuxPackages_latest; boot.initrd.systemd.storePaths = [ pkgs.util-linux pkgs.e2fsprogs ]; - # Ensures, that all state paths will be persisted across reboots - # - Mounts the state.qcow2 disk to /vmstate. - # - Binds directories from /vmstate/{some-path} to /{some-path}. - boot.initrd.systemd.services.rw-etc-pre = { - unitConfig = { - DefaultDependencies = false; - RequiresMountsFor = "/sysroot /dev"; + boot.initrd.systemd.emergencyAccess = true; + + boot.initrd.kernelModules = [ "virtiofs" ]; + virtualisation.writableStore = false; + virtualisation.fileSystems = lib.mkForce ({ + "/nix/store" = { + device = "nix-store"; + options = [ "x-systemd.requires=systemd-modules-load.service" "ro" ]; + fsType = "virtiofs"; }; - wantedBy = [ "initrd.target" ]; - requiredBy = [ "rw-etc.service" ]; - before = [ "rw-etc.service" ]; - serviceConfig = { - Type = "oneshot"; + + "/" = { + device = "/dev/vda"; + fsType = "ext4"; + options = [ "defaults" "x-systemd.makefs" ]; }; - script = '' - set -x - mkdir -p -m 0755 \ - /sysroot/vmstate \ - /sysroot/.rw-etc \ - /sysroot/var/lib/nixos - ${pkgs.util-linux}/bin/blkid /dev/vdb || ${pkgs.e2fsprogs}/bin/mkfs.ext4 /dev/vdb - sync - mount /dev/vdb /sysroot/vmstate + "/vmstate" = { + device = "/dev/vdb"; + options = [ "x-systemd.makefs" ]; + noCheck = true; + fsType = "ext4"; + }; - mkdir -p -m 0755 /sysroot/vmstate/{.rw-etc,var/lib/nixos} - mount --bind /sysroot/vmstate/.rw-etc /sysroot/.rw-etc - mount --bind /sysroot/vmstate/var/lib/nixos /sysroot/var/lib/nixos - - for folder in "${lib.concatStringsSep ''" "'' sortedStateFolders}"; do - mkdir -p -m 0755 "/sysroot/vmstate/$folder" "/sysroot/$folder" - mount --bind "/sysroot/vmstate/$folder" "/sysroot/$folder" - done - ''; - }; - virtualisation.fileSystems = { - ${config.clanCore.secretsUploadDirectory} = lib.mkForce { + ${config.clanCore.secretsUploadDirectory} = { device = "secrets"; fsType = "9p"; neededForBoot = true; options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ]; }; - }; + + } // lib.listToAttrs (map + (folder: + lib.nameValuePair folder { + device = "/vmstate${folder}"; + fsType = "none"; + options = [ "bind" ]; + }) + stateFolders)); }; # We cannot simply merge the VM config into the current system config, because diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index 40a3454e6..489a9cb86 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -5,6 +5,7 @@ import json import logging import os import random +import shutil import socket import subprocess import time @@ -91,6 +92,7 @@ def qemu_command( secrets_dir: Path, rootfs_img: Path, state_img: Path, + virtiofsd_socket: Path, qmp_socket_file: Path, qga_socket_file: Path, ) -> QemuCommand: @@ -98,7 +100,7 @@ def qemu_command( (Path(nixos_config["toplevel"]) / "kernel-params").read_text(), f'init={nixos_config["toplevel"]}/init', f'regInfo={nixos_config["regInfo"]}/registration', - "console=ttyS0,115200n8", + "console=hvc0", ] if not vm.waypipe: kernel_cmdline.append("console=tty0") @@ -112,14 +114,15 @@ def qemu_command( "-smp", str(nixos_config["cores"]), "-cpu", "max", "-enable-kvm", + # speed-up boot by not waiting for the boot menu + "-boot", "menu=off,strict=on", "-device", "virtio-rng-pci", - "-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0", - "-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store", - "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared", - "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg", + "-device", "virtio-net-pci,netdev=user.0,romfile=", + "-chardev", f"socket,id=char1,path={virtiofsd_socket}", + "-device", "vhost-user-fs-pci,chardev=char1,tag=nix-store", "-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets", - "-drive", f"cache=writeback,file={rootfs_img},format=raw,id=drive1,if=none,index=1,werror=report", + "-drive", f"cache=writeback,file={rootfs_img},format=qcow2,id=drive1,if=none,index=1,werror=report", "-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root", "-drive", f"cache=writeback,file={state_img},format=qcow2,id=state,if=none,index=2,werror=report", "-device", "virtio-blk-pci,drive=state", @@ -133,6 +136,11 @@ def qemu_command( "-chardev", f"socket,path={qga_socket_file},server=on,wait=off,id=qga0", "-device", "virtio-serial", "-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0", + + "-serial", "null", + "-chardev", "stdio,mux=on,id=char0,signal=off", + "-mon", "chardev=char0,mode=readline", + "-device", "virtconsole,chardev=char0,nr=0", ] # fmt: on vsock_cid = None @@ -189,9 +197,7 @@ def get_secrets( def prepare_disk( directory: Path, - disk_format: str = "raw", size: str = "1024M", - label: str = "nixos", file_name: str = "disk.img", ) -> Path: disk_img = directory / file_name @@ -201,7 +207,7 @@ def prepare_disk( "qemu-img", "create", "-f", - disk_format, + "qcow2", str(disk_img), size, ], @@ -212,21 +218,6 @@ def prepare_disk( error_msg=f"Could not create disk image at {disk_img}", ) - if disk_format == "raw": - cmd = nix_shell( - ["nixpkgs#e2fsprogs"], - [ - "mkfs.ext4", - "-L", - label, - str(disk_img), - ], - ) - run( - cmd, - log=Log.BOTH, - error_msg=f"Could not create ext4 filesystem at {disk_img}", - ) return disk_img @@ -263,6 +254,42 @@ def start_waypipe(cid: int | None, title_prefix: str) -> Iterator[None]: with subprocess.Popen(waypipe) as proc: try: while not test_vsock_port(3049): + rc = proc.poll() + if rc is not None: + msg = f"waypipe exited unexpectedly with code {rc}" + raise ClanError(msg) + time.sleep(0.1) + yield + finally: + proc.kill() + + +@contextlib.contextmanager +def start_virtiofsd(socket_path: Path) -> Iterator[None]: + sandbox = "namespace" + if shutil.which("newuidmap") is None: + sandbox = "none" + virtiofsd = nix_shell( + ["nixpkgs#virtiofsd"], + [ + "virtiofsd", + "--socket-path", + str(socket_path), + "--cache", + "always", + "--sandbox", + sandbox, + "--shared-dir", + "/nix/store", + ], + ) + with subprocess.Popen(virtiofsd) as proc: + try: + while not socket_path.exists(): + rc = proc.poll() + if rc is not None: + msg = f"virtiofsd exited unexpectedly with code {rc}" + raise ClanError(msg) time.sleep(0.1) yield finally: @@ -314,10 +341,9 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None: state_img = prepare_disk( directory=state_dir, file_name="state.qcow2", - disk_format="qcow2", size="50G", - label="state", ) + virtiofsd_socket = Path(sockets) / "virtiofsd.sock" qemu_cmd = qemu_command( vm, nixos_config, @@ -325,6 +351,7 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None: secrets_dir=secrets_dir, rootfs_img=rootfs_img, state_img=state_img, + virtiofsd_socket=virtiofsd_socket, qmp_socket_file=qmp_socket_file, qga_socket_file=qga_socket_file, ) @@ -339,7 +366,9 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None: "XDG_DATA_DIRS" ] = f"{remote_viewer_mimetypes}:{env.get('XDG_DATA_DIRS', '')}" - with start_waypipe(qemu_cmd.vsock_cid, f"[{vm.machine_name}] "): + with start_waypipe( + qemu_cmd.vsock_cid, f"[{vm.machine_name}] " + ), start_virtiofsd(virtiofsd_socket): run( nix_shell(packages, qemu_cmd.args), env=env, diff --git a/pkgs/clan-cli/tests/test_vms_cli.py b/pkgs/clan-cli/tests/test_vms_cli.py index 4345ca89d..4805ee63a 100644 --- a/pkgs/clan-cli/tests/test_vms_cli.py +++ b/pkgs/clan-cli/tests/test_vms_cli.py @@ -266,13 +266,6 @@ def test_vm_persistence( # connect second time qga = qga_connect(state_dir) - # ensure that either /var/lib/nixos or /etc gets persisted - # (depending on if system.etc.overlay.enable is set or not) - exitcode, out, err = qga.run( - "ls /vmstate/var/lib/nixos/gid-map || ls /vmstate/.rw-etc/upper" - ) - assert exitcode == 0, err - # ensure that the file created by the service is still there and has the expected content exitcode, out, err = qga.run("cat /var/my-state/test") assert exitcode == 0, err diff --git a/pkgs/clan-vm-manager/demo.sh b/pkgs/clan-vm-manager/demo.sh index 1bf573579..48a4830e0 100755 --- a/pkgs/clan-vm-manager/demo.sh +++ b/pkgs/clan-vm-manager/demo.sh @@ -1,21 +1,29 @@ #!/usr/bin/env bash +set -eux -o pipefail + rm -r ~/.config/clan -clan history add "clan://~/Projects/democlan#syncthing-peer1" -clan history add "clan://~/Projects/democlan#syncthing-peer2" +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi -clan history add "clan://~/Projects/democlan#moonlight-peer1" -clan history add "clan://~/Projects/democlan#moonlight-peer2" +democlan="$1" + +clan history add "clan://$democlan#syncthing-peer1" +clan history add "clan://$democlan#syncthing-peer2" + +clan history add "clan://$democlan#moonlight-peer1" +clan history add "clan://$democlan#moonlight-peer2" clear cat << EOF Open up this link in a browser: -"clan://~/Projects/democlan#syncthing-introducer" +"clan://$democlan#syncthing-introducer" EOF - cat << EOF Execute this command to show waypipe windows: -$ clan --flake ~/Projects/democlan/ vms run --wayland wayland +$ clan --flake $democlan vms run --wayland wayland EOF