VMs: persist state folders on host

Done:
- move vm inspect attrs from system.clan.vm.config to clanCore.vm.inspect. This gives us proper name and type checking. everything in `system` is basically freeform, so the previous option definitions were never enforced
- when running VMs, mount state directory from ~/.config/clan/vmstate/{...} from the host to /var/vmstate inside the vm
- create bind mount inside the VM from /var/vmstate/{folder} to / for all folders defined in clanCore.state.<name>.folders

TODOs:
- make sure directories in ~/.config/clan/vmstate never collide (include hash of clan-url, etc.)
- port impure test to python

checks/impure/flake-module.nix

@@ -1,5 +1,5 @@
-{ ... }: {
+{ self, ... }: {
-  perSystem = { pkgs, lib, ... }: {
+  perSystem = { pkgs, lib, self', ... }: {
     packages = rec {
       # a script that executes all other checks
       impure-checks = pkgs.writeShellScriptBin "impure-checks" ''
@@ -13,9 +13,86 @@
         ]}"
         ROOT=$(git rev-parse --show-toplevel)
         cd "$ROOT/pkgs/clan-cli"
+        ${self'.packages.vm-persistence}/bin/vm-persistence
         nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -m impure ./tests $@"
       '';
 
+      # TODO: port this to python and make it pure
+      vm-persistence =
+        let
+          machineConfigFile = builtins.toFile "vm-config.json" (builtins.toJSON {
+            clanCore.state.my-state = {
+              folders = [ "/var/my-state" ];
+            };
+            # powers off the machine after the state is created
+            systemd.services.poweroff = {
+              description = "Poweroff the machine";
+              wantedBy = [ "multi-user.target" ];
+              after = [ "my-state.service" ];
+              script = ''
+                echo "Powering off the machine"
+                poweroff
+              '';
+            };
+            # creates a file in the state folder
+            systemd.services.my-state = {
+              description = "Create a file in the state folder";
+              wantedBy = [ "multi-user.target" ];
+              script = ''
+                echo "Creating a file in the state folder"
+                echo "dream2nix" > /var/my-state/test
+              '';
+              serviceConfig.Type = "oneshot";
+            };
+            clan.virtualisation.graphics = false;
+            users.users.root.password = "root";
+          });
+        in
+        pkgs.writeShellScriptBin "vm-persistence" ''
+          #!${pkgs.bash}/bin/bash
+          set -euo pipefail
+
+          export PATH="${lib.makeBinPath [
+            pkgs.coreutils
+            pkgs.gitMinimal
+            pkgs.jq
+            pkgs.nix
+            pkgs.gnused
+            self'.packages.clan-cli
+          ]}"
+
+          clanName=_test_vm_persistence
+          testFile=~/".config/clan/vmstate/$clanName/my-machine/var/my-state/test"
+
+          export TMPDIR=$(${pkgs.coreutils}/bin/mktemp -d)
+          trap "${pkgs.coreutils}/bin/chmod -R +w '$TMPDIR'; ${pkgs.coreutils}/bin/rm -rf '$TMPDIR'" EXIT
+
+          # clean up vmstate after test
+          trap "${pkgs.coreutils}/bin/rm -rf ~/.config/clan/vmstate/$clanName" EXIT
+
+          cd $TMPDIR
+          mkdir ./clan
+          cd ./clan
+          nix flake init -t ${self}#templates.new-clan
+          nix flake lock --override-input clan-core ${self}
+          sed -i "s/__CHANGE_ME__/$clanName/g" flake.nix
+          clan machines create my-machine
+
+          cat ${machineConfigFile} | jq > ./machines/my-machine/settings.json
+
+          # clear state from previous runs
+          rm -rf "$testFile"
+
+          # machine will automatically shutdown due to the shutdown service above
+          clan vms run my-machine
+
+          set -x
+          if ! test -e "$testFile"; then
+            echo "failed: file "$testFile" was not created"
+            exit 1
+          fi
+        '';
+
       runMockApi = pkgs.writeShellScriptBin "run-mock-api" ''
         #!${pkgs.bash}/bin/bash
         set -euo pipefail

nixosModules/clanCore/outputs.nix

@@ -45,12 +45,6 @@
           '';
           default = "${pkgs.coreutils}/bin/true";
         };
-        vm.config = lib.mkOption {
-          type = lib.types.attrs;
-          description = ''
-            the vm config
-          '';
-        };
         vm.create = lib.mkOption {
           type = lib.types.path;
           description = ''

nixosModules/clanCore/vm.nix

@@ -1,19 +1,59 @@
 { lib, config, pkgs, options, extendModules, modulesPath, ... }:
 let
-  vmConfig = extendModules {
+  # Generates a fileSystems entry for bind mounting a given state folder path
-    modules = [
+  # It binds directories from /var/clanstate/{some-path} to /{some-path}.
+  # As a result, all state paths will be persisted across reboots, because
+  #   the state folder is mounted from the host system.
+  mkBindMount = path: {
+    name = path;
+    value = {
+      device = "/var/clanstate/${path}";
+      options = [ "bind" ];
+    };
+  };
+
+  # Flatten the list of state folders into a single list
+  stateFolders = lib.flatten (
+    lib.mapAttrsToList
+      (_item: attrs: attrs.folders)
+      config.clanCore.state
+  );
+
+  # A module setting up bind mounts for all state folders
+  stateMounts = {
+    virtualisation.fileSystems =
+      lib.listToAttrs
+        (map mkBindMount stateFolders);
+  };
+
+  vmModule = {
+    imports = [
       (modulesPath + "/virtualisation/qemu-vm.nix")
       ./serial.nix
-      {
+      stateMounts
-        virtualisation.fileSystems.${config.clanCore.secretsUploadDirectory} = lib.mkForce {
-          device = "secrets";
-          fsType = "9p";
-          neededForBoot = true;
-          options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
-        };
-        boot.initrd.systemd.enable = true;
-      }
     ];
+    virtualisation.fileSystems = {
+      ${config.clanCore.secretsUploadDirectory} = lib.mkForce {
+        device = "secrets";
+        fsType = "9p";
+        neededForBoot = true;
+        options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
+      };
+      "/var/clanstate" = {
+        device = "state";
+        fsType = "9p";
+        options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
+      };
+    };
+    boot.initrd.systemd.enable = true;
+  };
+
+  # We cannot simply merge the VM config into the current system config, because
+  # it is not necessarily a VM.
+  # Instead we use extendModules to create a second instance of the current
+  # system configuration, and then merge the VM config into that.
+  vmConfig = extendModules {
+    modules = [ vmModule stateMounts ];
   };
 in
 {
@@ -47,17 +87,54 @@ in
         '';
       };
     };
+    # All important VM config variables needed by the vm runner
+    # this is really just a remapping of values defined elsewhere
+    # and therefore not intended to be set by the user
+    clanCore.vm.inspect = {
+      clan_name = lib.mkOption {
+        type = lib.types.str;
+        internal = true;
+        readOnly = true;
+        description = ''
+          the name of the clan
+        '';
+      };
+      memory_size = lib.mkOption {
+        type = lib.types.int;
+        internal = true;
+        readOnly = true;
+        description = ''
+          the amount of memory to allocate to the vm
+        '';
+      };
+      cores = lib.mkOption {
+        type = lib.types.int;
+        internal = true;
+        readOnly = true;
+        description = ''
+          the number of cores to allocate to the vm
+        '';
+      };
+      graphics = lib.mkOption {
+        type = lib.types.bool;
+        internal = true;
+        readOnly = true;
+        description = ''
+          whether to enable graphics for the vm
+        '';
+      };
+    };
   };
 
   config = {
+    # for clan vm inspect
+    clanCore.vm.inspect = {
+      clan_name = config.clanCore.clanName;
+      memory_size = config.clan.virtualisation.memorySize;
+      inherit (config.clan.virtualisation) cores graphics;
+    };
+    # for clan vm create
     system.clan.vm = {
-      # for clan vm inspect
-      config = {
-        clan_name = config.clanCore.clanName;
-        memory_size = config.clan.virtualisation.memorySize;
-        inherit (config.clan.virtualisation) cores graphics;
-      };
-      # for clan vm create
       create = pkgs.writeText "vm.json" (builtins.toJSON {
         initrd = "${vmConfig.config.system.build.initialRamdisk}/${vmConfig.config.system.boot.loader.initrdFile}";
         toplevel = vmConfig.config.system.build.toplevel;

pkgs/clan-cli/clan_cli/dirs.py

@@ -56,6 +56,10 @@ def user_history_file() -> Path:
     return user_config_dir() / "clan" / "history"
 
 
+def vm_state_dir(clan_name: str, vm_name: str) -> Path:
+    return user_config_dir() / "clan" / "vmstate" / clan_name / vm_name
+
+
 def machines_dir(flake_dir: Path) -> Path:
     return flake_dir / "machines"
 

pkgs/clan-cli/clan_cli/vms/inspect.py

@@ -25,7 +25,7 @@ def inspect_vm(flake_url: str | Path, flake_attr: str) -> VmConfig:
 
     cmd = nix_eval(
         [
-            f'{flake_url}#clanInternals.machines."{system}"."{flake_attr}".config.system.clan.vm.config'
+            f'{flake_url}#clanInternals.machines."{system}"."{flake_attr}".config.clanCore.vm.inspect'
         ]
     )
 

pkgs/clan-cli/clan_cli/vms/run.py

@@ -11,7 +11,7 @@ from pathlib import Path
 from typing import IO
 
 from ..cmd import run
-from ..dirs import module_root, specific_groot_dir
+from ..dirs import module_root, specific_groot_dir, vm_state_dir
 from ..errors import ClanError
 from ..nix import nix_build, nix_config, nix_shell
 from .inspect import VmConfig, inspect_vm
@@ -82,6 +82,7 @@ def qemu_command(
     nixos_config: dict[str, str],
     xchg_dir: Path,
     secrets_dir: Path,
+    state_dir: Path,
     disk_img: Path,
 ) -> list[str]:
     kernel_cmdline = [
@@ -107,6 +108,7 @@ def qemu_command(
         "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
         "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
         "-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
+        "-virtfs", f"local,path={state_dir},security_model=none,mount_tag=state",
         "-drive", f"cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report",
         "-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
         "-device", "virtio-keyboard",
@@ -253,11 +255,15 @@ def run_vm(
         secrets_dir = generate_secrets(vm, nixos_config, tmpdir, log_fd)
         disk_img = prepare_disk(tmpdir, log_fd)
 
+        state_dir = vm_state_dir(vm.clan_name, machine)
+        state_dir.mkdir(parents=True, exist_ok=True)
+
         qemu_cmd = qemu_command(
             vm,
             nixos_config,
             xchg_dir=xchg_dir,
             secrets_dir=secrets_dir,
+            state_dir=state_dir,
             disk_img=disk_img,
         )