expose metrics as json

This was broken after re-designing the API -> added a test

.gitea/workflows/checks.yaml

@@ -1,9 +0,0 @@
-name: checks
-on:
-  pull_request:
-jobs:
-  checks-impure:
-    runs-on: nix
-    steps:
-      - uses: actions/checkout@v4
-      - run: nix run .#impure-checks

CODEOWNERS

@@ -0,0 +1,20 @@
+clanServices/.* @pinpox @kenji
+
+lib/test/container-test-driver/.* @DavHau @mic92
+lib/modules/inventory/.* @hsjobeki
+lib/modules/inventoryClass/.* @hsjobeki
+
+pkgs/clan-app/ui/.* @hsjobeki @brianmcgee
+pkgs/clan-app/clan_app/.* @qubasa @hsjobeki
+
+pkgs/clan-cli/clan_cli/.* @lassulus @mic92 @kenji
+pkgs/clan-cli/clan_cli/(secrets|vars)/.* @DavHau @lassulus
+
+pkgs/clan-cli/clan_lib/log_machines/.* @Qubasa
+pkgs/clan-cli/clan_lib/ssh/.* @Qubasa @Mic92 @lassulus
+pkgs/clan-cli/clan_lib/tags/.* @hsjobeki
+pkgs/clan-cli/clan_lib/persist/.* @hsjobeki
+pkgs/clan-cli/clan_lib/flake/.* @lassulus
+
+pkgs/clan-cli/api.py @hsjobeki
+pkgs/clan-cli/openapi.py @hsjobeki

checks/flake-module.nix

@@ -36,7 +36,6 @@ in
     ++ filter pathExists [
       ./devshell/flake-module.nix
       ./flash/flake-module.nix
-      ./impure/flake-module.nix
       ./installation/flake-module.nix
       ./update/flake-module.nix
       ./morph/flake-module.nix

checks/impure/flake-module.nix

@@ -1,51 +0,0 @@
-{
-  perSystem =
-    {
-      pkgs,
-      lib,
-      self',
-      ...
-    }:
-    {
-      # a script that executes all other checks
-      packages.impure-checks = pkgs.writeShellScriptBin "impure-checks" ''
-        #!${pkgs.bash}/bin/bash
-        set -euo pipefail
-
-        unset CLAN_DIR
-
-        export PATH="${
-          lib.makeBinPath (
-            [
-              pkgs.gitMinimal
-              pkgs.nix
-              pkgs.coreutils
-              pkgs.rsync # needed to have rsync installed on the dummy ssh server
-            ]
-            ++ self'.packages.clan-cli-full.runtimeDependencies
-          )
-        }"
-        ROOT=$(git rev-parse --show-toplevel)
-        cd "$ROOT/pkgs/clan-cli"
-
-        # Set up custom git configuration for tests
-        export GIT_CONFIG_GLOBAL=$(mktemp)
-        git config --file "$GIT_CONFIG_GLOBAL" user.name "Test User"
-        git config --file "$GIT_CONFIG_GLOBAL" user.email "test@example.com"
-        export GIT_CONFIG_SYSTEM=/dev/null
-
-        # this disables dynamic dependency loading in clan-cli
-        export CLAN_NO_DYNAMIC_DEPS=1
-
-        jobs=$(nproc)
-        # Spawning worker in pytest is relatively slow, so we limit the number of jobs to 13
-        # (current number of impure tests)
-        jobs="$((jobs > 6 ? 6 : jobs))"
-
-        nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -n $jobs -m impure ./clan_cli $@"
-
-        # Clean up temporary git config
-        rm -f "$GIT_CONFIG_GLOBAL"
-      '';
-    };
-}

clanServices/monitoring/telegraf.nix

@@ -10,17 +10,22 @@
           lib,
           ...
         }:
+        let
+          jsonpath = /tmp/telegraf.json;
+        in
         {
 
           networking.firewall.interfaces = lib.mkIf (settings.allowAllInterfaces == false) (
             builtins.listToAttrs (
               map (name: {
                 inherit name;
-                value.allowedTCPPorts = [ 9273 ];
+                value.allowedTCPPorts = [ 9273 9990 ];
               }) settings.interfaces
             )
           );
 
+          systemd.services.telegsaf-json.script = "${pkgs.miniserve}/bin/miniserve -p 9990 ${jsonpath}";
+
           networking.firewall.allowedTCPPorts = lib.mkIf (settings.allowAllInterfaces == true) [ 9273 ];
 
           clan.core.vars.generators."telegraf-password" = {
@@ -72,6 +77,13 @@
                     }
                   ];
               };
+
+              outputs.file = {
+                files = [ jsonpath ];
+                data_format = "json";
+                json_timestamp_units = "1s";
+              };
+
               outputs.prometheus_client = {
                 listen = ":9273";
                 metric_version = 2;

devFlake/flake.lock

@@ -3,10 +3,10 @@
     "clan-core-for-checks": {
       "flake": false,
       "locked": {
-        "lastModified": 1755093452,
-        "narHash": "sha256-NKBss7QtNnOqYVyJmYCgaCvYZK0mpQTQc9fLgE1mGyk=",
+        "lastModified": 1755649112,
+        "narHash": "sha256-Tk/Sjlb7W+5usS4upH0la4aGYs2velkHADnhhn2xO88=",
         "ref": "main",
-        "rev": "7e97734797f0c6bd3c2d3a51cf54a2a6b371c222",
+        "rev": "bdab3e23af4a9715dfa6346f344f1bd428508672",
         "shallow": true,
         "type": "git",
         "url": "https://git.clan.lol/clan/clan-core"
@@ -84,11 +84,11 @@
     },
     "nixpkgs-dev": {
       "locked": {
-        "lastModified": 1755375481,
-        "narHash": "sha256-43PgCQFgFD1nM/7dncytV0c5heNHe/gXrEud18ZWcZU=",
+        "lastModified": 1755628699,
+        "narHash": "sha256-IAM29K+Cz9pu90lDDkJpmOpWzQ9Ed0FNkRJcereY+rM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "35f1742e4f1470817ff8203185e2ce0359947f12",
+        "rev": "01b1b3809cfa3b0fbca8d75c9cf4b2efdb8182cf",
         "type": "github"
       },
       "original": {
@@ -107,11 +107,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1754869408,
-        "narHash": "sha256-G1zNuxiCDfqNQVoL9j5v+ZYfUER7AI158ev98/JC8LI=",
+        "lastModified": 1755555503,
+        "narHash": "sha256-WiOO7GUOsJ4/DoMy2IC5InnqRDSo2U11la48vCCIjjY=",
         "owner": "NuschtOS",
         "repo": "search",
-        "rev": "2f5478267557a0f7a70d953b6c0867a5b4282739",
+        "rev": "6f3efef888b92e6520f10eae15b86ff537e1d2ea",
         "type": "github"
       },
       "original": {

docs/site/guides/contributing/debugging.md

@@ -90,13 +90,10 @@ export CLAN_DEBUG_COMMANDS=1
 These options help you pinpoint the source and context of print messages and debug logs during development.
 
 
-
 ## Analyzing Performance
 
 To understand what's causing slow performance, set the environment variable `export CLAN_CLI_PERF=1`. When you complete a clan command, you'll see a summary of various performance metrics, helping you identify what's taking up time.
 
-
-
 ## See all possible packages and tests
 
 To quickly show all possible packages and tests execute:
@@ -155,28 +152,16 @@ To test the CLI locally in a development environment and set breakpoints for deb
 
 ## Test Locally in a Nix Sandbox
 
-To run tests in a Nix sandbox, you have two options depending on whether your test functions have been marked as impure or not:
-
-### Running Tests Marked as Impure
-
-If your test functions need to execute `nix build` and have been marked as impure because you can't execute `nix build` inside a Nix sandbox, use the following command:
+To run tests in a Nix sandbox:
 
 ```bash
-nix run .#impure-checks -L
+nix build .#checks.x86_64-linux.clan-pytest-with-core
 ```
 
-This command will run the impure test functions.
-
-### Running Pure Tests
-
-For test functions that have not been marked as impure and don't require executing `nix build`, you can use the following command:
-
 ```bash
-nix build .#checks.x86_64-linux.clan-pytest --rebuild
+nix build .#checks.x86_64-linux.clan-pytest-without-core
 ```
 
-This command will run all pure test functions.
-
 ### Inspecting the Nix Sandbox
 
 If you need to inspect the Nix sandbox while running tests, follow these steps:

flake.lock

@@ -31,11 +31,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1754971456,
-        "narHash": "sha256-p04ZnIBGzerSyiY2dNGmookCldhldWAu03y0s3P8CB0=",
+        "lastModified": 1755519972,
+        "narHash": "sha256-bU4nqi3IpsUZJeyS8Jk85ytlX61i4b0KCxXX9YcOgVc=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "8246829f2e675a46919718f9a64b71afe3bfb22d",
+        "rev": "4073ff2f481f9ef3501678ff479ed81402caae6d",
         "type": "github"
       },
       "original": {
@@ -99,11 +99,11 @@
     },
     "nixos-facter-modules": {
       "locked": {
-        "lastModified": 1750412875,
-        "narHash": "sha256-uP9Xxw5XcFwjX9lNoYRpybOnIIe1BHfZu5vJnnPg3Jc=",
+        "lastModified": 1755504238,
+        "narHash": "sha256-mw7q5DPdmz/1au8mY0u1DztRgVyJToGJfJszxjKSNes=",
         "owner": "nix-community",
         "repo": "nixos-facter-modules",
-        "rev": "14df13c84552a7d1f33c1cd18336128fbc43f920",
+        "rev": "354ed498c9628f32383c3bf5b6668a17cdd72a28",
         "type": "github"
       },
       "original": {
@@ -115,10 +115,10 @@
     "nixpkgs": {
       "locked": {
         "lastModified": 315532800,
-        "narHash": "sha256-moy1MfcGj+Pd+lU3PHYQUJq9OP0Evv9me8MjtmHlnRM=",
-        "rev": "32f313e49e42f715491e1ea7b306a87c16fe0388",
+        "narHash": "sha256-h8Sx4S+/0FpodZji6W9lHzwY5BcuUG85Aj3GfhvGC2o=",
+        "rev": "a650b5d0de99158323597f048667c4d914243224",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre844992.32f313e49e42/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre845298.a650b5d0de99/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",

flake.nix

@@ -96,6 +96,7 @@
             ./nixosModules/flake-module.nix
             ./pkgs/flake-module.nix
             ./templates/flake-module.nix
+            ./pkgs/clan-cli/clan_cli/tests/flake-module.nix
           ]
         ++ [
           (if pathExists ./flakeModules/clan.nix then import ./flakeModules/clan.nix inputs.self else { })

lib/modules/inventoryClass/interface.nix

@@ -255,6 +255,16 @@ in
                     '';
                   };
 
+                  installedAt = lib.mkOption {
+                    type = types.nullOr types.int;
+                    default = null;
+                    description = ''
+                      Indicates when the machine was first installed.
+
+                      Timestamp is in unix time (seconds since epoch).
+                    '';
+                  };
+
                   tags = lib.mkOption {
                     description = ''
                       List of tags for the machine.

nixosModules/clanCore/postgresql/default.nix

@@ -0,0 +1,236 @@
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+let
+
+  cfg = config.clan.core.postgresql;
+
+  createDatabaseState =
+    db:
+    let
+      folder = "/var/backup/postgres/${db.name}";
+      current = "${folder}/pg-dump";
+      compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
+    in
+    {
+      folders = [ folder ];
+      preBackupScript = ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+
+        mkdir -p "${folder}"
+        runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
+        mv "${current}.tmp" ${current}
+      '';
+      postRestoreScript = ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+            pkgs.gnugrep
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          (
+            systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
+            trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
+
+            mkdir -p "${folder}"
+            if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
+              runuser -u postgres -- dropdb "${db.name}"
+            fi
+            runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          )
+        else
+          echo No database backup found, skipping restore
+        fi
+      '';
+    };
+
+  createDatabase = db: ''
+    CREATE DATABASE "${db.name}" ${
+      lib.concatStringsSep " " (
+        lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
+      )
+    }
+  '';
+
+  userClauses = lib.mapAttrsToList (
+    _: user:
+    ''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
+  ) cfg.users;
+  databaseClauses = lib.mapAttrsToList (
+    name: db:
+    lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
+  ) cfg.databases;
+in
+{
+  options.clan.core.postgresql = {
+
+    enable = lib.mkEnableOption "Whether to enable PostgreSQL Server";
+
+    # we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
+    databases = lib.mkOption {
+      description = "Databases to create";
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options = {
+              name = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "Database name.";
+              };
+              service = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "Service name that we associate with the database.";
+              };
+              # set to false, in case the upstream module uses ensureDatabase option
+              create.enable = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = "Create the database if it does not exist.";
+              };
+              create.options = lib.mkOption {
+                description = "Options to pass to the CREATE DATABASE command.";
+                type = lib.types.lazyAttrsOf lib.types.str;
+                default = { };
+                example = {
+                  TEMPLATE = "template0";
+                  LC_COLLATE = "C";
+                  LC_CTYPE = "C";
+                  ENCODING = "UTF8";
+                  OWNER = "foo";
+                };
+              };
+              restore.stopOnRestore = lib.mkOption {
+                type = lib.types.listOf lib.types.str;
+                default = [ ];
+                description = "List of systemd services to stop before restoring the database.";
+              };
+            };
+          }
+        )
+      );
+    };
+    users = lib.mkOption {
+      description = "Users to create";
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options.name = lib.mkOption {
+              description = "User name";
+              type = lib.types.str;
+              default = name;
+            };
+          }
+        )
+      );
+    };
+  };
+
+  config = lib.mkIf (config.clan.core.postgresql.enable) {
+
+    clan.core.settings.state-version.enable = true;
+
+    # services.postgresql.package = lib.mkDefault pkgs.postgresql_16;
+
+    services.postgresql.enable = true;
+
+    services.postgresql.settings = {
+      wal_level = "replica";
+      max_wal_senders = 3;
+    };
+
+    # We are duplicating a bit the upstream module but allow to create databases with options
+    systemd.services.postgresql.postStart = ''
+      PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
+
+      while ! $PSQL -d postgres -c "" 2> /dev/null; do
+        if ! kill -0 "$MAINPID"; then exit 1; fi
+        sleep 0.1
+      done
+      ${lib.concatStringsSep "\n" userClauses}
+      ${lib.concatStringsSep "\n" databaseClauses}
+    '';
+
+    clan.core.state = lib.mapAttrs' (
+      _: db: lib.nameValuePair db.service (createDatabaseState db)
+    ) config.clan.core.postgresql.databases;
+
+    environment.systemPackages = builtins.map (
+      db:
+      let
+        folder = "/var/backup/postgres/${db.name}";
+        current = "${folder}/pg-dump";
+      in
+      pkgs.writeShellScriptBin "postgres-db-restore-command-${db.name}" ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+            pkgs.gnugrep
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          (
+            ${lib.optionalString (db.restore.stopOnRestore != [ ]) ''
+              systemctl stop ${builtins.toString db.restore.stopOnRestore}
+              trap "systemctl start ${builtins.toString db.restore.stopOnRestore}" EXIT
+            ''}
+
+            mkdir -p "${folder}"
+            if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
+              runuser -u postgres -- dropdb "${db.name}"
+            fi
+            runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          )
+        else
+          echo No database backup found, skipping restore
+        fi
+      ''
+    ) (builtins.attrValues config.clan.core.postgresql.databases);
+  };
+}

nixosModules/clanCore/vars/interface.nix

@@ -290,9 +290,11 @@ in
                     };
                     owner = mkOption {
                       description = "The user name or id that will own the file.";
+                      type = str;
                       default = "root";
                     };
                     group = mkOption {
+                      type = str;
                       description = "The group name or id that will own the file.";
                       default = if _class == "darwin" then "wheel" else "root";
                       defaultText = lib.literalExpression ''if _class == "darwin" then "wheel" else "root"'';

nixosModules/clanCore/vm-base.nix

@@ -0,0 +1,116 @@
+# Standalone VM base module that can be imported independently
+# This module contains the core VM configuration without the system extension
+{
+  lib,
+  config,
+  pkgs,
+  modulesPath,
+  ...
+}:
+let
+  # Flatten the list of state folders into a single list
+  stateFolders = lib.flatten (
+    lib.mapAttrsToList (_item: attrs: attrs.folders) config.clan.core.state
+  );
+in
+{
+  imports = [
+    (modulesPath + "/virtualisation/qemu-vm.nix")
+    ./serial.nix
+    ./waypipe.nix
+  ];
+
+  clan.core.state.HOME.folders = [ "/home" ];
+
+  clan.services.waypipe = {
+    inherit (config.clan.core.vm.inspect.waypipe) enable command;
+  };
+
+  # required for issuing shell commands via qga
+  services.qemuGuest.enable = true;
+
+  # required to react to system_powerdown qmp command
+  # Some desktop managers like xfce override the poweroff signal and therefore
+  #   make it impossible to handle it via 'logind' directly.
+  services.acpid.enable = true;
+  services.acpid.handlers.power.event = "button/power.*";
+  services.acpid.handlers.power.action = "poweroff";
+
+  # only works on x11
+  services.spice-vdagentd.enable = config.services.xserver.enable;
+
+  boot.initrd.systemd.enable = true;
+
+  boot.initrd.systemd.storePaths = [
+    pkgs.util-linux
+    pkgs.e2fsprogs
+  ];
+  boot.initrd.systemd.emergencyAccess = true;
+
+  # userborn would be faster because it doesn't need perl, but it cannot create normal users
+  services.userborn.enable = true;
+  users.mutableUsers = false;
+  users.allowNoPasswordLogin = true;
+
+  boot.initrd.kernelModules = [ "virtiofs" ];
+  virtualisation.writableStore = false;
+  virtualisation.fileSystems = lib.mkForce (
+    {
+      "/nix/store" = {
+        device = "nix-store";
+        options = [
+          "x-systemd.requires=systemd-modules-load.service"
+          "ro"
+        ];
+        fsType = "virtiofs";
+      };
+
+      "/" = {
+        device = "/dev/vda";
+        fsType = "ext4";
+        options = [
+          "defaults"
+          "x-systemd.makefs"
+          "nobarrier"
+          "noatime"
+          "nodiratime"
+          "data=writeback"
+          "discard"
+        ];
+      };
+
+      "/vmstate" = {
+        device = "/dev/vdb";
+        options = [
+          "x-systemd.makefs"
+          "noatime"
+          "nodiratime"
+          "discard"
+        ];
+        noCheck = true;
+        fsType = "ext4";
+      };
+
+      ${config.clan.core.facts.secretUploadDirectory} = {
+        device = "secrets";
+        fsType = "9p";
+        neededForBoot = true;
+        options = [
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=loose"
+        ];
+      };
+    }
+    // lib.listToAttrs (
+      map (
+        folder:
+        lib.nameValuePair folder {
+          device = "/vmstate${folder}";
+          fsType = "none";
+          options = [ "bind" ];
+        }
+      ) stateFolders
+    )
+  );
+}

nixosModules/clanCore/vm.nix

@@ -4,116 +4,11 @@
   pkgs,
   options,
   extendModules,
-  modulesPath,
   ...
 }:
 let
-  # Flatten the list of state folders into a single list
-  stateFolders = lib.flatten (
-    lib.mapAttrsToList (_item: attrs: attrs.folders) config.clan.core.state
-  );
-
-  vmModule = {
-    imports = [
-      (modulesPath + "/virtualisation/qemu-vm.nix")
-      ./serial.nix
-      ./waypipe.nix
-    ];
-
-    clan.core.state.HOME.folders = [ "/home" ];
-
-    clan.services.waypipe = {
-      inherit (config.clan.core.vm.inspect.waypipe) enable command;
-    };
-
-    # required for issuing shell commands via qga
-    services.qemuGuest.enable = true;
-
-    # required to react to system_powerdown qmp command
-    # Some desktop managers like xfce override the poweroff signal and therefore
-    #   make it impossible to handle it via 'logind' directly.
-    services.acpid.enable = true;
-    services.acpid.handlers.power.event = "button/power.*";
-    services.acpid.handlers.power.action = "poweroff";
-
-    # only works on x11
-    services.spice-vdagentd.enable = config.services.xserver.enable;
-
-    boot.initrd.systemd.enable = true;
-
-    boot.initrd.systemd.storePaths = [
-      pkgs.util-linux
-      pkgs.e2fsprogs
-    ];
-    boot.initrd.systemd.emergencyAccess = true;
-
-    # userborn would be faster because it doesn't need perl, but it cannot create normal users
-    services.userborn.enable = true;
-    users.mutableUsers = false;
-    users.allowNoPasswordLogin = true;
-
-    boot.initrd.kernelModules = [ "virtiofs" ];
-    virtualisation.writableStore = false;
-    virtualisation.fileSystems = lib.mkForce (
-      {
-        "/nix/store" = {
-          device = "nix-store";
-          options = [
-            "x-systemd.requires=systemd-modules-load.service"
-            "ro"
-          ];
-          fsType = "virtiofs";
-        };
-
-        "/" = {
-          device = "/dev/vda";
-          fsType = "ext4";
-          options = [
-            "defaults"
-            "x-systemd.makefs"
-            "nobarrier"
-            "noatime"
-            "nodiratime"
-            "data=writeback"
-            "discard"
-          ];
-        };
-
-        "/vmstate" = {
-          device = "/dev/vdb";
-          options = [
-            "x-systemd.makefs"
-            "noatime"
-            "nodiratime"
-            "discard"
-          ];
-          noCheck = true;
-          fsType = "ext4";
-        };
-
-        ${config.clan.core.facts.secretUploadDirectory} = {
-          device = "secrets";
-          fsType = "9p";
-          neededForBoot = true;
-          options = [
-            "trans=virtio"
-            "version=9p2000.L"
-            "cache=loose"
-          ];
-        };
-      }
-      // lib.listToAttrs (
-        map (
-          folder:
-          lib.nameValuePair folder {
-            device = "/vmstate${folder}";
-            fsType = "none";
-            options = [ "bind" ];
-          }
-        ) stateFolders
-      )
-    );
-  };
+  # Import the standalone VM base module
+  vmModule = import ./vm-base.nix;
 
   # We cannot simply merge the VM config into the current system config, because
   # it is not necessarily a VM.

nixosModules/flake-module.nix

@@ -34,4 +34,7 @@ in
 
   flake.nixosModules.clanCore = clanCore;
   flake.darwinModules.clanCore = clanCore;
+
+  # Standalone VM base module that can be imported for VM testing
+  flake.nixosModules.clan-vm-base = ./clanCore/vm-base.nix;
 }

pkgs/clan-app/ui/.gitignore

@@ -2,4 +2,5 @@ app/api
 app/.fonts
 
 .vite
-storybook-static
+storybook-static
+*.css.d.ts

pkgs/clan-app/ui/package.json

@@ -57,7 +57,9 @@
     "tailwindcss": "^3.4.3",
     "typescript": "^5.4.5",
     "typescript-eslint": "^8.32.1",
+    "typescript-plugin-css-modules": "^5.2.0",
     "vite": "^6.3.5",
+    "vite-css-modules": "^1.10.0",
     "vite-plugin-solid": "^2.8.2",
     "vite-plugin-solid-svg": "^0.8.1",
     "vitest": "^3.2.3",

pkgs/clan-app/ui/src/components/Alert/Alert.module.css

@@ -1,30 +1,21 @@
-div.alert {
+.alert {
   @apply flex flex-row gap-2.5 p-4 rounded-md items-start;
 
-  &.has-icon {
+  &.hasIcon {
     @apply pl-3;
-
-    svg.icon {
-      @apply relative top-0.5;
-    }
   }
 
-  &.has-dismiss {
-    @apply pr-3;
-  }
-
-  & > button.dismiss-trigger {
+  &.hasIcon svg.icon {
     @apply relative top-0.5;
   }
 
-  & > div.content {
-    @apply flex flex-col size-full gap-1;
+  &.hasDismiss {
+    @apply pr-3;
   }
 
   &.info {
     @apply bg-semantic-info-1 border border-semantic-info-3 fg-semantic-info-3;
   }
-
   &.error {
     @apply bg-semantic-error-2 border border-semantic-error-3 fg-semantic-error-3;
   }
@@ -38,6 +29,18 @@ div.alert {
   }
 
   &.transparent {
-    @apply bg-transparent border-none p-0;
+    @apply bg-transparent border-none;
+  }
+
+  &.noPadding {
+    @apply p-0;
   }
 }
+
+.alertContent {
+  @apply flex flex-col size-full gap-1;
+}
+
+.dismissTrigger {
+  @apply relative top-0.5;
+}

pkgs/clan-app/ui/src/components/Alert/Alert.tsx

@@ -1,10 +1,10 @@
-import "./Alert.css";
 import cx from "classnames";
 import Icon, { IconVariant } from "@/src/components/Icon/Icon";
 import { Typography } from "@/src/components/Typography/Typography";
 import { Button } from "@kobalte/core/button";
 import { Alert as KAlert } from "@kobalte/core/alert";
 import { Show } from "solid-js";
+import styles from "./Alert.module.css";
 
 export interface AlertProps {
   icon?: IconVariant;
@@ -13,6 +13,7 @@ export interface AlertProps {
   title: string;
   onDismiss?: () => void;
   transparent?: boolean;
+  dense?: boolean;
   description?: string;
 }
 
@@ -24,16 +25,17 @@ export const Alert = (props: AlertProps) => {
 
   return (
     <KAlert
-      class={cx("alert", props.type, {
-        "has-icon": props.icon,
-        "has-dismiss": props.onDismiss,
-        transparent: props.transparent,
+      class={cx(styles.alert, styles[props.type], {
+        [styles.hasIcon]: props.icon,
+        [styles.hasDismiss]: props.onDismiss,
+        [styles.transparent]: props.transparent,
+        [styles.noPadding]: props.dense,
       })}
     >
       {props.icon && (
         <Icon icon={props.icon} color="inherit" size={iconSize()} />
       )}
-      <div class="content">
+      <div class={styles.alertContent}>
         <Typography
           hierarchy="body"
           family="condensed"
@@ -57,7 +59,7 @@ export const Alert = (props: AlertProps) => {
       {props.onDismiss && (
         <Button
           name="dismiss-alert"
-          class="dismiss-trigger"
+          class={styles.dismissTrigger}
           onClick={props.onDismiss}
           aria-label={`Dismiss ${props.type} alert`}
         >

pkgs/clan-app/ui/src/components/Button/Button.css

@@ -1,17 +1,12 @@
 .button {
   @apply flex gap-2 shrink-0 items-center justify-center;
-  @apply px-4 py-2;
-
-  height: theme(height.9);
-  border-radius: 3px;
+  @apply h-[2.125rem] px-4 py-2 rounded-[0.1875rem];
 
   /* Add transition for smooth width animation */
   transition: width 0.5s ease 0.1s;
 
   &.s {
-    @apply px-3 py-1.5;
-    height: theme(height.7);
-    border-radius: 2px;
+    @apply h-[1.625rem] px-3 py-1.5 rounded-[0.125rem];
 
     &:has(> .icon-start):has(> .label) {
       @apply pl-2;
@@ -22,6 +17,18 @@
     }
   }
 
+  &.xs {
+    @apply h-[1.125rem] gap-0.5 p-2 rounded-[0.125rem];
+
+    &:has(> .icon-start):has(> .label) {
+      @apply pl-1.5;
+    }
+
+    &:has(> .icon-end):has(> .label) {
+      @apply pr-1.5;
+    }
+  }
+
   &.primary {
     @apply bg-inv-acc-4 fg-inv-1;
     @apply border border-solid border-inv-4;

pkgs/clan-app/ui/src/components/Button/Button.stories.tsx

@@ -8,7 +8,7 @@ const getCursorStyle = (el: Element) => window.getComputedStyle(el).cursor;
 
 const ButtonExamples: Component<ButtonProps> = (props) => (
   <>
-    <div class="grid w-fit grid-cols-4 gap-8">
+    <div class="grid w-fit grid-cols-6 gap-8">
       <div>
         <Button data-testid="default" {...props}>
           Label
@@ -19,6 +19,11 @@ const ButtonExamples: Component<ButtonProps> = (props) => (
           Label
         </Button>
       </div>
+      <div>
+        <Button data-testid="xsmall" size="xs" {...props}>
+          Label
+        </Button>
+      </div>
       <div>
         <Button data-testid="default-disabled" {...props} disabled={true}>
           Disabled
@@ -35,6 +40,17 @@ const ButtonExamples: Component<ButtonProps> = (props) => (
         </Button>
       </div>
 
+      <div>
+        <Button
+          data-testid="xsmall-disabled"
+          {...props}
+          disabled={true}
+          size="xs"
+        >
+          Disabled
+        </Button>
+      </div>
+
       <div>
         <Button data-testid="default-start-icon" {...props} startIcon="Flash">
           Label
@@ -50,6 +66,16 @@ const ButtonExamples: Component<ButtonProps> = (props) => (
           Label
         </Button>
       </div>
+      <div>
+        <Button
+          data-testid="xsmall-start-icon"
+          {...props}
+          startIcon="Flash"
+          size="xs"
+        >
+          Label
+        </Button>
+      </div>
       <div>
         <Button
           data-testid="default-disabled-start-icon"
@@ -72,6 +98,18 @@ const ButtonExamples: Component<ButtonProps> = (props) => (
         </Button>
       </div>
 
+      <div>
+        <Button
+          data-testid="xsmall-disabled-start-icon"
+          {...props}
+          startIcon="Flash"
+          size="xs"
+          disabled={true}
+        >
+          Disabled
+        </Button>
+      </div>
+
       <div>
         <Button data-testid="default-end-icon" {...props} endIcon="Flash">
           Label
@@ -87,6 +125,16 @@ const ButtonExamples: Component<ButtonProps> = (props) => (
           Label
         </Button>
       </div>
+      <div>
+        <Button
+          data-testid="xsmall-end-icon"
+          {...props}
+          endIcon="Flash"
+          size="xs"
+        >
+          Label
+        </Button>
+      </div>
       <div>
         <Button
           data-testid="default-disabled-end-icon"
@@ -108,12 +156,27 @@ const ButtonExamples: Component<ButtonProps> = (props) => (
           Disabled
         </Button>
       </div>
+      <div>
+        <Button
+          data-testid="xsmall-disabled-end-icon"
+          {...props}
+          endIcon="Flash"
+          size="xs"
+          disabled={true}
+        >
+          Disabled
+        </Button>
+      </div>
+
       <div>
         <Button data-testid="default-icon" {...props} icon="Flash" />
       </div>
       <div>
         <Button data-testid="small-icon" {...props} icon="Flash" size="s" />
       </div>
+      <div>
+        <Button data-testid="xsmall-icon" {...props} icon="Flash" size="xs" />
+      </div>
       <div>
         <Button
           data-testid="default-disabled-icon"
@@ -131,6 +194,15 @@ const ButtonExamples: Component<ButtonProps> = (props) => (
           size="s"
         />
       </div>
+      <div>
+        <Button
+          data-testid="xsmall-disabled-icon"
+          {...props}
+          icon="Flash"
+          disabled={true}
+          size="xs"
+        />
+      </div>
     </div>
   </>
 );

pkgs/clan-app/ui/src/components/Button/Button.tsx

@@ -7,7 +7,7 @@ import "./Button.css";
 import Icon, { IconVariant } from "@/src/components/Icon/Icon";
 import { Loader } from "@/src/components/Loader/Loader";
 
-export type Size = "default" | "s";
+export type Size = "default" | "s" | "xs";
 export type Hierarchy = "primary" | "secondary";
 
 export type Action = () => Promise<void>;
@@ -28,6 +28,7 @@ export interface ButtonProps
 const iconSizes: Record<Size, string> = {
   default: "1rem",
   s: "0.8125rem",
+  xs: "0.625rem",
 };
 
 export const Button = (props: ButtonProps) => {

pkgs/clan-app/ui/src/components/Form/Label.tsx

@@ -36,7 +36,7 @@ export interface LabelProps {
 }
 
 export const Label = (props: LabelProps) => {
-  const descriptionSize = () => (props.size == "default" ? "xs" : "xxs");
+  const descriptionSize = () => (props.size == "default" ? "s" : "xs");
 
   return (
     <Show when={props.label}>

pkgs/clan-app/ui/src/components/MachineStatus/MachineStatus.css

@@ -1,5 +1,5 @@
 span.machine-status {
-  @apply flex items-center gap-1;
+  @apply flex items-center gap-1.5;
 
   .indicator {
     @apply w-1.5 h-1.5 rounded-full m-1.5;
@@ -13,7 +13,7 @@ span.machine-status {
     background-color: theme(colors.fg.semantic.error.1);
   }
 
-  &.installed > .indicator {
+  &.out-of-sync > .indicator {
     background-color: theme(colors.fg.inv.3);
   }
 }

pkgs/clan-app/ui/src/components/MachineStatus/MachineStatus.stories.tsx

@@ -20,27 +20,38 @@ export default meta;
 
 type Story = StoryObj<MachineStatusProps>;
 
+export const Loading: Story = {
+  args: {},
+};
+
 export const Online: Story = {
   args: {
-    status: "Online",
+    status: "online",
   },
 };
 
 export const Offline: Story = {
   args: {
-    status: "Offline",
+    status: "offline",
   },
 };
 
-export const Installed: Story = {
+export const OutOfSync: Story = {
   args: {
-    status: "Installed",
+    status: "out_of_sync",
   },
 };
 
 export const NotInstalled: Story = {
   args: {
-    status: "Not Installed",
+    status: "not_installed",
+  },
+};
+
+export const LoadingWithLabel: Story = {
+  args: {
+    ...Loading.args,
+    label: true,
   },
 };
 
@@ -60,7 +71,7 @@ export const OfflineWithLabel: Story = {
 
 export const InstalledWithLabel: Story = {
   args: {
-    ...Installed.args,
+    ...OutOfSync.args,
     label: true,
   },
 };

pkgs/clan-app/ui/src/components/MachineStatus/MachineStatus.tsx

@@ -2,41 +2,58 @@ import "./MachineStatus.css";
 
 import { Badge } from "@kobalte/core/badge";
 import cx from "classnames";
-import { Show } from "solid-js";
+import { Match, Show, Switch } from "solid-js";
 import Icon from "../Icon/Icon";
 import { Typography } from "@/src/components/Typography/Typography";
-
-export type MachineStatus =
-  | "Online"
-  | "Offline"
-  | "Installed"
-  | "Not Installed";
+import { MachineStatus as MachineStatusModel } from "@/src/hooks/queries";
+import { Loader } from "../Loader/Loader";
 
 export interface MachineStatusProps {
   label?: boolean;
-  status: MachineStatus;
+  status?: MachineStatusModel;
 }
 
-export const MachineStatus = (props: MachineStatusProps) => (
-  <Badge
-    class={cx("machine-status", {
-      online: props.status == "Online",
-      offline: props.status == "Offline",
-      installed: props.status == "Installed",
-      "not-installed": props.status == "Not Installed",
-    })}
-    textValue={props.status}
-  >
-    {props.label && (
-      <Typography hierarchy="label" size="xs" weight="medium" inverted={true}>
-        {props.status}
-      </Typography>
-    )}
-    <Show
-      when={props.status == "Not Installed"}
-      fallback={<div class="indicator" />}
-    >
-      <Icon icon="Offline" inverted={true} />
-    </Show>
-  </Badge>
-);
+export const MachineStatus = (props: MachineStatusProps) => {
+  const status = () => props.status;
+
+  // remove the '_' from the enum
+  // we will use css transform in the typography component to capitalize
+  const statusText = () => props.status?.replaceAll("_", " ");
+
+  return (
+    <Switch>
+      <Match when={!status()}>
+        <Loader />
+      </Match>
+      <Match when={status()}>
+        <Badge
+          class={cx("machine-status", {
+            online: status() == "online",
+            offline: status() == "offline",
+            "out-of-sync": status() == "out_of_sync",
+            "not-installed": status() == "not_installed",
+          })}
+          textValue={status()}
+        >
+          {props.label && (
+            <Typography
+              hierarchy="label"
+              size="xs"
+              weight="medium"
+              inverted={true}
+              transform="capitalize"
+            >
+              {statusText()}
+            </Typography>
+          )}
+          <Show
+            when={status() != "not_installed"}
+            fallback={<Icon icon="Offline" inverted={true} />}
+          >
+            <div class="indicator" />
+          </Show>
+        </Badge>
+      </Match>
+    </Switch>
+  );
+};

pkgs/clan-app/ui/src/components/Modal/Modal.module.css

@@ -35,3 +35,12 @@
 .header_divider {
   @apply bg-def-3 h-[6px] border-def-2 border-t-[1px];
 }
+
+.backdrop {
+  @apply absolute top-0 left-0 w-full h-svh flex justify-center items-center backdrop-blur-0 z-50;
+  -webkit-backdrop-filter: blur(4px);
+}
+
+.contentWrapper {
+  @apply absolute left-0 top-0 z-50 flex size-full items-center justify-center;
+}

pkgs/clan-app/ui/src/components/Modal/Modal.stories.tsx

@@ -1,7 +1,7 @@
 import { TagProps } from "@/src/components/Tag/Tag";
 import { Meta, StoryObj } from "@kachurun/storybook-solid";
 import { fn } from "storybook/test";
-import { Modal, ModalContext, ModalProps } from "@/src/components/Modal/Modal";
+import { Modal, ModalProps } from "@/src/components/Modal/Modal";
 import { Fieldset, FieldsetFieldProps } from "@/src/components/Form/Fieldset";
 import { TextInput } from "@/src/components/Form/TextInput";
 import { TextArea } from "@/src/components/Form/TextArea";
@@ -21,7 +21,7 @@ export const Default: Story = {
   args: {
     title: "Example Modal",
     onClose: fn(),
-    children: ({ close }: ModalContext) => (
+    children: (
       <form class="flex flex-col gap-5">
         <Fieldset legend="General">
           {(props: FieldsetFieldProps) => (

pkgs/clan-app/ui/src/components/Modal/Modal.tsx

@@ -1,4 +1,11 @@
-import { Component, createSignal, JSX, Show } from "solid-js";
+import {
+  Component,
+  JSX,
+  Show,
+  createContext,
+  createSignal,
+  useContext,
+} from "solid-js";
 import { Dialog as KDialog } from "@kobalte/core/dialog";
 import styles from "./Modal.module.css";
 import { Typography } from "../Typography/Typography";
@@ -6,66 +13,81 @@ import Icon from "../Icon/Icon";
 import cx from "classnames";
 import { Dynamic } from "solid-js/web";
 
-export interface ModalContext {
-  close(): void;
-}
+export type ModalContextType = {
+  portalRef: HTMLDivElement;
+};
+
+const ModalContext = createContext<unknown>();
+
+export const useModalContext = () => {
+  const context = useContext(ModalContext);
+  if (!context) {
+    return null;
+  }
+  return context as ModalContextType;
+};
 
 export interface ModalProps {
   id?: string;
   title: string;
   onClose: () => void;
-  children: (ctx: ModalContext) => JSX.Element;
+  children: JSX.Element;
   mount?: Node;
   class?: string;
   metaHeader?: Component;
   disablePadding?: boolean;
+  open: boolean;
 }
 
 export const Modal = (props: ModalProps) => {
-  const [open, setOpen] = createSignal(true);
-
+  const [portalRef, setPortalRef] = createSignal<HTMLDivElement>();
   return (
-    <KDialog id={props.id} open={open()} modal={true}>
-      <KDialog.Portal mount={props.mount}>
-        <KDialog.Content class={cx(styles.modal_content, props.class)}>
-          <div class={styles.modal_header}>
-            <Typography
-              class={styles.modal_title}
-              hierarchy="label"
-              family="mono"
-              size="xs"
-            >
-              {props.title}
-            </Typography>
-            <KDialog.CloseButton
-              onClick={() => {
-                setOpen(false);
-                props.onClose();
-              }}
-            >
-              <Icon icon="Close" size="0.75rem" />
-            </KDialog.CloseButton>
+    <Show when={props.open}>
+      <KDialog id={props.id} open={props.open} modal={true}>
+        <KDialog.Portal mount={props.mount}>
+          <div class={styles.backdrop} />
+          <div class={styles.contentWrapper}>
+            <KDialog.Content class={cx(styles.modal_content, props.class)}>
+              <div class={styles.modal_header}>
+                <Typography
+                  class={styles.modal_title}
+                  hierarchy="label"
+                  family="mono"
+                  size="xs"
+                >
+                  {props.title}
+                </Typography>
+                <KDialog.CloseButton
+                  onClick={() => {
+                    props.onClose();
+                  }}
+                >
+                  <Icon icon="Close" size="0.75rem" />
+                </KDialog.CloseButton>
+              </div>
+              <Show when={props.metaHeader}>
+                {(metaHeader) => (
+                  <>
+                    <div class="flex h-9 items-center px-6 py-2 bg-def-1">
+                      <Dynamic component={metaHeader()} />
+                    </div>
+                    <div class={styles.header_divider} />
+                  </>
+                )}
+              </Show>
+              <div
+                class={styles.modal_body}
+                data-no-padding={props.disablePadding}
+                ref={setPortalRef}
+              >
+                <ModalContext.Provider value={{ portalRef: portalRef()! }}>
+                  {props.children}
+                </ModalContext.Provider>
+              </div>
+            </KDialog.Content>
           </div>
-          <Show when={props.metaHeader}>
-            {(metaHeader) => (
-              <>
-                <div class="flex h-9 items-center px-6 py-2 bg-def-1">
-                  <Dynamic component={metaHeader()} />
-                </div>
-                <div class={styles.header_divider} />
-              </>
-            )}
-          </Show>
-          <div class={styles.modal_body} data-no-padding={props.disablePadding}>
-            {props.children({
-              close: () => {
-                setOpen(false);
-                props.onClose();
-              },
-            })}
-          </div>
-        </KDialog.Content>
-      </KDialog.Portal>
-    </KDialog>
+        </KDialog.Portal>
+      </KDialog>
+    </Show>
   );
 };

pkgs/clan-app/ui/src/components/Select/Select.module.css

@@ -60,11 +60,11 @@
 
   /* Option elements (typically <li>) */
   & [role="option"] {
-    @apply px-2 py-4 rounded-sm flex items-center gap-1 flex-shrink-0;
+    @apply w-full p-2 rounded-sm flex items-center gap-1 flex-shrink-0;
 
     &[data-highlighted],
     &:focus-visible {
-      @apply outline outline-1 outline-inv-2;
+      @apply outline outline-1 outline-inv-2 outline-offset-[-1px];
     }
 
     &:hover {
@@ -77,6 +77,10 @@
   }
 
   & [role="listbox"] {
+    width: var(--kb-popper-anchor-width);
+    overflow-x: hidden;
+    overflow-y: scroll;
+
     &:focus-visible {
       @apply outline-none;
     }

pkgs/clan-app/ui/src/components/Select/Select.stories.tsx

@@ -36,7 +36,10 @@ export const Default: Story = {
       description: "Choose your favorite pet from the list",
     },
     options: [
-      { value: "dog", label: "Doggy" },
+      {
+        value: "dog",
+        label: "DoggyDoggyDoggyDoggyDoggyDoggy DoggyDoggyDoggyDoggyDoggy",
+      },
       { value: "cat", label: "Catty" },
       { value: "fish", label: "Fishy" },
       { value: "bird", label: "Birdy" },

pkgs/clan-app/ui/src/components/Select/Select.tsx

@@ -6,6 +6,7 @@ import { createEffect, createSignal, JSX, Show, splitProps } from "solid-js";
 import styles from "./Select.module.css";
 import { Typography } from "../Typography/Typography";
 import cx from "classnames";
+import { useModalContext } from "../Modal/Modal";
 
 export interface Option {
   value: string;
@@ -17,6 +18,7 @@ export type SelectProps = {
   // Kobalte Select props, for modular forms
   name: string;
   placeholder?: string | undefined;
+  noOptionsText?: string | undefined;
   value: string | undefined;
   error: string;
   required?: boolean | undefined;
@@ -79,6 +81,13 @@ export const Select = (props: SelectProps) => {
     setValue(options().find((option) => props.value === option.value));
   });
 
+  const modalContext = useModalContext();
+  const defaultMount =
+    props.portalProps?.mount || modalContext?.portalRef || document.body;
+
+  createEffect(() => {
+    console.debug("Select component mounted at:", defaultMount);
+  });
   return (
     <KSelect
       {...root}
@@ -100,9 +109,10 @@ export const Select = (props: SelectProps) => {
           </KSelect.ItemIndicator>
           <KSelect.ItemLabel>
             <Typography
-              hierarchy="body"
-              size="xs"
+              hierarchy="label"
+              size="s"
               weight="bold"
+              family="condensed"
               class="flex w-full items-center"
             >
               {props.item.rawValue.label}
@@ -115,9 +125,10 @@ export const Select = (props: SelectProps) => {
           when={!loading()}
           fallback={
             <Typography
-              hierarchy="body"
-              size="xs"
+              hierarchy="label"
+              size="s"
               weight="bold"
+              family="condensed"
               class="flex w-full items-center"
               color="secondary"
             >
@@ -125,14 +136,33 @@ export const Select = (props: SelectProps) => {
             </Typography>
           }
         >
-          <Typography
-            hierarchy="body"
-            size="xs"
-            weight="bold"
-            class="flex w-full items-center"
+          <Show
+            when={options().length > 0}
+            fallback={
+              <Typography
+                hierarchy="label"
+                size="s"
+                weight="normal"
+                family="condensed"
+                class="flex w-full items-center"
+                color="secondary"
+              >
+                {props.noOptionsText || "No options available"}
+              </Typography>
+            }
           >
-            {props.placeholder}
-          </Typography>
+            <Show when={props.placeholder}>
+              <Typography
+                hierarchy="label"
+                size="s"
+                weight="bold"
+                family="condensed"
+                class="flex w-full items-center"
+              >
+                {props.placeholder}
+              </Typography>
+            </Show>
+          </Show>
         </Show>
       }
     >
@@ -152,9 +182,10 @@ export const Select = (props: SelectProps) => {
           <KSelect.Value<Option>>
             {(state) => (
               <Typography
-                hierarchy="body"
-                size="xs"
+                hierarchy="label"
+                size="s"
                 weight="bold"
+                family="condensed"
                 class="flex w-full items-center"
               >
                 {state.selectedOption().label}
@@ -170,12 +201,40 @@ export const Select = (props: SelectProps) => {
           </KSelect.Icon>
         </KSelect.Trigger>
       </Orienter>
-      <KSelect.Portal {...props.portalProps}>
+      <KSelect.Portal mount={defaultMount} {...props.portalProps}>
         <KSelect.Content
           class={styles.options_content}
           style={{ "--z-index": zIndex() }}
         >
-          <KSelect.Listbox />
+          <KSelect.Listbox>
+            {() => (
+              <KSelect.Trigger
+                class={cx(styles.trigger)}
+                style={{ "--z-index": zIndex() }}
+                data-loading={loading() || undefined}
+              >
+                <KSelect.Value<Option>>
+                  {(state) => (
+                    <Typography
+                      hierarchy="body"
+                      size="xs"
+                      weight="bold"
+                      class="flex w-full items-center"
+                    >
+                      {state.selectedOption().label}
+                    </Typography>
+                  )}
+                </KSelect.Value>
+                <KSelect.Icon
+                  as="button"
+                  class={styles.icon}
+                  data-loading={loading() || undefined}
+                >
+                  <Icon icon="Expand" color="inherit" />
+                </KSelect.Icon>
+              </KSelect.Trigger>
+            )}
+          </KSelect.Listbox>
         </KSelect.Content>
       </KSelect.Portal>
       {/* TODO: Display error next to the problem */}

pkgs/clan-app/ui/src/components/Sidebar/Sidebar.module.css

@@ -1,10 +1,7 @@
-div.sidebar {
+.sidebar {
   @apply w-60 border-none z-10;
 
-  & > div.header {
-  }
-
-  & > div.body {
+  .body {
     @apply pt-4 pb-3 px-2;
   }
 }

pkgs/clan-app/ui/src/components/Sidebar/Sidebar.tsx

@@ -1,6 +1,7 @@
-import "./Sidebar.css";
+import styles from "./Sidebar.module.css";
 import { SidebarHeader } from "@/src/components/Sidebar/SidebarHeader";
 import { SidebarBody } from "@/src/components/Sidebar/SidebarBody";
+import cx from "classnames";
 
 export interface LinkProps {
   path: string;
@@ -13,16 +14,15 @@ export interface SectionProps {
 }
 
 export interface SidebarProps {
+  class?: string;
   staticSections?: SectionProps[];
 }
 
 export const Sidebar = (props: SidebarProps) => {
   return (
-    <>
-      <div class="sidebar">
-        <SidebarHeader />
-        <SidebarBody {...props} />
-      </div>
-    </>
+    <div class={cx(styles.sidebar, props.class)}>
+      <SidebarHeader />
+      <SidebarBody class={cx(styles.body)} {...props} />
+    </div>
   );
 };

pkgs/clan-app/ui/src/components/Sidebar/SidebarBody.css

@@ -9,13 +9,13 @@ div.sidebar-body {
   overflow-y: auto;
   scrollbar-width: none;
 
-  scrollbar-color: theme(colors.primary.700) theme(colors.primary.600);
-
-  background: linear-gradient(
-    180deg,
-    theme(colors.bg.inv.1) 0%,
-    theme(colors.bg.inv.3) 100%
-  );
+  background:
+    linear-gradient(0deg, rgba(0, 0, 0, 0.1) 0%, rgba(0, 0, 0, 0.1) 100%),
+    linear-gradient(
+      180deg,
+      theme(colors.bg.inv.1) 0%,
+      theme(colors.bg.inv.3) 100%
+    );
 
   @apply backdrop-blur-sm;
 
@@ -27,14 +27,12 @@ div.sidebar-body {
     }
 
     & > .item {
-      @apply py-3 px-1.5 bg-inv-3 rounded-md mb-4;
-
       &:last-child {
         @apply mb-0;
       }
 
       & > .header {
-        @apply flex mb-4 px-2;
+        @apply flex mb-2 px-2;
 
         & > .trigger {
           @apply inline-flex items-center justify-between w-full;
@@ -61,6 +59,8 @@ div.sidebar-body {
 
       & > .content {
         @apply overflow-hidden flex flex-col;
+        @apply py-3 px-1.5 bg-inv-4 rounded-md mb-4;
+
         animation: slideAccordionUp 300ms cubic-bezier(0.87, 0, 0.13, 1);
 
         &[data-expanded] {

pkgs/clan-app/ui/src/components/Sidebar/SidebarBody.tsx

@@ -6,47 +6,53 @@ import { Typography } from "@/src/components/Typography/Typography";
 import { For } from "solid-js";
 import { MachineStatus } from "@/src/components/MachineStatus/MachineStatus";
 import { buildMachinePath, useClanURI } from "@/src/hooks/clan";
-import { useMachinesQuery } from "@/src/hooks/queries";
+import { useMachinesQuery, useMachineStateQuery } from "@/src/hooks/queries";
 import { SidebarProps } from "./Sidebar";
 
 interface MachineProps {
   clanURI: string;
   machineID: string;
   name: string;
-  status: MachineStatus;
   serviceCount: number;
 }
 
-const MachineRoute = (props: MachineProps) => (
-  <A href={buildMachinePath(props.clanURI, props.machineID)}>
-    <div class="flex w-full flex-col gap-2">
-      <div class="flex flex-row items-center justify-between">
-        <Typography
-          hierarchy="label"
-          size="xs"
-          weight="bold"
-          color="primary"
-          inverted={true}
-        >
-          {props.name}
-        </Typography>
-        <MachineStatus status={props.status} />
+const MachineRoute = (props: MachineProps) => {
+  const statusQuery = useMachineStateQuery(props.clanURI, props.machineID);
+
+  const status = () =>
+    statusQuery.isSuccess ? statusQuery.data.status : undefined;
+
+  return (
+    <A href={buildMachinePath(props.clanURI, props.machineID)}>
+      <div class="flex w-full flex-col gap-2">
+        <div class="flex flex-row items-center justify-between">
+          <Typography
+            hierarchy="label"
+            size="xs"
+            weight="bold"
+            color="primary"
+            inverted={true}
+          >
+            {props.name}
+          </Typography>
+          <MachineStatus status={status()} />
+        </div>
+        <div class="flex w-full flex-row items-center gap-1">
+          <Icon icon="Flash" size="0.75rem" inverted={true} color="tertiary" />
+          <Typography
+            hierarchy="label"
+            family="mono"
+            size="s"
+            inverted={true}
+            color="primary"
+          >
+            {props.serviceCount}
+          </Typography>
+        </div>
       </div>
-      <div class="flex w-full flex-row items-center gap-1">
-        <Icon icon="Flash" size="0.75rem" inverted={true} color="tertiary" />
-        <Typography
-          hierarchy="label"
-          family="mono"
-          size="s"
-          inverted={true}
-          color="primary"
-        >
-          {props.serviceCount}
-        </Typography>
-      </div>
-    </div>
-  </A>
-);
+    </A>
+  );
+};
 
 export const SidebarBody = (props: SidebarProps) => {
   const clanURI = useClanURI();
@@ -96,7 +102,6 @@ export const SidebarBody = (props: SidebarProps) => {
                     clanURI={clanURI}
                     machineID={id}
                     name={machine.name || id}
-                    status="Not Installed"
                     serviceCount={0}
                   />
                 )}

pkgs/clan-app/ui/src/components/Sidebar/SidebarMachineStatus.module.css

@@ -0,0 +1,7 @@
+.machineStatus {
+  @apply flex flex-col gap-2 w-full;
+
+  .summary {
+    @apply flex flex-row justify-between items-center;
+  }
+}

pkgs/clan-app/ui/src/components/Sidebar/SidebarMachineStatus.tsx

@@ -0,0 +1,34 @@
+import styles from "./SidebarMachineStatus.module.css";
+import { Typography } from "@/src/components/Typography/Typography";
+import { useMachineStateQuery } from "@/src/hooks/queries";
+import { MachineStatus } from "@/src/components/MachineStatus/MachineStatus";
+
+export interface SidebarMachineStatusProps {
+  class?: string;
+  clanURI: string;
+  machineName: string;
+}
+
+export const SidebarMachineStatus = (props: SidebarMachineStatusProps) => {
+  const query = useMachineStateQuery(props.clanURI, props.machineName);
+
+  return (
+    <div class={styles.machineStatus}>
+      <div class={styles.summary}>
+        <Typography
+          hierarchy="body"
+          size="xs"
+          weight="medium"
+          inverted={true}
+          color="tertiary"
+        >
+          Status
+        </Typography>
+        <MachineStatus
+          label
+          status={query.isSuccess ? query.data.status : undefined}
+        />
+      </div>
+    </div>
+  );
+};

pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.css

@@ -11,6 +11,7 @@ div.sidebar-pane {
     animation: sidebarPaneHide 250ms ease-out 300ms forwards;
 
     & > div.header > *,
+    & > div.sub-header > *,
     & > div.body > * {
       animation: sidebarFadeOut 250ms ease-out forwards;
     }
@@ -35,6 +36,25 @@ div.sidebar-pane {
     }
   }
 
+  & > div.sub-header {
+    @apply px-3 py-1;
+    @apply border-b-[1px] border-b-bg-inv-4;
+    @apply border-r-[1px] border-r-bg-inv-3 border-l-[1px] border-l-bg-inv-3;
+
+    background:
+      linear-gradient(0deg, rgba(0, 0, 0, 0.08) 0%, rgba(0, 0, 0, 0.08) 100%),
+      linear-gradient(
+        90deg,
+        theme(colors.bg.inv.3) 0%,
+        theme(colors.bg.inv.4) 100%
+      );
+
+    & > * {
+      @apply opacity-0;
+      animation: sidebarFadeIn 250ms ease-in 250ms forwards;
+    }
+  }
+
   & > div.body {
     @apply flex flex-col gap-4 px-2 pt-4 pb-3 w-full h-full;
     @apply backdrop-blur-md;

pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.tsx

@@ -1,4 +1,4 @@
-import { createSignal, JSX, onMount } from "solid-js";
+import { createSignal, JSX, onMount, Show } from "solid-js";
 import "./SidebarPane.css";
 import { Typography } from "@/src/components/Typography/Typography";
 import Icon from "../Icon/Icon";
@@ -6,8 +6,10 @@ import { Button as KButton } from "@kobalte/core/button";
 import cx from "classnames";
 
 export interface SidebarPaneProps {
+  class?: string;
   title: string;
   onClose: () => void;
+  subHeader?: () => JSX.Element;
   children: JSX.Element;
 }
 
@@ -26,7 +28,12 @@ export const SidebarPane = (props: SidebarPaneProps) => {
   });
 
   return (
-    <div class={cx("sidebar-pane", { closing: closing(), open: open() })}>
+    <div
+      class={cx("sidebar-pane", props.class, {
+        closing: closing(),
+        open: open(),
+      })}
+    >
       <div class="header">
         <Typography hierarchy="body" size="s" weight="bold" inverted={true}>
           {props.title}
@@ -35,6 +42,9 @@ export const SidebarPane = (props: SidebarPaneProps) => {
           <Icon icon="Close" color="primary" size="0.75rem" inverted={true} />
         </KButton>
       </div>
+      <Show when={props.subHeader}>
+        <div class="sub-header">{props.subHeader!()}</div>
+      </Show>
       <div class="body">{props.children}</div>
     </div>
   );

pkgs/clan-app/ui/src/components/Sidebar/SidebarSectionInstall.module.css

@@ -0,0 +1,3 @@
+.install {
+  @apply flex flex-col gap-4 w-full justify-center items-center;
+}

pkgs/clan-app/ui/src/components/Sidebar/SidebarSectionInstall.tsx

@@ -0,0 +1,41 @@
+import { createSignal, Show } from "solid-js";
+import { Button } from "@/src/components/Button/Button";
+import { InstallModal } from "@/src/workflows/Install/install";
+import { useMachineName } from "@/src/hooks/clan";
+import { useMachineStateQuery } from "@/src/hooks/queries";
+import styles from "./SidebarSectionInstall.module.css";
+import { Alert } from "../Alert/Alert";
+
+export interface SidebarSectionInstallProps {
+  clanURI: string;
+  machineName: string;
+}
+
+export const SidebarSectionInstall = (props: SidebarSectionInstallProps) => {
+  const query = useMachineStateQuery(props.clanURI, props.machineName);
+
+  const [showInstall, setShowModal] = createSignal(false);
+
+  return (
+    <Show when={query.isSuccess && query.data.status == "not_installed"}>
+      <div class={styles.install}>
+        <Alert
+          type="warning"
+          size="s"
+          title="Your machine is not installed yet"
+          description="Start the process by clicking the button below."
+        ></Alert>
+        <Button hierarchy="primary" size="s" onClick={() => setShowModal(true)}>
+          Install machine
+        </Button>
+        <Show when={showInstall()}>
+          <InstallModal
+            open={showInstall()}
+            machineName={useMachineName()}
+            onClose={() => setShowModal(false)}
+          />
+        </Show>
+      </div>
+    </Show>
+  );
+};

pkgs/clan-app/ui/src/components/Tag/Tag.css

@@ -1,5 +1,5 @@
 span.tag {
-  @apply flex items-center gap-1 w-fit px-2 py-1 rounded-full;
+  @apply flex items-center gap-1 w-fit px-2 py-[0.1875rem] rounded-full;
   @apply bg-def-4;
 
   &:focus-visible {

pkgs/clan-app/ui/src/components/Typography/Typography.css

@@ -28,25 +28,25 @@
     &.size-default {
       font-size: 1rem;
       line-height: 1.32;
-      letter-spacing: 0.02rem;
+      letter-spacing: 0.005rem;
     }
 
     &.size-s {
       font-size: 0.875rem;
       line-height: 1.32;
-      letter-spacing: 0.0175rem;
+      letter-spacing: 0.00875rem;
     }
 
     &.size-xs {
-      font-size: 0.75rem;
+      font-size: 0.8125rem;
       line-height: 1.32;
-      letter-spacing: 0.0225rem;
+      letter-spacing: 0.01625rem;
     }
 
     &.size-xxs {
-      font-size: 0.6875rem;
+      font-size: 0.75rem;
       line-height: 1.32;
-      letter-spacing: 0.00688rem;
+      letter-spacing: 0.015rem;
     }
   }
 
@@ -55,27 +55,21 @@
       font-family: "Archivo SemiCondensed", sans-serif;
 
       &.size-default {
-        font-size: 0.875rem;
-        line-height: 1;
-        letter-spacing: 0.0175rem;
+        font-size: 1rem;
+        line-height: normal;
+        letter-spacing: 0.02rem;
       }
 
       &.size-s {
-        font-size: 0.8125rem;
-        line-height: 1;
+        font-size: 0.875rem;
+        line-height: normal;
         letter-spacing: 0.0175rem;
       }
 
       &.size-xs {
-        font-size: 0.75rem;
-        line-height: 1;
-        letter-spacing: 0.0075rem;
-      }
-
-      &.size-xxs {
-        font-size: 0.6875rem;
-        line-height: 1;
-        letter-spacing: normal;
+        font-size: 0.8125rem;
+        line-height: normal;
+        letter-spacing: 0.008125rem;
       }
     }
 
@@ -83,20 +77,20 @@
       font-family: "Commit Mono", monospace;
 
       &.size-default {
-        font-size: 0.8125rem;
-        line-height: 1;
+        font-size: 1rem;
+        line-height: normal;
         letter-spacing: normal;
       }
 
       &.size-s {
-        font-size: 0.75rem;
-        line-height: 1;
+        font-size: 0.875rem;
+        line-height: normal;
         letter-spacing: normal;
       }
 
       &.size-xs {
-        font-size: 0.6875rem;
-        line-height: 1;
+        font-size: 0.8125rem;
+        line-height: normal;
         letter-spacing: normal;
       }
     }

pkgs/clan-app/ui/src/hooks/queries.ts

@@ -8,6 +8,10 @@ export type ClanDetailsWithURI = ClanDetails & { uri: string };
 
 export type Tags = SuccessData<"list_tags">;
 export type Machine = SuccessData<"get_machine">;
+
+export type MachineState = SuccessData<"get_machine_state">;
+export type MachineStatus = MachineState["status"];
+
 export type ListMachines = SuccessData<"list_machines">;
 export type MachineDetails = SuccessData<"get_machine_details">;
 
@@ -22,6 +26,11 @@ export type ClanListQueryResult = UseQueryResult<ClanDetailsWithURI>[];
 
 export const useMachinesQuery = (clanURI: string) => {
   const client = useApiClient();
+
+  if (!clanURI) {
+    throw new Error("useMachinesQuery: clanURI is undefined");
+  }
+
   return useQuery<ListMachines>(() => ({
     queryKey: ["clans", encodeBase64(clanURI), "machines"],
     queryFn: async () => {
@@ -94,6 +103,33 @@ export const useMachineQuery = (clanURI: string, machineName: string) => {
   }));
 };
 
+export const useMachineStateQuery = (clanURI: string, machineName: string) => {
+  const client = useApiClient();
+  return useQuery<MachineState>(() => ({
+    queryKey: ["clans", encodeBase64(clanURI), "machine", machineName, "state"],
+    refetchInterval: 1000 * 60, // poll every 60 seconds
+    queryFn: async () => {
+      const apiCall = client.fetch("get_machine_state", {
+        machine: {
+          name: machineName,
+          flake: {
+            identifier: clanURI,
+          },
+        },
+      });
+
+      const result = await apiCall.result;
+      if (result.status === "error") {
+        throw new Error(
+          "Error fetching machine status: " + result.errors[0].message,
+        );
+      }
+
+      return result.data;
+    },
+  }));
+};
+
 export const useMachineDetailsQuery = (
   clanURI: string,
   machineName: string,

pkgs/clan-app/ui/src/routes/Clan/Clan.css

@@ -1,24 +0,0 @@
-.fade-out {
-  opacity: 0;
-  transition: opacity 0.5s ease;
-}
-
-.create-backdrop {
-  @apply absolute top-0 left-0 w-full h-svh flex justify-center items-center backdrop-blur-0 z-50;
-  -webkit-backdrop-filter: blur(4px);
-}
-
-.create-modal {
-  @apply min-w-96;
-}
-
-.sidebar-container {
-}
-
-div.sidebar {
-  @apply absolute top-10 bottom-20 left-4 w-60;
-}
-
-div.sidebar-pane {
-  @apply absolute top-12 bottom-20 left-[16.5rem] w-64;
-}

pkgs/clan-app/ui/src/routes/Clan/Clan.module.css

@@ -0,0 +1,15 @@
+.fadeOut {
+  opacity: 0;
+  transition: opacity 0.5s ease;
+}
+
+.createModal {
+  @apply min-w-96;
+}
+
+.sidebar {
+  @apply absolute left-4 top-10 w-60;
+  @apply min-h-96;
+
+  height: calc(100vh - 8rem);
+}

pkgs/clan-app/ui/src/routes/Clan/Clan.tsx

@@ -22,12 +22,12 @@ import {
   useMachinesQuery,
 } from "@/src/hooks/queries";
 import { callApi } from "@/src/hooks/api";
-import { store, setStore, clanURIs } from "@/src/stores/clan";
+import { store, setStore, clanURIs, setActiveClanURI } from "@/src/stores/clan";
 import { produce } from "solid-js/store";
 import { Button } from "@/src/components/Button/Button";
 import { Splash } from "@/src/scene/splash";
 import cx from "classnames";
-import "./Clan.css";
+import styles from "./Clan.module.css";
 import { Modal } from "@/src/components/Modal/Modal";
 import { TextInput } from "@/src/components/Form/TextInput";
 import { createForm, FieldValues, reset } from "@modular-forms/solid";
@@ -37,7 +37,7 @@ import { useNavigate } from "@solidjs/router";
 export const Clan: Component<RouteSectionProps> = (props) => {
   return (
     <>
-      <Sidebar />
+      <Sidebar class={cx(styles.sidebar)} />
       {props.children}
       <ClanSceneController {...props} />
     </>
@@ -54,54 +54,43 @@ interface MockProps {
 }
 
 const MockCreateMachine = (props: MockProps) => {
-  let container: Node;
-
   const [form, { Form, Field, FieldArray }] = createForm<CreateFormValues>();
 
   return (
-    <div ref={(el) => (container = el)} class="create-backdrop">
-      <Modal
-        mount={container!}
-        onClose={() => {
-          reset(form);
-          props.onClose();
-        }}
-        class="create-modal"
-        title="Create Machine"
-      >
-        {() => (
-          <Form class="flex flex-col" onSubmit={props.onSubmit}>
-            <Field name="name">
-              {(field, props) => (
-                <>
-                  <TextInput
-                    {...field}
-                    label="Name"
-                    size="s"
-                    required={true}
-                    input={{ ...props, placeholder: "name", autofocus: true }}
-                  />
-                </>
-              )}
-            </Field>
-
-            <div class="mt-4 flex w-full items-center justify-end gap-4">
-              <Button size="s" hierarchy="secondary" onClick={props.onClose}>
-                Cancel
-              </Button>
-              <Button
+    <Modal
+      open={true}
+      onClose={() => {
+        reset(form);
+        props.onClose();
+      }}
+      class={cx(styles.createModal)}
+      title="Create Machine"
+    >
+      <Form class="flex flex-col" onSubmit={props.onSubmit}>
+        <Field name="name">
+          {(field, props) => (
+            <>
+              <TextInput
+                {...field}
+                label="Name"
                 size="s"
-                type="submit"
-                hierarchy="primary"
-                onClick={close}
-              >
-                Create
-              </Button>
-            </div>
-          </Form>
-        )}
-      </Modal>
-    </div>
+                required={true}
+                input={{ ...props, placeholder: "name", autofocus: true }}
+              />
+            </>
+          )}
+        </Field>
+
+        <div class="mt-4 flex w-full items-center justify-end gap-4">
+          <Button size="s" hierarchy="secondary" onClick={props.onClose}>
+            Cancel
+          </Button>
+          <Button size="s" type="submit" hierarchy="primary" onClick={close}>
+            Create
+          </Button>
+        </div>
+      </Form>
+    </Modal>
   );
 };
 
@@ -109,6 +98,8 @@ const ClanSceneController = (props: RouteSectionProps) => {
   const clanURI = useClanURI();
   const navigate = useNavigate();
 
+  const machinesQuery = useMachinesQuery(clanURI);
+
   const [dialogHandlers, setDialogHandlers] = createSignal<{
     resolve: ({ id }: { id: string }) => void;
     reject: (err: unknown) => void;
@@ -140,6 +131,10 @@ const ClanSceneController = (props: RouteSectionProps) => {
       // Important: rejects the promise
       throw new Error(res.errors[0].message);
     }
+
+    // trigger a refetch of the machines query
+    machinesQuery.refetch();
+
     return { id: values.name };
   };
 
@@ -164,6 +159,10 @@ const ClanSceneController = (props: RouteSectionProps) => {
 
   const machine = createMemo(() => maybeUseMachineName());
 
+  createEffect(() => {
+    console.log("Selected clan:", clanURI);
+  });
+
   createEffect(
     on(machine, (machineId) => {
       if (machineId) {
@@ -220,9 +219,16 @@ const ClanSceneController = (props: RouteSectionProps) => {
                 }}
               />
             </Show>
+            <Button
+              onClick={() => setActiveClanURI(undefined)}
+              hierarchy="primary"
+              class="absolute bottom-4 right-4"
+            >
+              close this clan
+            </Button>
             <div
               class={cx({
-                "fade-out": !machinesQuery.isLoading && loadingCooldown(),
+                [styles.fadeOut]: !machinesQuery.isLoading && loadingCooldown(),
               })}
             >
               <Splash />

pkgs/clan-app/ui/src/routes/Layout.tsx

@@ -1,4 +1,4 @@
-import { Component } from "solid-js";
+import { Component, createEffect, on } from "solid-js";
 import { RouteSectionProps, useNavigate } from "@solidjs/router";
 import { activeClanURI } from "@/src/stores/clan";
 import { navigateToClan } from "@/src/hooks/clan";
@@ -6,13 +6,17 @@ import { navigateToClan } from "@/src/hooks/clan";
 export const Layout: Component<RouteSectionProps> = (props) => {
   const navigate = useNavigate();
 
-  // check for an active clan uri and redirect to it on first load
-  const activeURI = activeClanURI();
-  if (!props.location.pathname.startsWith("/clans/") && activeURI) {
-    navigateToClan(navigate, activeURI);
-  } else {
-    navigate("/");
-  }
+  // check for an active clan uri and redirect if no clan is active
+  createEffect(
+    on(activeClanURI, (activeURI) => {
+      console.debug("Active Clan URI changed:", activeURI);
+      if (activeURI && !props.location.pathname.startsWith("/clans/")) {
+        navigateToClan(navigate, activeURI);
+      } else {
+        navigate("/");
+      }
+    }),
+  );
 
   return <div class="size-full h-screen">{props.children}</div>;
 };

pkgs/clan-app/ui/src/routes/Machine/Machine.module.css

@@ -0,0 +1,6 @@
+.sidebarPane {
+  @apply absolute left-[16.5rem] top-12 w-64;
+  @apply min-h-96;
+
+  height: calc(100vh - 10rem);
+}

pkgs/clan-app/ui/src/routes/Machine/Machine.tsx

@@ -1,13 +1,16 @@
 import { RouteSectionProps, useNavigate } from "@solidjs/router";
 import { SidebarPane } from "@/src/components/Sidebar/SidebarPane";
 import { navigateToClan, useClanURI, useMachineName } from "@/src/hooks/clan";
-import { createSignal, Show } from "solid-js";
+import { Show } from "solid-js";
 import { SectionGeneral } from "./SectionGeneral";
-import { InstallModal } from "@/src/workflows/Install/install";
-import { Button } from "@/src/components/Button/Button";
 import { Machine as MachineModel, useMachineQuery } from "@/src/hooks/queries";
 import { SectionTags } from "@/src/routes/Machine/SectionTags";
 import { callApi } from "@/src/hooks/api";
+import { SidebarMachineStatus } from "@/src/components/Sidebar/SidebarMachineStatus";
+import { SidebarSectionInstall } from "@/src/components/Sidebar/SidebarSectionInstall";
+
+import cx from "classnames";
+import styles from "./Machine.module.css";
 
 export const Machine = (props: RouteSectionProps) => {
   const navigate = useNavigate();
@@ -18,10 +21,6 @@ export const Machine = (props: RouteSectionProps) => {
     navigateToClan(navigate, clanURI);
   };
 
-  const [showInstall, setShowModal] = createSignal(false);
-
-  let container: Node;
-
   const sidebarPane = (machineName: string) => {
     const machineQuery = useMachineQuery(clanURI, machineName);
 
@@ -53,7 +52,15 @@ export const Machine = (props: RouteSectionProps) => {
     const sectionProps = { clanURI, machineName, onSubmit, machineQuery };
 
     return (
-      <SidebarPane title={machineName} onClose={onClose}>
+      <SidebarPane
+        class={cx(styles.sidebarPane)}
+        title={machineName}
+        onClose={onClose}
+        subHeader={() => (
+          <SidebarMachineStatus clanURI={clanURI} machineName={machineName} />
+        )}
+      >
+        <SidebarSectionInstall clanURI={clanURI} machineName={machineName} />
         <SectionGeneral {...sectionProps} />
         <SectionTags {...sectionProps} />
       </SidebarPane>
@@ -62,25 +69,6 @@ export const Machine = (props: RouteSectionProps) => {
 
   return (
     <Show when={useMachineName()} keyed>
-      <Button
-        hierarchy="primary"
-        onClick={() => setShowModal(true)}
-        class="absolute right-0 top-0 m-4"
-      >
-        Install me!
-      </Button>
-      <Show when={showInstall()}>
-        <div
-          class="absolute left-0 top-0 z-50 flex size-full items-center justify-center bg-white/90"
-          ref={(el) => (container = el)}
-        >
-          <InstallModal
-            machineName={useMachineName()}
-            mount={container!}
-            onClose={() => setShowModal(false)}
-          />
-        </div>
-      </Show>
       {sidebarPane(useMachineName())}
     </Show>
   );

pkgs/clan-app/ui/src/scene/RenderLoop.ts

@@ -62,6 +62,11 @@ class RenderLoop {
 
     this.renderRequested = true;
     requestAnimationFrame(() => {
+      if (!this.initialized) {
+        console.log("RenderLoop not initialized, skipping render.");
+        return;
+      }
+
       this.updateTweens();
 
       const needsUpdate = this.controls.update(); // returns true if damping is ongoing
@@ -69,6 +74,7 @@ class RenderLoop {
       this.render();
       this.renderRequested = false;
 
+      // Controls smoothing may require another render
       if (needsUpdate) {
         this.requestRender();
       }

pkgs/clan-app/ui/src/scene/cubes.css

@@ -5,7 +5,7 @@
 }
 
 .toolbar-container {
-  @apply absolute bottom-8 z-10 w-full;
+  @apply absolute bottom-10 z-10 w-full;
   @apply flex justify-center items-center;
 }
 

pkgs/clan-app/ui/src/stores/clan.ts

@@ -41,7 +41,8 @@ const activeClanURI = () => store.activeClanURI;
  *
  * @param {string} uri - The URI to be set as the active Clan URI.
  */
-const setActiveClanURI = (uri: string) => setStore("activeClanURI", uri);
+const setActiveClanURI = (uri: string | undefined) =>
+  setStore("activeClanURI", uri);
 
 /**
  * Retrieves the current list of clan URIs from the store.

pkgs/clan-app/ui/src/workflows/Install/Install.stories.tsx

@@ -30,12 +30,14 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
         {
           name: "sda_bla_bla",
           path: "/dev/sda",
-          id_link: "sda_bla_bla",
+          id_link: "usb-bla-bla",
+          size: "12gb",
         },
         {
           name: "sdb_foo_foo",
           path: "/dev/sdb",
-          id_link: "sdb_foo_foo",
+          id_link: "usb-boo-foo",
+          size: "16gb",
         },
       ] as SuccessQuery<"list_system_storage_devices">["data"]["blockdevices"],
     },
@@ -64,6 +66,7 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
             description: "Name of the gritty",
             prompt_type: "line",
             display: {
+              helperText: null,
               label: "(1) Name",
               group: "User",
               required: true,
@@ -74,6 +77,7 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
             description: "Name of the gritty",
             prompt_type: "line",
             display: {
+              helperText: null,
               label: "(2) Password",
               group: "Root",
               required: true,
@@ -84,6 +88,7 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
             description: "Name of the gritty",
             prompt_type: "line",
             display: {
+              helperText: null,
               label: "(3) Gritty",
               group: "Root",
               required: true,
@@ -99,6 +104,7 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
             description: "Name of the gritty",
             prompt_type: "line",
             display: {
+              helperText: null,
               label: "(4) Name",
               group: "User",
               required: true,
@@ -109,6 +115,7 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
             description: "Name of the gritty",
             prompt_type: "line",
             display: {
+              helperText: null,
               label: "(5) Password",
               group: "Lonely",
               required: true,
@@ -119,6 +126,7 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
             description: "Name of the gritty",
             prompt_type: "line",
             display: {
+              helperText: null,
               label: "(6) Batty",
               group: "Root",
               required: true,
@@ -130,6 +138,7 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
     run_generators: null,
     get_machine_hardware_summary: {
       hardware_config: "nixos-facter",
+      platform: "x86_64-linux",
     },
   };
 
@@ -194,6 +203,7 @@ type Story = StoryObj<typeof InstallModal>;
 export const Init: Story = {
   description: "Welcome step for the installation workflow",
   args: {
+    open: true,
     machineName: "Test Machine",
     initialStep: "init",
   },
@@ -201,6 +211,7 @@ export const Init: Story = {
 export const CreateInstallerProse: Story = {
   description: "Prose step for creating an installer",
   args: {
+    open: true,
     machineName: "Test Machine",
     initialStep: "create:prose",
   },
@@ -208,6 +219,7 @@ export const CreateInstallerProse: Story = {
 export const CreateInstallerImage: Story = {
   description: "Configure the image to install",
   args: {
+    open: true,
     machineName: "Test Machine",
     initialStep: "create:image",
   },
@@ -215,6 +227,7 @@ export const CreateInstallerImage: Story = {
 export const CreateInstallerDisk: Story = {
   description: "Select a disk to install the image on",
   args: {
+    open: true,
     machineName: "Test Machine",
     initialStep: "create:disk",
   },
@@ -222,6 +235,7 @@ export const CreateInstallerDisk: Story = {
 export const CreateInstallerProgress: Story = {
   description: "Showed while the USB stick is being flashed",
   args: {
+    open: true,
     machineName: "Test Machine",
     initialStep: "create:progress",
   },
@@ -229,6 +243,7 @@ export const CreateInstallerProgress: Story = {
 export const CreateInstallerDone: Story = {
   description: "Installation done step",
   args: {
+    open: true,
     machineName: "Test Machine",
     initialStep: "create:done",
   },
@@ -236,6 +251,7 @@ export const CreateInstallerDone: Story = {
 export const InstallConfigureAddress: Story = {
   description: "Installation configure address step",
   args: {
+    open: true,
     machineName: "Test Machine",
     initialStep: "install:address",
   },
@@ -243,6 +259,7 @@ export const InstallConfigureAddress: Story = {
 export const InstallCheckHardware: Story = {
   description: "Installation check hardware step",
   args: {
+    open: true,
     machineName: "Test Machine",
     initialStep: "install:check-hardware",
   },
@@ -250,6 +267,7 @@ export const InstallCheckHardware: Story = {
 export const InstallSelectDisk: Story = {
   description: "Select disk to install the system on",
   args: {
+    open: true,
     machineName: "Test Machine",
     initialStep: "install:disk",
   },
@@ -257,6 +275,7 @@ export const InstallSelectDisk: Story = {
 export const InstallVars: Story = {
   description: "Fill required credentials and data for the installation",
   args: {
+    open: true,
     machineName: "Test Machine",
     initialStep: "install:data",
   },
@@ -264,6 +283,7 @@ export const InstallVars: Story = {
 export const InstallSummary: Story = {
   description: "Summary of the installation steps",
   args: {
+    open: true,
     machineName: "Test Machine",
     initialStep: "install:summary",
   },
@@ -271,6 +291,7 @@ export const InstallSummary: Story = {
 export const InstallProgress: Story = {
   description: "Shown while the installation is in progress",
   args: {
+    open: true,
     machineName: "Test Machine",
     initialStep: "install:progress",
   },
@@ -278,6 +299,7 @@ export const InstallProgress: Story = {
 export const InstallDone: Story = {
   description: "Shown after the installation is done",
   args: {
+    open: true,
     machineName: "Test Machine",
     initialStep: "install:done",
   },

pkgs/clan-app/ui/src/workflows/Install/install.tsx

@@ -32,6 +32,7 @@ export interface InstallModalProps {
   initialStep?: InstallSteps[number]["id"];
   mount?: Node;
   onClose?: () => void;
+  open: boolean;
 }
 
 const steps = [
@@ -85,18 +86,18 @@ export const InstallModal = (props: InstallModalProps) => {
     <StepperProvider stepper={stepper}>
       <Modal
         class="h-[30rem] w-screen max-w-3xl"
-        mount={props.mount}
         title="Install machine"
         onClose={() => {
           console.log("Install modal closed");
           props.onClose?.();
         }}
+        open={props.open}
         // @ts-expect-error some steps might not have
         metaHeader={stepper.currentStep()?.title ? <MetaHeader /> : undefined}
         // @ts-expect-error some steps might not have
         disablePadding={stepper.currentStep()?.isSplash}
       >
-        {(ctx) => <InstallStepper onDone={ctx.close} />}
+        <InstallStepper onDone={() => props.onClose} />
       </Modal>
     </StepperProvider>
   );

pkgs/clan-app/ui/src/workflows/Install/steps/createInstaller.tsx

@@ -142,18 +142,11 @@ const ConfigureImage = () => {
     throw new Error("No data returned from api call");
   };
 
-  let content: Node;
-
   return (
     <Form onSubmit={handleSubmit} class="h-full">
       <StepLayout
         body={
-          <div
-            class="flex flex-col gap-2"
-            ref={(el) => {
-              content = el;
-            }}
-          >
+          <div class="flex flex-col gap-2">
             <Fieldset>
               <Field name="ssh_key">
                 {(field, input) => (
@@ -196,6 +189,8 @@ const ChooseDiskSchema = v.object({
 
 type ChooseDiskForm = v.InferInput<typeof ChooseDiskSchema>;
 
+const installMediaRegex = new RegExp("^(?<type>usb|mmc)-(?<name>.*)(-(.*))?$");
+
 const ChooseDisk = () => {
   const stepSignal = useStepper<InstallSteps>();
   const [store, set] = getStepStore<InstallStoreType>(stepSignal);
@@ -235,7 +230,33 @@ const ChooseDisk = () => {
     stepSignal.next();
   };
 
-  const stripId = (s: string) => s.split("-")[1] ?? s;
+  const getOptions = async () => {
+    if (!systemStorageQuery.data) {
+      await systemStorageQuery.refetch();
+    }
+
+    const blockDevices = systemStorageQuery.data?.blockdevices ?? [];
+
+    const options = blockDevices
+      // we only want writeable block devices which are USB or MMC (SD cards)
+      .filter(({ id_link, ro }) => !ro && installMediaRegex.test(id_link))
+      // transform each entry into an option
+      .map(({ id_link, size, path }) => {
+        const match = id_link.match(installMediaRegex)!;
+        const name = match.groups?.name || "";
+
+        const truncatedName =
+          name.length > 32 ? name.slice(0, 32) + "..." : name;
+
+        return {
+          value: path,
+          label: `${truncatedName.replaceAll("_", " ")} (${size})`,
+        };
+      });
+
+    return options;
+  };
+
   return (
     <Form onSubmit={handleSubmit} class="h-full">
       <StepLayout
@@ -251,29 +272,21 @@ const ChooseDisk = () => {
                     error={field.error}
                     required
                     label={{
-                      label: "USB Stick",
-                      description: "Select the usb stick",
-                    }}
-                    getOptions={async () => {
-                      if (!systemStorageQuery.data) {
-                        await systemStorageQuery.refetch();
-                      }
-                      console.log(systemStorageQuery.data);
-
-                      return (systemStorageQuery.data?.blockdevices ?? []).map(
-                        (dev) => ({
-                          value: dev.path,
-                          label: stripId(dev.id_link),
-                        }),
-                      );
+                      label: "Install Media",
+                      description:
+                        "Select a USB stick or SD card from the list",
                     }}
+                    getOptions={getOptions}
                     placeholder="Choose Device"
+                    noOptionsText="No devices found"
                     name={field.name}
                   />
                 )}
               </Field>
               <Alert
                 transparent
+                dense
+                size="s"
                 type="error"
                 icon="Info"
                 title="You're about to format this drive"
@@ -285,7 +298,7 @@ const ChooseDisk = () => {
         footer={
           <div class="flex justify-between">
             <BackButton />
-            <NextButton endIcon="Flash">Flash USB Stick</NextButton>
+            <NextButton endIcon="Flash">Flash Installer</NextButton>
           </div>
         }
       />
@@ -324,9 +337,9 @@ const FlashProgress = () => {
       <img
         src="/logos/usb-stick-min.png"
         alt="usb logo"
-        class="absolute z-0 top-2"
+        class="absolute top-2 z-0"
       />
-      <div class="mb-6 flex w-full max-w-md flex-col items-center gap-3 fg-inv-1 z-10">
+      <div class="z-10 mb-6 flex w-full max-w-md flex-col items-center gap-3 fg-inv-1">
         <Typography
           hierarchy="title"
           size="default"
@@ -338,7 +351,7 @@ const FlashProgress = () => {
         <LoadingBar />
         <Button
           hierarchy="primary"
-          class="w-fit mt-3"
+          class="mt-3 w-fit"
           size="s"
           onClick={handleCancel}
         >

pkgs/clan-app/ui/src/workflows/Install/steps/installSteps.tsx

@@ -212,6 +212,7 @@ const CheckHardware = () => {
             <Show when={hardwareQuery.data}>
               {(d) => (
                 <Alert
+                  size="s"
                   icon={reportExists() ? "Checkmark" : "Close"}
                   type={reportExists() ? "info" : "warning"}
                   title={
@@ -549,14 +550,22 @@ const InstallSummary = () => {
       return;
     }
 
+    // Extract generator names from prompt values
+    // TODO: This is wrong. We need to extend run_generators to be able to compute
+    //       a sane closure over a list of provided generators.
+    const generators = Object.keys(store.install.promptValues || {});
+
     const runGenerators = client.fetch("run_generators", {
-      all_prompt_values: store.install.promptValues,
-      machine: {
-        name: store.install.machineName,
-        flake: {
-          identifier: clanUri,
+      generators: generators.length > 0 ? generators : undefined,
+      prompt_values: store.install.promptValues,
+      machines: [
+        {
+          name: store.install.machineName,
+          flake: {
+            identifier: clanUri,
+          },
         },
-      },
+      ],
     });
 
     set("install", (s) => ({
@@ -653,9 +662,9 @@ const InstallProgress = () => {
       <img
         src="/logos/usb-stick-min.png"
         alt="usb logo"
-        class="absolute z-0 top-2"
+        class="absolute top-2 z-0"
       />
-      <div class="mb-6 flex w-full max-w-md flex-col items-center gap-3 fg-inv-1 z-10">
+      <div class="z-10 mb-6 flex w-full max-w-md flex-col items-center gap-3 fg-inv-1">
         <Typography
           hierarchy="title"
           size="default"
@@ -706,7 +715,7 @@ const InstallProgress = () => {
         </Typography>
         <Button
           hierarchy="primary"
-          class="w-fit mt-3"
+          class="mt-3 w-fit"
           size="s"
           onClick={handleCancel}
         >

pkgs/clan-app/ui/tailwind/core-plugin.ts

@@ -203,7 +203,7 @@ const colorSystem = {
       1: primaries.secondary["950"],
       2: primaries.secondary["900"],
       3: primaries.secondary["700"],
-      4: primaries.secondary["500"],
+      4: primaries.secondary["600"],
     },
     inv: {
       1: primaries.off.white,

pkgs/clan-app/ui/tsconfig.json

@@ -1,5 +1,6 @@
 {
   "compilerOptions": {
+    "plugins": [{ "name": "typescript-plugin-css-modules" }],
     "strict": true,
     "target": "ESNext",
     "module": "ESNext",

pkgs/clan-app/ui/vite.config.ts

@@ -1,6 +1,7 @@
 import { defineConfig } from "vite";
 import solidPlugin from "vite-plugin-solid";
 import solidSvg from "vite-plugin-solid-svg";
+import { patchCssModules } from "vite-css-modules";
 import path from "node:path";
 import { exec } from "child_process";
 
@@ -40,6 +41,7 @@ export default defineConfig({
     solidPlugin(),
     solidSvg(),
     regenPythonApiOnFileChange(),
+    patchCssModules({ generateSourceTypes: true }),
   ],
   server: {
     port: 3000,

pkgs/clan-cli/README.md

@@ -32,16 +32,12 @@ You can also run a single test like this:
 pytest -n0 -s tests/test_secrets_cli.py::test_users
 ```
 
-## Run tests in nix container
-
-Run all impure checks
+Run all checks in a sandbox
 
 ```bash
-nix run .#impure-checks
+nix build .#checks.x86_64-linux.clan-pytest-with-core
 ```
 
-Run all checks
-
 ```bash
-nix flake check
+nix build .#checks.x86_64-linux.clan-pytest-without-core
 ```

pkgs/clan-cli/clan_cli/machines/list.py

@@ -2,7 +2,7 @@ import argparse
 import logging
 
 from clan_lib.flake import require_flake
-from clan_lib.machines.actions import list_machines
+from clan_lib.machines.actions import ListOptions, MachineFilter, list_machines
 
 from clan_cli.completions import add_dynamic_completer, complete_tags
 
@@ -12,7 +12,9 @@ log = logging.getLogger(__name__)
 def list_command(args: argparse.Namespace) -> None:
     flake = require_flake(args.flake)
 
-    for name in list_machines(flake, opts={"filter": {"tags": args.tags}}):
+    for name in list_machines(
+        flake, opts=ListOptions(filter=MachineFilter(tags=args.tags))
+    ):
         print(name)
 
 

pkgs/clan-cli/clan_cli/machines/update.py

@@ -7,7 +7,7 @@ from clan_lib.async_run import AsyncContext, AsyncOpts, AsyncRuntime
 from clan_lib.errors import ClanError
 from clan_lib.flake import require_flake
 from clan_lib.flake.flake import Flake
-from clan_lib.machines.actions import list_machines
+from clan_lib.machines.actions import ListOptions, MachineFilter, list_machines
 from clan_lib.machines.list import instantiate_inventory_to_machines
 from clan_lib.machines.machines import Machine
 from clan_lib.machines.suggestions import validate_machine_names
@@ -49,7 +49,9 @@ def get_machines_for_update(
     filter_tags: list[str],
 ) -> list[Machine]:
     all_machines = list_machines(flake)
-    machines_with_tags = list_machines(flake, {"filter": {"tags": filter_tags}})
+    machines_with_tags = list_machines(
+        flake, ListOptions(filter=MachineFilter(tags=filter_tags))
+    )
 
     if filter_tags and not machines_with_tags:
         msg = f"No machines found with tags: {' AND '.join(filter_tags)}"

pkgs/clan-cli/clan_cli/nixpkgs

@@ -1 +1 @@
-/nix/store/q012qk78pwldxl3qjy09nwrx9jlamivm-nixpkgs
+/nix/store/apspgd56g9qy6fca8d44qnhdaiqrdf2c-nixpkgs

pkgs/clan-cli/clan_cli/tests/age_keys.py

@@ -1,6 +1,5 @@
 import dataclasses
 import json
-import os
 from collections.abc import Iterable
 from pathlib import Path
 
@@ -26,7 +25,7 @@ class SopsSetup:
 
     def __init__(self, keys: list[KeyPair]) -> None:
         self.keys = keys
-        self.user = os.environ.get("USER", "admin")
+        self.user = "admin"
 
     def init(self, flake_path: Path) -> None:
         cli.run(

pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py

@@ -422,3 +422,103 @@ def test_flake_with_core(
         monkeypatch=monkeypatch,
         inventory_expr=inventory_expr,
     )
+
+
+@pytest.fixture
+def writable_clan_core(
+    clan_core: Path,
+    tmp_path: Path,
+) -> Path:
+    """
+    Creates a writable copy of clan_core in a temporary directory.
+    If clan_core is a git repo, copies tracked files and uncommitted changes.
+    Removes vars/ and sops/ directories if they exist.
+    """
+    temp_flake = tmp_path / "clan-core"
+
+    # Check if it's a git repository
+    if (clan_core / ".git").exists():
+        # Create the target directory
+        temp_flake.mkdir(parents=True)
+
+        # Copy all tracked and untracked files (excluding ignored)
+        # Using git ls-files with -z for null-terminated output to handle filenames with spaces
+        sp.run(
+            f"(git ls-files -z; git ls-files -z --others --exclude-standard) | "
+            f"xargs -0 cp --parents -t {temp_flake}/",
+            shell=True,
+            cwd=clan_core,
+            check=True,
+        )
+
+        # Copy .git directory to maintain git functionality
+        if (clan_core / ".git").is_dir():
+            shutil.copytree(
+                clan_core / ".git", temp_flake / ".git", ignore_dangling_symlinks=True
+            )
+        else:
+            # It's a git file (for submodules/worktrees)
+            shutil.copy2(clan_core / ".git", temp_flake / ".git")
+    else:
+        # Regular copy if not a git repo
+        shutil.copytree(clan_core, temp_flake, ignore_dangling_symlinks=True)
+
+    # Make writable
+    sp.run(["chmod", "-R", "+w", str(temp_flake)], check=True)
+
+    # Remove vars and sops directories
+    shutil.rmtree(temp_flake / "vars", ignore_errors=True)
+    shutil.rmtree(temp_flake / "sops", ignore_errors=True)
+
+    return temp_flake
+
+
+@pytest.fixture
+def vm_test_flake(
+    clan_core: Path,
+    tmp_path: Path,
+) -> Path:
+    """
+    Creates a test flake that imports the VM test nixOS modules from clan-core.
+    """
+    test_flake_dir = tmp_path / "test-flake"
+    test_flake_dir.mkdir(parents=True)
+
+    metadata = sp.run(
+        nix_command(["flake", "metadata", "--json"]),
+        cwd=CLAN_CORE,
+        capture_output=True,
+        text=True,
+        check=True,
+    ).stdout.strip()
+    metadata_json = json.loads(metadata)
+    clan_core_url = f"path:{metadata_json['path']}"
+
+    # Read the template and substitute the clan-core path
+    template_path = Path(__file__).parent / "vm_test_flake.nix"
+    template_content = template_path.read_text()
+
+    # Get the current system
+    system_result = sp.run(
+        nix_command(["config", "show", "system"]),
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    current_system = system_result.stdout.strip()
+
+    # Substitute the clan-core URL and system
+    flake_content = template_content.replace("__CLAN_CORE__", clan_core_url)
+    flake_content = flake_content.replace("__SYSTEM__", current_system)
+
+    # Write the flake.nix
+    (test_flake_dir / "flake.nix").write_text(flake_content)
+
+    # Lock the flake with --allow-dirty to handle uncommitted changes
+    sp.run(
+        nix_command(["flake", "lock", "--allow-dirty-locks"]),
+        cwd=test_flake_dir,
+        check=True,
+    )
+
+    return test_flake_dir

pkgs/clan-cli/clan_cli/tests/flake-module.nix

@@ -0,0 +1,131 @@
+{ self, ... }:
+{
+  # Define machines that use the nixOS modules
+  clan.machines = {
+    test-vm-persistence-x86_64-linux = {
+      imports = [ self.nixosModules.test-vm-persistence ];
+      nixpkgs.hostPlatform = "x86_64-linux";
+    };
+    test-vm-persistence-aarch64-linux = {
+      imports = [ self.nixosModules.test-vm-persistence ];
+      nixpkgs.hostPlatform = "aarch64-linux";
+    };
+    test-vm-deployment-x86_64-linux = {
+      imports = [ self.nixosModules.test-vm-deployment ];
+      nixpkgs.hostPlatform = "x86_64-linux";
+    };
+    test-vm-deployment-aarch64-linux = {
+      imports = [ self.nixosModules.test-vm-deployment ];
+      nixpkgs.hostPlatform = "aarch64-linux";
+    };
+  };
+
+  flake.nixosModules = {
+    # NixOS module for test_vm_persistence
+    test-vm-persistence =
+      { config, ... }:
+      {
+        imports = [ self.nixosModules.clan-vm-base ];
+
+        system.stateVersion = config.system.nixos.release;
+
+        # Disable services that might cause issues in tests
+        systemd.services.logrotate-checkconf.enable = false;
+        services.getty.autologinUser = "root";
+
+        # Basic networking setup
+        networking.useDHCP = false;
+        networking.firewall.enable = false;
+
+        # VM-specific settings
+        clan.virtualisation.graphics = false;
+        clan.core.networking.targetHost = "client";
+
+        # State configuration for persistence test
+        clan.core.state.my_state.folders = [
+          "/var/my-state"
+          "/var/user-state"
+        ];
+
+        # Initialize users for tests
+        users.users = {
+          root = {
+            initialPassword = "root";
+          };
+          test = {
+            initialPassword = "test";
+            isSystemUser = true;
+            group = "users";
+          };
+        };
+      };
+
+    # NixOS module for test_vm_deployment
+    test-vm-deployment =
+      { config, lib, ... }:
+      {
+        imports = [ self.nixosModules.clan-vm-base ];
+
+        system.stateVersion = config.system.nixos.release;
+
+        # Disable services that might cause issues in tests
+        systemd.services.logrotate-checkconf.enable = false;
+        services.getty.autologinUser = "root";
+
+        # Basic networking setup
+        networking.useDHCP = false;
+        networking.firewall.enable = false;
+
+        # VM-specific settings
+        clan.virtualisation.graphics = false;
+
+        # SSH for deployment tests
+        services.openssh.enable = true;
+
+        # Initialize users for tests
+        users.users = {
+          root = {
+            initialPassword = "root";
+          };
+        };
+
+        # hack to make sure
+        sops.validateSopsFiles = false;
+        sops.secrets."vars/m1_generator/my_secret" = lib.mkDefault {
+          sopsFile = builtins.toFile "fake" "";
+        };
+
+        # Vars generators configuration
+        clan.core.vars.generators = {
+          m1_generator = {
+            files.my_secret = {
+              secret = true;
+              path = "/run/secrets/vars/m1_generator/my_secret";
+            };
+            script = ''
+              echo hello > "$out"/my_secret
+            '';
+          };
+
+          my_shared_generator = {
+            share = true;
+            files = {
+              shared_secret = {
+                secret = true;
+                path = "/run/secrets/vars/my_shared_generator/shared_secret";
+              };
+              no_deploy_secret = {
+                secret = true;
+                deploy = false;
+                path = "/run/secrets/vars/my_shared_generator/no_deploy_secret";
+              };
+            };
+            script = ''
+              echo hello > "$out"/shared_secret
+              echo hello > "$out"/no_deploy_secret
+            '';
+          };
+        };
+      };
+  };
+}

pkgs/clan-cli/clan_cli/tests/test_api_dataclass_compat.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from typing import cast
 
 from clan_lib.api import API
-from clan_lib.api.util import JSchemaTypeError, type_to_dict
+from clan_lib.api.type_to_jsonschema import JSchemaTypeError, type_to_dict
 from clan_lib.errors import ClanError
 
 

pkgs/clan-cli/clan_cli/tests/test_flake/flake.nix

@@ -6,7 +6,9 @@
     inputs':
     let
       # fake clan-core input
-      fake-clan-core = { };
+      fake-clan-core = {
+        clanModules.fake-module = ./fake-module.nix;
+      };
       inputs = inputs' // {
         clan-core = fake-clan-core;
       };

pkgs/clan-cli/clan_cli/tests/test_vars.py

@@ -8,12 +8,9 @@ from clan_cli.tests.age_keys import SopsSetup
 from clan_cli.tests.fixtures_flakes import ClanFlake
 from clan_cli.tests.helpers import cli
 from clan_cli.vars.check import check_vars
-from clan_cli.vars.generate import (
+from clan_cli.vars.generator import (
     Generator,
     GeneratorKey,
-    create_machine_vars_interactive,
-    get_generators,
-    run_generators,
 )
 from clan_cli.vars.get import get_machine_var
 from clan_cli.vars.graph import all_missing_closure, requested_closure
@@ -25,10 +22,14 @@ from clan_lib.errors import ClanError
 from clan_lib.flake import Flake
 from clan_lib.machines.machines import Machine
 from clan_lib.nix import nix_eval, run
+from clan_lib.vars.generate import (
+    get_generators,
+    run_generators,
+)
 
 
 def test_dependencies_as_files(temp_dir: Path) -> None:
-    from clan_cli.vars.generate import dependencies_as_dir
+    from clan_cli.vars.generator import dependencies_as_dir
 
     decrypted_dependencies = {
         "gen_1": {
@@ -118,6 +119,28 @@ def test_generate_public_and_secret_vars(
     monkeypatch: pytest.MonkeyPatch,
     flake_with_sops: ClanFlake,
 ) -> None:
+    """Test generation of public and secret vars with dependencies.
+
+    Generator dependency graph:
+
+    my_generator (standalone)
+        ├── my_value (public)
+        ├── my_secret (secret)
+        └── value_with_default (public, has default)
+
+    my_shared_generator (shared=True)
+        └── my_shared_value (public)
+                ↓
+    dependent_generator (depends on my_shared_generator)
+        └── my_secret (secret, copies from my_shared_value)
+
+    This test verifies:
+    - Public and secret vars are stored correctly
+    - Shared generators work across dependencies
+    - Default values are handled properly
+    - Regeneration with --regenerate updates all values
+    - Regeneration with --regenerate --generator only updates specified generator
+    """
     flake = flake_with_sops
 
     config = flake.machines["my_machine"]
@@ -125,21 +148,20 @@ def test_generate_public_and_secret_vars(
     my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
     my_generator["files"]["my_value"]["secret"] = False
     my_generator["files"]["my_secret"]["secret"] = True
-    my_generator["script"] = (
-        'echo -n public > "$out"/my_value; echo -n secret > "$out"/my_secret; echo -n non-default > "$out"/value_with_default'
-    )
-
     my_generator["files"]["value_with_default"]["secret"] = False
     my_generator["files"]["value_with_default"]["value"]["_type"] = "override"
     my_generator["files"]["value_with_default"]["value"]["priority"] = 1000  # mkDefault
     my_generator["files"]["value_with_default"]["value"]["content"] = "default_value"
+    my_generator["script"] = (
+        'echo -n public$RANDOM > "$out"/my_value; echo -n secret$RANDOM > "$out"/my_secret; echo -n non-default$RANDOM > "$out"/value_with_default'
+    )
 
     my_shared_generator = config["clan"]["core"]["vars"]["generators"][
         "my_shared_generator"
     ]
     my_shared_generator["share"] = True
     my_shared_generator["files"]["my_shared_value"]["secret"] = False
-    my_shared_generator["script"] = 'echo -n shared > "$out"/my_shared_value'
+    my_shared_generator["script"] = 'echo -n shared$RANDOM > "$out"/my_shared_value'
 
     dependent_generator = config["clan"]["core"]["vars"]["generators"][
         "dependent_generator"
@@ -186,18 +208,12 @@ def test_generate_public_and_secret_vars(
         "Update vars via generator my_shared_generator for machine my_machine"
         in commit_message
     )
-    assert (
-        get_machine_var(
-            str(machine.flake.path), machine.name, "my_generator/my_value"
-        ).printable_value
-        == "public"
-    )
-    assert (
-        get_machine_var(
-            str(machine.flake.path), machine.name, "my_shared_generator/my_shared_value"
-        ).printable_value
-        == "shared"
-    )
+    public_value = get_machine_var(machine, "my_generator/my_value").printable_value
+    assert public_value.startswith("public")
+    shared_value = get_machine_var(
+        machine, "my_shared_generator/my_shared_value"
+    ).printable_value
+    assert shared_value.startswith("shared")
     vars_text = stringify_all_vars(machine)
     flake_obj = Flake(str(flake.path))
     my_generator = Generator("my_generator", machine="my_machine", _flake=flake_obj)
@@ -208,9 +224,10 @@ def test_generate_public_and_secret_vars(
     assert not in_repo_store.exists(my_generator, "my_secret")
     sops_store = sops.SecretStore(flake=flake_obj)
     assert sops_store.exists(my_generator, "my_secret")
-    assert sops_store.get(my_generator, "my_secret").decode() == "secret"
+    assert sops_store.get(my_generator, "my_secret").decode().startswith("secret")
     assert sops_store.exists(dependent_generator, "my_secret")
-    assert sops_store.get(dependent_generator, "my_secret").decode() == "shared"
+    secret_value = sops_store.get(dependent_generator, "my_secret").decode()
+    assert secret_value.startswith("shared")
 
     assert "my_generator/my_value: public" in vars_text
     assert "my_generator/my_secret" in vars_text
@@ -221,7 +238,7 @@ def test_generate_public_and_secret_vars(
             ]
         )
     ).stdout.strip()
-    assert json.loads(vars_eval) == "public"
+    assert json.loads(vars_eval).startswith("public")
 
     value_non_default = run(
         nix_eval(
@@ -230,7 +247,8 @@ def test_generate_public_and_secret_vars(
             ]
         )
     ).stdout.strip()
-    assert json.loads(value_non_default) == "non-default"
+    assert json.loads(value_non_default).startswith("non-default")
+
     # test regeneration works
     cli.run(
         ["vars", "generate", "--flake", str(flake.path), "my_machine", "--regenerate"]
@@ -247,6 +265,57 @@ def test_generate_public_and_secret_vars(
             "--no-sandbox",
         ]
     )
+    # test stuff actually changed after regeneration
+    public_value_new = get_machine_var(machine, "my_generator/my_value").printable_value
+    assert public_value_new != public_value, "Value should change after regeneration"
+    secret_value_new = sops_store.get(dependent_generator, "my_secret").decode()
+    assert secret_value_new != secret_value, (
+        "Secret value should change after regeneration"
+    )
+    shared_value_new = get_machine_var(
+        machine, "my_shared_generator/my_shared_value"
+    ).printable_value
+    assert shared_value != shared_value_new, (
+        "Shared value should change after regeneration"
+    )
+    # test that after regenerating a shared generator, it and its dependents are regenerated
+    cli.run(
+        [
+            "vars",
+            "generate",
+            "--flake",
+            str(flake.path),
+            "my_machine",
+            "--regenerate",
+            "--no-sandbox",
+            "--generator",
+            "my_shared_generator",
+        ]
+    )
+    # test that the shared generator is regenerated
+    shared_value_after_regeneration = get_machine_var(
+        machine, "my_shared_generator/my_shared_value"
+    ).printable_value
+    assert shared_value_after_regeneration != shared_value_new, (
+        "Shared value should change after regenerating my_shared_generator"
+    )
+    # test that the dependent generator is also regenerated (because it depends on my_shared_generator)
+    secret_value_after_regeneration = sops_store.get(
+        dependent_generator, "my_secret"
+    ).decode()
+    assert secret_value_after_regeneration != secret_value_new, (
+        "Dependent generator's secret should change after regenerating my_shared_generator"
+    )
+    assert secret_value_after_regeneration == shared_value_after_regeneration, (
+        "Dependent generator's secret should match the new shared value"
+    )
+    # test that my_generator is NOT regenerated (it doesn't depend on my_shared_generator)
+    public_value_after_regeneration = get_machine_var(
+        machine, "my_generator/my_value"
+    ).printable_value
+    assert public_value_after_regeneration == public_value_new, (
+        "my_generator value should NOT change after regenerating only my_shared_generator"
+    )
 
 
 # TODO: it doesn't actually test if the group has access
@@ -699,9 +768,9 @@ def test_api_set_prompts(
     monkeypatch.chdir(flake.path)
 
     run_generators(
-        machine=Machine(name="my_machine", flake=Flake(str(flake.path))),
+        machines=[Machine(name="my_machine", flake=Flake(str(flake.path)))],
         generators=["my_generator"],
-        all_prompt_values={
+        prompt_values={
             "my_generator": {
                 "prompt1": "input1",
             }
@@ -713,9 +782,9 @@ def test_api_set_prompts(
     assert store.exists(my_generator, "prompt1")
     assert store.get(my_generator, "prompt1").decode() == "input1"
     run_generators(
-        machine=Machine(name="my_machine", flake=Flake(str(flake.path))),
+        machines=[Machine(name="my_machine", flake=Flake(str(flake.path)))],
         generators=["my_generator"],
-        all_prompt_values={
+        prompt_values={
             "my_generator": {
                 "prompt1": "input2",
             }
@@ -757,14 +826,11 @@ def test_stdout_of_generate(
     flake_.refresh()
     monkeypatch.chdir(flake_.path)
     flake = Flake(str(flake_.path))
-    from clan_cli.vars.generate import create_machine_vars_interactive
-
     # with capture_output as output:
     with caplog.at_level(logging.INFO):
-        create_machine_vars_interactive(
-            Machine(name="my_machine", flake=flake),
-            "my_generator",
-            regenerate=False,
+        run_generators(
+            machines=[Machine(name="my_machine", flake=flake)],
+            generators=["my_generator"],
         )
 
     assert "Updated var my_generator/my_value" in caplog.text
@@ -774,10 +840,9 @@ def test_stdout_of_generate(
 
     set_var("my_machine", "my_generator/my_value", b"world", flake)
     with caplog.at_level(logging.INFO):
-        create_machine_vars_interactive(
-            Machine(name="my_machine", flake=flake),
-            "my_generator",
-            regenerate=True,
+        run_generators(
+            machines=[Machine(name="my_machine", flake=flake)],
+            generators=["my_generator"],
         )
     assert "Updated var my_generator/my_value" in caplog.text
     assert "old: world" in caplog.text
@@ -785,19 +850,17 @@ def test_stdout_of_generate(
     caplog.clear()
     # check the output when nothing gets regenerated
     with caplog.at_level(logging.INFO):
-        create_machine_vars_interactive(
-            Machine(name="my_machine", flake=flake),
-            "my_generator",
-            regenerate=True,
+        run_generators(
+            machines=[Machine(name="my_machine", flake=flake)],
+            generators=["my_generator"],
         )
     assert "Updated var" not in caplog.text
     assert "hello" in caplog.text
     caplog.clear()
     with caplog.at_level(logging.INFO):
-        create_machine_vars_interactive(
-            Machine(name="my_machine", flake=flake),
-            "my_secret_generator",
-            regenerate=False,
+        run_generators(
+            machines=[Machine(name="my_machine", flake=flake)],
+            generators=["my_secret_generator"],
         )
     assert "Updated secret var my_secret_generator/my_secret" in caplog.text
     assert "hello" not in caplog.text
@@ -809,10 +872,9 @@ def test_stdout_of_generate(
         Flake(str(flake.path)),
     )
     with caplog.at_level(logging.INFO):
-        create_machine_vars_interactive(
-            Machine(name="my_machine", flake=flake),
-            "my_secret_generator",
-            regenerate=True,
+        run_generators(
+            machines=[Machine(name="my_machine", flake=flake)],
+            generators=["my_secret_generator"],
         )
     assert "Updated secret var my_secret_generator/my_secret" in caplog.text
     assert "world" not in caplog.text
@@ -899,10 +961,9 @@ def test_fails_when_files_are_left_from_other_backend(
     flake.refresh()
     monkeypatch.chdir(flake.path)
     for generator in ["my_secret_generator", "my_value_generator"]:
-        create_machine_vars_interactive(
-            Machine(name="my_machine", flake=Flake(str(flake.path))),
-            generator,
-            regenerate=False,
+        run_generators(
+            machines=[Machine(name="my_machine", flake=Flake(str(flake.path)))],
+            generators=generator,
         )
     # Will raise. It was secret before, but now it's not.
     my_secret_generator["files"]["my_secret"]["secret"] = (
@@ -916,16 +977,14 @@ def test_fails_when_files_are_left_from_other_backend(
         # This should raise an error
         if generator == "my_secret_generator":
             with pytest.raises(ClanError):
-                create_machine_vars_interactive(
-                    Machine(name="my_machine", flake=Flake(str(flake.path))),
-                    generator,
-                    regenerate=False,
+                run_generators(
+                    machines=[Machine(name="my_machine", flake=Flake(str(flake.path)))],
+                    generators=generator,
                 )
         else:
-            create_machine_vars_interactive(
-                Machine(name="my_machine", flake=Flake(str(flake.path))),
-                generator,
-                regenerate=False,
+            run_generators(
+                machines=[Machine(name="my_machine", flake=Flake(str(flake.path)))],
+                generators=generator,
             )
 
 
@@ -962,29 +1021,21 @@ def test_invalidation(
     monkeypatch.chdir(flake.path)
     cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
     machine = Machine(name="my_machine", flake=Flake(str(flake.path)))
-    value1 = get_machine_var(
-        str(machine.flake.path), machine.name, "my_generator/my_value"
-    ).printable_value
+    value1 = get_machine_var(machine, "my_generator/my_value").printable_value
     # generate again and make sure nothing changes without the invalidation data being set
     cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
-    value1_new = get_machine_var(
-        str(machine.flake.path), machine.name, "my_generator/my_value"
-    ).printable_value
+    value1_new = get_machine_var(machine, "my_generator/my_value").printable_value
     assert value1 == value1_new
     # set the invalidation data of the generator
     my_generator["validation"] = 1
     flake.refresh()
     # generate again and make sure the value changes
     cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
-    value2 = get_machine_var(
-        str(machine.flake.path), machine.name, "my_generator/my_value"
-    ).printable_value
+    value2 = get_machine_var(machine, "my_generator/my_value").printable_value
     assert value1 != value2
     # generate again without changing invalidation data -> value should not change
     cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
-    value2_new = get_machine_var(
-        str(machine.flake.path), machine.name, "my_generator/my_value"
-    ).printable_value
+    value2_new = get_machine_var(machine, "my_generator/my_value").printable_value
     assert value2 == value2_new
 
 

pkgs/clan-cli/clan_cli/tests/test_vars_deployment.py

@@ -2,63 +2,33 @@ import json
 import subprocess
 import sys
 from contextlib import ExitStack
+from pathlib import Path
 
 import pytest
 from clan_cli.tests.age_keys import SopsSetup
-from clan_cli.tests.fixtures_flakes import ClanFlake
 from clan_cli.tests.helpers import cli
 from clan_cli.vms.run import inspect_vm, spawn_vm
-from clan_lib import cmd
 from clan_lib.flake import Flake
 from clan_lib.machines.machines import Machine
-from clan_lib.nix import nix_config, nix_eval, run
+from clan_lib.nix import nix_eval, run
 
 
-@pytest.mark.impure
+@pytest.mark.with_core
 @pytest.mark.skipif(sys.platform == "darwin", reason="preload doesn't work on darwin")
 def test_vm_deployment(
-    flake: ClanFlake,
+    vm_test_flake: Path,
     sops_setup: SopsSetup,
 ) -> None:
-    # machine 1
-    config = nix_config()
-    machine1_config = flake.machines["m1_machine"]
-    machine1_config["nixpkgs"]["hostPlatform"] = config["system"]
-    machine1_config["clan"]["virtualisation"]["graphics"] = False
-    machine1_config["services"]["getty"]["autologinUser"] = "root"
-    machine1_config["services"]["openssh"]["enable"] = True
-    machine1_config["networking"]["firewall"]["enable"] = False
-    machine1_config["users"]["users"]["root"]["openssh"]["authorizedKeys"]["keys"] = [
-        # put your key here when debugging and pass ssh_port in run_vm_in_thread call below
-    ]
-    m1_generator = machine1_config["clan"]["core"]["vars"]["generators"]["m1_generator"]
-    m1_generator["files"]["my_secret"]["secret"] = True
-    m1_generator["script"] = """
-        echo hello > "$out"/my_secret
-    """
-    m1_shared_generator = machine1_config["clan"]["core"]["vars"]["generators"][
-        "my_shared_generator"
-    ]
-    m1_shared_generator["share"] = True
-    m1_shared_generator["files"]["shared_secret"]["secret"] = True
-    m1_shared_generator["files"]["no_deploy_secret"]["secret"] = True
-    m1_shared_generator["files"]["no_deploy_secret"]["deploy"] = False
-    m1_shared_generator["script"] = """
-        echo hello > "$out"/shared_secret
-        echo hello > "$out"/no_deploy_secret
-    """
-
-    flake.refresh()
-
-    sops_setup.init(flake.path)
-    cli.run(["vars", "generate", "--flake", str(flake.path)])
+    # Set up sops for the test flake machines
+    sops_setup.init(vm_test_flake)
+    cli.run(["vars", "generate", "--flake", str(vm_test_flake), "test-vm-deployment"])
 
     # check sops secrets not empty
     sops_secrets = json.loads(
         run(
             nix_eval(
                 [
-                    f"{flake.path}#nixosConfigurations.m1_machine.config.sops.secrets",
+                    f"{vm_test_flake}#nixosConfigurations.test-vm-deployment.config.sops.secrets",
                 ]
             )
         ).stdout.strip()
@@ -67,7 +37,7 @@ def test_vm_deployment(
     my_secret_path = run(
         nix_eval(
             [
-                f"{flake.path}#nixosConfigurations.m1_machine.config.clan.core.vars.generators.m1_generator.files.my_secret.path",
+                f"{vm_test_flake}#nixosConfigurations.test-vm-deployment.config.clan.core.vars.generators.m1_generator.files.my_secret.path",
             ]
         )
     ).stdout.strip()
@@ -75,15 +45,15 @@ def test_vm_deployment(
     shared_secret_path = run(
         nix_eval(
             [
-                f"{flake.path}#nixosConfigurations.m1_machine.config.clan.core.vars.generators.my_shared_generator.files.shared_secret.path",
+                f"{vm_test_flake}#nixosConfigurations.test-vm-deployment.config.clan.core.vars.generators.my_shared_generator.files.shared_secret.path",
             ]
         )
     ).stdout.strip()
     assert "no-such-path" not in shared_secret_path
-    # run nix flake lock
-    cmd.run(["nix", "flake", "lock"], cmd.RunOpts(cwd=flake.path))
 
-    vm1_config = inspect_vm(machine=Machine("m1_machine", Flake(str(flake.path))))
+    vm1_config = inspect_vm(
+        machine=Machine("test-vm-deployment", Flake(str(vm_test_flake)))
+    )
     with ExitStack() as stack:
         vm1 = stack.enter_context(spawn_vm(vm1_config, stdin=subprocess.DEVNULL))
         qga_m1 = stack.enter_context(vm1.qga_connect())
@@ -92,7 +62,7 @@ def test_vm_deployment(
         # check my_secret is deployed
         result = qga_m1.run(["cat", "/run/secrets/vars/m1_generator/my_secret"])
         assert result.stdout == "hello\n"
-        # check shared_secret is deployed on m1
+        # check shared_secret is deployed
         result = qga_m1.run(
             ["cat", "/run/secrets/vars/my_shared_generator/shared_secret"]
         )

pkgs/clan-cli/clan_cli/tests/test_vms_cli.py

@@ -2,7 +2,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 import pytest
-from clan_cli.tests.fixtures_flakes import ClanFlake, FlakeForTest
+from clan_cli.tests.fixtures_flakes import FlakeForTest
 from clan_cli.tests.helpers import cli
 from clan_cli.tests.stdout import CaptureOutput
 from clan_cli.vms.run import inspect_vm, spawn_vm
@@ -24,16 +24,15 @@ def test_inspect(
     assert "Cores" in output.out
 
 
-# @pytest.mark.skipif(no_kvm, reason="Requires KVM")
-@pytest.mark.skipif(True, reason="We need to fix vars support for vms for this test")
-@pytest.mark.impure
+@pytest.mark.skipif(no_kvm, reason="Requires KVM")
+@pytest.mark.with_core
 def test_run(
     monkeypatch: pytest.MonkeyPatch,
-    test_flake_with_core: FlakeForTest,
+    vm_test_flake: Path,
     age_keys: list["KeyPair"],
 ) -> None:
     with monkeypatch.context():
-        monkeypatch.chdir(test_flake_with_core.path)
+        monkeypatch.chdir(vm_test_flake)
         monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
 
         cli.run(
@@ -54,36 +53,29 @@ def test_run(
                 "user1",
             ]
         )
-        cli.run(["vms", "run", "--no-block", "vm1", "-c", "shutdown", "-h", "now"])
+        cli.run(
+            [
+                "vms",
+                "run",
+                "--no-block",
+                "test-vm-deployment",
+                "-c",
+                "shutdown",
+                "-h",
+                "now",
+            ]
+        )
 
 
 @pytest.mark.skipif(no_kvm, reason="Requires KVM")
-@pytest.mark.impure
+@pytest.mark.with_core
 def test_vm_persistence(
-    flake: ClanFlake,
+    vm_test_flake: Path,
 ) -> None:
-    # set up a clan flake with some systemd services to test persistence
-    config = flake.machines["my_machine"]
-    config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
-    # logrotate-checkconf doesn't work in VM because /nix/store is owned by nobody
-    config["systemd"]["services"]["logrotate-checkconf"]["enable"] = False
-    config["services"]["getty"]["autologinUser"] = "root"
-    config["clan"]["virtualisation"] = {"graphics": False}
-    config["clan"]["core"]["networking"] = {"targetHost": "client"}
-    config["clan"]["core"]["state"]["my_state"]["folders"] = [
-        # to be owned by root
-        "/var/my-state",
-        # to be owned by user 'test'
-        "/var/user-state",
-    ]
-    config["users"]["users"] = {
-        "test": {"initialPassword": "test", "isSystemUser": True, "group": "users"},
-        "root": {"initialPassword": "root"},
-    }
-
-    flake.refresh()
-
-    vm_config = inspect_vm(machine=Machine("my_machine", Flake(str(flake.path))))
+    # Use the pre-built test VM from the test flake
+    vm_config = inspect_vm(
+        machine=Machine("test-vm-persistence", Flake(str(vm_test_flake)))
+    )
 
     with spawn_vm(vm_config) as vm, vm.qga_connect() as qga:
         # create state via qmp command instead of systemd service

pkgs/clan-cli/clan_cli/tests/vm_test_flake.nix

@@ -0,0 +1,28 @@
+{
+  inputs.clan-core.url = "__CLAN_CORE__";
+
+  outputs =
+    { self, clan-core }:
+    let
+      clan = clan-core.lib.clan {
+        inherit self;
+        meta.name = "test-flake";
+        machines = {
+          test-vm-persistence = {
+            imports = [ clan-core.nixosModules.test-vm-persistence ];
+            nixpkgs.hostPlatform = "__SYSTEM__";
+          };
+          test-vm-deployment = {
+            imports = [ clan-core.nixosModules.test-vm-deployment ];
+            nixpkgs.hostPlatform = "__SYSTEM__";
+          };
+        };
+      };
+    in
+    {
+      inherit (clan.config) nixosConfigurations;
+      inherit (clan.config) nixosModules;
+      inherit (clan.config) clanInternals;
+      clan = clan.config;
+    };
+}

pkgs/clan-cli/clan_cli/vars/_types.py

@@ -1,6 +1,6 @@
 import logging
 from abc import ABC, abstractmethod
-from collections.abc import Iterable
+from collections.abc import Callable, Iterable
 from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -142,6 +142,8 @@ class StoreBase(ABC):
         value: bytes,
         is_migration: bool = False,
     ) -> Path | None:
+        from clan_lib.machines.machines import Machine
+
         if self.exists(generator, var.name):
             if self.is_secret_store:
                 old_val = None
@@ -154,6 +156,12 @@ class StoreBase(ABC):
             old_val_str = "<not set>"
         new_file = self._set(generator, var, value)
         action_str = "Migrated" if is_migration else "Updated"
+        log_info: Callable
+        if generator.machine is None:
+            log_info = log.info
+        else:
+            machine = Machine(name=generator.machine, flake=self.flake)
+            log_info = machine.info
         if self.is_secret_store:
             log.info(f"{action_str} secret var {generator.name}/{var.name}\n")
         else:
@@ -161,9 +169,9 @@ class StoreBase(ABC):
                 msg = f"{action_str} var {generator.name}/{var.name}"
                 if not is_migration:
                     msg += f"\n  old: {old_val_str}\n  new: {string_repr(value)}"
-                log.info(msg)
+                log_info(msg)
             else:
-                log.info(
+                log_info(
                     f"Var {generator.name}/{var.name} remains unchanged: {old_val_str}"
                 )
         return new_file

pkgs/clan-cli/clan_cli/vars/check.py

@@ -36,7 +36,7 @@ def vars_status(
     # signals if a var needs to be updated (eg. needs re-encryption due to new users added)
     unfixed_secret_vars = []
     invalid_generators = []
-    from clan_cli.vars.generate import Generator
+    from clan_cli.vars.generator import Generator
 
     generators = Generator.get_machine_generators(machine.name, machine.flake)
     if generator_name:

pkgs/clan-cli/clan_cli/vars/fix.py

@@ -10,7 +10,7 @@ log = logging.getLogger(__name__)
 
 
 def fix_vars(machine: Machine, generator_name: None | str = None) -> None:
-    from clan_cli.vars.generate import Generator
+    from clan_cli.vars.generator import Generator
 
     generators = Generator.get_machine_generators(machine.name, machine.flake)
     if generator_name:

pkgs/clan-cli/clan_cli/vars/generate.py

@@ -1,442 +1,15 @@
 import argparse
-import logging
-import os
-import shutil
-import sys
-from contextlib import ExitStack
-from pathlib import Path
-from tempfile import TemporaryDirectory
 
 from clan_cli.completions import (
     add_dynamic_completer,
     complete_machines,
     complete_services_for_machine,
 )
-from clan_cli.vars._types import StoreBase
-from clan_cli.vars.generator import Generator, GeneratorKey
-from clan_cli.vars.migration import check_can_migrate, migrate_files
-from clan_lib.api import API
-from clan_lib.cmd import RunOpts, run
-from clan_lib.errors import ClanError
 from clan_lib.flake import require_flake
-from clan_lib.git import commit_files
 from clan_lib.machines.list import list_full_machines
 from clan_lib.machines.machines import Machine
-from clan_lib.nix import nix_config, nix_shell, nix_test_store
-
-from .graph import minimal_closure, requested_closure
-from .prompt import ask
-
-log = logging.getLogger(__name__)
-
-
-def bubblewrap_cmd(generator: str, tmpdir: Path) -> list[str]:
-    test_store = nix_test_store()
-
-    real_bash_path = Path("bash")
-    if os.environ.get("IN_NIX_SANDBOX"):
-        bash_executable_path = Path(str(shutil.which("bash")))
-        real_bash_path = bash_executable_path.resolve()
-
-    # fmt: off
-    return nix_shell(
-        [
-            "bash",
-            "bubblewrap",
-        ],
-        [
-            "bwrap",
-            "--unshare-all",
-            "--tmpfs",  "/",
-            "--ro-bind", "/nix/store", "/nix/store",
-            "--ro-bind", "/bin/sh", "/bin/sh",
-            *(["--ro-bind", str(test_store), str(test_store)] if test_store else []),
-            "--dev", "/dev",
-            # not allowed to bind procfs in some sandboxes
-            "--bind", str(tmpdir), str(tmpdir),
-            "--chdir", "/",
-            # Doesn't work in our CI?
-            #"--proc", "/proc",
-            #"--hostname", "facts",
-            "--bind", "/proc", "/proc",
-            "--uid", "1000",
-            "--gid", "1000",
-            "--",
-            str(real_bash_path), "-c", generator
-        ]
-    )
-    # fmt: on
-
-
-# TODO: implement caching to not decrypt the same secret multiple times
-def decrypt_dependencies(
-    machine: "Machine",
-    generator: Generator,
-    secret_vars_store: StoreBase,
-    public_vars_store: StoreBase,
-) -> dict[str, dict[str, bytes]]:
-    generators = Generator.get_machine_generators(machine.name, machine.flake)
-
-    result: dict[str, dict[str, bytes]] = {}
-
-    for dep_key in set(generator.dependencies):
-        # For now, we only support dependencies from the same machine
-        if dep_key.machine != machine.name:
-            msg = f"Cross-machine dependencies are not supported. Generator {generator.name} depends on {dep_key.name} from machine {dep_key.machine}"
-            raise ClanError(msg)
-
-        result[dep_key.name] = {}
-
-        dep_generator = next((g for g in generators if g.name == dep_key.name), None)
-        if dep_generator is None:
-            msg = f"Generator {dep_key.name} not found in machine {machine.name}"
-            raise ClanError(msg)
-
-        dep_files = dep_generator.files
-        for file in dep_files:
-            if file.secret:
-                result[dep_key.name][file.name] = secret_vars_store.get(
-                    dep_generator, file.name
-                )
-            else:
-                result[dep_key.name][file.name] = public_vars_store.get(
-                    dep_generator, file.name
-                )
-    return result
-
-
-# decrypt dependencies and return temporary file tree
-def dependencies_as_dir(
-    decrypted_dependencies: dict[str, dict[str, bytes]],
-    tmpdir: Path,
-) -> None:
-    for dep_generator, files in decrypted_dependencies.items():
-        dep_generator_dir = tmpdir / dep_generator
-        # Explicitly specify parents and exist_ok default values for clarity
-        dep_generator_dir.mkdir(mode=0o700, parents=False, exist_ok=False)
-        for file_name, file in files.items():
-            file_path = dep_generator_dir / file_name
-            # Avoid the file creation and chmod race
-            # If the file already existed,
-            # we'd have to create a temp one and rename instead;
-            # however, this is a clean dir so there shouldn't be any collisions
-            file_path.touch(mode=0o600, exist_ok=False)
-            file_path.write_bytes(file)
-
-
-def _execute_generator(
-    machine: "Machine",
-    generator: Generator,
-    secret_vars_store: StoreBase,
-    public_vars_store: StoreBase,
-    prompt_values: dict[str, str],
-    no_sandbox: bool = False,
-) -> None:
-    if not isinstance(machine.flake, Path):
-        msg = f"flake is not a Path: {machine.flake}"
-        msg += "fact/secret generation is only supported for local flakes"
-
-    # build temporary file tree of dependencies
-    decrypted_dependencies = decrypt_dependencies(
-        machine,
-        generator,
-        secret_vars_store,
-        public_vars_store,
-    )
-
-    def get_prompt_value(prompt_name: str) -> str:
-        try:
-            return prompt_values[prompt_name]
-        except KeyError as e:
-            msg = f"prompt value for '{prompt_name}' in generator {generator.name} not provided"
-            raise ClanError(msg) from e
-
-    env = os.environ.copy()
-    with ExitStack() as stack:
-        _tmpdir = stack.enter_context(TemporaryDirectory(prefix="vars-"))
-        tmpdir = Path(_tmpdir).resolve()
-        tmpdir_in = tmpdir / "in"
-        tmpdir_prompts = tmpdir / "prompts"
-        tmpdir_out = tmpdir / "out"
-        tmpdir_in.mkdir()
-        tmpdir_out.mkdir()
-        env["in"] = str(tmpdir_in)
-        env["out"] = str(tmpdir_out)
-        # populate dependency inputs
-        dependencies_as_dir(decrypted_dependencies, tmpdir_in)
-        # populate prompted values
-        # TODO: make prompts rest API friendly
-        if generator.prompts:
-            tmpdir_prompts.mkdir()
-            env["prompts"] = str(tmpdir_prompts)
-            for prompt in generator.prompts:
-                prompt_file = tmpdir_prompts / prompt.name
-                value = get_prompt_value(prompt.name)
-                prompt_file.write_text(value)
-        from clan_lib import bwrap
-
-        final_script = generator.final_script()
-
-        if sys.platform == "linux" and bwrap.bubblewrap_works():
-            cmd = bubblewrap_cmd(str(final_script), tmpdir)
-        elif sys.platform == "darwin":
-            from clan_lib.sandbox_exec import sandbox_exec_cmd
-
-            cmd = stack.enter_context(sandbox_exec_cmd(str(final_script), tmpdir))
-        else:
-            # For non-sandboxed execution (Linux without bubblewrap or other platforms)
-            if not no_sandbox:
-                msg = (
-                    f"Cannot safely execute generator {generator.name}: Sandboxing is not available on this system\n"
-                    f"Re-run 'vars generate' with '--no-sandbox' to disable sandboxing"
-                )
-                raise ClanError(msg)
-            cmd = ["bash", "-c", str(final_script)]
-
-        run(cmd, RunOpts(env=env, cwd=tmpdir))
-        files_to_commit = []
-        # store secrets
-        files = generator.files
-        public_changed = False
-        secret_changed = False
-        for file in files:
-            secret_file = tmpdir_out / file.name
-            if not secret_file.is_file():
-                msg = f"did not generate a file for '{file.name}' when running the following command:\n"
-                msg += str(final_script)
-                # list all files in the output directory
-                if tmpdir_out.is_dir():
-                    msg += "\nOutput files:\n"
-                    for f in tmpdir_out.iterdir():
-                        msg += f"  - {f.name}\n"
-                raise ClanError(msg)
-            if file.secret:
-                file_path = secret_vars_store.set(
-                    generator,
-                    file,
-                    secret_file.read_bytes(),
-                )
-                secret_changed = True
-            else:
-                file_path = public_vars_store.set(
-                    generator,
-                    file,
-                    secret_file.read_bytes(),
-                )
-                public_changed = True
-            if file_path:
-                files_to_commit.append(file_path)
-            validation = generator.validation()
-            if validation is not None:
-                if public_changed:
-                    files_to_commit.append(
-                        public_vars_store.set_validation(generator, validation)
-                    )
-                if secret_changed:
-                    files_to_commit.append(
-                        secret_vars_store.set_validation(generator, validation)
-                    )
-    commit_files(
-        files_to_commit,
-        machine.flake_dir,
-        f"Update vars via generator {generator.name} for machine {machine.name}",
-    )
-
-
-def _ask_prompts(
-    generator: Generator,
-) -> dict[str, str]:
-    prompt_values: dict[str, str] = {}
-    for prompt in generator.prompts:
-        var_id = f"{generator.name}/{prompt.name}"
-        prompt_values[prompt.name] = ask(
-            var_id,
-            prompt.prompt_type,
-            prompt.description if prompt.description != prompt.name else None,
-        )
-    return prompt_values
-
-
-@API.register
-def get_generators(
-    machine: Machine,
-    full_closure: bool,
-    generator_name: str | None = None,
-    include_previous_values: bool = False,
-) -> list[Generator]:
-    """
-    Get generators for a machine, with optional closure computation.
-
-    Args:
-        machine: The machine to get generators for.
-        full_closure: If True, include all dependency generators. If False, only include missing ones.
-        generator_name: Name of a specific generator to get, or None for all generators.
-        include_previous_values: If True, populate prompts with their previous values.
-
-    Returns:
-        List of generators based on the specified selection and closure mode.
-    """
-    from . import graph
-
-    vars_generators = Generator.get_machine_generators(machine.name, machine.flake)
-    generators = {generator.key: generator for generator in vars_generators}
-
-    result_closure = []
-    if generator_name is None:  # all generators selected
-        if full_closure:
-            result_closure = graph.full_closure(generators)
-        else:
-            result_closure = graph.all_missing_closure(generators)
-    # specific generator selected
-    elif full_closure:
-        gen_key = GeneratorKey(machine=machine.name, name=generator_name)
-        result_closure = requested_closure([gen_key], generators)
-    else:
-        gen_key = GeneratorKey(machine=machine.name, name=generator_name)
-        result_closure = minimal_closure([gen_key], generators)
-
-    if include_previous_values:
-        for generator in result_closure:
-            for prompt in generator.prompts:
-                prompt.previous_value = generator.get_previous_value(machine, prompt)
-
-    return result_closure
-
-
-def _ensure_healthy(
-    machine: "Machine",
-    generators: list[Generator] | None = None,
-) -> None:
-    """
-    Run health checks on the provided generators.
-    Fails if any of the generators' health checks fail.
-    """
-    if generators is None:
-        generators = Generator.get_machine_generators(machine.name, machine.flake)
-
-    pub_healtcheck_msg = machine.public_vars_store.health_check(
-        machine.name, generators
-    )
-    sec_healtcheck_msg = machine.secret_vars_store.health_check(
-        machine.name, generators
-    )
-
-    if pub_healtcheck_msg or sec_healtcheck_msg:
-        msg = f"Health check failed for machine {machine.name}:\n"
-        if pub_healtcheck_msg:
-            msg += f"Public vars store: {pub_healtcheck_msg}\n"
-        if sec_healtcheck_msg:
-            msg += f"Secret vars store: {sec_healtcheck_msg}"
-        raise ClanError(msg)
-
-
-def _generate_vars_for_machine(
-    machine: "Machine",
-    generators: list[Generator],
-    all_prompt_values: dict[str, dict[str, str]],
-    no_sandbox: bool = False,
-) -> None:
-    _ensure_healthy(machine=machine, generators=generators)
-    for generator in generators:
-        if check_can_migrate(machine, generator):
-            migrate_files(machine, generator)
-        else:
-            _execute_generator(
-                machine=machine,
-                generator=generator,
-                secret_vars_store=machine.secret_vars_store,
-                public_vars_store=machine.public_vars_store,
-                prompt_values=all_prompt_values.get(generator.name, {}),
-                no_sandbox=no_sandbox,
-            )
-
-
-@API.register
-def run_generators(
-    machine: Machine,
-    all_prompt_values: dict[str, dict[str, str]],
-    generators: list[str] | None = None,
-    no_sandbox: bool = False,
-) -> None:
-    """Run the specified generators for a machine.
-    Args:
-        machine_name (str): The name of the machine.
-        generators (list[str]): The list of generator names to run.
-        all_prompt_values (dict[str, dict[str, str]]): A dictionary mapping generator names
-            to their prompt values.
-        base_dir (Path): The base directory of the flake.
-        no_sandbox (bool): Whether to disable sandboxing when executing the generator.
-    Returns:
-        bool: True if any variables were generated, False otherwise.
-    Raises:
-        ClanError: If the machine or generator is not found, or if there are issues with
-        executing the generator.
-    """
-
-    if not generators:
-        generator_objects = Generator.get_machine_generators(
-            machine.name, machine.flake
-        )
-    else:
-        generators_set = set(generators)
-        generator_objects = [
-            g
-            for g in Generator.get_machine_generators(machine.name, machine.flake)
-            if g.name in generators_set
-        ]
-    _generate_vars_for_machine(
-        machine=machine,
-        generators=generator_objects,
-        all_prompt_values=all_prompt_values,
-        no_sandbox=no_sandbox,
-    )
-
-
-def create_machine_vars_interactive(
-    machine: "Machine",
-    generator_name: str | None,
-    regenerate: bool,
-    no_sandbox: bool = False,
-) -> None:
-    generators = get_generators(machine, regenerate, generator_name)
-    if len(generators) == 0:
-        return
-    all_prompt_values = {}
-    for generator in generators:
-        all_prompt_values[generator.name] = _ask_prompts(generator)
-    _generate_vars_for_machine(
-        machine,
-        generators,
-        all_prompt_values,
-        no_sandbox=no_sandbox,
-    )
-
-
-def generate_vars(
-    machines: list["Machine"],
-    generator_name: str | None = None,
-    regenerate: bool = False,
-    no_sandbox: bool = False,
-) -> None:
-    for machine in machines:
-        errors = []
-        try:
-            create_machine_vars_interactive(
-                machine,
-                generator_name,
-                regenerate,
-                no_sandbox=no_sandbox,
-            )
-            machine.info("All vars are up to date")
-        except Exception as exc:
-            errors += [(machine, exc)]
-        if len(errors) == 1:
-            raise errors[0][1]
-        if len(errors) > 1:
-            msg = f"Failed to generate vars for {len(errors)} hosts:"
-            for machine, error in errors:
-                msg += f"\n{machine}: {error}"
-            raise ClanError(msg) from errors[0][1]
+from clan_lib.nix import nix_config
+from clan_lib.vars.generate import run_generators
 
 
 def generate_command(args: argparse.Namespace) -> None:
@@ -461,10 +34,11 @@ def generate_command(args: argparse.Namespace) -> None:
             f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.validationHash",
         ]
     )
-    generate_vars(
+
+    run_generators(
         machines,
-        args.generator,
-        args.regenerate,
+        generators=args.generator,
+        full_closure=args.regenerate if args.regenerate is not None else False,
         no_sandbox=args.no_sandbox,
     )
 

pkgs/clan-cli/clan_cli/vars/generator.py

@@ -2,6 +2,7 @@ import logging
 from dataclasses import dataclass, field
 from functools import cached_property
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from clan_lib.flake import Flake
 from clan_lib.machines.machines import Machine
@@ -11,9 +12,25 @@ from .check import check_vars
 from .prompt import Prompt
 from .var import Var
 
+if TYPE_CHECKING:
+    from ._types import StoreBase
+
 log = logging.getLogger(__name__)
 
 
+def dependencies_as_dir(
+    decrypted_dependencies: dict[str, dict[str, bytes]], tmpdir: Path
+) -> None:
+    """Helper function to create directory structure from decrypted dependencies."""
+    for dep_generator, files in decrypted_dependencies.items():
+        dep_generator_dir = tmpdir / dep_generator
+        dep_generator_dir.mkdir(mode=0o700, parents=False, exist_ok=False)
+        for file_name, file in files.items():
+            file_path = dep_generator_dir / file_name
+            file_path.touch(mode=0o600, exist_ok=False)
+            file_path.write_bytes(file)
+
+
 @dataclass(frozen=True)
 class GeneratorKey:
     """A key uniquely identifying a generator within a clan."""
@@ -174,3 +191,237 @@ class Generator:
         return machine.select(
             f'config.clan.core.vars.generators."{self.name}".validationHash'
         )
+
+    def decrypt_dependencies(
+        self,
+        machine: "Machine",
+        secret_vars_store: "StoreBase",
+        public_vars_store: "StoreBase",
+    ) -> dict[str, dict[str, bytes]]:
+        """Decrypt and retrieve all dependency values for this generator.
+
+        Args:
+            machine: The machine context
+            secret_vars_store: Store for secret variables
+            public_vars_store: Store for public variables
+
+        Returns:
+            Dictionary mapping generator names to their variable values
+        """
+        from clan_lib.errors import ClanError
+
+        generators = self.get_machine_generators(machine.name, machine.flake)
+        result: dict[str, dict[str, bytes]] = {}
+
+        for dep_key in set(self.dependencies):
+            # For now, we only support dependencies from the same machine
+            if dep_key.machine != machine.name:
+                msg = f"Cross-machine dependencies are not supported. Generator {self.name} depends on {dep_key.name} from machine {dep_key.machine}"
+                raise ClanError(msg)
+
+            result[dep_key.name] = {}
+
+            dep_generator = next(
+                (g for g in generators if g.name == dep_key.name), None
+            )
+            if dep_generator is None:
+                msg = f"Generator {dep_key.name} not found in machine {machine.name}"
+                raise ClanError(msg)
+
+            dep_files = dep_generator.files
+            for file in dep_files:
+                if file.secret:
+                    result[dep_key.name][file.name] = secret_vars_store.get(
+                        dep_generator, file.name
+                    )
+                else:
+                    result[dep_key.name][file.name] = public_vars_store.get(
+                        dep_generator, file.name
+                    )
+        return result
+
+    def ask_prompts(self) -> dict[str, str]:
+        """Interactively ask for all prompt values for this generator.
+
+        Returns:
+            Dictionary mapping prompt names to their values
+        """
+        from .prompt import ask
+
+        prompt_values: dict[str, str] = {}
+        for prompt in self.prompts:
+            var_id = f"{self.name}/{prompt.name}"
+            prompt_values[prompt.name] = ask(
+                var_id,
+                prompt.prompt_type,
+                prompt.description if prompt.description != prompt.name else None,
+            )
+        return prompt_values
+
+    def execute(
+        self,
+        machine: "Machine",
+        prompt_values: dict[str, str] | None = None,
+        no_sandbox: bool = False,
+    ) -> None:
+        """Execute this generator to produce its output files.
+
+        Args:
+            machine: The machine to execute the generator for
+            prompt_values: Optional dictionary of prompt values. If not provided, prompts will be asked interactively.
+            no_sandbox: Whether to disable sandboxing when executing the generator
+        """
+        import os
+        import sys
+        from contextlib import ExitStack
+        from pathlib import Path
+        from tempfile import TemporaryDirectory
+
+        from clan_lib import bwrap
+        from clan_lib.cmd import RunOpts, run
+        from clan_lib.errors import ClanError
+        from clan_lib.git import commit_files
+
+        if prompt_values is None:
+            prompt_values = self.ask_prompts()
+
+        # build temporary file tree of dependencies
+        decrypted_dependencies = self.decrypt_dependencies(
+            machine,
+            machine.secret_vars_store,
+            machine.public_vars_store,
+        )
+
+        def get_prompt_value(prompt_name: str) -> str:
+            try:
+                return prompt_values[prompt_name]
+            except KeyError as e:
+                msg = f"prompt value for '{prompt_name}' in generator {self.name} not provided"
+                raise ClanError(msg) from e
+
+        def bubblewrap_cmd(generator: str, tmpdir: Path) -> list[str]:
+            """Helper function to create bubblewrap command."""
+            import shutil
+
+            from clan_lib.nix import nix_shell, nix_test_store
+
+            test_store = nix_test_store()
+            real_bash_path = Path("bash")
+            if os.environ.get("IN_NIX_SANDBOX"):
+                bash_executable_path = Path(str(shutil.which("bash")))
+                real_bash_path = bash_executable_path.resolve()
+
+            # fmt: off
+            return nix_shell(
+                ["bash", "bubblewrap"],
+                [
+                    "bwrap",
+                    "--unshare-all",
+                    "--tmpfs",  "/",
+                    "--ro-bind", "/nix/store", "/nix/store",
+                    "--ro-bind", "/bin/sh", "/bin/sh",
+                    *(["--ro-bind", str(test_store), str(test_store)] if test_store else []),
+                    "--dev", "/dev",
+                    "--bind", str(tmpdir), str(tmpdir),
+                    "--chdir", "/",
+                    "--bind", "/proc", "/proc",
+                    "--uid", "1000",
+                    "--gid", "1000",
+                    "--",
+                    str(real_bash_path), "-c", generator
+                ]
+            )
+            # fmt: on
+
+        env = os.environ.copy()
+        with ExitStack() as stack:
+            _tmpdir = stack.enter_context(TemporaryDirectory(prefix="vars-"))
+            tmpdir = Path(_tmpdir).resolve()
+            tmpdir_in = tmpdir / "in"
+            tmpdir_prompts = tmpdir / "prompts"
+            tmpdir_out = tmpdir / "out"
+            tmpdir_in.mkdir()
+            tmpdir_out.mkdir()
+            env["in"] = str(tmpdir_in)
+            env["out"] = str(tmpdir_out)
+
+            # populate dependency inputs
+            dependencies_as_dir(decrypted_dependencies, tmpdir_in)
+
+            # populate prompted values
+            if self.prompts:
+                tmpdir_prompts.mkdir()
+                env["prompts"] = str(tmpdir_prompts)
+                for prompt in self.prompts:
+                    prompt_file = tmpdir_prompts / prompt.name
+                    value = get_prompt_value(prompt.name)
+                    prompt_file.write_text(value)
+
+            final_script = self.final_script()
+
+            if sys.platform == "linux" and bwrap.bubblewrap_works():
+                cmd = bubblewrap_cmd(str(final_script), tmpdir)
+            elif sys.platform == "darwin":
+                from clan_lib.sandbox_exec import sandbox_exec_cmd
+
+                cmd = stack.enter_context(sandbox_exec_cmd(str(final_script), tmpdir))
+            else:
+                # For non-sandboxed execution
+                if not no_sandbox:
+                    msg = (
+                        f"Cannot safely execute generator {self.name}: Sandboxing is not available on this system\n"
+                        f"Re-run 'vars generate' with '--no-sandbox' to disable sandboxing"
+                    )
+                    raise ClanError(msg)
+                cmd = ["bash", "-c", str(final_script)]
+
+            run(cmd, RunOpts(env=env, cwd=tmpdir))
+            files_to_commit = []
+
+            # store secrets
+            public_changed = False
+            secret_changed = False
+            for file in self.files:
+                secret_file = tmpdir_out / file.name
+                if not secret_file.is_file():
+                    msg = f"did not generate a file for '{file.name}' when running the following command:\n"
+                    msg += str(final_script)
+                    # list all files in the output directory
+                    if tmpdir_out.is_dir():
+                        msg += "\nOutput files:\n"
+                        for f in tmpdir_out.iterdir():
+                            msg += f"  - {f.name}\n"
+                    raise ClanError(msg)
+                if file.secret:
+                    file_path = machine.secret_vars_store.set(
+                        self,
+                        file,
+                        secret_file.read_bytes(),
+                    )
+                    secret_changed = True
+                else:
+                    file_path = machine.public_vars_store.set(
+                        self,
+                        file,
+                        secret_file.read_bytes(),
+                    )
+                    public_changed = True
+                if file_path:
+                    files_to_commit.append(file_path)
+
+            validation = self.validation()
+            if validation is not None:
+                if public_changed:
+                    files_to_commit.append(
+                        machine.public_vars_store.set_validation(self, validation)
+                    )
+                if secret_changed:
+                    files_to_commit.append(
+                        machine.secret_vars_store.set_validation(self, validation)
+                    )
+
+        commit_files(
+            files_to_commit,
+            machine.flake_dir,
+            f"Update vars via generator {self.name} for machine {machine.name}",
+        )

pkgs/clan-cli/clan_cli/vars/get.py

@@ -9,6 +9,7 @@ from clan_cli.completions import (
 )
 from clan_lib.errors import ClanError
 from clan_lib.flake import Flake, require_flake
+from clan_lib.machines.machines import Machine
 
 from .generator import Var
 from .list import get_machine_vars
@@ -16,9 +17,9 @@ from .list import get_machine_vars
 log = logging.getLogger(__name__)
 
 
-def get_machine_var(base_dir: str, machine_name: str, var_id: str) -> Var:
-    log.debug(f"getting var: {var_id} from machine: {machine_name}")
-    vars_ = get_machine_vars(base_dir=base_dir, machine_name=machine_name)
+def get_machine_var(machine: Machine, var_id: str) -> Var:
+    log.debug(f"getting var: {var_id} from machine: {machine.name}")
+    vars_ = get_machine_vars(machine)
     results = []
     for var in vars_:
         if var.id == var_id:
@@ -44,7 +45,8 @@ def get_machine_var(base_dir: str, machine_name: str, var_id: str) -> Var:
 
 
 def get_command(machine_name: str, var_id: str, flake: Flake) -> None:
-    var = get_machine_var(str(flake.path), machine_name, var_id)
+    machine = Machine(name=machine_name, flake=flake)
+    var = get_machine_var(machine, var_id)
     if not var.exists:
         msg = f"Var {var.id} has not been generated yet"
         raise ClanError(msg)

pkgs/clan-cli/clan_cli/vars/graph.py

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING
 from clan_lib.errors import ClanError
 
 if TYPE_CHECKING:
-    from .generate import Generator, GeneratorKey
+    from .generator import Generator, GeneratorKey
 
 
 class GeneratorNotFoundError(ClanError):

pkgs/clan-cli/clan_cli/vars/list.py

@@ -2,19 +2,18 @@ import argparse
 import logging
 
 from clan_cli.completions import add_dynamic_completer, complete_machines
-from clan_lib.flake import Flake, require_flake
+from clan_lib.flake import require_flake
 from clan_lib.machines.machines import Machine
+from clan_lib.vars.generate import get_generators
 
-from .generate import get_generators
 from .generator import Var
 
 log = logging.getLogger(__name__)
 
 
-def get_machine_vars(base_dir: str, machine_name: str) -> list[Var]:
+def get_machine_vars(machine: Machine) -> list[Var]:
     # TODO: We dont have machine level store / this granularity yet
     # We should move the store definition to the flake, as there can be only one store per clan
-    machine = Machine(name=machine_name, flake=Flake(base_dir))
     pub_store = machine.public_vars_store
     sec_store = machine.secret_vars_store
 
@@ -37,7 +36,7 @@ def stringify_vars(_vars: list[Var]) -> str:
 
 
 def stringify_all_vars(machine: Machine) -> str:
-    return stringify_vars(get_machine_vars(str(machine.flake), machine.name))
+    return stringify_vars(get_machine_vars(machine))
 
 
 def list_command(args: argparse.Namespace) -> None:

pkgs/clan-cli/clan_cli/vars/migration.py

@@ -8,7 +8,7 @@ from clan_lib.git import commit_files
 log = logging.getLogger(__name__)
 
 if TYPE_CHECKING:
-    from clan_cli.vars.generate import Generator
+    from clan_cli.vars.generator import Generator
     from clan_lib.machines.machines import Machine
 
 

pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py

@@ -146,7 +146,7 @@ class SecretStore(StoreBase):
         if not git_hash:
             return b""
 
-        from clan_cli.vars.generate import Generator
+        from clan_cli.vars.generator import Generator
 
         manifest = []
         generators = Generator.get_machine_generators(machine, self.flake)
@@ -178,7 +178,7 @@ class SecretStore(StoreBase):
         return local_hash != remote_hash.encode()
 
     def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None:
-        from clan_cli.vars.generate import Generator
+        from clan_cli.vars.generator import Generator
 
         vars_generators = Generator.get_machine_generators(machine, self.flake)
         if "users" in phases:

pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py

@@ -23,7 +23,7 @@ from clan_cli.secrets.secrets import (
 )
 from clan_cli.secrets.sops import load_age_plugins
 from clan_cli.vars._types import StoreBase
-from clan_cli.vars.generate import Generator
+from clan_cli.vars.generator import Generator
 from clan_cli.vars.var import Var
 from clan_lib.errors import ClanError
 from clan_lib.flake import Flake
@@ -54,7 +54,7 @@ class SecretStore(StoreBase):
     def ensure_machine_key(self, machine: str) -> None:
         """Ensure machine has sops keys initialized."""
         # no need to generate keys if we don't manage secrets
-        from clan_cli.vars.generate import Generator
+        from clan_cli.vars.generator import Generator
 
         vars_generators = Generator.get_machine_generators(machine, self.flake)
         if not vars_generators:
@@ -135,7 +135,7 @@ class SecretStore(StoreBase):
         """
 
         if generators is None:
-            from clan_cli.vars.generate import Generator
+            from clan_cli.vars.generator import Generator
 
             generators = Generator.get_machine_generators(machine, self.flake)
         file_found = False
@@ -212,7 +212,7 @@ class SecretStore(StoreBase):
         return [store_folder]
 
     def populate_dir(self, machine: str, output_dir: Path, phases: list[str]) -> None:
-        from clan_cli.vars.generate import Generator
+        from clan_cli.vars.generator import Generator
 
         vars_generators = Generator.get_machine_generators(machine, self.flake)
         if "users" in phases or "services" in phases:
@@ -347,7 +347,7 @@ class SecretStore(StoreBase):
         from clan_cli.secrets.secrets import update_keys
 
         if generators is None:
-            from clan_cli.vars.generate import Generator
+            from clan_cli.vars.generator import Generator
 
             generators = Generator.get_machine_generators(machine, self.flake)
         file_found = False

pkgs/clan-cli/clan_cli/vars/set.py

@@ -25,7 +25,7 @@ def set_var(machine: str | Machine, var: str | Var, value: bytes, flake: Flake)
     else:
         _machine = machine
     if isinstance(var, str):
-        _var = get_machine_var(str(flake.path), _machine.name, var)
+        _var = get_machine_var(_machine, var)
     else:
         _var = var
     path = _var.set(value)
@@ -39,7 +39,7 @@ def set_var(machine: str | Machine, var: str | Var, value: bytes, flake: Flake)
 
 def set_via_stdin(machine_name: str, var_id: str, flake: Flake) -> None:
     machine = Machine(name=machine_name, flake=flake)
-    var = get_machine_var(str(flake.path), machine_name, var_id)
+    var = get_machine_var(machine, var_id)
     if sys.stdin.isatty():
         new_value = ask(
             var.id,

pkgs/clan-cli/clan_cli/vars/var.py

@@ -3,7 +3,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from clan_cli.vars.generate import Generator
+    from clan_cli.vars.generator import Generator
 
     from ._types import StoreBase
 

pkgs/clan-cli/clan_cli/vms/qemu.py

@@ -1,3 +1,4 @@
+import platform
 import random
 from collections.abc import Generator
 from contextlib import contextmanager
@@ -5,6 +6,7 @@ from dataclasses import dataclass
 from pathlib import Path
 
 from clan_lib.errors import ClanError
+from clan_lib.nix import nix_test_store
 
 from clan_cli.qemu.qmp import QEMUMonitorProtocol
 
@@ -84,6 +86,44 @@ class QemuCommand:
     vsock_cid: int | None = None
 
 
+def get_machine_options() -> str:
+    """Get appropriate QEMU machine options for host architecture."""
+    arch = platform.machine().lower()
+    system = platform.system().lower()
+
+    # Determine accelerator based on OS
+    if system == "darwin":
+        # macOS uses Hypervisor.framework
+        accel = "hvf"
+    else:
+        # Linux and others use KVM
+        accel = "kvm"
+
+    if arch in ("x86_64", "amd64", "i386", "i686"):
+        # For x86_64, use q35 for modern PCIe support
+        return f"q35,memory-backend=mem,accel={accel}"
+    if arch in ("aarch64", "arm64"):
+        # Use virt machine type for ARM64
+        if system == "darwin":
+            # macOS ARM uses GIC version 2
+            return f"virt,gic-version=2,memory-backend=mem,accel={accel}"
+        # Linux ARM uses max GIC version
+        return f"virt,gic-version=max,memory-backend=mem,accel={accel}"
+    if arch == "armv7l":
+        # 32-bit ARM
+        return f"virt,memory-backend=mem,accel={accel}"
+    if arch in ("riscv64", "riscv32"):
+        # RISC-V architectures
+        return f"virt,memory-backend=mem,accel={accel}"
+    if arch in ("powerpc64le", "powerpc64", "ppc64le", "ppc64"):
+        # PowerPC architectures
+        return f"powernv,memory-backend=mem,accel={accel}"
+
+    # No fallback - raise an error for unsupported architectures
+    msg = f"Unsupported architecture: {arch} on {system}. Supported architectures are: x86_64, aarch64, armv7l, riscv64, riscv32, powerpc64"
+    raise ClanError(msg)
+
+
 def qemu_command(
     vm: VmConfig,
     nixos_config: dict[str, str],
@@ -98,22 +138,31 @@ def qemu_command(
 ) -> QemuCommand:
     if portmap is None:
         portmap = {}
+
+    toplevel = Path(nixos_config["toplevel"])
+    chroot_toplevel = toplevel
+    initrd = Path(nixos_config["initrd"])
+    if tmp_store := nix_test_store():
+        chroot_toplevel = tmp_store / toplevel.relative_to("/")
+        initrd = tmp_store / initrd.relative_to("/")
+
     kernel_cmdline = [
-        (Path(nixos_config["toplevel"]) / "kernel-params").read_text(),
-        f"init={nixos_config['toplevel']}/init",
+        (chroot_toplevel / "kernel-params").read_text(),
+        f"init={toplevel}/init",
         f"regInfo={nixos_config['regInfo']}/registration",
         "console=hvc0",
     ]
     if not vm.waypipe.enable:
         kernel_cmdline.append("console=tty0")
     hostfwd = ",".join(f"hostfwd=tcp::{h}-:{g}" for h, g in portmap.items())
+    machine_options = get_machine_options()
     # fmt: off
     command = [
         "qemu-kvm",
         "-name", vm.machine_name,
         "-m", f'{nixos_config["memorySize"]}M',
         "-object", f"memory-backend-memfd,id=mem,size={nixos_config['memorySize']}M",
-        "-machine", "pc,memory-backend=mem,accel=kvm",
+        "-machine", machine_options,
         "-smp", str(nixos_config["cores"]),
         "-cpu", "max",
         "-enable-kvm",
@@ -130,9 +179,8 @@ def qemu_command(
         "-drive", f"cache=writeback,file={state_img},format=qcow2,id=state,if=none,index=2,werror=report",
         "-device", "virtio-blk-pci,drive=state",
         "-device", "virtio-keyboard",
-        "-usb", "-device", "usb-tablet,bus=usb-bus.0",
-        "-kernel", f'{nixos_config["toplevel"]}/kernel',
-        "-initrd", nixos_config["initrd"],
+        "-kernel", f"{chroot_toplevel}/kernel",
+        "-initrd", str(initrd),
         "-append", " ".join(kernel_cmdline),
         # qmp & qga setup
         "-qmp", f"unix:{qmp_socket_file},server,wait=off",
@@ -140,6 +188,11 @@ def qemu_command(
         "-device", "virtio-serial",
         "-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0",
     ]
+    # USB tablet only works reliably on x86_64 Linux for now, not aarch64-linux.
+    # TODO: Fix USB tablet support for ARM architectures and test macOS
+    if platform.system().lower() == "linux" and platform.machine().lower() in ("x86_64", "amd64"):
+        command.extend(["-usb", "-device", "usb-tablet,bus=usb-bus.0"])
+
     if interactive:
         command.extend(
             [

pkgs/clan-cli/clan_cli/vms/run.py

@@ -16,13 +16,13 @@ from clan_lib.cmd import CmdOut, Log, RunOpts, handle_io, run
 from clan_lib.dirs import module_root, user_cache_dir, vm_state_dir
 from clan_lib.errors import ClanCmdError, ClanError
 from clan_lib.machines.machines import Machine
-from clan_lib.nix import nix_shell
+from clan_lib.nix import nix_shell, nix_test_store
+from clan_lib.vars.generate import run_generators
 
 from clan_cli.completions import add_dynamic_completer, complete_machines
 from clan_cli.facts.generate import generate_facts
 from clan_cli.qemu.qga import QgaSession
 from clan_cli.qemu.qmp import QEMUMonitorProtocol
-from clan_cli.vars.generate import generate_vars
 from clan_cli.vars.upload import populate_secret_vars
 
 from .inspect import VmConfig, inspect_vm
@@ -57,8 +57,6 @@ def build_vm(
         nix_options = []
     secrets_dir = get_secrets(machine, tmpdir)
 
-    from clan_lib.nix import nix_test_store
-
     output = Path(
         machine.select(
             "config.system.clan.vm.create",
@@ -84,11 +82,9 @@ def get_secrets(
     secrets_dir = tmpdir / "secrets"
     secrets_dir.mkdir(parents=True, exist_ok=True)
 
-    generate_facts([machine])
-    generate_vars([machine])
-
     machine.secret_facts_store.upload(secrets_dir)
     populate_secret_vars(machine, secrets_dir)
+
     return secrets_dir
 
 
@@ -386,6 +382,9 @@ def run_command(
 ) -> None:
     machine_obj: Machine = Machine(args.machine, args.flake)
 
+    generate_facts([machine_obj])
+    run_generators([machine_obj], generators=None, full_closure=False)
+
     vm: VmConfig = inspect_vm(machine=machine_obj)
 
     if not os.environ.get("WAYLAND_DISPLAY"):

pkgs/clan-cli/clan_cli/vms/virtiofsd.py

@@ -1,4 +1,5 @@
 import contextlib
+import logging
 import shutil
 import subprocess
 import time
@@ -6,7 +7,9 @@ from collections.abc import Iterator
 from pathlib import Path
 
 from clan_lib.errors import ClanError
-from clan_lib.nix import nix_shell
+from clan_lib.nix import nix_shell, nix_test_store
+
+log = logging.getLogger(__name__)
 
 
 @contextlib.contextmanager
@@ -14,6 +17,9 @@ def start_virtiofsd(socket_path: Path) -> Iterator[None]:
     sandbox = "namespace"
     if shutil.which("newuidmap") is None:
         sandbox = "none"
+    store_root = nix_test_store() or Path("/")
+    store = store_root / "nix" / "store"
+
     virtiofsd = nix_shell(
         ["virtiofsd"],
         [
@@ -25,9 +31,10 @@ def start_virtiofsd(socket_path: Path) -> Iterator[None]:
             "--sandbox",
             sandbox,
             "--shared-dir",
-            "/nix/store",
+            str(store),
         ],
     )
+    log.debug("$ {}".format(" ".join(virtiofsd)))
     with subprocess.Popen(virtiofsd) as proc:
         try:
             while not socket_path.exists():

pkgs/clan-cli/clan_lib/api/__init__.py

@@ -16,7 +16,6 @@ from typing import (
     get_type_hints,
 )
 
-from clan_lib.api.util import JSchemaTypeError
 from clan_lib.async_run import get_current_thread_opkey
 from clan_lib.errors import ClanError
 
@@ -204,7 +203,7 @@ API.register(get_system_file)
     def to_json_schema(self) -> dict[str, Any]:
         from typing import get_type_hints
 
-        from .util import type_to_dict
+        from .type_to_jsonschema import JSchemaTypeError, type_to_dict
 
         api_schema: dict[str, Any] = {
             "$comment": "An object containing API methods. ",
@@ -221,7 +220,9 @@ API.register(get_system_file)
             try:
                 serialized_hints = {
                     key: type_to_dict(
-                        value, scope=name + " argument" if key != "return" else "return"
+                        value,
+                        scope=name + " argument" if key != "return" else "return",
+                        narrow_unsupported_union_types=True,
                     )
                     for key, value in hints.items()
                 }

pkgs/clan-cli/clan_lib/api/type_to_jsonschema.py

@@ -104,7 +104,10 @@ def is_total(typed_dict_class: type) -> bool:
 
 
 def type_to_dict(
-    t: Any, scope: str = "", type_map: dict[TypeVar, type] | None = None
+    t: Any,
+    scope: str = "",
+    type_map: dict[TypeVar, type] | None = None,
+    narrow_unsupported_union_types: bool = False,
 ) -> dict:
     if type_map is None:
         type_map = {}
@@ -148,13 +151,13 @@ def type_to_dict(
             if f.default is MISSING and f.default_factory is MISSING
         }
 
-        # Find intersection
-        intersection = required & required_fields
+        # TODO: figure out why we needed to do this
+        # intersection = required_fields & required
 
         return {
             "type": "object",
             "properties": properties,
-            "required": list(intersection),
+            "required": sorted(required_fields),
             # Dataclasses can only have the specified properties
             "additionalProperties": False,
         }
@@ -162,28 +165,59 @@ def type_to_dict(
     if is_typed_dict(t):
         dict_fields = get_typed_dict_fields(t, scope)
         dict_properties: dict = {}
-        dict_required: list[str] = []
+        explicit_optional: set[str] = set()
+        explicit_required: set[str] = set()
         for field_name, field_type in dict_fields.items():
-            if (
-                not is_type_in_union(field_type, type(None))
-                and get_origin(field_type) is not NotRequired
-            ) or get_origin(field_type) is Required:
-                dict_required.append(field_name)
+            # Unwrap special case for "NotRequired" and "Required"
+            # A field type that only exist for TypedDicts
+            if get_origin(field_type) is NotRequired:
+                explicit_optional.add(field_name)
+
+            if get_origin(field_type) is Required:
+                explicit_required.add(field_name)
 
             dict_properties[field_name] = type_to_dict(
                 field_type, f"{scope} {t.__name__}.{field_name}", type_map
             )
 
+        optional = set(dict_fields) - explicit_optional
         return {
             "type": "object",
             "properties": dict_properties,
-            "required": dict_required if is_total(t) else [],
+            "required": sorted(optional) if is_total(t) else sorted(explicit_required),
             "additionalProperties": False,
         }
 
-    if type(t) is UnionType:
+    origin = get_origin(t)
+    # UnionTypes
+    if type(t) is UnionType or origin is Union:
+        supported = []
+        for arg in get_args(t):
+            try:
+                supported.append(
+                    type_to_dict(arg, scope, type_map, narrow_unsupported_union_types)
+                )
+            except JSchemaTypeError:
+                if narrow_unsupported_union_types:
+                    # If we are narrowing unsupported union types, we skip the error
+                    continue
+                raise
+
+        if len(supported) == 0:
+            msg = f"{scope} - No supported types in Union {t!s}, type_map: {type_map}"
+            raise JSchemaTypeError(msg)
+
+        if len(supported) == 1:
+            # If there's only one supported type, return it directly
+            return supported[0]
+
+        # TODO: it would maybe be better to return 'anyOf' this should work for typescript
+        # But is more correct for JSON Schema validation
+        # i.e. 42 would match all of "int | float" which would be an invalid value for that using "oneOf"
+
+        # If there are multiple supported types, return them as oneOf
         return {
-            "oneOf": [type_to_dict(arg, scope, type_map) for arg in t.__args__],
+            "oneOf": supported,
         }
 
     if isinstance(t, TypeVar):
@@ -221,12 +255,6 @@ def type_to_dict(
             schema = type_to_dict(base_type, scope)  # Generate schema for the base type
             return apply_annotations(schema, metadata)
 
-        if origin is Union:
-            union_types = [type_to_dict(arg, scope, type_map) for arg in t.__args__]
-            return {
-                "oneOf": union_types,
-            }
-
         if origin in {list, set, frozenset, tuple}:
             return {
                 "type": "array",

pkgs/clan-cli/clan_lib/api/type_to_jsonschema_test.py

@@ -0,0 +1,331 @@
+from dataclasses import dataclass, field
+from typing import Any, NotRequired, Required
+
+import pytest
+
+from .type_to_jsonschema import JSchemaTypeError, type_to_dict
+
+
+def test_simple_primitives() -> None:
+    assert type_to_dict(int) == {
+        "type": "integer",
+    }
+    assert type_to_dict(float) == {
+        "type": "number",
+    }
+
+    assert type_to_dict(str) == {
+        "type": "string",
+    }
+
+    assert type_to_dict(bool) == {
+        "type": "boolean",
+    }
+    assert type_to_dict(object) == {
+        "type": "object",
+    }
+
+
+def test_enum_type() -> None:
+    from enum import Enum
+
+    class Color(Enum):
+        RED = "red"
+        GREEN = "green"
+        BLUE = "blue"
+
+    assert type_to_dict(Color) == {
+        "type": "string",
+        "enum": ["red", "green", "blue"],
+    }
+
+
+def test_unsupported_any_types() -> None:
+    with pytest.raises(JSchemaTypeError) as exc_info:
+        type_to_dict(Any)
+    assert "Usage of the Any type is not supported" in str(exc_info.value)
+
+    with pytest.raises(JSchemaTypeError) as exc_info:
+        type_to_dict(list[Any])
+    assert "Usage of the Any type is not supported" in str(exc_info.value)
+
+    # TBD.
+    # with pytest.raises(JSchemaTypeError) as exc_info:
+    #     type_to_dict(dict[str, Any])
+    # assert "Usage of the Any type is not supported" in str(exc_info.value)
+
+    with pytest.raises(JSchemaTypeError) as exc_info:
+        type_to_dict(tuple[Any, ...])
+    assert "Usage of the Any type is not supported" in str(exc_info.value)
+
+    with pytest.raises(JSchemaTypeError) as exc_info:
+        type_to_dict(set[Any])
+    assert "Usage of the Any type is not supported" in str(exc_info.value)
+
+    with pytest.raises(JSchemaTypeError) as exc_info:
+        type_to_dict(str | Any)
+    assert "Usage of the Any type is not supported" in str(exc_info.value)
+
+
+def test_allowed_any_types() -> None:
+    # Object with arbitrary keys
+    assert type_to_dict(dict[str, Any]) == {
+        "type": "object",
+        "additionalProperties": True,
+    }
+
+    # Union where Any is discarded
+    assert type_to_dict(str | Any, narrow_unsupported_union_types=True) == {
+        "type": "string",
+    }
+
+
+def test_simple_union_types() -> None:
+    assert type_to_dict(int | str) == {
+        "oneOf": [
+            {"type": "integer"},
+            {"type": "string"},
+        ]
+    }
+
+    assert type_to_dict(int | str | float) == {
+        "oneOf": [
+            {"type": "integer"},
+            {"type": "string"},
+            {"type": "number"},
+        ]
+    }
+
+    assert type_to_dict(int | str | None) == {
+        "oneOf": [
+            {"type": "integer"},
+            {"type": "string"},
+            {"type": "null"},
+        ]
+    }
+
+
+def test_complex_union_types() -> None:
+    @dataclass
+    class Foo:
+        foo: str
+
+    @dataclass
+    class Bar:
+        bar: str
+
+    assert type_to_dict(Foo | Bar | None) == {
+        "oneOf": [
+            {
+                "type": "object",
+                "properties": {
+                    "foo": {"type": "string"},
+                },
+                "additionalProperties": False,
+                "required": ["foo"],
+            },
+            {
+                "type": "object",
+                "properties": {
+                    "bar": {"type": "string"},
+                },
+                "additionalProperties": False,
+                "required": ["bar"],
+            },
+            {"type": "null"},
+        ]
+    }
+
+
+def test_dataclasses() -> None:
+    # @dataclass
+    # class Example:
+    #     name: str
+    #     value: bool
+
+    # assert type_to_dict(Example) == {
+    #     "type": "object",
+    #     "properties": {
+    #         "name": {"type": "string"},
+    #         "value": {"type": "boolean"},
+    #     },
+    #     "additionalProperties": False,
+    #     "required": [
+    #         "name",
+    #         "value",
+    #     ],
+    # }
+
+    @dataclass
+    class ExampleWithNullable:
+        name: str
+        value: int | None
+
+    assert type_to_dict(ExampleWithNullable) == {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string"},
+            "value": {"oneOf": [{"type": "integer"}, {"type": "null"}]},
+        },
+        "additionalProperties": False,
+        "required": [
+            "name",
+            "value",
+        ],  # value is required because it has no default value
+    }
+
+    @dataclass
+    class ExampleWithOptional:
+        name: str
+        value: int | None = None
+
+    assert type_to_dict(ExampleWithOptional) == {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string"},
+            "value": {"oneOf": [{"type": "integer"}, {"type": "null"}]},
+        },
+        "additionalProperties": False,
+        "required": [
+            "name"
+        ],  # value is optional because it has a default value of None
+    }
+
+
+def test_dataclass_with_optional_fields() -> None:
+    @dataclass
+    class Example:
+        value: dict[str, Any] = field(default_factory=dict)
+
+    assert type_to_dict(Example) == {
+        "type": "object",
+        "properties": {
+            "value": {
+                "type": "object",
+                "additionalProperties": True,
+            },
+        },
+        "additionalProperties": False,
+        "required": [],  # value is optional because it has default factory
+    }
+
+
+def test_nested_open_dicts() -> None:
+    assert type_to_dict(dict[str, dict[str, list[str]]]) == {
+        "type": "object",
+        "additionalProperties": {
+            "type": "object",
+            "additionalProperties": {
+                "type": "array",
+                "items": {"type": "string"},
+            },
+        },
+    }
+
+
+def test_type_variables() -> None:
+    from typing import Generic, TypeVar
+
+    T = TypeVar("T")
+
+    @dataclass
+    class Wrapper(Generic[T]):
+        value: T
+
+    assert type_to_dict(Wrapper[int]) == {
+        "type": "object",
+        "properties": {
+            "value": {"type": "integer"},
+        },
+        "additionalProperties": False,
+        "required": ["value"],
+    }
+
+    assert type_to_dict(Wrapper[str]) == {
+        "type": "object",
+        "properties": {
+            "value": {"type": "string"},
+        },
+        "additionalProperties": False,
+        "required": ["value"],
+    }
+
+
+def test_type_variable_nested_scopes() -> None:
+    # Define two type variables with the same name "T" but in different scopes
+
+    from typing import Generic, TypeVar
+
+    T = TypeVar("T")
+
+    @dataclass
+    class Outer(Generic[T]):
+        foo: T
+
+    @dataclass
+    class Inner(Generic[T]):
+        bar: T
+
+    assert type_to_dict(Outer[Inner[int]]) == {
+        "type": "object",
+        "properties": {
+            "foo": {
+                "type": "object",
+                "properties": {
+                    "bar": {"type": "integer"},
+                },
+                "additionalProperties": False,
+                "required": ["bar"],
+            },
+        },
+        "additionalProperties": False,
+        "required": ["foo"],
+    }
+
+
+def test_total_typed_dict() -> None:
+    from typing import TypedDict
+
+    class ExampleTypedDict(TypedDict):
+        name: str
+        value: NotRequired[int]
+        bar: int | None
+
+    assert type_to_dict(ExampleTypedDict) == {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string"},
+            "value": {"type": "integer"},
+            "bar": {
+                "oneOf": [
+                    {
+                        "type": "integer",
+                    },
+                    {
+                        "type": "null",
+                    },
+                ],
+            },
+        },
+        "additionalProperties": False,
+        # bar is required because it's not explicitly marked as 'NotRequired'
+        "required": ["bar", "name"],
+    }
+
+
+def test_open_typed_dict() -> None:
+    from typing import TypedDict
+
+    class ExampleTypedDict(TypedDict, total=False):
+        name: Required[str]
+        value: int
+
+    assert type_to_dict(ExampleTypedDict) == {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string"},
+            "value": {"type": "integer"},
+        },
+        "additionalProperties": False,
+        "required": ["name"],
+    }

pkgs/clan-cli/clan_lib/flash/flash.py

@@ -7,7 +7,6 @@ from tempfile import TemporaryDirectory
 from typing import Any, Literal
 
 from clan_cli.facts.generate import generate_facts
-from clan_cli.vars.generate import generate_vars
 from clan_cli.vars.upload import populate_secret_vars
 
 from clan_lib.api import API
@@ -17,6 +16,7 @@ from clan_lib.errors import ClanError
 from clan_lib.flake.flake import Flake
 from clan_lib.machines.machines import Machine
 from clan_lib.nix import nix_shell
+from clan_lib.vars.generate import run_generators
 
 from .automount import pause_automounting
 from .list import list_keymaps, list_languages
@@ -78,7 +78,7 @@ def run_machine_flash(
         system_config_nix: dict[str, Any] = {}
 
         generate_facts([machine])
-        generate_vars([machine])
+        run_generators([machine], generators=None, full_closure=False)
 
         if system_config.language:
             if system_config.language not in list_languages():
@@ -113,7 +113,7 @@ def run_machine_flash(
                 "users": {"root": {"openssh": {"authorizedKeys": {"keys": root_keys}}}}
             }
 
-        from clan_cli.vars.generate import Generator
+        from clan_cli.vars.generator import Generator
 
         for generator in Generator.get_machine_generators(machine.name, machine.flake):
             for file in generator.files:

pkgs/clan-cli/clan_lib/machines/actions.py

@@ -1,3 +1,5 @@
+from dataclasses import dataclass, field
+from enum import StrEnum
 from typing import TypedDict
 
 from clan_lib.api import API
@@ -17,12 +19,26 @@ from clan_lib.persist.util import (
 )
 
 
-class MachineFilter(TypedDict):
-    tags: list[str]
+@dataclass
+class MachineFilter:
+    tags: list[str] | None = None
 
 
-class ListOptions(TypedDict):
-    filter: MachineFilter
+@dataclass
+class ListOptions:
+    filter: MachineFilter = field(default_factory=MachineFilter)
+
+
+class MachineStatus(StrEnum):
+    NOT_INSTALLED = "not_installed"
+    OFFLINE = "offline"
+    OUT_OF_SYNC = "out_of_sync"
+    ONLINE = "online"
+
+
+class MachineState(TypedDict):
+    status: MachineStatus
+    # add more info later when retrieving remote state
 
 
 @API.register
@@ -31,26 +47,18 @@ def list_machines(
 ) -> dict[str, InventoryMachine]:
     """
     List machines of a clan
-
-    Usage Example:
-
-    machines = list_machines(flake, {"filter": {"tags": ["foo" "bar"]}})
-
-    lists only machines that include both "foo" AND "bar"
-
     """
     inventory_store = InventoryStore(flake=flake)
     inventory = inventory_store.read()
 
     machines = inventory.get("machines", {})
 
-    if opts and opts.get("filter"):
+    if opts and opts.filter.tags is not None:
         filtered_machines = {}
-        filter_tags = opts.get("filter", {}).get("tags", [])
 
         for machine_name, machine in machines.items():
             machine_tags = machine.get("tags", [])
-            if all(ft in machine_tags for ft in filter_tags):
+            if all(ft in machine_tags for ft in opts.filter.tags):
                 filtered_machines[machine_name] = machine
 
         return filtered_machines
@@ -154,3 +162,47 @@ def get_machine_fields_schema(machine: Machine) -> dict[str, FieldSchema]:
         }
         for field in field_names
     }
+
+
+@API.register
+def list_machine_state(flake: Flake) -> dict[str, MachineState]:
+    """
+    Retrieve the current state of all machines in the clan.
+
+    Args:
+        flake (Flake): The flake object representing the configuration source.
+    """
+    inventory_store = InventoryStore(flake=flake)
+    inventory = inventory_store.read()
+
+    # todo integrate with remote state when implementing https://git.clan.lol/clan/clan-core/issues/4748
+    machines = inventory.get("machines", {})
+
+    return {
+        machine_name: MachineState(
+            status=MachineStatus.OFFLINE
+            if get_value_by_path(machine, "installedAt", None)
+            else MachineStatus.NOT_INSTALLED
+        )
+        for machine_name, machine in machines.items()
+    }
+
+
+@API.register
+def get_machine_state(machine: Machine) -> MachineState:
+    """
+    Retrieve the current state of the machine.
+
+    Args:
+        machine (Machine): The machine object for which we want to retrieve the latest state.
+    """
+    inventory_store = InventoryStore(flake=machine.flake)
+    inventory = inventory_store.read()
+
+    # todo integrate with remote state when implementing https://git.clan.lol/clan/clan-core/issues/4748
+
+    return MachineState(
+        status=MachineStatus.OFFLINE
+        if get_value_by_path(inventory, f"machines.{machine.name}.installedAt", None)
+        else MachineStatus.NOT_INSTALLED
+    )

pkgs/clan-cli/clan_lib/machines/actions_test.py

@@ -1,3 +1,4 @@
+import time
 from collections.abc import Callable
 from typing import cast
 from unittest.mock import ANY, patch
@@ -12,7 +13,16 @@ from clan_lib.nix_models.clan import Clan, InventoryMachine, Unknown
 from clan_lib.persist.inventory_store import InventoryStore
 from clan_lib.persist.util import get_value_by_path, set_value_by_path
 
-from .actions import get_machine, get_machine_fields_schema, list_machines, set_machine
+from .actions import (
+    MachineState,
+    MachineStatus,
+    get_machine,
+    get_machine_fields_schema,
+    get_machine_state,
+    list_machine_state,
+    list_machines,
+    set_machine,
+)
 
 
 @pytest.mark.with_core
@@ -219,7 +229,44 @@ def test_get_machine_writeability(clan_flake: Callable[..., Flake]) -> None:
         "deploy.buildHost",
         "description",
         "icon",
+        "installedAt",
     }
     assert read_only_fields == {"machineClass", "name"}
 
     assert write_info["tags"]["readonly_members"] == ["nix1", "all", "nixos"]
+
+
+@pytest.mark.with_core
+def test_machine_state(clan_flake: Callable[..., Flake]) -> None:
+    now = int(time.time())
+    yesterday = now - 86400
+    last_week = now - 604800
+
+    flake = clan_flake(
+        # clan.nix, cannot be changed
+        clan={
+            "inventory": {
+                "machines": {
+                    "jon": {},
+                    "sara": {"installedAt": yesterday},
+                    "bob": {"installedAt": last_week},
+                },
+            }
+        },
+    )
+
+    assert list_machine_state(flake) == {
+        "jon": MachineState(status=MachineStatus.NOT_INSTALLED),
+        "sara": MachineState(status=MachineStatus.OFFLINE),
+        "bob": MachineState(status=MachineStatus.OFFLINE),
+    }
+
+    assert get_machine_state(Machine("jon", flake)) == MachineState(
+        status=MachineStatus.NOT_INSTALLED
+    )
+    assert get_machine_state(Machine("sara", flake)) == MachineState(
+        status=MachineStatus.OFFLINE
+    )
+    assert get_machine_state(Machine("bob", flake)) == MachineState(
+        status=MachineStatus.OFFLINE
+    )

pkgs/clan-cli/clan_lib/machines/install.py

@@ -7,7 +7,6 @@ from typing import Literal
 
 from clan_cli.facts.generate import generate_facts
 from clan_cli.machines.hardware import HardwareConfig
-from clan_cli.vars.generate import generate_vars
 
 from clan_lib.api import API, message_queue
 from clan_lib.cmd import Log, RunOpts, run
@@ -15,6 +14,7 @@ from clan_lib.machines.machines import Machine
 from clan_lib.nix import nix_config, nix_shell
 from clan_lib.ssh.create import create_secret_key_nixos_anywhere
 from clan_lib.ssh.remote import Remote
+from clan_lib.vars.generate import run_generators
 
 log = logging.getLogger(__name__)
 
@@ -88,7 +88,7 @@ def run_machine_install(opts: InstallOptions, target_host: Remote) -> None:
     # Notify the UI about what we are doing
     notify_install_step("generators")
     generate_facts([machine])
-    generate_vars([machine])
+    run_generators([machine], generators=None, full_closure=False)
 
     with (
         TemporaryDirectory(prefix="nixos-install-") as _base_directory,

pkgs/clan-cli/clan_lib/machines/morph.py

@@ -7,7 +7,6 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 
 from clan_cli.machines.create import CreateOptions, create_machine
-from clan_cli.vars.generate import generate_vars
 
 from clan_lib.cmd import Log, RunOpts, run
 from clan_lib.dirs import specific_machine_dir
@@ -16,6 +15,7 @@ from clan_lib.machines.actions import list_machines
 from clan_lib.machines.machines import Machine
 from clan_lib.nix import nix_build, nix_command
 from clan_lib.nix_models.clan import InventoryMachine
+from clan_lib.vars.generate import run_generators
 
 log = logging.getLogger(__name__)
 
@@ -78,7 +78,7 @@ def morph_machine(
 
         machine = Machine(name=name, flake=Flake(str(flakedir)))
 
-        generate_vars([machine], generator_name=None, regenerate=False)
+        run_generators([machine], generators=None, full_closure=False)
 
         machine.secret_vars_store.populate_dir(
             machine.name,