From 8fe7cb1b3dd6db6095a56eeb71d643dcb3a1fd1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 18 Aug 2025 12:32:47 +0200 Subject: [PATCH 1/9] virtiofsd: fix nix chroot store support --- flake.nix | 1 + pkgs/clan-cli/clan_cli/vms/virtiofsd.py | 11 +++++++++-- pkgs/clan-cli/flake-module.nix | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 7e92e3ec1..8f7356fdb 100644 --- a/flake.nix +++ b/flake.nix @@ -96,6 +96,7 @@ ./nixosModules/flake-module.nix ./pkgs/flake-module.nix ./templates/flake-module.nix + ./pkgs/clan-cli/clan_cli/tests/flake-module.nix ] ++ [ (if pathExists ./flakeModules/clan.nix then import ./flakeModules/clan.nix inputs.self else { }) diff --git a/pkgs/clan-cli/clan_cli/vms/virtiofsd.py b/pkgs/clan-cli/clan_cli/vms/virtiofsd.py index f010ebeed..9a5dccfdd 100644 --- a/pkgs/clan-cli/clan_cli/vms/virtiofsd.py +++ b/pkgs/clan-cli/clan_cli/vms/virtiofsd.py @@ -1,4 +1,5 @@ import contextlib +import logging import shutil import subprocess import time @@ -6,7 +7,9 @@ from collections.abc import Iterator from pathlib import Path from clan_lib.errors import ClanError -from clan_lib.nix import nix_shell +from clan_lib.nix import nix_shell, nix_test_store + +log = logging.getLogger(__name__) @contextlib.contextmanager @@ -14,6 +17,9 @@ def start_virtiofsd(socket_path: Path) -> Iterator[None]: sandbox = "namespace" if shutil.which("newuidmap") is None: sandbox = "none" + store_root = nix_test_store() or Path("/") + store = store_root / "nix" / "store" + virtiofsd = nix_shell( ["virtiofsd"], [ @@ -25,9 +31,10 @@ def start_virtiofsd(socket_path: Path) -> Iterator[None]: "--sandbox", sandbox, "--shared-dir", - "/nix/store", + str(store), ], ) + log.debug("$ {}".format(" ".join(virtiofsd))) with subprocess.Popen(virtiofsd) as proc: try: while not socket_path.exists(): diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index d2b045875..3213e7991 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -25,6 +25,7 @@ "clanServices" "pkgs/zerotierone" "pkgs/minifakeroot" + "pkgs/clan-cli/clan_cli/tests/flake-module.nix" ]; }; }; From 6fe2b06f09de5157ab95fb0c98516a21c0632bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 18 Aug 2025 12:33:17 +0200 Subject: [PATCH 2/9] qemu: fix nix chroot store support --- pkgs/clan-cli/clan_cli/vms/qemu.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/vms/qemu.py b/pkgs/clan-cli/clan_cli/vms/qemu.py index bb4c6b977..63b3f4136 100644 --- a/pkgs/clan-cli/clan_cli/vms/qemu.py +++ b/pkgs/clan-cli/clan_cli/vms/qemu.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from pathlib import Path from clan_lib.errors import ClanError +from clan_lib.nix import nix_test_store from clan_cli.qemu.qmp import QEMUMonitorProtocol @@ -98,9 +99,17 @@ def qemu_command( ) -> QemuCommand: if portmap is None: portmap = {} + + toplevel = Path(nixos_config["toplevel"]) + chroot_toplevel = toplevel + initrd = Path(nixos_config["initrd"]) + if tmp_store := nix_test_store(): + chroot_toplevel = tmp_store / toplevel.relative_to("/") + initrd = tmp_store / initrd.relative_to("/") + kernel_cmdline = [ - (Path(nixos_config["toplevel"]) / "kernel-params").read_text(), - f"init={nixos_config['toplevel']}/init", + (chroot_toplevel / "kernel-params").read_text(), + f"init={toplevel}/init", f"regInfo={nixos_config['regInfo']}/registration", "console=hvc0", ] @@ -131,8 +140,8 @@ def qemu_command( "-device", "virtio-blk-pci,drive=state", "-device", "virtio-keyboard", "-usb", "-device", "usb-tablet,bus=usb-bus.0", - "-kernel", f'{nixos_config["toplevel"]}/kernel', - "-initrd", nixos_config["initrd"], + "-kernel", f"{chroot_toplevel}/kernel", + "-initrd", str(initrd), "-append", " ".join(kernel_cmdline), # qmp & qga setup "-qmp", f"unix:{qmp_socket_file},server,wait=off", From 4074a184b28a8e597b93eaf477a6a0d6bff4433b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Aug 2025 15:54:25 +0200 Subject: [PATCH 3/9] make vm test pure --- pkgs/clan-cli/clan_cli/tests/age_keys.py | 3 +- .../clan_cli/tests/fixtures_flakes.py | 90 +++++++++++++ pkgs/clan-cli/clan_cli/tests/flake-module.nix | 119 ++++++++++++++++++ .../clan_cli/tests/test_vars_deployment.py | 56 ++------- pkgs/clan-cli/clan_cli/tests/test_vms_cli.py | 33 ++--- .../clan-cli/clan_cli/tests/vm_test_flake.nix | 26 ++++ pkgs/clan-cli/clan_cli/vms/run.py | 2 +- pkgs/clan-cli/default.nix | 21 ++++ pkgs/clan-cli/flake-module.nix | 4 + 9 files changed, 282 insertions(+), 72 deletions(-) create mode 100644 pkgs/clan-cli/clan_cli/tests/flake-module.nix create mode 100644 pkgs/clan-cli/clan_cli/tests/vm_test_flake.nix diff --git a/pkgs/clan-cli/clan_cli/tests/age_keys.py b/pkgs/clan-cli/clan_cli/tests/age_keys.py index ef10335f6..5c38ad35a 100644 --- a/pkgs/clan-cli/clan_cli/tests/age_keys.py +++ b/pkgs/clan-cli/clan_cli/tests/age_keys.py @@ -1,6 +1,5 @@ import dataclasses import json -import os from collections.abc import Iterable from pathlib import Path @@ -26,7 +25,7 @@ class SopsSetup: def __init__(self, keys: list[KeyPair]) -> None: self.keys = keys - self.user = os.environ.get("USER", "admin") + self.user = "admin" def init(self, flake_path: Path) -> None: cli.run( diff --git a/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py b/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py index 060776503..be434265a 100644 --- a/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py +++ b/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py @@ -422,3 +422,93 @@ def test_flake_with_core( monkeypatch=monkeypatch, inventory_expr=inventory_expr, ) + + +@pytest.fixture +def writable_clan_core( + clan_core: Path, + tmp_path: Path, +) -> Path: + """ + Creates a writable copy of clan_core in a temporary directory. + If clan_core is a git repo, copies tracked files and uncommitted changes. + Removes vars/ and sops/ directories if they exist. + """ + temp_flake = tmp_path / "clan-core" + + # Check if it's a git repository + if (clan_core / ".git").exists(): + # Create the target directory + temp_flake.mkdir(parents=True) + + # Copy all tracked and untracked files (excluding ignored) + # Using git ls-files with -z for null-terminated output to handle filenames with spaces + sp.run( + f"(git ls-files -z; git ls-files -z --others --exclude-standard) | " + f"xargs -0 cp --parents -t {temp_flake}/", + shell=True, + cwd=clan_core, + check=True, + ) + + # Copy .git directory to maintain git functionality + if (clan_core / ".git").is_dir(): + shutil.copytree( + clan_core / ".git", temp_flake / ".git", ignore_dangling_symlinks=True + ) + else: + # It's a git file (for submodules/worktrees) + shutil.copy2(clan_core / ".git", temp_flake / ".git") + else: + # Regular copy if not a git repo + shutil.copytree(clan_core, temp_flake, ignore_dangling_symlinks=True) + + # Make writable + sp.run(["chmod", "-R", "+w", str(temp_flake)], check=True) + + # Remove vars and sops directories + shutil.rmtree(temp_flake / "vars", ignore_errors=True) + shutil.rmtree(temp_flake / "sops", ignore_errors=True) + + return temp_flake + + +@pytest.fixture +def vm_test_flake( + clan_core: Path, + tmp_path: Path, +) -> Path: + """ + Creates a test flake that imports the VM test nixOS modules from clan-core. + """ + test_flake_dir = tmp_path / "test-flake" + test_flake_dir.mkdir(parents=True) + + metadata = sp.run( + nix_command(["flake", "metadata", "--json"]), + cwd=CLAN_CORE, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + metadata_json = json.loads(metadata) + clan_core_url = f"path:{metadata_json['path']}" + + # Read the template and substitute the clan-core path + template_path = Path(__file__).parent / "vm_test_flake.nix" + template_content = template_path.read_text() + + # Substitute the clan-core URL + flake_content = template_content.replace("__CLAN_CORE__", clan_core_url) + + # Write the flake.nix + (test_flake_dir / "flake.nix").write_text(flake_content) + + # Lock the flake with --allow-dirty to handle uncommitted changes + sp.run( + nix_command(["flake", "lock", "--allow-dirty-locks"]), + cwd=test_flake_dir, + check=True, + ) + + return test_flake_dir diff --git a/pkgs/clan-cli/clan_cli/tests/flake-module.nix b/pkgs/clan-cli/clan_cli/tests/flake-module.nix new file mode 100644 index 000000000..c8835a0e7 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/tests/flake-module.nix @@ -0,0 +1,119 @@ +{ self, ... }: +{ + # Define machines that use the nixOS modules + clan.machines = { + test-vm-persistence = { + imports = [ self.nixosModules.test-vm-persistence ]; + }; + test-vm-deployment = { + imports = [ self.nixosModules.test-vm-deployment ]; + }; + }; + + flake.nixosModules = { + # NixOS module for test_vm_persistence + test-vm-persistence = + { config, ... }: + { + nixpkgs.hostPlatform = "x86_64-linux"; + system.stateVersion = config.system.nixos.release; + + # Disable services that might cause issues in tests + systemd.services.logrotate-checkconf.enable = false; + services.getty.autologinUser = "root"; + + # Basic networking setup + networking.useDHCP = false; + networking.firewall.enable = false; + + # VM-specific settings + clan.virtualisation.graphics = false; + clan.core.networking.targetHost = "client"; + + # State configuration for persistence test + clan.core.state.my_state.folders = [ + "/var/my-state" + "/var/user-state" + ]; + + # Initialize users for tests + users.users = { + root = { + initialPassword = "root"; + }; + test = { + initialPassword = "test"; + isSystemUser = true; + group = "users"; + }; + }; + }; + + # NixOS module for test_vm_deployment + test-vm-deployment = + { config, lib, ... }: + { + nixpkgs.hostPlatform = "x86_64-linux"; + system.stateVersion = config.system.nixos.release; + + # Disable services that might cause issues in tests + systemd.services.logrotate-checkconf.enable = false; + services.getty.autologinUser = "root"; + + # Basic networking setup + networking.useDHCP = false; + networking.firewall.enable = false; + + # VM-specific settings + clan.virtualisation.graphics = false; + + # SSH for deployment tests + services.openssh.enable = true; + + # Initialize users for tests + users.users = { + root = { + initialPassword = "root"; + }; + }; + + # hack to make sure + sops.validateSopsFiles = false; + sops.secrets."vars/m1_generator/my_secret" = lib.mkDefault { + sopsFile = builtins.toFile "fake" ""; + }; + + # Vars generators configuration + clan.core.vars.generators = { + m1_generator = { + files.my_secret = { + secret = true; + path = "/run/secrets/vars/m1_generator/my_secret"; + }; + script = '' + echo hello > "$out"/my_secret + ''; + }; + + my_shared_generator = { + share = true; + files = { + shared_secret = { + secret = true; + path = "/run/secrets/vars/my_shared_generator/shared_secret"; + }; + no_deploy_secret = { + secret = true; + deploy = false; + path = "/run/secrets/vars/my_shared_generator/no_deploy_secret"; + }; + }; + script = '' + echo hello > "$out"/shared_secret + echo hello > "$out"/no_deploy_secret + ''; + }; + }; + }; + }; +} diff --git a/pkgs/clan-cli/clan_cli/tests/test_vars_deployment.py b/pkgs/clan-cli/clan_cli/tests/test_vars_deployment.py index ae17ffff8..a5d726c9d 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_vars_deployment.py +++ b/pkgs/clan-cli/clan_cli/tests/test_vars_deployment.py @@ -2,63 +2,33 @@ import json import subprocess import sys from contextlib import ExitStack +from pathlib import Path import pytest from clan_cli.tests.age_keys import SopsSetup -from clan_cli.tests.fixtures_flakes import ClanFlake from clan_cli.tests.helpers import cli from clan_cli.vms.run import inspect_vm, spawn_vm -from clan_lib import cmd from clan_lib.flake import Flake from clan_lib.machines.machines import Machine -from clan_lib.nix import nix_config, nix_eval, run +from clan_lib.nix import nix_eval, run @pytest.mark.impure @pytest.mark.skipif(sys.platform == "darwin", reason="preload doesn't work on darwin") def test_vm_deployment( - flake: ClanFlake, + vm_test_flake: Path, sops_setup: SopsSetup, ) -> None: - # machine 1 - config = nix_config() - machine1_config = flake.machines["m1_machine"] - machine1_config["nixpkgs"]["hostPlatform"] = config["system"] - machine1_config["clan"]["virtualisation"]["graphics"] = False - machine1_config["services"]["getty"]["autologinUser"] = "root" - machine1_config["services"]["openssh"]["enable"] = True - machine1_config["networking"]["firewall"]["enable"] = False - machine1_config["users"]["users"]["root"]["openssh"]["authorizedKeys"]["keys"] = [ - # put your key here when debugging and pass ssh_port in run_vm_in_thread call below - ] - m1_generator = machine1_config["clan"]["core"]["vars"]["generators"]["m1_generator"] - m1_generator["files"]["my_secret"]["secret"] = True - m1_generator["script"] = """ - echo hello > "$out"/my_secret - """ - m1_shared_generator = machine1_config["clan"]["core"]["vars"]["generators"][ - "my_shared_generator" - ] - m1_shared_generator["share"] = True - m1_shared_generator["files"]["shared_secret"]["secret"] = True - m1_shared_generator["files"]["no_deploy_secret"]["secret"] = True - m1_shared_generator["files"]["no_deploy_secret"]["deploy"] = False - m1_shared_generator["script"] = """ - echo hello > "$out"/shared_secret - echo hello > "$out"/no_deploy_secret - """ - - flake.refresh() - - sops_setup.init(flake.path) - cli.run(["vars", "generate", "--flake", str(flake.path)]) + # Set up sops for the test flake machines + sops_setup.init(vm_test_flake) + cli.run(["vars", "generate", "--flake", str(vm_test_flake), "test-vm-deployment"]) # check sops secrets not empty sops_secrets = json.loads( run( nix_eval( [ - f"{flake.path}#nixosConfigurations.m1_machine.config.sops.secrets", + f"{vm_test_flake}#nixosConfigurations.test-vm-deployment.config.sops.secrets", ] ) ).stdout.strip() @@ -67,7 +37,7 @@ def test_vm_deployment( my_secret_path = run( nix_eval( [ - f"{flake.path}#nixosConfigurations.m1_machine.config.clan.core.vars.generators.m1_generator.files.my_secret.path", + f"{vm_test_flake}#nixosConfigurations.test-vm-deployment.config.clan.core.vars.generators.m1_generator.files.my_secret.path", ] ) ).stdout.strip() @@ -75,15 +45,15 @@ def test_vm_deployment( shared_secret_path = run( nix_eval( [ - f"{flake.path}#nixosConfigurations.m1_machine.config.clan.core.vars.generators.my_shared_generator.files.shared_secret.path", + f"{vm_test_flake}#nixosConfigurations.test-vm-deployment.config.clan.core.vars.generators.my_shared_generator.files.shared_secret.path", ] ) ).stdout.strip() assert "no-such-path" not in shared_secret_path - # run nix flake lock - cmd.run(["nix", "flake", "lock"], cmd.RunOpts(cwd=flake.path)) - vm1_config = inspect_vm(machine=Machine("m1_machine", Flake(str(flake.path)))) + vm1_config = inspect_vm( + machine=Machine("test-vm-deployment", Flake(str(vm_test_flake))) + ) with ExitStack() as stack: vm1 = stack.enter_context(spawn_vm(vm1_config, stdin=subprocess.DEVNULL)) qga_m1 = stack.enter_context(vm1.qga_connect()) @@ -92,7 +62,7 @@ def test_vm_deployment( # check my_secret is deployed result = qga_m1.run(["cat", "/run/secrets/vars/m1_generator/my_secret"]) assert result.stdout == "hello\n" - # check shared_secret is deployed on m1 + # check shared_secret is deployed result = qga_m1.run( ["cat", "/run/secrets/vars/my_shared_generator/shared_secret"] ) diff --git a/pkgs/clan-cli/clan_cli/tests/test_vms_cli.py b/pkgs/clan-cli/clan_cli/tests/test_vms_cli.py index 913c98a80..a39b2a309 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_vms_cli.py +++ b/pkgs/clan-cli/clan_cli/tests/test_vms_cli.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import TYPE_CHECKING import pytest -from clan_cli.tests.fixtures_flakes import ClanFlake, FlakeForTest +from clan_cli.tests.fixtures_flakes import FlakeForTest from clan_cli.tests.helpers import cli from clan_cli.tests.stdout import CaptureOutput from clan_cli.vms.run import inspect_vm, spawn_vm @@ -24,8 +24,7 @@ def test_inspect( assert "Cores" in output.out -# @pytest.mark.skipif(no_kvm, reason="Requires KVM") -@pytest.mark.skipif(True, reason="We need to fix vars support for vms for this test") +@pytest.mark.skipif(no_kvm, reason="Requires KVM") @pytest.mark.impure def test_run( monkeypatch: pytest.MonkeyPatch, @@ -60,30 +59,12 @@ def test_run( @pytest.mark.skipif(no_kvm, reason="Requires KVM") @pytest.mark.impure def test_vm_persistence( - flake: ClanFlake, + vm_test_flake: Path, ) -> None: - # set up a clan flake with some systemd services to test persistence - config = flake.machines["my_machine"] - config["nixpkgs"]["hostPlatform"] = "x86_64-linux" - # logrotate-checkconf doesn't work in VM because /nix/store is owned by nobody - config["systemd"]["services"]["logrotate-checkconf"]["enable"] = False - config["services"]["getty"]["autologinUser"] = "root" - config["clan"]["virtualisation"] = {"graphics": False} - config["clan"]["core"]["networking"] = {"targetHost": "client"} - config["clan"]["core"]["state"]["my_state"]["folders"] = [ - # to be owned by root - "/var/my-state", - # to be owned by user 'test' - "/var/user-state", - ] - config["users"]["users"] = { - "test": {"initialPassword": "test", "isSystemUser": True, "group": "users"}, - "root": {"initialPassword": "root"}, - } - - flake.refresh() - - vm_config = inspect_vm(machine=Machine("my_machine", Flake(str(flake.path)))) + # Use the pre-built test VM from the test flake + vm_config = inspect_vm( + machine=Machine("test-vm-persistence", Flake(str(vm_test_flake))) + ) with spawn_vm(vm_config) as vm, vm.qga_connect() as qga: # create state via qmp command instead of systemd service diff --git a/pkgs/clan-cli/clan_cli/tests/vm_test_flake.nix b/pkgs/clan-cli/clan_cli/tests/vm_test_flake.nix new file mode 100644 index 000000000..7a6aa25a8 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/tests/vm_test_flake.nix @@ -0,0 +1,26 @@ +{ + inputs.clan-core.url = "__CLAN_CORE__"; + + outputs = + { self, clan-core }: + let + clan = clan-core.lib.clan { + inherit self; + meta.name = "test-flake"; + machines = { + test-vm-persistence = { + imports = [ clan-core.nixosModules.test-vm-persistence ]; + }; + test-vm-deployment = { + imports = [ clan-core.nixosModules.test-vm-deployment ]; + }; + }; + }; + in + { + inherit (clan.config) nixosConfigurations; + inherit (clan.config) nixosModules; + inherit (clan.config) clanInternals; + clan = clan.config; + }; +} diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index 467b5f6b2..07d25b628 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -16,7 +16,7 @@ from clan_lib.cmd import CmdOut, Log, RunOpts, handle_io, run from clan_lib.dirs import module_root, user_cache_dir, vm_state_dir from clan_lib.errors import ClanCmdError, ClanError from clan_lib.machines.machines import Machine -from clan_lib.nix import nix_shell +from clan_lib.nix import nix_shell, nix_test_store from clan_lib.vars.generate import run_generators from clan_cli.completions import add_dynamic_completer, complete_machines diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 425076dd6..1bf64f382 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -19,6 +19,7 @@ templateDerivation, zerotierone, minifakeroot, + nixosConfigurations, }: let pyDeps = ps: [ @@ -225,6 +226,26 @@ pythonRuntime.pkgs.buildPythonApplication { # needed by flash list tests nixpkgs.legacyPackages.x86_64-linux.kbd nixpkgs.legacyPackages.x86_64-linux.glibcLocales + + # Pre-built VMs for impure tests + pkgs.stdenv.drvPath + pkgs.bash.drvPath + pkgs.buildPackages.xorg.lndir + (pkgs.perl.withPackages ( + p: with p; [ + ConfigIniFiles + FileSlurp + ] + )) + (pkgs.closureInfo { rootPaths = [ ]; }).drvPath + pkgs.desktop-file-utils + pkgs.dbus + pkgs.unzip + pkgs.libxslt + pkgs.getconf + + nixosConfigurations.test-vm-persistence.config.system.clan.vm.create + nixosConfigurations.test-vm-deployment.config.system.clan.vm.create ]; }; } diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index 3213e7991..d9cb14b71 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -5,6 +5,8 @@ ... }: { + imports = [ ./clan_cli/tests/flake-module.nix ]; + perSystem = { self', @@ -55,6 +57,7 @@ "age" "git" ]; + inherit (self) nixosConfigurations; }; clan-cli-full = pkgs.callPackage ./default.nix { inherit (inputs) nixpkgs nix-select; @@ -64,6 +67,7 @@ templateDerivation = templateDerivation; pythonRuntime = pkgs.python3; includedRuntimeDeps = lib.importJSON ./clan_lib/nix/allowed-packages.json; + inherit (self) nixosConfigurations; }; clan-cli-docs = pkgs.stdenv.mkDerivation { name = "clan-cli-docs"; From ed503f64da73fa559bbd67088b820754f39d9d72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 18 Aug 2025 12:34:02 +0200 Subject: [PATCH 4/9] vms/run: move python import to the top. --- pkgs/clan-cli/clan_cli/vms/run.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index 07d25b628..e348457f4 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -57,8 +57,6 @@ def build_vm( nix_options = [] secrets_dir = get_secrets(machine, tmpdir) - from clan_lib.nix import nix_test_store - output = Path( machine.select( "config.system.clan.vm.create", From 1850abdd0d65a91c5e988a211419bbdefea3e08d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 18 Aug 2025 13:49:17 +0200 Subject: [PATCH 5/9] clan-cli/vms/run: generate secret before inspect_vm inspect_vm does some caching, which lead to secrets not beeing found. --- pkgs/clan-cli/clan_cli/vms/run.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index e348457f4..acfb12838 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -82,11 +82,9 @@ def get_secrets( secrets_dir = tmpdir / "secrets" secrets_dir.mkdir(parents=True, exist_ok=True) - generate_facts([machine]) - run_generators([machine]) - machine.secret_facts_store.upload(secrets_dir) populate_secret_vars(machine, secrets_dir) + return secrets_dir @@ -384,6 +382,9 @@ def run_command( ) -> None: machine_obj: Machine = Machine(args.machine, args.flake) + generate_facts([machine_obj]) + run_generators([machine_obj]) + vm: VmConfig = inspect_vm(machine=machine_obj) if not os.environ.get("WAYLAND_DISPLAY"): From 5b1a9d6848b2b594c7048937552d8c5afd0ba7cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 18 Aug 2025 15:16:07 +0200 Subject: [PATCH 6/9] vms: also prebuild for aarch64 --- pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py | 12 +++++++++++- pkgs/clan-cli/clan_cli/tests/flake-module.nix | 16 ++++++++++++---- pkgs/clan-cli/clan_cli/tests/vm_test_flake.nix | 2 ++ pkgs/clan-cli/default.nix | 4 ++-- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py b/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py index be434265a..e79371074 100644 --- a/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py +++ b/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py @@ -498,8 +498,18 @@ def vm_test_flake( template_path = Path(__file__).parent / "vm_test_flake.nix" template_content = template_path.read_text() - # Substitute the clan-core URL + # Get the current system + system_result = sp.run( + nix_command(["config", "show", "system"]), + capture_output=True, + text=True, + check=True, + ) + current_system = system_result.stdout.strip() + + # Substitute the clan-core URL and system flake_content = template_content.replace("__CLAN_CORE__", clan_core_url) + flake_content = flake_content.replace("__SYSTEM__", current_system) # Write the flake.nix (test_flake_dir / "flake.nix").write_text(flake_content) diff --git a/pkgs/clan-cli/clan_cli/tests/flake-module.nix b/pkgs/clan-cli/clan_cli/tests/flake-module.nix index c8835a0e7..a60bf8e37 100644 --- a/pkgs/clan-cli/clan_cli/tests/flake-module.nix +++ b/pkgs/clan-cli/clan_cli/tests/flake-module.nix @@ -2,11 +2,21 @@ { # Define machines that use the nixOS modules clan.machines = { - test-vm-persistence = { + test-vm-persistence-x86_64-linux = { imports = [ self.nixosModules.test-vm-persistence ]; + nixpkgs.hostPlatform = "x86_64-linux"; }; - test-vm-deployment = { + test-vm-persistence-aarch64-linux = { + imports = [ self.nixosModules.test-vm-persistence ]; + nixpkgs.hostPlatform = "aarch64-linux"; + }; + test-vm-deployment-x86_64-linux = { imports = [ self.nixosModules.test-vm-deployment ]; + nixpkgs.hostPlatform = "x86_64-linux"; + }; + test-vm-deployment-aarch64-linux = { + imports = [ self.nixosModules.test-vm-deployment ]; + nixpkgs.hostPlatform = "aarch64-linux"; }; }; @@ -15,7 +25,6 @@ test-vm-persistence = { config, ... }: { - nixpkgs.hostPlatform = "x86_64-linux"; system.stateVersion = config.system.nixos.release; # Disable services that might cause issues in tests @@ -53,7 +62,6 @@ test-vm-deployment = { config, lib, ... }: { - nixpkgs.hostPlatform = "x86_64-linux"; system.stateVersion = config.system.nixos.release; # Disable services that might cause issues in tests diff --git a/pkgs/clan-cli/clan_cli/tests/vm_test_flake.nix b/pkgs/clan-cli/clan_cli/tests/vm_test_flake.nix index 7a6aa25a8..10676624d 100644 --- a/pkgs/clan-cli/clan_cli/tests/vm_test_flake.nix +++ b/pkgs/clan-cli/clan_cli/tests/vm_test_flake.nix @@ -10,9 +10,11 @@ machines = { test-vm-persistence = { imports = [ clan-core.nixosModules.test-vm-persistence ]; + nixpkgs.hostPlatform = "__SYSTEM__"; }; test-vm-deployment = { imports = [ clan-core.nixosModules.test-vm-deployment ]; + nixpkgs.hostPlatform = "__SYSTEM__"; }; }; }; diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 1bf64f382..ae76a006a 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -244,8 +244,8 @@ pythonRuntime.pkgs.buildPythonApplication { pkgs.libxslt pkgs.getconf - nixosConfigurations.test-vm-persistence.config.system.clan.vm.create - nixosConfigurations.test-vm-deployment.config.system.clan.vm.create + nixosConfigurations."test-vm-persistence-${stdenv.hostPlatform.system}".config.system.clan.vm.create + nixosConfigurations."test-vm-deployment-${stdenv.hostPlatform.system}".config.system.clan.vm.create ]; }; } From 3e664255d65b04d380bd594c2072985994e77e43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 18 Aug 2025 16:50:20 +0200 Subject: [PATCH 7/9] speed up tests by doing reflink copies --- pkgs/testing/flake-module.nix | 5 ++--- pkgs/testing/nixos_test_lib/nix_setup.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pkgs/testing/flake-module.nix b/pkgs/testing/flake-module.nix index 235514393..3772259e9 100644 --- a/pkgs/testing/flake-module.nix +++ b/pkgs/testing/flake-module.nix @@ -15,8 +15,7 @@ mkdir -p "$CLAN_TEST_STORE/nix/store" mkdir -p "$CLAN_TEST_STORE/nix/var/nix/gcroots" if [[ -n "''${closureInfo-}" ]]; then - # ${pkgs.findutils}/bin/xargs ${pkgs.xcp}/bin/xcp --recursive --target-directory "$CLAN_TEST_STORE/nix/store" < "$closureInfo/store-paths" - ${pkgs.findutils}/bin/xargs ${pkgs.coreutils}/bin/cp --recursive --target "$CLAN_TEST_STORE/nix/store" < "$closureInfo/store-paths" + ${pkgs.findutils}/bin/xargs ${pkgs.xcp}/bin/xcp --recursive --target-directory "$CLAN_TEST_STORE/nix/store" < "$closureInfo/store-paths" ${pkgs.nix}/bin/nix-store --load-db --store "$CLAN_TEST_STORE" < "$closureInfo/registration" fi ''; @@ -39,7 +38,7 @@ ]; postPatch = '' substituteInPlace nixos_test_lib/nix_setup.py \ - --replace '@cp@' '${pkgs.coreutils}/bin/cp' \ + --replace '@xcp@' '${pkgs.xcp}/bin/xcp' \ --replace '@nix-store@' '${pkgs.nix}/bin/nix-store' \ --replace '@xargs@' '${pkgs.findutils}/bin/xargs' ''; diff --git a/pkgs/testing/nixos_test_lib/nix_setup.py b/pkgs/testing/nixos_test_lib/nix_setup.py index 5e0ebc8dd..e5fe50d9b 100644 --- a/pkgs/testing/nixos_test_lib/nix_setup.py +++ b/pkgs/testing/nixos_test_lib/nix_setup.py @@ -5,7 +5,7 @@ import subprocess from pathlib import Path # These paths will be substituted during package build -CP_BIN = "@cp@" +XCP_BIN = "@xcp@" NIX_STORE_BIN = "@nix-store@" XARGS_BIN = "@xargs@" @@ -52,7 +52,7 @@ def setup_nix_in_nix(closure_info: str | None) -> None: subprocess.run( # noqa: S603 [ XARGS_BIN, - CP_BIN, + XCP_BIN, "--recursive", "--target-directory", f"{tmpdir}/store/nix/store", From 2ce5388a75ba62476422245266b7e52f9d9c899a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 19 Aug 2025 12:06:52 +0200 Subject: [PATCH 8/9] qemu: fix machine types for various platforms --- pkgs/clan-cli/clan_cli/vms/qemu.py | 42 +++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_cli/vms/qemu.py b/pkgs/clan-cli/clan_cli/vms/qemu.py index 63b3f4136..e760c9dc7 100644 --- a/pkgs/clan-cli/clan_cli/vms/qemu.py +++ b/pkgs/clan-cli/clan_cli/vms/qemu.py @@ -1,3 +1,4 @@ +import platform import random from collections.abc import Generator from contextlib import contextmanager @@ -85,6 +86,44 @@ class QemuCommand: vsock_cid: int | None = None +def get_machine_options() -> str: + """Get appropriate QEMU machine options for host architecture.""" + arch = platform.machine().lower() + system = platform.system().lower() + + # Determine accelerator based on OS + if system == "darwin": + # macOS uses Hypervisor.framework + accel = "hvf" + else: + # Linux and others use KVM + accel = "kvm" + + if arch in ("x86_64", "amd64", "i386", "i686"): + # For x86_64, use q35 for modern PCIe support + return f"q35,memory-backend=mem,accel={accel}" + if arch in ("aarch64", "arm64"): + # Use virt machine type for ARM64 + if system == "darwin": + # macOS ARM uses GIC version 2 + return f"virt,gic-version=2,memory-backend=mem,accel={accel}" + # Linux ARM uses max GIC version + return f"virt,gic-version=max,memory-backend=mem,accel={accel}" + if arch == "armv7l": + # 32-bit ARM + return f"virt,memory-backend=mem,accel={accel}" + if arch in ("riscv64", "riscv32"): + # RISC-V architectures + return f"virt,memory-backend=mem,accel={accel}" + if arch in ("powerpc64le", "powerpc64", "ppc64le", "ppc64"): + # PowerPC architectures + return f"powernv,memory-backend=mem,accel={accel}" + + # No fallback - raise an error for unsupported architectures + msg = f"Unsupported architecture: {arch} on {system}. Supported architectures are: x86_64, aarch64, armv7l, riscv64, riscv32, powerpc64" + raise ClanError(msg) + + def qemu_command( vm: VmConfig, nixos_config: dict[str, str], @@ -116,13 +155,14 @@ def qemu_command( if not vm.waypipe.enable: kernel_cmdline.append("console=tty0") hostfwd = ",".join(f"hostfwd=tcp::{h}-:{g}" for h, g in portmap.items()) + machine_options = get_machine_options() # fmt: off command = [ "qemu-kvm", "-name", vm.machine_name, "-m", f'{nixos_config["memorySize"]}M', "-object", f"memory-backend-memfd,id=mem,size={nixos_config['memorySize']}M", - "-machine", "pc,memory-backend=mem,accel=kvm", + "-machine", machine_options, "-smp", str(nixos_config["cores"]), "-cpu", "max", "-enable-kvm", From 699c56c721464d5ded1ad18ecc44b3f756223373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 19 Aug 2025 12:43:30 +0000 Subject: [PATCH 9/9] qemu: enable usb tablet option only on x86_64-linux at least on aarch64-linux this locks up the hypervisor --- pkgs/clan-cli/clan_cli/vms/qemu.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_cli/vms/qemu.py b/pkgs/clan-cli/clan_cli/vms/qemu.py index e760c9dc7..61f83c3f6 100644 --- a/pkgs/clan-cli/clan_cli/vms/qemu.py +++ b/pkgs/clan-cli/clan_cli/vms/qemu.py @@ -179,7 +179,6 @@ def qemu_command( "-drive", f"cache=writeback,file={state_img},format=qcow2,id=state,if=none,index=2,werror=report", "-device", "virtio-blk-pci,drive=state", "-device", "virtio-keyboard", - "-usb", "-device", "usb-tablet,bus=usb-bus.0", "-kernel", f"{chroot_toplevel}/kernel", "-initrd", str(initrd), "-append", " ".join(kernel_cmdline), @@ -189,6 +188,11 @@ def qemu_command( "-device", "virtio-serial", "-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0", ] + # USB tablet only works reliably on x86_64 Linux for now, not aarch64-linux. + # TODO: Fix USB tablet support for ARM architectures and test macOS + if platform.system().lower() == "linux" and platform.machine().lower() in ("x86_64", "amd64"): + command.extend(["-usb", "-device", "usb-tablet,bus=usb-bus.0"]) + if interactive: command.extend( [