yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'Make most vm tests pure.' (#4796) from no-impure into main

Browse Source 
Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4796
This commit is contained in:
Mic92
2025-08-19 16:10:08 +00:00
parent 099f4c2b8b 899dba5a08
commit 26b31e24a3
12 changed files with 149 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.gitea/workflows/checks.yaml
View File

@@ -1,9 +0,0 @@
name: checks
on:
pull_request:
jobs:
checks-impure:
runs-on: nix
steps:
- uses: actions/checkout@v4
- run: nix run .#impure-checks

1
checks/flake-module.nix
View File

@@ -36,7 +36,6 @@ in
++ filter pathExists [
./devshell/flake-module.nix
./flash/flake-module.nix
./impure/flake-module.nix
./installation/flake-module.nix
./update/flake-module.nix
./morph/flake-module.nix

51
checks/impure/flake-module.nix
View File

@@ -1,51 +0,0 @@
{
perSystem =
{
pkgs,
lib,
self',
...
}:
{
# a script that executes all other checks
packages.impure-checks = pkgs.writeShellScriptBin "impure-checks" ''
#!${pkgs.bash}/bin/bash
set -euo pipefail
unset CLAN_DIR
export PATH="${
lib.makeBinPath (
[
pkgs.gitMinimal
pkgs.nix
pkgs.coreutils
pkgs.rsync # needed to have rsync installed on the dummy ssh server
]
++ self'.packages.clan-cli-full.runtimeDependencies
)
}"
ROOT=$(git rev-parse --show-toplevel)
cd "$ROOT/pkgs/clan-cli"
# Set up custom git configuration for tests
export GIT_CONFIG_GLOBAL=$(mktemp)
git config --file "$GIT_CONFIG_GLOBAL" user.name "Test User"
git config --file "$GIT_CONFIG_GLOBAL" user.email "test@example.com"
export GIT_CONFIG_SYSTEM=/dev/null
# this disables dynamic dependency loading in clan-cli
export CLAN_NO_DYNAMIC_DEPS=1
jobs=$(nproc)
# Spawning worker in pytest is relatively slow, so we limit the number of jobs to 13
# (current number of impure tests)
jobs="$((jobs > 6 ? 6 : jobs))"
nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -n $jobs -m impure ./clan_cli $@"
# Clean up temporary git config
rm -f "$GIT_CONFIG_GLOBAL"
'';
};
}

21
docs/site/guides/contributing/debugging.md
View File

@@ -90,13 +90,10 @@ export CLAN_DEBUG_COMMANDS=1
These options help you pinpoint the source and context of print messages and debug logs during development.
## Analyzing Performance
To understand what's causing slow performance, set the environment variable `export CLAN_CLI_PERF=1`. When you complete a clan command, you'll see a summary of various performance metrics, helping you identify what's taking up time.
## See all possible packages and tests
To quickly show all possible packages and tests execute:
@@ -155,28 +152,16 @@ To test the CLI locally in a development environment and set breakpoints for deb
## Test Locally in a Nix Sandbox
To run tests in a Nix sandbox, you have two options depending on whether your test functions have been marked as impure or not:
### Running Tests Marked as Impure
If your test functions need to execute `nix build` and have been marked as impure because you can't execute `nix build` inside a Nix sandbox, use the following command:
To run tests in a Nix sandbox:
```bash
nix run .#impure-checks -L
nix build .#checks.x86_64-linux.clan-pytest-with-core
```
This command will run the impure test functions.
### Running Pure Tests
For test functions that have not been marked as impure and don't require executing `nix build`, you can use the following command:
```bash
nix build .#checks.x86_64-linux.clan-pytest --rebuild
nix build .#checks.x86_64-linux.clan-pytest-without-core
```
This command will run all pure test functions.
### Inspecting the Nix Sandbox
If you need to inspect the Nix sandbox while running tests, follow these steps:

116
nixosModules/clanCore/vm-base.nix Normal file
View File

@@ -0,0 +1,116 @@
# Standalone VM base module that can be imported independently
# This module contains the core VM configuration without the system extension
{
lib,
config,
pkgs,
modulesPath,
...
}:
let
# Flatten the list of state folders into a single list
stateFolders = lib.flatten (
lib.mapAttrsToList (_item: attrs: attrs.folders) config.clan.core.state
);
in
{
imports = [
(modulesPath + "/virtualisation/qemu-vm.nix")
./serial.nix
./waypipe.nix
];
clan.core.state.HOME.folders = [ "/home" ];
clan.services.waypipe = {
inherit (config.clan.core.vm.inspect.waypipe) enable command;
};
# required for issuing shell commands via qga
services.qemuGuest.enable = true;
# required to react to system_powerdown qmp command
# Some desktop managers like xfce override the poweroff signal and therefore
# make it impossible to handle it via 'logind' directly.
services.acpid.enable = true;
services.acpid.handlers.power.event = "button/power.*";
services.acpid.handlers.power.action = "poweroff";
# only works on x11
services.spice-vdagentd.enable = config.services.xserver.enable;
boot.initrd.systemd.enable = true;
boot.initrd.systemd.storePaths = [
pkgs.util-linux
pkgs.e2fsprogs
];
boot.initrd.systemd.emergencyAccess = true;
# userborn would be faster because it doesn't need perl, but it cannot create normal users
services.userborn.enable = true;
users.mutableUsers = false;
users.allowNoPasswordLogin = true;
boot.initrd.kernelModules = [ "virtiofs" ];
virtualisation.writableStore = false;
virtualisation.fileSystems = lib.mkForce (
{
"/nix/store" = {
device = "nix-store";
options = [
"x-systemd.requires=systemd-modules-load.service"
"ro"
];
fsType = "virtiofs";
};
"/" = {
device = "/dev/vda";
fsType = "ext4";
options = [
"defaults"
"x-systemd.makefs"
"nobarrier"
"noatime"
"nodiratime"
"data=writeback"
"discard"
];
};
"/vmstate" = {
device = "/dev/vdb";
options = [
"x-systemd.makefs"
"noatime"
"nodiratime"
"discard"
];
noCheck = true;
fsType = "ext4";
};
${config.clan.core.facts.secretUploadDirectory} = {
device = "secrets";
fsType = "9p";
neededForBoot = true;
options = [
"trans=virtio"
"version=9p2000.L"
"cache=loose"
];
};
}
// lib.listToAttrs (
map (
folder:
lib.nameValuePair folder {
device = "/vmstate${folder}";
fsType = "none";
options = [ "bind" ];
}
) stateFolders
)
);
}

109
nixosModules/clanCore/vm.nix
View File

@@ -4,116 +4,11 @@
pkgs,
options,
extendModules,
modulesPath,
...
}:
let
# Flatten the list of state folders into a single list
stateFolders = lib.flatten (
lib.mapAttrsToList (_item: attrs: attrs.folders) config.clan.core.state
);
vmModule = {
imports = [
(modulesPath + "/virtualisation/qemu-vm.nix")
./serial.nix
./waypipe.nix
];
clan.core.state.HOME.folders = [ "/home" ];
clan.services.waypipe = {
inherit (config.clan.core.vm.inspect.waypipe) enable command;
};
# required for issuing shell commands via qga
services.qemuGuest.enable = true;
# required to react to system_powerdown qmp command
# Some desktop managers like xfce override the poweroff signal and therefore
# make it impossible to handle it via 'logind' directly.
services.acpid.enable = true;
services.acpid.handlers.power.event = "button/power.*";
services.acpid.handlers.power.action = "poweroff";
# only works on x11
services.spice-vdagentd.enable = config.services.xserver.enable;
boot.initrd.systemd.enable = true;
boot.initrd.systemd.storePaths = [
pkgs.util-linux
pkgs.e2fsprogs
];
boot.initrd.systemd.emergencyAccess = true;
# userborn would be faster because it doesn't need perl, but it cannot create normal users
services.userborn.enable = true;
users.mutableUsers = false;
users.allowNoPasswordLogin = true;
boot.initrd.kernelModules = [ "virtiofs" ];
virtualisation.writableStore = false;
virtualisation.fileSystems = lib.mkForce (
{
"/nix/store" = {
device = "nix-store";
options = [
"x-systemd.requires=systemd-modules-load.service"
"ro"
];
fsType = "virtiofs";
};
"/" = {
device = "/dev/vda";
fsType = "ext4";
options = [
"defaults"
"x-systemd.makefs"
"nobarrier"
"noatime"
"nodiratime"
"data=writeback"
"discard"
];
};
"/vmstate" = {
device = "/dev/vdb";
options = [
"x-systemd.makefs"
"noatime"
"nodiratime"
"discard"
];
noCheck = true;
fsType = "ext4";
};
${config.clan.core.facts.secretUploadDirectory} = {
device = "secrets";
fsType = "9p";
neededForBoot = true;
options = [
"trans=virtio"
"version=9p2000.L"
"cache=loose"
];
};
}
// lib.listToAttrs (
map (
folder:
lib.nameValuePair folder {
device = "/vmstate${folder}";
fsType = "none";
options = [ "bind" ];
}
) stateFolders
)
);
};
# Import the standalone VM base module
vmModule = import ./vm-base.nix;
# We cannot simply merge the VM config into the current system config, because
# it is not necessarily a VM.

3
nixosModules/flake-module.nix
View File

@@ -34,4 +34,7 @@ in
flake.nixosModules.clanCore = clanCore;
flake.darwinModules.clanCore = clanCore;
# Standalone VM base module that can be imported for VM testing
flake.nixosModules.clan-vm-base = ./clanCore/vm-base.nix;
}

10
pkgs/clan-cli/README.md
View File

@@ -32,16 +32,12 @@ You can also run a single test like this:
pytest -n0 -s tests/test_secrets_cli.py::test_users
```
## Run tests in nix container
Run all impure checks
Run all checks in a sandbox
```bash
nix run .#impure-checks
nix build .#checks.x86_64-linux.clan-pytest-with-core
```
Run all checks
```bash
nix flake check
nix build .#checks.x86_64-linux.clan-pytest-without-core
```

4
pkgs/clan-cli/clan_cli/tests/flake-module.nix
View File

@@ -25,6 +25,8 @@
test-vm-persistence =
{ config, ... }:
{
imports = [ self.nixosModules.clan-vm-base ];
system.stateVersion = config.system.nixos.release;
# Disable services that might cause issues in tests
@@ -62,6 +64,8 @@
test-vm-deployment =
{ config, lib, ... }:
{
imports = [ self.nixosModules.clan-vm-base ];
system.stateVersion = config.system.nixos.release;
# Disable services that might cause issues in tests

2
pkgs/clan-cli/clan_cli/tests/test_vars_deployment.py
View File

@@ -13,7 +13,7 @@ from clan_lib.machines.machines import Machine
from clan_lib.nix import nix_eval, run
@pytest.mark.impure
@pytest.mark.with_core
@pytest.mark.skipif(sys.platform == "darwin", reason="preload doesn't work on darwin")
def test_vm_deployment(
vm_test_flake: Path,

21
pkgs/clan-cli/clan_cli/tests/test_vms_cli.py
View File

@@ -25,14 +25,14 @@ def test_inspect(
@pytest.mark.skipif(no_kvm, reason="Requires KVM")
@pytest.mark.impure
@pytest.mark.with_core
def test_run(
monkeypatch: pytest.MonkeyPatch,
test_flake_with_core: FlakeForTest,
vm_test_flake: Path,
age_keys: list["KeyPair"],
) -> None:
with monkeypatch.context():
monkeypatch.chdir(test_flake_with_core.path)
monkeypatch.chdir(vm_test_flake)
monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
cli.run(
@@ -53,11 +53,22 @@ def test_run(
"user1",
]
)
cli.run(["vms", "run", "--no-block", "vm1", "-c", "shutdown", "-h", "now"])
cli.run(
[
"vms",
"run",
"--no-block",
"test-vm-deployment",
"-c",
"shutdown",
"-h",
"now",
]
)
@pytest.mark.skipif(no_kvm, reason="Requires KVM")
@pytest.mark.impure
@pytest.mark.with_core
def test_vm_persistence(
vm_test_flake: Path,
) -> None:

1
pkgs/clan-cli/default.nix
View File

@@ -243,6 +243,7 @@ pythonRuntime.pkgs.buildPythonApplication {
pkgs.unzip
pkgs.libxslt
pkgs.getconf
pkgs.chroot-realpath
nixosConfigurations."test-vm-persistence-${stdenv.hostPlatform.system}".config.system.clan.vm.create
nixosConfigurations."test-vm-deployment-${stdenv.hostPlatform.system}".config.system.clan.vm.create