VMs: port vm_persistence test to python

checks/impure/flake-module.nix

@@ -1,5 +1,5 @@
-{ self, ... }: {
+{ ... }: {
-  perSystem = { pkgs, lib, self', ... }: {
+  perSystem = { pkgs, lib, ... }: {
     packages = rec {
       # a script that executes all other checks
       impure-checks = pkgs.writeShellScriptBin "impure-checks" ''
@@ -13,86 +13,9 @@
         ]}"
         ROOT=$(git rev-parse --show-toplevel)
         cd "$ROOT/pkgs/clan-cli"
-        ${self'.packages.vm-persistence}/bin/vm-persistence
         nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -m impure ./tests $@"
       '';
 
-      # TODO: port this to python and make it pure
-      vm-persistence =
-        let
-          machineConfigFile = builtins.toFile "vm-config.json" (builtins.toJSON {
-            clanCore.state.my-state = {
-              folders = [ "/var/my-state" ];
-            };
-            # powers off the machine after the state is created
-            systemd.services.poweroff = {
-              description = "Poweroff the machine";
-              wantedBy = [ "multi-user.target" ];
-              after = [ "my-state.service" ];
-              script = ''
-                echo "Powering off the machine"
-                poweroff
-              '';
-            };
-            # creates a file in the state folder
-            systemd.services.my-state = {
-              description = "Create a file in the state folder";
-              wantedBy = [ "multi-user.target" ];
-              script = ''
-                echo "Creating a file in the state folder"
-                echo "dream2nix" > /var/my-state/test
-              '';
-              serviceConfig.Type = "oneshot";
-            };
-            clan.virtualisation.graphics = false;
-            users.users.root.password = "root";
-          });
-        in
-        pkgs.writeShellScriptBin "vm-persistence" ''
-          #!${pkgs.bash}/bin/bash
-          set -euo pipefail
-
-          export PATH="${lib.makeBinPath [
-            pkgs.coreutils
-            pkgs.gitMinimal
-            pkgs.jq
-            pkgs.nix
-            pkgs.gnused
-            self'.packages.clan-cli
-          ]}"
-
-          clanName=_test_vm_persistence
-          testFile=~/".config/clan/vmstate/$clanName/my-machine/var/my-state/test"
-
-          export TMPDIR=$(${pkgs.coreutils}/bin/mktemp -d)
-          trap "${pkgs.coreutils}/bin/chmod -R +w '$TMPDIR'; ${pkgs.coreutils}/bin/rm -rf '$TMPDIR'" EXIT
-
-          # clean up vmstate after test
-          trap "${pkgs.coreutils}/bin/rm -rf ~/.config/clan/vmstate/$clanName" EXIT
-
-          cd $TMPDIR
-          mkdir ./clan
-          cd ./clan
-          nix flake init -t ${self}#templates.new-clan
-          nix flake lock --override-input clan-core ${self}
-          sed -i "s/__CHANGE_ME__/$clanName/g" flake.nix
-          clan machines create my-machine
-
-          cat ${machineConfigFile} | jq > ./machines/my-machine/settings.json
-
-          # clear state from previous runs
-          rm -rf "$testFile"
-
-          # machine will automatically shutdown due to the shutdown service above
-          clan vms run my-machine
-
-          set -x
-          if ! test -e "$testFile"; then
-            echo "failed: file "$testFile" was not created"
-            exit 1
-          fi
-        '';
-
       runMockApi = pkgs.writeShellScriptBin "run-mock-api" ''
         #!${pkgs.bash}/bin/bash
         set -euo pipefail

pkgs/clan-cli/tests/fixtures_flakes.py

@@ -1,4 +1,5 @@
 import fileinput
+import json
 import logging
 import os
 import shutil
@@ -37,22 +38,88 @@ class FlakeForTest(NamedTuple):
     path: Path
 
 
+def generate_flake(
+    temporary_home: Path,
+    flake_template: Path,
+    substitutions: dict[str, str] = {},
+    # define the machines directly including their config
+    machine_configs: dict[str, dict] = {},
+) -> FlakeForTest:
+    """
+    Creates a clan flake with the given name.
+    Machines are fully generated from the machine_configs.
+
+    Example:
+        machine_configs = dict(
+            my_machine=dict(
+                clanCore=dict(
+                    backups=dict(
+                        ...
+                    )
+                )
+            )
+        )
+    """
+
+    # copy the template to a new temporary location
+    flake = temporary_home / "flake"
+    shutil.copytree(flake_template, flake)
+
+    # substitute `substitutions` in all files of the template
+    for file in flake.rglob("*"):
+        if file.is_file():
+            print(f"Final Content of {file}:")
+            for line in fileinput.input(file, inplace=True):
+                for key, value in substitutions.items():
+                    line = line.replace(key, value)
+                print(line, end="")
+
+    # generate machines from machineConfigs
+    for machine_name, machine_config in machine_configs.items():
+        settings_path = flake / "machines" / machine_name / "settings.json"
+        settings_path.parent.mkdir(parents=True, exist_ok=True)
+        settings_path.write_text(json.dumps(machine_config, indent=2))
+
+    if "/tmp" not in str(os.environ.get("HOME")):
+        log.warning(
+            f"!! $HOME does not point to a temp directory!! HOME={os.environ['HOME']}"
+        )
+
+    # TODO: Find out why test_vms_api.py fails in nix build
+    # but works in pytest when this bottom line is commented out
+    sp.run(
+        ["git", "config", "--global", "init.defaultBranch", "main"],
+        cwd=flake,
+        check=True,
+    )
+    sp.run(["git", "init"], cwd=flake, check=True)
+    sp.run(["git", "add", "."], cwd=flake, check=True)
+    sp.run(["git", "config", "user.name", "clan-tool"], cwd=flake, check=True)
+    sp.run(["git", "config", "user.email", "clan@example.com"], cwd=flake, check=True)
+    sp.run(["git", "commit", "-a", "-m", "Initial commit"], cwd=flake, check=True)
+
+    return FlakeForTest(flake)
+
+
 def create_flake(
     monkeypatch: pytest.MonkeyPatch,
     temporary_home: Path,
-    flake_name: str,
+    flake_template_name: str,
     clan_core_flake: Path | None = None,
+    # names referring to pre-defined machines from ../machines
     machines: list[str] = [],
+    # alternatively specify the machines directly including their config
+    machine_configs: dict[str, dict] = {},
     remote: bool = False,
 ) -> Iterator[FlakeForTest]:
     """
     Creates a flake with the given name and machines.
     The machine names map to the machines in ./test_machines
     """
-    template = Path(__file__).parent / flake_name
+    template = Path(__file__).parent / flake_template_name
 
     # copy the template to a new temporary location
-    flake = temporary_home / flake_name
+    flake = temporary_home / flake_template_name
     shutil.copytree(template, flake)
 
     # lookup the requested machines in ./test_machines and include them
@@ -62,6 +129,13 @@ def create_flake(
         machine_path = Path(__file__).parent / "machines" / machine_name
         shutil.copytree(machine_path, flake / "machines" / machine_name)
         substitute(flake / "machines" / machine_name / "default.nix", flake)
+
+    # generate machines from machineConfigs
+    for machine_name, machine_config in machine_configs.items():
+        settings_path = flake / "machines" / machine_name / "settings.json"
+        settings_path.parent.mkdir(parents=True, exist_ok=True)
+        settings_path.write_text(json.dumps(machine_config, indent=2))
+
     # in the flake.nix file replace the string __CLAN_URL__ with the the clan flake
     # provided by get_test_flake_toplevel
     flake_nix = flake / "flake.nix"

pkgs/clan-cli/tests/test_vms_cli.py

@@ -1,9 +1,11 @@
 import os
+from pathlib import Path
 from typing import TYPE_CHECKING
 
 import pytest
 from cli import Cli
-from fixtures_flakes import FlakeForTest
+from fixtures_flakes import FlakeForTest, generate_flake
+from root import CLAN_CORE
 
 if TYPE_CHECKING:
     from age_keys import KeyPair
@@ -41,3 +43,63 @@ def test_run(
         ]
     )
     cli.run(["vms", "run", "vm1"])
+
+
+@pytest.mark.skipif(no_kvm, reason="Requires KVM")
+@pytest.mark.impure
+def test_vm_persistence(
+    monkeypatch: pytest.MonkeyPatch,
+    temporary_home: Path,
+) -> None:
+    flake = generate_flake(
+        temporary_home,
+        flake_template=CLAN_CORE / "templates" / "new-clan",
+        substitutions=dict(
+            __CHANGE_ME__="_test_vm_persistence",
+        ),
+        machine_configs=dict(
+            my_machine=dict(
+                clanCore=dict(state=dict(my_state=dict(folders=["/var/my-state"]))),
+                systemd=dict(
+                    services=dict(
+                        poweroff=dict(
+                            description="Poweroff the machine",
+                            wantedBy=["multi-user.target"],
+                            after=["my-state.service"],
+                            script="""
+                        echo "Powering off the machine"
+                        poweroff
+                        """,
+                        ),
+                        my_state=dict(
+                            description="Create a file in the state folder",
+                            wantedBy=["multi-user.target"],
+                            script="""
+                        echo "Creating a file in the state folder"
+                        echo "dream2nix" > /var/my-state/test
+                        """,
+                            serviceConfig=dict(Type="oneshot"),
+                        ),
+                    )
+                ),
+                clan=dict(virtualisation=dict(graphics=False)),
+                users=dict(users=dict(root=dict(password="root"))),
+            )
+        ),
+    )
+    monkeypatch.chdir(flake.path)
+    Cli().run(["vms", "run", "my_machine"])
+
+    test_file = (
+        temporary_home
+        / ".config"
+        / "clan"
+        / "vmstate"
+        / "_test_vm_persistence"
+        / "my_machine"
+        / "var"
+        / "my-state"
+        / "test"
+    )
+    assert test_file.exists()
+    assert test_file.read_text() == "dream2nix\n"