tests: test vars generation for all clan service tests

lib/test/default.nix

@@ -1,4 +1,7 @@
-{ lib, clanLib }:
+{
+  lib,
+  clanLib,
+}:
 let
   inherit (lib)
     mkOption
@@ -16,6 +19,7 @@ in
   minifyModule = ./minify.nix;
   sopsModule = ./sops.nix;
   # A function that returns an extension to runTest
+  # TODO: remove this from clanLib, add to legacyPackages, simplify signature
   makeTestClan =
     {
       nixosTest,
@@ -23,13 +27,64 @@ in
       self,
       useContainers ? true,
       # Displayed for better error messages, otherwise the placeholder
-      system ? "<system>",
       attrName ? "<check_name>",
       ...
     }:
     let
       nixos-lib = import (pkgs.path + "/nixos/lib") { };
-    in
+
+      testName = test.config.name;
+
+      update-vars-script = "${self.packages.${pkgs.system}.generate-test-vars}/bin/generate-test-vars";
+
+      update-vars = pkgs.writeShellScriptBin "update-vars" ''
+        ${update-vars-script} $PRJ_ROOT/checks/${testName} ${testName}
+      '';
+
+      testSrc = lib.cleanSource (self + "/checks/${testName}");
+
+      vars-check =
+        pkgs.runCommand "update-vars-check"
+          {
+            nativeBuildInputs = [
+              pkgs.nix
+              pkgs.git
+              pkgs.age
+              pkgs.sops
+              pkgs.bubblewrap
+            ];
+            closureInfo = pkgs.closureInfo {
+              rootPaths = [
+                pkgs.bash
+                pkgs.coreutils
+                pkgs.jq.dev
+                pkgs.stdenv
+                pkgs.stdenvNoCC
+                pkgs.shellcheck-minimal
+                pkgs.age
+                pkgs.sops
+              ];
+            };
+          }
+          ''
+            # make the test depend on its vars-check derivation
+            echo ${vars-check} >/dev/null
+
+            ${self.legacyPackages.${pkgs.system}.setupNixInNix}
+            cp -r ${testSrc} ./src
+            chmod +w -R ./src
+            find ./src/sops ./src/vars | sort > filesBefore
+            ${update-vars-script} ./src ${testName} --repo-root ${self.packages.${pkgs.system}.clan-core-flake}
+            find ./src/sops ./src/vars | sort > filesAfter
+            if ! diff -q filesBefore filesAfter; then
+              echo "The update-vars script changed the files in ${testSrc}."
+              echo "Diff:"
+              diff filesBefore filesAfter || true
+              exit 1
+            fi
+            touch $out
+          '';
+      test =
         (nixos-lib.runTest (
           { config, ... }:
           let
@@ -66,7 +121,7 @@ in
                     {
                       _prefix = [
                         "checks"
-                    system
+                        "<system>"
                         attrName
                         "config"
                         "clan"
@@ -138,4 +193,9 @@ in
             };
           }
         )).config.result;
+    in
+    test
+    // {
+      inherit update-vars vars-check;
+    };
 }

pkgs/clan-cli/default.nix

@@ -10,10 +10,11 @@
   stdenv,
   # custom args
   clan-core-path,
-  nixpkgs,
-  nix-select,
   includedRuntimeDeps,
+  nix-select,
+  nixpkgs,
   pythonRuntime,
+  setupNixInNix,
   templateDerivation,
 }:
 let
@@ -211,17 +212,10 @@ pythonRuntime.pkgs.buildPythonApplication {
             chmod +w -R ./src
             cd ./src
 
+            ${setupNixInNix}
+
             export CLAN_CORE_PATH=${clan-core-path}
-            export NIX_STATE_DIR=$TMPDIR/nix
-            export IN_NIX_SANDBOX=1
             export PYTHONWARNINGS=error
-            export CLAN_TEST_STORE=$TMPDIR/store
-            # required to prevent concurrent 'nix flake lock' operations
-            export LOCK_NIX=$TMPDIR/nix_lock
-            mkdir -p "$CLAN_TEST_STORE/nix/store"
-            mkdir -p "$CLAN_TEST_STORE/nix/var/nix/gcroots"
-            xargs cp --recursive --target "$CLAN_TEST_STORE/nix/store"  < "$closureInfo/store-paths"
-            nix-store --load-db --store "$CLAN_TEST_STORE" < "$closureInfo/registration"
 
             # limit build cores to 16
             jobs="$((NIX_BUILD_CORES>16 ? 16 : NIX_BUILD_CORES))"
@@ -264,17 +258,10 @@ pythonRuntime.pkgs.buildPythonApplication {
             chmod +w -R ./src
             cd ./src
 
+            ${setupNixInNix}
+
             export CLAN_CORE_PATH=${clan-core-path}
-            export NIX_STATE_DIR=$TMPDIR/nix
-            export IN_NIX_SANDBOX=1
             export PYTHONWARNINGS=error
-            export CLAN_TEST_STORE=$TMPDIR/store
-            # required to prevent concurrent 'nix flake lock' operations
-            export LOCK_NIX=$TMPDIR/nix_lock
-            mkdir -p "$CLAN_TEST_STORE/nix/store"
-            mkdir -p "$CLAN_TEST_STORE/nix/var/nix/gcroots"
-            xargs cp --recursive --target "$CLAN_TEST_STORE/nix/store"  < "$closureInfo/store-paths"
-            nix-store --load-db --store "$CLAN_TEST_STORE" < "$closureInfo/registration"
 
             # used for tests without flakes
             export NIXPKGS=${nixpkgs}

pkgs/clan-cli/flake-module.nix

@@ -9,6 +9,7 @@
     {
       self',
       pkgs,
+      system,
       ...
     }:
     let
@@ -41,6 +42,7 @@
       packages = {
         clan-cli = pkgs.callPackage ./default.nix {
           inherit (inputs) nixpkgs nix-select;
+          inherit (self.legacyPackages.${system}) setupNixInNix;
           templateDerivation = templateDerivation;
           pythonRuntime = pkgs.python3;
           clan-core-path = clanCoreWithVendoredDeps;
@@ -51,6 +53,7 @@
         };
         clan-cli-full = pkgs.callPackage ./default.nix {
           inherit (inputs) nixpkgs nix-select;
+          inherit (self.legacyPackages.${system}) setupNixInNix;
           clan-core-path = clanCoreWithVendoredDeps;
           templateDerivation = templateDerivation;
           pythonRuntime = pkgs.python3;

pkgs/flake-module.nix

@@ -10,6 +10,7 @@
     ./generate-test-vars/flake-module.nix
     ./clan-core-flake/flake-module.nix
     ./clan-app/flake-module.nix
+    ./testing/flake-module.nix
   ];
 
   flake.packages.x86_64-linux =

pkgs/generate-test-vars/generate_test_vars/cli.py

@@ -14,7 +14,7 @@ from clan_cli.vars.generate import generate_vars
 from clan_lib.dirs import find_git_repo_root
 from clan_lib.flake.flake import Flake
 from clan_lib.machines.machines import Machine
-from clan_lib.nix import nix_config, nix_eval
+from clan_lib.nix import nix_config, nix_eval, nix_test_store
 
 sops_priv_key = (
     "AGE-SECRET-KEY-1PL0M9CWRCG3PZ9DXRTTLMCVD57U6JDFE8K7DNVQ35F4JENZ6G3MQ0RQLRV"
@@ -26,11 +26,15 @@ def get_machine_names(repo_root: Path, check_attr: str) -> list[str]:
     """
     Get the machine names from the test flake
     """
+    nix_options = []
+    if tmp_store := nix_test_store():
+        nix_options += ["--store", str(tmp_store)]
     cmd = nix_eval(
         [
-            f"git+file://{repo_root}#checks.{nix_config()['system']}.{check_attr}.nodes",
+            f"path://{repo_root}#checks.{nix_config()['system']}.{check_attr}.nodes",
             "--apply",
             "builtins.attrNames",
+            *nix_options,
         ]
     )
     out = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE)
@@ -93,7 +97,7 @@ def parse_args() -> Options:
         """,
     )
     parser.add_argument(
-        "--repo_root",
+        "--repo-root",
         type=Path,
         help="""
             Should be an absolute path to the repo root.
@@ -127,7 +131,7 @@ def parse_args() -> Options:
 def main() -> None:
     os.environ["CLAN_NO_COMMIT"] = "1"
     opts = parse_args()
-    test_dir = opts.repo_root / opts.test_dir
+    test_dir = opts.test_dir
 
     shutil.rmtree(test_dir / "vars", ignore_errors=True)
     shutil.rmtree(test_dir / "sops", ignore_errors=True)

pkgs/testing/flake-module.nix

@@ -0,0 +1,18 @@
+{
+  perSystem = {
+    legacyPackages.setupNixInNix = ''
+      export HOME=$TMPDIR
+      export NIX_STATE_DIR=$TMPDIR/nix
+      export IN_NIX_SANDBOX=1
+      export CLAN_TEST_STORE=$TMPDIR/store
+      # required to prevent concurrent 'nix flake lock' operations
+      export LOCK_NIX=$TMPDIR/nix_lock
+      mkdir -p "$CLAN_TEST_STORE/nix/store"
+      mkdir -p "$CLAN_TEST_STORE/nix/var/nix/gcroots"
+      if [[ -n "''${closureInfo-}" ]]; then
+        xargs cp --recursive --target "$CLAN_TEST_STORE/nix/store"  < "$closureInfo/store-paths"
+        nix-store --load-db --store "$CLAN_TEST_STORE" < "$closureInfo/registration"
+      fi
+    '';
+  };
+}