tests: test vars generation for all clan service tests

lib/test/default.nix

@@ -1,4 +1,7 @@
-{ lib, clanLib }:
+{
+  lib,
+  clanLib,
+}:
 let
   inherit (lib)
     mkOption
@@ -16,6 +19,7 @@ in
   minifyModule = ./minify.nix;
   sopsModule = ./sops.nix;
   # A function that returns an extension to runTest
+  # TODO: remove this from clanLib, add to legacyPackages, simplify signature
   makeTestClan =
     {
       nixosTest,
@@ -23,119 +27,175 @@ in
       self,
       useContainers ? true,
       # Displayed for better error messages, otherwise the placeholder
-      system ? "<system>",
       attrName ? "<check_name>",
       ...
     }:
     let
       nixos-lib = import (pkgs.path + "/nixos/lib") { };
-    in
-    (nixos-lib.runTest (
-      { config, ... }:
-      let
-        clanFlakeResult = config.clan;
-      in
-      {
-        imports = [
-          nixosTest
-        ] ++ lib.optionals useContainers [ ./container-test-driver/driver-module.nix ];
-        options = {
-          clanSettings = mkOption {
-            default = { };
-            type = types.submodule {
-              options = {
-                clan-core = mkOption { default = self; };
-                nixpkgs = mkOption { default = self.inputs.nixpkgs; };
-                nix-darwin = mkOption { default = self.inputs.nix-darwin; };
-              };
-            };
-          };
 
-          clan = mkOption {
+      testName = test.config.name;
-            default = { };
+
-            type = types.submoduleWith {
+      update-vars-script = "${self.packages.${pkgs.system}.generate-test-vars}/bin/generate-test-vars";
-              specialArgs = {
+
-                inherit (config.clanSettings)
+      update-vars = pkgs.writeShellScriptBin "update-vars" ''
-                  clan-core
+        ${update-vars-script} $PRJ_ROOT/checks/${testName} ${testName}
-                  nixpkgs
+      '';
-                  nix-darwin
+
-                  ;
+      testSrc = lib.cleanSource (self + "/checks/${testName}");
+
+      vars-check =
+        pkgs.runCommand "update-vars-check"
+          {
+            nativeBuildInputs = [
+              pkgs.nix
+              pkgs.git
+              pkgs.age
+              pkgs.sops
+              pkgs.bubblewrap
+            ];
+            closureInfo = pkgs.closureInfo {
+              rootPaths = [
+                pkgs.bash
+                pkgs.coreutils
+                pkgs.jq.dev
+                pkgs.stdenv
+                pkgs.stdenvNoCC
+                pkgs.shellcheck-minimal
+                pkgs.age
+                pkgs.sops
+              ];
+            };
+          }
+          ''
+            # make the test depend on its vars-check derivation
+            echo ${vars-check} >/dev/null
+
+            ${self.legacyPackages.${pkgs.system}.setupNixInNix}
+            cp -r ${testSrc} ./src
+            chmod +w -R ./src
+            find ./src/sops ./src/vars | sort > filesBefore
+            ${update-vars-script} ./src ${testName} --repo-root ${self.packages.${pkgs.system}.clan-core-flake}
+            find ./src/sops ./src/vars | sort > filesAfter
+            if ! diff -q filesBefore filesAfter; then
+              echo "The update-vars script changed the files in ${testSrc}."
+              echo "Diff:"
+              diff filesBefore filesAfter || true
+              exit 1
+            fi
+            touch $out
+          '';
+      test =
+        (nixos-lib.runTest (
+          { config, ... }:
+          let
+            clanFlakeResult = config.clan;
+          in
+          {
+            imports = [
+              nixosTest
+            ] ++ lib.optionals useContainers [ ./container-test-driver/driver-module.nix ];
+            options = {
+              clanSettings = mkOption {
+                default = { };
+                type = types.submodule {
+                  options = {
+                    clan-core = mkOption { default = self; };
+                    nixpkgs = mkOption { default = self.inputs.nixpkgs; };
+                    nix-darwin = mkOption { default = self.inputs.nix-darwin; };
+                  };
+                };
               };
-              modules = [
+
-                clanLib.buildClanModule.flakePartsModule
+              clan = mkOption {
-                {
+                default = { };
-                  _prefix = [
+                type = types.submoduleWith {
-                    "checks"
+                  specialArgs = {
-                    system
+                    inherit (config.clanSettings)
-                    attrName
+                      clan-core
-                    "config"
+                      nixpkgs
-                    "clan"
+                      nix-darwin
+                      ;
+                  };
+                  modules = [
+                    clanLib.buildClanModule.flakePartsModule
+                    {
+                      _prefix = [
+                        "checks"
+                        "<system>"
+                        attrName
+                        "config"
+                        "clan"
+                      ];
+                    }
                   ];
-                }
+                };
-              ];
+              };
             };
-          };
+            config = {
-        };
+              # Inherit all nodes from the clan
-        config = {
+              # i.e. nodes.jon <- clan.machines.jon
-          # Inherit all nodes from the clan
+              # clanInternals.nixosModules contains nixosModules per node
-          # i.e. nodes.jon <- clan.machines.jon
+              nodes = clanFlakeResult.clanInternals.nixosModules;
-          # clanInternals.nixosModules contains nixosModules per node
-          nodes = clanFlakeResult.clanInternals.nixosModules;
 
-          hostPkgs = pkgs;
+              hostPkgs = pkgs;
 
-          # !WARNING: Write a detailed comment if adding new options here
+              # !WARNING: Write a detailed comment if adding new options here
-          # We should be very careful about adding new options here because it affects all tests
+              # We should be very careful about adding new options here because it affects all tests
-          # Keep in mind:
+              # Keep in mind:
-          # - tests should be close to the real world as possible
+              # - tests should be close to the real world as possible
-          # - ensure stability: in clan-core and downstream
+              # - ensure stability: in clan-core and downstream
-          # - ensure that the tests are fast and reliable
+              # - ensure that the tests are fast and reliable
-          defaults = (
+              defaults = (
-            { config, ... }:
+                { config, ... }:
-            {
+                {
-              imports = [
+                  imports = [
-                # Speed up evaluation
+                    # Speed up evaluation
-                clanLib.test.minifyModule
+                    clanLib.test.minifyModule
 
-                # Setup for sops during tests
+                    # Setup for sops during tests
-                # configures a static age-key to skip the age-key generation
+                    # configures a static age-key to skip the age-key generation
-                clanLib.test.sopsModule
+                    clanLib.test.sopsModule
-              ];
+                  ];
 
-              # Disable documentation
+                  # Disable documentation
-              # This is nice to speed up the evaluation
+                  # This is nice to speed up the evaluation
-              # And also suppresses any warnings or errors about the documentation
+                  # And also suppresses any warnings or errors about the documentation
-              documentation.enable = lib.mkDefault false;
+                  documentation.enable = lib.mkDefault false;
 
-              # Disable garbage collection during the test
+                  # Disable garbage collection during the test
-              # https://nix.dev/manual/nix/2.28/command-ref/conf-file.html?highlight=min-free#available-settings
+                  # https://nix.dev/manual/nix/2.28/command-ref/conf-file.html?highlight=min-free#available-settings
-              nix.settings.min-free = 0;
+                  nix.settings.min-free = 0;
 
-              # This is typically set once via vars generate for a machine
+                  # This is typically set once via vars generate for a machine
-              # Since we have ephemeral machines, we set it here for the test
+                  # Since we have ephemeral machines, we set it here for the test
-              system.stateVersion = config.system.nixos.release;
+                  system.stateVersion = config.system.nixos.release;
 
-              # Currently this is the default in NixOS, but we set it explicitly to avoid surprises
+                  # Currently this is the default in NixOS, but we set it explicitly to avoid surprises
-              # Disable the initrd systemd service which has the following effect
+                  # Disable the initrd systemd service which has the following effect
-              #
+                  #
-              # With the below on 'false' initrd runs a 'minimal shell script', called the stage-1 init.
+                  # With the below on 'false' initrd runs a 'minimal shell script', called the stage-1 init.
-              # Benefits:
+                  # Benefits:
-              #     Simple and fast.
+                  #     Simple and fast.
-              #     Easier to debug for very minimal or custom setups.
+                  #     Easier to debug for very minimal or custom setups.
-              # Drawbacks:
+                  # Drawbacks:
-              #     Limited flexibility.
+                  #     Limited flexibility.
-              #     Harder to handle advanced setups (like TPM, LUKS, or LVM-on-LUKS) but not needed since we are in a test
+                  #     Harder to handle advanced setups (like TPM, LUKS, or LVM-on-LUKS) but not needed since we are in a test
-              #     No systemd journal logs from initrd.
+                  #     No systemd journal logs from initrd.
-              boot.initrd.systemd.enable = false;
+                  boot.initrd.systemd.enable = false;
-            }
+                }
-          );
+              );
 
-          # TODO: figure out if we really need this
+              # TODO: figure out if we really need this
-          # I am proposing for less magic in the test-framework
+              # I am proposing for less magic in the test-framework
-          # People may add this in their own tests
+              # People may add this in their own tests
-          # _module.args = { inherit self; };
+              # _module.args = { inherit self; };
-          # node.specialArgs.self = self;
+              # node.specialArgs.self = self;
-        };
+            };
-      }
+          }
-    )).config.result;
+        )).config.result;
+    in
+    test
+    // {
+      inherit update-vars vars-check;
+    };
 }

pkgs/clan-cli/default.nix

@@ -10,10 +10,11 @@
   stdenv,
   # custom args
   clan-core-path,
-  nixpkgs,
-  nix-select,
   includedRuntimeDeps,
+  nix-select,
+  nixpkgs,
   pythonRuntime,
+  setupNixInNix,
   templateDerivation,
 }:
 let
@@ -211,17 +212,10 @@ pythonRuntime.pkgs.buildPythonApplication {
             chmod +w -R ./src
             cd ./src
 
+            ${setupNixInNix}
+
             export CLAN_CORE_PATH=${clan-core-path}
-            export NIX_STATE_DIR=$TMPDIR/nix
-            export IN_NIX_SANDBOX=1
             export PYTHONWARNINGS=error
-            export CLAN_TEST_STORE=$TMPDIR/store
-            # required to prevent concurrent 'nix flake lock' operations
-            export LOCK_NIX=$TMPDIR/nix_lock
-            mkdir -p "$CLAN_TEST_STORE/nix/store"
-            mkdir -p "$CLAN_TEST_STORE/nix/var/nix/gcroots"
-            xargs cp --recursive --target "$CLAN_TEST_STORE/nix/store"  < "$closureInfo/store-paths"
-            nix-store --load-db --store "$CLAN_TEST_STORE" < "$closureInfo/registration"
 
             # limit build cores to 16
             jobs="$((NIX_BUILD_CORES>16 ? 16 : NIX_BUILD_CORES))"
@@ -264,17 +258,10 @@ pythonRuntime.pkgs.buildPythonApplication {
             chmod +w -R ./src
             cd ./src
 
+            ${setupNixInNix}
+
             export CLAN_CORE_PATH=${clan-core-path}
-            export NIX_STATE_DIR=$TMPDIR/nix
-            export IN_NIX_SANDBOX=1
             export PYTHONWARNINGS=error
-            export CLAN_TEST_STORE=$TMPDIR/store
-            # required to prevent concurrent 'nix flake lock' operations
-            export LOCK_NIX=$TMPDIR/nix_lock
-            mkdir -p "$CLAN_TEST_STORE/nix/store"
-            mkdir -p "$CLAN_TEST_STORE/nix/var/nix/gcroots"
-            xargs cp --recursive --target "$CLAN_TEST_STORE/nix/store"  < "$closureInfo/store-paths"
-            nix-store --load-db --store "$CLAN_TEST_STORE" < "$closureInfo/registration"
 
             # used for tests without flakes
             export NIXPKGS=${nixpkgs}

pkgs/clan-cli/flake-module.nix

@@ -9,6 +9,7 @@
     {
       self',
       pkgs,
+      system,
       ...
     }:
     let
@@ -41,6 +42,7 @@
       packages = {
         clan-cli = pkgs.callPackage ./default.nix {
           inherit (inputs) nixpkgs nix-select;
+          inherit (self.legacyPackages.${system}) setupNixInNix;
           templateDerivation = templateDerivation;
           pythonRuntime = pkgs.python3;
           clan-core-path = clanCoreWithVendoredDeps;
@@ -51,6 +53,7 @@
         };
         clan-cli-full = pkgs.callPackage ./default.nix {
           inherit (inputs) nixpkgs nix-select;
+          inherit (self.legacyPackages.${system}) setupNixInNix;
           clan-core-path = clanCoreWithVendoredDeps;
           templateDerivation = templateDerivation;
           pythonRuntime = pkgs.python3;

pkgs/flake-module.nix

@@ -10,6 +10,7 @@
     ./generate-test-vars/flake-module.nix
     ./clan-core-flake/flake-module.nix
     ./clan-app/flake-module.nix
+    ./testing/flake-module.nix
   ];
 
   flake.packages.x86_64-linux =

pkgs/generate-test-vars/generate_test_vars/cli.py

@@ -14,7 +14,7 @@ from clan_cli.vars.generate import generate_vars
 from clan_lib.dirs import find_git_repo_root
 from clan_lib.flake.flake import Flake
 from clan_lib.machines.machines import Machine
-from clan_lib.nix import nix_config, nix_eval
+from clan_lib.nix import nix_config, nix_eval, nix_test_store
 
 sops_priv_key = (
     "AGE-SECRET-KEY-1PL0M9CWRCG3PZ9DXRTTLMCVD57U6JDFE8K7DNVQ35F4JENZ6G3MQ0RQLRV"
@@ -26,11 +26,15 @@ def get_machine_names(repo_root: Path, check_attr: str) -> list[str]:
     """
     Get the machine names from the test flake
     """
+    nix_options = []
+    if tmp_store := nix_test_store():
+        nix_options += ["--store", str(tmp_store)]
     cmd = nix_eval(
         [
-            f"git+file://{repo_root}#checks.{nix_config()['system']}.{check_attr}.nodes",
+            f"path://{repo_root}#checks.{nix_config()['system']}.{check_attr}.nodes",
             "--apply",
             "builtins.attrNames",
+            *nix_options,
         ]
     )
     out = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE)
@@ -93,7 +97,7 @@ def parse_args() -> Options:
         """,
     )
     parser.add_argument(
-        "--repo_root",
+        "--repo-root",
         type=Path,
         help="""
             Should be an absolute path to the repo root.
@@ -127,7 +131,7 @@ def parse_args() -> Options:
 def main() -> None:
     os.environ["CLAN_NO_COMMIT"] = "1"
     opts = parse_args()
-    test_dir = opts.repo_root / opts.test_dir
+    test_dir = opts.test_dir
 
     shutil.rmtree(test_dir / "vars", ignore_errors=True)
     shutil.rmtree(test_dir / "sops", ignore_errors=True)

pkgs/testing/flake-module.nix

@@ -0,0 +1,18 @@
+{
+  perSystem = {
+    legacyPackages.setupNixInNix = ''
+      export HOME=$TMPDIR
+      export NIX_STATE_DIR=$TMPDIR/nix
+      export IN_NIX_SANDBOX=1
+      export CLAN_TEST_STORE=$TMPDIR/store
+      # required to prevent concurrent 'nix flake lock' operations
+      export LOCK_NIX=$TMPDIR/nix_lock
+      mkdir -p "$CLAN_TEST_STORE/nix/store"
+      mkdir -p "$CLAN_TEST_STORE/nix/var/nix/gcroots"
+      if [[ -n "''${closureInfo-}" ]]; then
+        xargs cp --recursive --target "$CLAN_TEST_STORE/nix/store"  < "$closureInfo/store-paths"
+        nix-store --load-db --store "$CLAN_TEST_STORE" < "$closureInfo/registration"
+      fi
+    '';
+  };
+}