clan tests: convert makeTestClan to a module

Let's not put yet another wrapper around runTest, instead expose our logic as a module that can be imported into any nixos-vm-test

checks/admin/default.nix

@@ -1,20 +1,22 @@
 {
   pkgs,
-  self,
+  nixosLib,
-  clanLib,
+  clan-core,
   ...
 }:
 
 let
   public-key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII6zj7ubTg6z/aDwRNwvM/WlQdUocMprQ8E92NWxl6t+ test@test";
 in
-
+nixosLib.runTest (
-clanLib.test.makeTestClan {
-  inherit pkgs self;
-  nixosTest = (
   { ... }:
-
   {
+    imports = [
+      clan-core.modules.nixosVmTest.clanTest
+    ];
+
+    hostPkgs = pkgs;
+
     name = "admin";
 
     clan = {
@@ -57,5 +59,4 @@ clanLib.test.makeTestClan {
       client.succeed(f"ssh -F /dev/null -i /etc/private-test-key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes root@server true &>/dev/null")
     '';
   }
-  );
+)
-}

checks/borgbackup/default.nix

@@ -1,22 +1,23 @@
 {
   pkgs,
-  self,
+  nixosLib,
-  clanLib,
+  clan-core,
   ...
 }:
-
+nixosLib.runTest (
-clanLib.test.makeTestClan {
-  inherit pkgs self;
-  useContainers = true;
-
-  nixosTest = (
   { ... }:
-
   {
+    imports = [
+      clan-core.modules.nixosVmTest.clanTest
+    ];
+
+    hostPkgs = pkgs;
+
     name = "borgbackup";
 
     clan = {
       directory = ./.;
+      test.useContainers = true;
       modules."@clan/borgbackup" = ../../clanServices/borgbackup/default.nix;
       inventory = {
 
@@ -47,9 +48,9 @@ clanLib.test.makeTestClan {
         { config, pkgs, ... }:
         let
           dependencies = [
-              self
+            clan-core
             pkgs.stdenv.drvPath
-            ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
+          ] ++ builtins.map (i: i.outPath) (builtins.attrValues clan-core.inputs);
           closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
 
         in
@@ -61,7 +62,7 @@ clanLib.test.makeTestClan {
 
           clan.core.networking.targetHost = config.networking.hostName;
 
-            environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
+          environment.systemPackages = [ clan-core.packages.${pkgs.system}.clan-cli ];
 
           environment.etc.install-closure.source = "${closureInfo}/store-paths";
           nix.settings = {
@@ -114,5 +115,4 @@ clanLib.test.makeTestClan {
       assert clientone.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
     '';
   }
-  );
+)
-}

checks/data-mesher/default.nix

@@ -1,13 +1,10 @@
 {
   pkgs,
-  self,
+  nixosLib,
-  clanLib,
+  clan-core,
+  lib,
   ...
 }:
-clanLib.test.makeTestClan {
-  inherit pkgs self;
-  nixosTest = (
-    { lib, ... }:
 let
   machines = [
     "admin"
@@ -15,7 +12,14 @@ clanLib.test.makeTestClan {
     "signer"
   ];
 in
+nixosLib.runTest (
+  { ... }:
   {
+    imports = [
+      clan-core.modules.nixosVmTest.clanTest
+    ];
+
+    hostPkgs = pkgs;
     name = "data-mesher";
 
     clan = {
@@ -82,5 +86,4 @@ clanLib.test.makeTestClan {
         })
     '';
   }
-  );
+)
-}

checks/dummy-inventory-test/default.nix

@@ -1,14 +1,18 @@
 {
   pkgs,
-  self,
+  nixosLib,
-  clanLib,
+  clan-core,
   ...
 }:
-clanLib.test.makeTestClan {
+nixosLib.runTest (
-  inherit pkgs self;
-  nixosTest = (
   { ... }:
   {
+    imports = [
+      clan-core.modules.nixosVmTest.clanTest
+    ];
+
+    hostPkgs = pkgs;
+
     # This tests the compatibility of the inventory
     # With the test framework
     # - legacy-modules
@@ -89,5 +93,4 @@ clanLib.test.makeTestClan {
         assert "-rw-r--r--" in ls_out, f"File is not in the '0644' mode: {ls_out}"
       '';
   }
-  );
+)
-}

checks/flake-module.nix

@@ -4,6 +4,7 @@ let
     filter
     pathExists
     ;
+  nixosLib = import (self.inputs.nixpkgs + "/nixos/lib") { };
 in
 {
   imports = filter pathExists [
@@ -29,10 +30,11 @@ in
         let
           nixosTestArgs = {
             # reference to nixpkgs for the current system
-            inherit pkgs lib;
+            inherit pkgs lib nixosLib;
             # this gives us a reference to our flake but also all flake inputs
             inherit self;
             inherit (self) clanLib;
+            clan-core = self;
           };
           nixosTests =
             lib.optionalAttrs (pkgs.stdenv.isLinux) {

checks/mumble/default.nix

@@ -1,15 +1,28 @@
 {
   pkgs,
-  self,
+  nixosLib,
-  clanLib,
+  clan-core,
+  lib,
   ...
 }:
-clanLib.test.makeTestClan {
+nixosLib.runTest (
-  inherit pkgs self;
+  { ... }:
-  nixosTest = (
-    { lib, ... }:
   let
-      common =
+    machines = [
+      "peer1"
+      "peer2"
+    ];
+  in
+  {
+    imports = [
+      clan-core.modules.nixosVmTest.clanTest
+    ];
+
+    hostPkgs = pkgs;
+
+    name = "mumble";
+
+    defaults =
       { pkgs, modulesPath, ... }:
       {
         imports = [
@@ -19,13 +32,6 @@ clanLib.test.makeTestClan {
         clan.services.mumble.user = "alice";
         environment.systemPackages = [ pkgs.killall ];
       };
-      machines = [
-        "peer1"
-        "peer2"
-      ];
-    in
-    {
-      name = "mumble";
 
     clan = {
       directory = ./.;
@@ -43,9 +49,6 @@ clanLib.test.makeTestClan {
 
     enableOCR = true;
 
-      nodes.peer1 = common;
-      nodes.peer2 = common;
-
     testScript = ''
       import time
       import re
@@ -126,5 +129,4 @@ clanLib.test.makeTestClan {
           raise Exception("Timeout waiting for certificate creation")
     '';
   }
-  );
+)
-}

checks/syncthing/default.nix

@@ -1,14 +1,19 @@
 {
   pkgs,
-  self,
+  nixosLib,
-  clanLib,
+  clan-core,
+  lib,
   ...
 }:
-clanLib.test.makeTestClan {
+nixosLib.runTest (
-  inherit pkgs self;
+  { ... }:
-  nixosTest = (
-    { lib, ... }:
   {
+    imports = [
+      clan-core.modules.nixosVmTest.clanTest
+    ];
+
+    hostPkgs = pkgs;
+
     name = "syncthing";
 
     clan = {
@@ -79,5 +84,4 @@ clanLib.test.makeTestClan {
       assert "hello" in out
     '';
   }
-  );
+)
-}

clanServices/hello-world/flake-module.nix

@@ -47,8 +47,8 @@ in
           hello-service = import ./tests/vm/default.nix {
             inherit module;
             inherit self inputs pkgs;
-            # clanLib is exposed from inputs.clan-core
+            nixosLib = import (self.inputs.nixpkgs + "/nixos/lib") { };
-            clanLib = self.clanLib;
+            clan-core = self;
           };
         };
     };

clanServices/hello-world/tests/vm/default.nix

@@ -1,15 +1,19 @@
 {
   pkgs,
-  self,
+  nixosLib,
-  clanLib,
+  clan-core,
   module,
   ...
 }:
-clanLib.test.makeTestClan {
+nixosLib.runTest (
-  inherit pkgs self;
-  nixosTest = (
   { ... }:
   {
+    imports = [
+      clan-core.modules.nixosVmTest.clanTest
+    ];
+
+    hostPkgs = pkgs;
+
     name = "hello-service";
 
     clan = {
@@ -37,5 +41,4 @@ clanLib.test.makeTestClan {
         assert value.strip() == "Hello world from peer1", value
       '';
   }
-  );
+)
-}

clanServices/wifi/flake-module.nix

@@ -28,8 +28,9 @@ in
         lib.optionalAttrs (pkgs.stdenv.isLinux) {
           wifi-service = import ./tests/vm/default.nix {
             inherit module;
-            inherit self inputs pkgs;
+            inherit inputs pkgs;
-            clanLib = self.clanLib;
+            clan-core = self;
+            nixosLib = import (self.inputs.nixpkgs + "/nixos/lib") { };
           };
         };
     };

clanServices/wifi/tests/vm/default.nix

@@ -1,15 +1,19 @@
 {
   pkgs,
-  self,
+  nixosLib,
-  clanLib,
+  clan-core,
   module,
   ...
 }:
-clanLib.test.makeTestClan {
+nixosLib.runTest (
-  inherit pkgs self;
-  nixosTest = (
   { ... }:
   {
+    imports = [
+      clan-core.modules.nixosVmTest.clanTest
+    ];
+
+    hostPkgs = pkgs;
+
     name = "wifi-service";
 
     clan = {
@@ -39,5 +43,4 @@ clanLib.test.makeTestClan {
       assert "password-eins" in psk, "Password is incorrect"
     '';
   }
-  );
+)
-}

lib/clanTest/flake-module.nix

@@ -0,0 +1,197 @@
+{
+  lib,
+  self,
+  config,
+  ...
+}:
+let
+  inherit (lib)
+    mkOption
+    removePrefix
+    types
+    mapAttrsToList
+    flip
+    unique
+    flatten
+    ;
+
+  clanLib = config.flake.clanLib;
+in
+{
+  # A function that returns an extension to runTest
+  # TODO: remove this from clanLib, add to legacyPackages, simplify signature
+  flake.modules.nixosVmTest.clanTest =
+    { config, hostPkgs, ... }:
+    let
+      clanFlakeResult = config.clan;
+      testName = config.name;
+
+      update-vars-script = "${
+        self.packages.${hostPkgs.system}.generate-test-vars
+      }/bin/generate-test-vars";
+
+      relativeDir = removePrefix ("${self}/") (toString config.clan.directory);
+
+      update-vars = hostPkgs.writeShellScriptBin "update-vars" ''
+        ${update-vars-script} $PRJ_ROOT/${relativeDir} ${testName}
+      '';
+
+      testSrc = lib.cleanSource config.clan.directory;
+
+      inputsForMachine =
+        machine:
+        flip mapAttrsToList machine.clan.core.vars.generators (_name: generator: generator.runtimeInputs);
+
+      generatorRuntimeInputs = unique (
+        flatten (flip mapAttrsToList config.nodes (_machineName: machine: inputsForMachine machine))
+      );
+
+      vars-check =
+        hostPkgs.runCommand "update-vars-check"
+          {
+            nativeBuildInputs = generatorRuntimeInputs ++ [
+              hostPkgs.nix
+              hostPkgs.git
+              hostPkgs.age
+              hostPkgs.sops
+              hostPkgs.bubblewrap
+            ];
+            closureInfo = hostPkgs.closureInfo {
+              rootPaths = generatorRuntimeInputs ++ [
+                hostPkgs.bash
+                hostPkgs.coreutils
+                hostPkgs.jq.dev
+                hostPkgs.stdenv
+                hostPkgs.stdenvNoCC
+                hostPkgs.shellcheck-minimal
+                hostPkgs.age
+                hostPkgs.sops
+              ];
+            };
+          }
+          ''
+            ${self.legacyPackages.${hostPkgs.system}.setupNixInNix}
+            cp -r ${testSrc} ./src
+            chmod +w -R ./src
+            find ./src/sops ./src/vars | sort > filesBefore
+            ${update-vars-script} ./src ${testName} \
+              --repo-root ${self.packages.${hostPkgs.system}.clan-core-flake} \
+              --clean
+            find ./src/sops ./src/vars | sort > filesAfter
+            if ! diff -q filesBefore filesAfter; then
+              echo "The update-vars script changed the files in ${testSrc}."
+              echo "Diff:"
+              diff filesBefore filesAfter || true
+              exit 1
+            fi
+            touch $out
+          '';
+    in
+    {
+      imports = [
+        ../test/container-test-driver/driver-module.nix
+      ];
+      options = {
+        clanSettings = mkOption {
+          default = { };
+          type = types.submodule {
+            options = {
+              clan-core = mkOption { default = self; };
+              nixpkgs = mkOption { default = self.inputs.nixpkgs; };
+              nix-darwin = mkOption { default = self.inputs.nix-darwin; };
+            };
+          };
+        };
+
+        clan = mkOption {
+          default = { };
+          type = types.submoduleWith {
+            specialArgs = {
+              inherit (config.clanSettings)
+                clan-core
+                nixpkgs
+                nix-darwin
+                ;
+            };
+            modules = [
+              clanLib.buildClanModule.flakePartsModule
+              {
+                _prefix = [
+                  "checks"
+                  "<system>"
+                  config.name
+                  "config"
+                  "clan"
+                ];
+              }
+              {
+                options = {
+                  test.useContainers = mkOption {
+                    default = true;
+                    type = types.bool;
+                    description = "Whether to use containers for the test.";
+                  };
+                };
+              }
+            ];
+          };
+        };
+      };
+      config = {
+        # Inherit all nodes from the clan
+        # i.e. nodes.jon <- clan.machines.jon
+        # clanInternals.nixosModules contains nixosModules per node
+        nodes = clanFlakeResult.clanInternals.nixosModules;
+
+        # !WARNING: Write a detailed comment if adding new options here
+        # We should be very careful about adding new options here because it affects all tests
+        # Keep in mind:
+        # - tests should be close to the real world as possible
+        # - ensure stability: in clan-core and downstream
+        # - ensure that the tests are fast and reliable
+        defaults = (
+          { config, ... }:
+          {
+            imports = [
+              # Speed up evaluation
+              clanLib.test.minifyModule
+
+              # Setup for sops during tests
+              # configures a static age-key to skip the age-key generation
+              clanLib.test.sopsModule
+            ];
+
+            # Disable documentation
+            # This is nice to speed up the evaluation
+            # And also suppresses any warnings or errors about the documentation
+            documentation.enable = lib.mkDefault false;
+
+            # Disable garbage collection during the test
+            # https://nix.dev/manual/nix/2.28/command-ref/conf-file.html?highlight=min-free#available-settings
+            nix.settings.min-free = 0;
+
+            # This is typically set once via vars generate for a machine
+            # Since we have ephemeral machines, we set it here for the test
+            system.stateVersion = config.system.nixos.release;
+
+            # Currently this is the default in NixOS, but we set it explicitly to avoid surprises
+            # Disable the initrd systemd service which has the following effect
+            #
+            # With the below on 'false' initrd runs a 'minimal shell script', called the stage-1 init.
+            # Benefits:
+            #     Simple and fast.
+            #     Easier to debug for very minimal or custom setups.
+            # Drawbacks:
+            #     Limited flexibility.
+            #     Harder to handle advanced setups (like TPM, LUKS, or LVM-on-LUKS) but not needed since we are in a test
+            #     No systemd journal logs from initrd.
+            boot.initrd.systemd.enable = false;
+            # make the test depend on its vars-check derivation
+            environment.variables.CLAN_VARS_CHECK = "${vars-check}";
+          }
+        );
+
+        result = { inherit update-vars vars-check; };
+      };
+    };
+}

lib/flake-module.nix

@@ -8,6 +8,7 @@ rec {
   # TODO: automatically generate this from the directory conventions
   imports = [
     ./build-clan/flake-module.nix
+    ./clanTest/flake-module.nix
     ./introspection/flake-module.nix
     ./inventory/flake-module.nix
     ./jsonschema/flake-module.nix

lib/inventory/distributed-service/tests/default.nix

@@ -8,7 +8,6 @@ let
     evalModules
     ;
 
-  # TODO: Use makeTestClan
   evalInventory =
     m:
     (evalModules {

lib/test/default.nix

@@ -1,19 +1,7 @@
 {
-  lib,
   clanLib,
+  ...
 }:
-let
-  inherit (lib)
-    mkOption
-    removePrefix
-    types
-    mapAttrsToList
-    flip
-    unique
-    flatten
-    ;
-
-in
 {
   containerTest = import ./container-test.nix;
   baseTest = import ./test-base.nix;
@@ -22,194 +10,4 @@ in
 
   minifyModule = ./minify.nix;
   sopsModule = ./sops.nix;
-  # A function that returns an extension to runTest
-  # TODO: remove this from clanLib, add to legacyPackages, simplify signature
-  makeTestClan =
-    {
-      nixosTest,
-      pkgs,
-      self,
-      ...
-    }:
-    let
-      nixos-lib = import (pkgs.path + "/nixos/lib") { };
-
-      test = (
-        nixos-lib.runTest (
-          { config, ... }:
-          let
-            clanFlakeResult = config.clan;
-            testName = config.name;
-
-            update-vars-script = "${self.packages.${pkgs.system}.generate-test-vars}/bin/generate-test-vars";
-
-            relativeDir = removePrefix ("${self}/") (toString config.clan.directory);
-
-            update-vars = pkgs.writeShellScriptBin "update-vars" ''
-              ${update-vars-script} $PRJ_ROOT/${relativeDir} ${testName}
-            '';
-
-            testSrc = lib.cleanSource config.clan.directory;
-
-            inputsForMachine =
-              machine:
-              flip mapAttrsToList machine.clan.core.vars.generators (_name: generator: generator.runtimeInputs);
-
-            generatorRuntimeInputs = unique (
-              flatten (flip mapAttrsToList config.nodes (_machineName: machine: inputsForMachine machine))
-            );
-
-            vars-check =
-              pkgs.runCommand "update-vars-check"
-                {
-                  nativeBuildInputs = generatorRuntimeInputs ++ [
-                    pkgs.nix
-                    pkgs.git
-                    pkgs.age
-                    pkgs.sops
-                    pkgs.bubblewrap
-                  ];
-                  closureInfo = pkgs.closureInfo {
-                    rootPaths = generatorRuntimeInputs ++ [
-                      pkgs.bash
-                      pkgs.coreutils
-                      pkgs.jq.dev
-                      pkgs.stdenv
-                      pkgs.stdenvNoCC
-                      pkgs.shellcheck-minimal
-                      pkgs.age
-                      pkgs.sops
-                    ];
-                  };
-                }
-                ''
-                  ${self.legacyPackages.${pkgs.system}.setupNixInNix}
-                  cp -r ${testSrc} ./src
-                  chmod +w -R ./src
-                  find ./src/sops ./src/vars | sort > filesBefore
-                  ${update-vars-script} ./src ${testName} \
-                    --repo-root ${self.packages.${pkgs.system}.clan-core-flake} \
-                    --clean
-                  find ./src/sops ./src/vars | sort > filesAfter
-                  if ! diff -q filesBefore filesAfter; then
-                    echo "The update-vars script changed the files in ${testSrc}."
-                    echo "Diff:"
-                    diff filesBefore filesAfter || true
-                    exit 1
-                  fi
-                  touch $out
-                '';
-          in
-          {
-            imports = [
-              nixosTest
-              ./container-test-driver/driver-module.nix
-            ];
-            options = {
-              clanSettings = mkOption {
-                default = { };
-                type = types.submodule {
-                  options = {
-                    clan-core = mkOption { default = self; };
-                    nixpkgs = mkOption { default = self.inputs.nixpkgs; };
-                    nix-darwin = mkOption { default = self.inputs.nix-darwin; };
-                  };
-                };
-              };
-
-              clan = mkOption {
-                default = { };
-                type = types.submoduleWith {
-                  specialArgs = {
-                    inherit (config.clanSettings)
-                      clan-core
-                      nixpkgs
-                      nix-darwin
-                      ;
-                  };
-                  modules = [
-                    clanLib.buildClanModule.flakePartsModule
-                    {
-                      _prefix = [
-                        "checks"
-                        "<system>"
-                        config.name
-                        "config"
-                        "clan"
-                      ];
-                      options = {
-                        test.useContainers = mkOption {
-                          default = true;
-                          type = types.bool;
-                          description = "Whether to use containers for the test.";
-                        };
-                      };
-                    }
-                  ];
-                };
-              };
-            };
-            config = {
-              # Inherit all nodes from the clan
-              # i.e. nodes.jon <- clan.machines.jon
-              # clanInternals.nixosModules contains nixosModules per node
-              nodes = clanFlakeResult.clanInternals.nixosModules;
-
-              hostPkgs = pkgs;
-
-              # !WARNING: Write a detailed comment if adding new options here
-              # We should be very careful about adding new options here because it affects all tests
-              # Keep in mind:
-              # - tests should be close to the real world as possible
-              # - ensure stability: in clan-core and downstream
-              # - ensure that the tests are fast and reliable
-              defaults = (
-                { config, ... }:
-                {
-                  imports = [
-                    # Speed up evaluation
-                    clanLib.test.minifyModule
-
-                    # Setup for sops during tests
-                    # configures a static age-key to skip the age-key generation
-                    clanLib.test.sopsModule
-                  ];
-
-                  # Disable documentation
-                  # This is nice to speed up the evaluation
-                  # And also suppresses any warnings or errors about the documentation
-                  documentation.enable = lib.mkDefault false;
-
-                  # Disable garbage collection during the test
-                  # https://nix.dev/manual/nix/2.28/command-ref/conf-file.html?highlight=min-free#available-settings
-                  nix.settings.min-free = 0;
-
-                  # This is typically set once via vars generate for a machine
-                  # Since we have ephemeral machines, we set it here for the test
-                  system.stateVersion = config.system.nixos.release;
-
-                  # Currently this is the default in NixOS, but we set it explicitly to avoid surprises
-                  # Disable the initrd systemd service which has the following effect
-                  #
-                  # With the below on 'false' initrd runs a 'minimal shell script', called the stage-1 init.
-                  # Benefits:
-                  #     Simple and fast.
-                  #     Easier to debug for very minimal or custom setups.
-                  # Drawbacks:
-                  #     Limited flexibility.
-                  #     Harder to handle advanced setups (like TPM, LUKS, or LVM-on-LUKS) but not needed since we are in a test
-                  #     No systemd journal logs from initrd.
-                  boot.initrd.systemd.enable = false;
-                  # make the test depend on its vars-check derivation
-                  environment.variables.CLAN_VARS_CHECK = "${vars-check}";
-                }
-              );
-
-              result = { inherit update-vars vars-check; };
-            };
-          }
-        )
-      );
-    in
-    test;
 }

pkgs/generate-test-vars/generate_test_vars/cli.py

@@ -93,8 +93,8 @@ class Options:
 def parse_args() -> Options:
     parser = argparse.ArgumentParser(
         description="""
-            Update the vars of a 'makeTestClan' integration test.
+            Update the vars of a 'clanTest' integration test.
-            See 'clanLib.test.makeTestClan' for more information on how to create such a test.
+            See 'clanLib.test.clanTest' for more information on how to create such a test.
         """,
     )
     parser.add_argument(