Merge pull request 'tests: actually execute vars checks in CI' (#3803) from DavHau/clan-core:vars2 into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3803

clanServices/hello-world/tests/vm/default.nix

@@ -10,7 +10,7 @@ clanLib.test.makeTestClan {
   nixosTest = (
     { ... }:
     {
-      name = "service-hello-test";
+      name = "hello-service";
 
       clan = {
         directory = ./.;

clanServices/wifi/tests/vm/default.nix

@@ -11,7 +11,7 @@ clanLib.test.makeTestClan {
   nixosTest = (
     { ... }:
     {
-      name = "wifi";
+      name = "wifi-service";
 
       clan = {
         directory = ./.;

lib/build-clan/interface.nix

@@ -34,7 +34,7 @@ in
         else if v == null then
           throw "Please set either clan.self or clan.directory"
         else
-          "${v}"
+          v
       ) lib.types.path;
       default = builtins.toString self;
       defaultText = "Root directory of the flake";

lib/test/default.nix

@@ -5,7 +5,12 @@
 let
   inherit (lib)
     mkOption
+    removePrefix
     types
+    mapAttrsToList
+    flip
+    unique
+    flatten
     ;
 
 in
@@ -37,16 +42,26 @@ in
 
       update-vars-script = "${self.packages.${pkgs.system}.generate-test-vars}/bin/generate-test-vars";
 
+      relativeDir = removePrefix ("${self}/") (toString test.config.clan.directory);
+
       update-vars = pkgs.writeShellScriptBin "update-vars" ''
-        ${update-vars-script} $PRJ_ROOT/checks/${testName} ${testName}
+        ${update-vars-script} $PRJ_ROOT/${relativeDir} ${testName}
       '';
 
-      testSrc = lib.cleanSource (self + "/checks/${testName}");
+      testSrc = lib.cleanSource test.config.clan.directory;
+
+      inputsForMachine =
+        machine:
+        flip mapAttrsToList machine.clan.core.vars.generators (_name: generator: generator.runtimeInputs);
+
+      generatorRuntimeInputs = unique (
+        flatten (flip mapAttrsToList test.config.nodes (_machineName: machine: inputsForMachine machine))
+      );
 
       vars-check =
         pkgs.runCommand "update-vars-check"
           {
-            nativeBuildInputs = [
+            nativeBuildInputs = generatorRuntimeInputs ++ [
               pkgs.nix
               pkgs.git
               pkgs.age
@@ -54,7 +69,7 @@ in
               pkgs.bubblewrap
             ];
             closureInfo = pkgs.closureInfo {
-              rootPaths = [
+              rootPaths = generatorRuntimeInputs ++ [
                 pkgs.bash
                 pkgs.coreutils
                 pkgs.jq.dev
@@ -67,14 +82,13 @@ in
             };
           }
           ''
-            # make the test depend on its vars-check derivation
-            echo ${vars-check} >/dev/null
-
             ${self.legacyPackages.${pkgs.system}.setupNixInNix}
             cp -r ${testSrc} ./src
             chmod +w -R ./src
             find ./src/sops ./src/vars | sort > filesBefore
-            ${update-vars-script} ./src ${testName} --repo-root ${self.packages.${pkgs.system}.clan-core-flake}
+            ${update-vars-script} ./src ${testName} \
+              --repo-root ${self.packages.${pkgs.system}.clan-core-flake} \
+              --clean
             find ./src/sops ./src/vars | sort > filesAfter
             if ! diff -q filesBefore filesAfter; then
               echo "The update-vars script changed the files in ${testSrc}."
@@ -182,6 +196,8 @@ in
                   #     Harder to handle advanced setups (like TPM, LUKS, or LVM-on-LUKS) but not needed since we are in a test
                   #     No systemd journal logs from initrd.
                   boot.initrd.systemd.enable = false;
+                  # make the test depend on its vars-check derivation
+                  environment.variables.CLAN_VARS_CHECK = "${vars-check}";
                 }
               );
 

nixosModules/clanCore/vars/interface.nix

@@ -246,7 +246,7 @@ in
                         The path to the file containing the content of the generated value.
                         This will be set automatically
                       '';
-                      type = nullOr str;
+                      type = nullOr path;
                       default = null;
                     };
                     path = lib.mkOption {

pkgs/clan-cli/clan_cli/vars/generate.py

@@ -29,7 +29,7 @@ from .graph import (
     minimal_closure,
     requested_closure,
 )
-from .prompt import Prompt, ask
+from .prompt import Prompt, PromptType, ask
 from .var import Var
 
 log = logging.getLogger(__name__)
@@ -295,6 +295,26 @@ def _ask_prompts(
     return prompt_values
 
 
+def _fake_prompts(
+    generator: Generator,
+) -> dict[str, str]:
+    prompt_values: dict[str, str] = {}
+    for prompt in generator.prompts:
+        var_id = f"{generator.name}/{prompt.name}"
+        if prompt.prompt_type == PromptType.HIDDEN:
+            prompt_values[prompt.name] = "fake_hidden_value"
+        elif prompt.prompt_type == PromptType.MULTILINE_HIDDEN:
+            prompt_values[prompt.name] = "fake\nmultiline\nhidden\nvalue"
+        elif prompt.prompt_type == PromptType.MULTILINE:
+            prompt_values[prompt.name] = "fake\nmultiline\nvalue"
+        elif prompt.prompt_type == PromptType.LINE:
+            prompt_values[prompt.name] = "fake_line_value"
+        else:
+            msg = f"Unknown prompt type {prompt.prompt_type} for prompt {var_id} in generator {generator.name}"
+            raise ClanError(msg)
+    return prompt_values
+
+
 def _get_previous_value(
     machine: "Machine",
     generator: Generator,
@@ -414,6 +434,7 @@ def generate_vars_for_machine_interactive(
     generator_name: str | None,
     regenerate: bool,
     no_sandbox: bool = False,
+    fake_prompts: bool = False,
 ) -> bool:
     _generator = None
     if generator_name:
@@ -438,6 +459,9 @@ def generate_vars_for_machine_interactive(
         return False
     all_prompt_values = {}
     for generator in generators:
+        if fake_prompts:
+            all_prompt_values[generator.name] = _fake_prompts(generator)
+        else:
             all_prompt_values[generator.name] = _ask_prompts(generator)
     return _generate_vars_for_machine(
         machine,
@@ -452,13 +476,18 @@ def generate_vars(
     generator_name: str | None = None,
     regenerate: bool = False,
     no_sandbox: bool = False,
+    fake_prompts: bool = False,
 ) -> bool:
     was_regenerated = False
     for machine in machines:
         errors = []
         try:
             was_regenerated |= generate_vars_for_machine_interactive(
-                machine, generator_name, regenerate, no_sandbox=no_sandbox
+                machine,
+                generator_name,
+                regenerate,
+                no_sandbox=no_sandbox,
+                fake_prompts=fake_prompts,
             )
         except Exception as exc:
             errors += [(machine, exc)]
@@ -504,7 +533,11 @@ def generate_command(args: argparse.Namespace) -> None:
         ]
     )
     has_changed = generate_vars(
-        machines, args.generator, args.regenerate, no_sandbox=args.no_sandbox
+        machines,
+        args.generator,
+        args.regenerate,
+        no_sandbox=args.no_sandbox,
+        fake_prompts=args.fake_prompts,
     )
     if has_changed:
         args.flake.invalidate_cache()
@@ -544,4 +577,11 @@ def register_generate_parser(parser: argparse.ArgumentParser) -> None:
         default=False,
     )
 
+    parser.add_argument(
+        "--fake-prompts",
+        action="store_true",
+        help="automatically fill prompt responses for testing (unsafe)",
+        default=False,
+    )
+
     parser.set_defaults(func=generate_command)

pkgs/generate-test-vars/generate_test_vars/cli.py

@@ -87,6 +87,7 @@ class Options:
     repo_root: Path
     test_dir: Path
     check_attr: str
+    clean: bool
 
 
 def parse_args() -> Options:
@@ -107,6 +108,13 @@ def parse_args() -> Options:
         required=False,
         default=os.environ.get("PRJ_ROOT", find_git_repo_root()),
     )
+    parser.add_argument(
+        "--clean",
+        help="wipe vars and sops directories before generating new vars",
+        action="store_true",
+        default=False,
+    )
+
     parser.add_argument(
         "test_dir",
         type=Path,
@@ -125,6 +133,7 @@ def parse_args() -> Options:
         repo_root=args.repo_root,
         test_dir=args.test_dir,
         check_attr=args.check_attr,
+        clean=args.clean,
     )
 
 
@@ -133,6 +142,7 @@ def main() -> None:
     opts = parse_args()
     test_dir = opts.test_dir
 
+    if opts.clean:
         shutil.rmtree(test_dir / "vars", ignore_errors=True)
         shutil.rmtree(test_dir / "sops", ignore_errors=True)
 
@@ -162,8 +172,10 @@ def main() -> None:
             {
                 "publickey": sops_pub_key,
                 "type": "age",
-            }
+            },
+            indent=2,
         )
+        + "\n"
     )
     with NamedTemporaryFile("w") as f:
         f.write("# created: 2023-07-17T10:51:45+02:00\n")
@@ -171,7 +183,7 @@ def main() -> None:
         f.write(sops_priv_key)
         f.seek(0)
         os.environ["SOPS_AGE_KEY_FILE"] = f.name
-        generate_vars(list(machines))
+        generate_vars(list(machines), fake_prompts=True)
 
 
 if __name__ == "__main__":