Merge pull request 'vars: use writeShellApplication to shellcheck generators' (#3270) from Qubasa/clan-core:rtunreal-shell_check_vargen into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3270

checks/dummy-inventory-test/legacy-module/shared.nix

@@ -27,8 +27,8 @@
     files.host-id.secret = false;
     files.generated-password.secret = true;
     script = ''
-      echo $RANDOM > $out/host-id
-      echo $RANDOM > $out/generated-password
+      echo $RANDOM > "$out"/host-id
+      echo $RANDOM > "$out"/generated-password
     '';
   };
 }

checks/installation/flake-module.nix

@@ -127,7 +127,7 @@ in
         clan.core.vars.generators.test = {
           files.test.neededFor = "partitioning";
           script = ''
-            echo "notok" > $out/test
+            echo "notok" > "$out"/test
           '';
         };
         disko.devices = {

clanModules/borgbackup/roles/client.nix

@@ -190,8 +190,8 @@ in
         pkgs.xkcdpass
       ];
       script = ''
-        ssh-keygen -t ed25519 -N "" -f $out/borgbackup.ssh
-        xkcdpass -n 4 -d - > $out/borgbackup.repokey
+        ssh-keygen -t ed25519 -N "" -f "$out"/borgbackup.ssh
+        xkcdpass -n 4 -d - > "$out"/borgbackup.repokey
       '';
     };
 

clanModules/data-mesher/shared.nix

@@ -117,8 +117,8 @@ in
 
       script = ''
         data-mesher generate keypair \
-            --public-key-path $out/public_key \
-            --private-key-path $out/private_key
+            --public-key-path "$out"/public_key \
+            --private-key-path "$out"/private_key
       '';
     };
 
@@ -146,8 +146,8 @@ in
 
       script = ''
         data-mesher generate keypair \
-            --public-key-path $out/public_key \
-            --private-key-path $out/private_key
+            --public-key-path "$out"/public_key \
+            --private-key-path "$out"/private_key
       '';
     };
   };

clanModules/garage/default.nix

@@ -22,8 +22,8 @@
       pkgs.openssl
     ];
     script = ''
-      openssl rand -base64 -out $out/admin_token 32
-      openssl rand -base64 -out $out/metrics_token 32
+      openssl rand -base64 -out "$out"/admin_token 32
+      openssl rand -base64 -out "$out"/metrics_token 32
     '';
   };
 
@@ -35,7 +35,7 @@
       pkgs.openssl
     ];
     script = ''
-      openssl rand -hex -out $out/rpc_secret 32
+      openssl rand -hex -out "$out"/rpc_secret 32
     '';
   };
 

clanModules/root-password/roles/default.nix

@@ -26,13 +26,13 @@
     prompts.password.description = "You can autogenerate a password, if you leave this prompt blank.";
 
     script = ''
-      prompt_value=$(cat $prompts/password)
-      if [[ -n ''${prompt_value-} ]]; then
-        echo $prompt_value | tr -d "\n" > $out/password
+      prompt_value="$(cat "$prompts"/password)"
+      if [[ -n "''${prompt_value-}" ]]; then
+        echo "$prompt_value" | tr -d "\n" > "$out"/password
       else
-        xkcdpass --numwords 3 --delimiter - --count 1 | tr -d "\n" > $out/password
+        xkcdpass --numwords 3 --delimiter - --count 1 | tr -d "\n" > "$out"/password
       fi
-      mkpasswd -s -m sha-512 < $out/password | tr -d "\n" > $out/password-hash
+      mkpasswd -s -m sha-512 < "$out"/password | tr -d "\n" > "$out"/password-hash
     '';
   };
 }

clanModules/sshd/roles/server.nix

@@ -47,7 +47,7 @@ in
         pkgs.openssh
       ];
       script = ''
-        ssh-keygen -t ed25519 -N "" -f $out/ssh.id_ed25519
+        ssh-keygen -t ed25519 -N "" -f "$out"/ssh.id_ed25519
       '';
     };
 
@@ -67,7 +67,7 @@ in
         pkgs.openssh
       ];
       script = ''
-        ssh-keygen -t rsa -b 4096 -N "" -f $out/ssh.id_rsa
+        ssh-keygen -t rsa -b 4096 -N "" -f "$out"/ssh.id_rsa
       '';
     };
 
@@ -92,7 +92,7 @@ in
           -h \
           -n ${lib.concatMapStringsSep "," (d: "${config.clan.core.settings.machine.name}.${d}") domains} \
           $in/openssh/ssh.id_ed25519.pub
-        mv $in/openssh/ssh.id_ed25519-cert.pub $out/ssh.id_ed25519-cert.pub
+        mv $in/openssh/ssh.id_ed25519-cert.pub "$out"/ssh.id_ed25519-cert.pub
       '';
     };
   };

clanModules/sshd/shared.nix

@@ -36,7 +36,7 @@
             pkgs.openssh
           ];
           script = ''
-            ssh-keygen -t ed25519 -N "" -f $out/id_ed25519
+            ssh-keygen -t ed25519 -N "" -f "$out"/id_ed25519
           '';
         };
 

clanModules/state-version/roles/default.nix

@@ -12,7 +12,7 @@ in
     };
     runtimeInputs = [ ];
     script = ''
-      echo -n ${lib.versions.majorMinor config.system.stateVersion} > $out/version
+      echo -n ${lib.versions.majorMinor config.system.stateVersion} > "$out"/version
     '';
   };
 }

clanModules/syncthing-static-peers/default.nix

@@ -99,11 +99,11 @@ in
           pkgs.syncthing
         ];
         script = ''
-          syncthing generate --config $out
-          mv $out/key.pem $out/key
-          mv $out/cert.pem $out/cert
-          cat $out/config.xml | grep -oP '(?<=<device id=")[^"]+' | uniq > $out/id
-          cat $out/config.xml | grep -oP '<apikey>\K[^<]+' | uniq > $out/api
+          syncthing generate --config "$out"
+          mv "$out"/key.pem "$out"/key
+          mv "$out"/cert.pem "$out"/cert
+          cat "$out"/config.xml | grep -oP '(?<=<device id=")[^"]+' | uniq > "$out"/id
+          cat "$out"/config.xml | grep -oP '<apikey>\K[^<]+' | uniq > "$out"/api
         '';
       };
     }

clanModules/user-password/roles/default.nix

@@ -44,13 +44,13 @@ in
         pkgs.mkpasswd
       ];
       script = ''
-        prompt_value=$(cat $prompts/user-password)
-        if [[ -n ''${prompt_value-} ]]; then
-          echo $prompt_value | tr -d "\n" > $out/user-password
+        prompt_value=$(cat "$prompts"/user-password)
+        if [[ -n "''${prompt_value-}" ]]; then
+          echo "$prompt_value" | tr -d "\n" > "$out"/user-password
         else
-          xkcdpass --numwords 3 --delimiter - --count 1 | tr -d "\n" > $out/user-password
+          xkcdpass --numwords 3 --delimiter - --count 1 | tr -d "\n" > "$out"/user-password
         fi
-        mkpasswd -s -m sha-512 < $out/user-password | tr -d "\n" > $out/user-password-hash
+        mkpasswd -s -m sha-512 < "$out"/user-password | tr -d "\n" > "$out"/user-password-hash
       '';
     };
   };

formatter.nix

@@ -37,6 +37,8 @@
         "vars/*"
         # prettier messes up our mkdocs flavoured markdown
         "*.md"
+        "**/node_modules/*"
+        "**/.mypy_cache/*"
 
         "checks/data-mesher/vars/*"
         "checks/lib/ssh/privkey"

nixosModules/clanCore/vars/generator.nix

@@ -16,7 +16,7 @@ let
     ;
 
   promptToFile = name: ''
-    cat "$prompts/${name}" > "$out/${name}"
+    cat "$prompts"/${name} > "$out"/${name}
   '';
 
   promptsToFilesScript = concatMapStrings promptToFile;
@@ -25,8 +25,13 @@ let
 in
 {
   finalScript = mkOptionDefault (
-    pkgs.writeScript "generator-${config.name}" ''
-      set -eu -o pipefail
+    lib.getExe (
+      pkgs.writeShellApplication {
+        name = "generator-${config.name}";
+        text = ''
+          in="''${in:?}"
+          out="''${out:?}"
+          ${if config.prompts == { } then "" else ''prompts="''${prompts:?}"''}
 
           export PATH="${makeBinPath config.runtimeInputs}:${pkgs.coreutils}/bin"
 
@@ -53,7 +58,9 @@ in
           ''}
           ${promptsToFilesScript filePromptNames}
           ${config.script}
-    ''
+        '';
+      }
+    )
   );
 
   files = genAttrs filePromptNames (_name: { });

pkgs/clan-cli/clan_cli/conftest.py

@@ -28,4 +28,4 @@ def pytest_sessionstart(session: pytest.Session) -> None:
     # You can access the session config, items, testsfailed, etc.
     print(f"Session config: {session.config}")
 
-    setup_logging(level="DEBUG")
+    setup_logging(level="INFO")

pkgs/clan-cli/clan_cli/nix/allowed-programs.json

@@ -15,6 +15,7 @@
   "tor",
   "virtiofsd",
   "zbar",
+  "shellcheck-minimal",
   "util-linux",
   "avahi",
   "gnupg"

pkgs/clan-cli/clan_cli/tests/test_machines_cli.py

@@ -100,9 +100,9 @@ def test_machine_delete(
     my_generator["files"]["my_value"]["secret"] = False
     my_generator["files"]["my_secret"]["secret"] = True
     my_generator["script"] = (
-        "echo -n public > $out/my_value;"
-        "echo -n secret > $out/my_secret;"
-        "echo -n non-default > $out/value_with_default"
+        'echo -n public > "$out"/my_value;'
+        'echo -n secret > "$out"/my_secret;'
+        'echo -n non-default > "$out"/value_with_default'
     )
     flake.refresh()  # saves "my_generator"
     monkeypatch.chdir(flake.path)

pkgs/clan-cli/clan_cli/tests/test_vars.py

@@ -106,7 +106,7 @@ def test_generate_public_and_secret_vars(
     my_generator["files"]["my_value"]["secret"] = False
     my_generator["files"]["my_secret"]["secret"] = True
     my_generator["script"] = (
-        "echo -n public > $out/my_value; echo -n secret > $out/my_secret; echo -n non-default > $out/value_with_default"
+        'echo -n public > "$out"/my_value; echo -n secret > "$out"/my_secret; echo -n non-default > "$out"/value_with_default'
     )
 
     my_generator["files"]["value_with_default"]["secret"] = False
@@ -119,7 +119,7 @@ def test_generate_public_and_secret_vars(
     ]
     my_shared_generator["share"] = True
     my_shared_generator["files"]["my_shared_value"]["secret"] = False
-    my_shared_generator["script"] = "echo -n shared > $out/my_shared_value"
+    my_shared_generator["script"] = 'echo -n shared > "$out"/my_shared_value'
 
     dependent_generator = config["clan"]["core"]["vars"]["generators"][
         "dependent_generator"
@@ -128,7 +128,7 @@ def test_generate_public_and_secret_vars(
     dependent_generator["files"]["my_secret"]["secret"] = True
     dependent_generator["dependencies"] = ["my_shared_generator"]
     dependent_generator["script"] = (
-        "cat $in/my_shared_generator/my_shared_value > $out/my_secret"
+        'cat "$in"/my_shared_generator/my_shared_value > "$out"/my_secret'
     )
 
     flake.refresh()
@@ -247,13 +247,13 @@ def test_generate_secret_var_sops_with_default_group(
     first_generator["files"]["my_secret"]["secret"] = True
     first_generator["files"]["my_public"]["secret"] = False
     first_generator["script"] = (
-        "echo hello > $out/my_secret && echo hello > $out/my_public"
+        'echo hello > "$out"/my_secret && echo hello > "$out"/my_public'
     )
     second_generator = config["clan"]["core"]["vars"]["generators"]["second_generator"]
     second_generator["files"]["my_secret"]["secret"] = True
     second_generator["files"]["my_public"]["secret"] = False
     second_generator["script"] = (
-        "echo hello > $out/my_secret && echo hello > $out/my_public"
+        'echo hello > "$out"/my_secret && echo hello > "$out"/my_public'
     )
     flake.refresh()
     monkeypatch.chdir(flake.path)
@@ -335,7 +335,7 @@ def test_generated_shared_secret_sops(
     ]
     shared_generator["share"] = True
     shared_generator["files"]["my_shared_secret"]["secret"] = True
-    shared_generator["script"] = "echo hello > $out/my_shared_secret"
+    shared_generator["script"] = 'echo hello > "$out"/my_shared_secret'
     m2_config = flake.machines["machine2"]
     m2_config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
     m2_config["clan"]["core"]["vars"]["generators"]["my_shared_generator"] = (
@@ -380,14 +380,14 @@ def test_generate_secret_var_password_store(
     # we still have the second one to test `delete_store`:
     my_generator = clan_vars["generators"]["my_generator"]
     my_generator["files"]["my_secret"]["secret"] = True
-    my_generator["script"] = "echo hello > $out/my_secret"
+    my_generator["script"] = 'echo hello > "$out"/my_secret'
     my_generator2 = clan_vars["generators"]["my_generator2"]
     my_generator2["files"]["my_secret2"]["secret"] = True
-    my_generator2["script"] = "echo world > $out/my_secret2"
+    my_generator2["script"] = 'echo world > "$out"/my_secret2'
     my_shared_generator = clan_vars["generators"]["my_shared_generator"]
     my_shared_generator["share"] = True
     my_shared_generator["files"]["my_shared_secret"]["secret"] = True
-    my_shared_generator["script"] = "echo hello > $out/my_shared_secret"
+    my_shared_generator["script"] = 'echo hello > "$out"/my_shared_secret'
     flake.refresh()
     monkeypatch.chdir(flake.path)
     gnupghome = flake.path / "gpg"
@@ -458,7 +458,7 @@ def test_generate_secret_for_multiple_machines(
     machine1_generator["files"]["my_secret"]["secret"] = True
     machine1_generator["files"]["my_value"]["secret"] = False
     machine1_generator["script"] = (
-        "echo machine1 > $out/my_secret && echo machine1 > $out/my_value"
+        'echo machine1 > "$out"/my_secret && echo machine1 > "$out"/my_value'
     )
     machine2_config = flake.machines["machine2"]
     # Test that we can generate secrets for other platforms
@@ -472,7 +472,7 @@ def test_generate_secret_for_multiple_machines(
     machine2_generator["files"]["my_secret"]["secret"] = True
     machine2_generator["files"]["my_value"]["secret"] = False
     machine2_generator["script"] = (
-        "echo machine2 > $out/my_secret && echo machine2 > $out/my_value"
+        'echo machine2 > "$out"/my_secret && echo machine2 > "$out"/my_value'
     )
     flake.refresh()
     monkeypatch.chdir(flake.path)
@@ -535,7 +535,7 @@ def test_prompt(
     my_generator["prompts"]["prompt_persist"]["persist"] = True
 
     my_generator["script"] = (
-        "cat $prompts/prompt1 > $out/line_value; cat $prompts/prompt2 > $out/multiline_value"
+        'cat "$prompts"/prompt1 > "$out"/line_value; cat "$prompts"/prompt2 > "$out"/multiline_value'
     )
     flake.refresh()
     monkeypatch.chdir(flake.path)
@@ -594,7 +594,7 @@ def test_multi_machine_shared_vars(
     shared_generator["files"]["my_secret"]["secret"] = True
     shared_generator["files"]["my_value"]["secret"] = False
     shared_generator["script"] = (
-        "echo $RANDOM > $out/my_value && echo $RANDOM > $out/my_secret"
+        'echo "$RANDOM" > "$out"/my_value && echo "$RANDOM" > "$out"/my_secret'
     )
     # machine 2 is equivalent to machine 1
     flake.machines["machine2"] = machine1_config
@@ -697,12 +697,12 @@ def test_stdout_of_generate(
     config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
     my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
     my_generator["files"]["my_value"]["secret"] = False
-    my_generator["script"] = "echo -n hello > $out/my_value"
+    my_generator["script"] = 'echo -n hello > "$out"/my_value'
     my_secret_generator = config["clan"]["core"]["vars"]["generators"][
         "my_secret_generator"
     ]
     my_secret_generator["files"]["my_secret"]["secret"] = True
-    my_secret_generator["script"] = "echo -n hello > $out/my_secret"
+    my_secret_generator["script"] = 'echo -n hello > "$out"/my_secret'
     flake.refresh()
     monkeypatch.chdir(flake.path)
     from clan_cli.vars.generate import generate_vars_for_machine
@@ -782,28 +782,29 @@ def test_migration(
     my_service["public"]["my_value"] = {}
     my_service["secret"]["my_secret"] = {}
     my_service["generator"]["script"] = (
-        "echo -n hello > $facts/my_value && echo -n hello > $secrets/my_secret"
+        'echo -n hello > "$facts"/my_value && echo -n hello > "$secrets"/my_secret'
     )
     my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
     my_generator["files"]["my_value"]["secret"] = False
     my_generator["files"]["my_secret"]["secret"] = True
     my_generator["migrateFact"] = "my_service"
-    my_generator["script"] = "echo -n other > $out/my_value"
+    my_generator["script"] = 'echo -n other > "$out"/my_value'
 
     other_service = config["clan"]["core"]["facts"]["services"]["other_service"]
     other_service["secret"]["other_value"] = {}
-    other_service["generator"]["script"] = "echo -n hello > $secrets/other_value"
+    other_service["generator"]["script"] = 'echo -n hello > "$secrets"/other_value'
     other_generator = config["clan"]["core"]["vars"]["generators"]["other_generator"]
     # the var to migrate to is mistakenly marked as not secret (migration should fail)
     other_generator["files"]["other_value"]["secret"] = False
     other_generator["migrateFact"] = "my_service"
-    other_generator["script"] = "echo -n value-from-vars > $out/other_value"
+    other_generator["script"] = 'echo -n value-from-vars > "$out"/other_value'
 
     flake.refresh()
     monkeypatch.chdir(flake.path)
     cli.run(["facts", "generate", "--flake", str(flake.path), "my_machine"])
     with caplog.at_level(logging.INFO):
         cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
+
     assert "Migrated var my_generator/my_value" in caplog.text
     assert "Migrated secret var my_generator/my_secret" in caplog.text
     in_repo_store = in_repo.FactStore(
@@ -837,12 +838,12 @@ def test_fails_when_files_are_left_from_other_backend(
         "my_secret_generator"
     ]
     my_secret_generator["files"]["my_secret"]["secret"] = True
-    my_secret_generator["script"] = "echo hello > $out/my_secret"
+    my_secret_generator["script"] = 'echo hello > "$out"/my_secret'
     my_value_generator = config["clan"]["core"]["vars"]["generators"][
         "my_value_generator"
     ]
     my_value_generator["files"]["my_value"]["secret"] = False
-    my_value_generator["script"] = "echo hello > $out/my_value"
+    my_value_generator["script"] = 'echo hello > "$out"/my_value'
     flake.refresh()
     monkeypatch.chdir(flake.path)
     for generator in ["my_secret_generator", "my_value_generator"]:
@@ -902,7 +903,7 @@ def test_invalidation(
     config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
     my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
     my_generator["files"]["my_value"]["secret"] = False
-    my_generator["script"] = "echo -n $RANDOM > $out/my_value"
+    my_generator["script"] = 'echo -n "$RANDOM" > "$out"/my_value'
     flake.refresh()
     monkeypatch.chdir(flake.path)
     cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
@@ -947,14 +948,14 @@ def test_dynamic_invalidation(
 
     my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
     my_generator["files"]["my_value"]["secret"] = False
-    my_generator["script"] = "echo -n $RANDOM > $out/my_value"
+    my_generator["script"] = 'echo -n "$RANDOM" > "$out"/my_value'
 
     dependent_generator = config["clan"]["core"]["vars"]["generators"][
         "dependent_generator"
     ]
     dependent_generator["files"]["my_value"]["secret"] = False
     dependent_generator["dependencies"] = ["my_generator"]
-    dependent_generator["script"] = "echo -n $RANDOM > $out/my_value"
+    dependent_generator["script"] = 'echo -n "$RANDOM" > "$out"/my_value'
 
     flake.refresh()
 

pkgs/clan-cli/clan_cli/tests/test_vars_deployment.py

@@ -33,7 +33,7 @@ def test_vm_deployment(
     m1_generator = machine1_config["clan"]["core"]["vars"]["generators"]["m1_generator"]
     m1_generator["files"]["my_secret"]["secret"] = True
     m1_generator["script"] = """
-        echo hello > $out/my_secret
+        echo hello > "$out"/my_secret
     """
     m1_shared_generator = machine1_config["clan"]["core"]["vars"]["generators"][
         "my_shared_generator"
@@ -43,8 +43,8 @@ def test_vm_deployment(
     m1_shared_generator["files"]["no_deploy_secret"]["secret"] = True
     m1_shared_generator["files"]["no_deploy_secret"]["deploy"] = False
     m1_shared_generator["script"] = """
-        echo hello > $out/shared_secret
-        echo hello > $out/no_deploy_secret
+        echo hello > "$out"/shared_secret
+        echo hello > "$out"/no_deploy_secret
     """
     # machine 2
     machine2_config = flake.machines["m2_machine"]

pkgs/clan-cli/default.nix

@@ -141,7 +141,6 @@ pythonRuntime.pkgs.buildPythonApplication {
                 templateDerivation
               ];
             };
-
           }
           ''
             set -u -o pipefail
@@ -188,6 +187,7 @@ pythonRuntime.pkgs.buildPythonApplication {
                 pkgs.jq.dev
                 pkgs.stdenv
                 pkgs.stdenvNoCC
+                pkgs.shellcheck-minimal
               ];
             };
           }

pkgs/clan-cli/flake-module.nix

@@ -88,12 +88,6 @@
         rootPaths =
           builtins.attrValues (self.clanLib.select "clan.templates.clan.*.path" self)
           ++ builtins.attrValues (self.clanLib.select "clan.templates.machine.*.path" self);
-
-        # FIXME: As the templates get modified in clanCoreWithVendoredDeps below, we need to add the modified version to the nix store too
-        # However it is not possible (or I don't know how) to add a nix path from a built derivation to the nix store
-        # rootPaths = [
-        #   clanCoreWithVendoredDeps.clan.templates.clan.minimal.path
-        # ];
       };
 
       clanCoreWithVendoredDeps =