vars: introduce deploy=true/false for generated files

nixosModules/clanCore/vars/default.nix

@@ -45,7 +45,11 @@ in
             prompts
             share
             ;
-          files = lib.flip lib.mapAttrs generator.files (_name: file: { inherit (file) secret; });
+          files = lib.flip lib.mapAttrs generator.files (
+            _name: file: {
+              inherit (file) deploy secret;
+            }
+          );
         }
       );
       inherit (config.clan.core.vars.settings) secretUploadDirectory secretModule publicModule;

nixosModules/clanCore/vars/interface.nix

@@ -74,6 +74,15 @@ in
                       readOnly = true;
                       default = generator.config._module.args.name;
                     };
+                    deploy = {
+                      description = ''
+                        Whether the file should be deployed to the target machine.
+
+                        Enable this if the generated file is only used as an input to other generators.
+                      '';
+                      type = bool;
+                      default = true;
+                    };
                     secret = {
                       description = ''
                         Whether the file should be treated as a secret.

nixosModules/clanCore/vars/secret/sops/default.nix

@@ -6,17 +6,26 @@
 }:
 let
 
-  inherit (lib) flip;
+  inherit (lib) importJSON flip;
+
+  inherit (builtins) dirOf pathExists;
 
   inherit (import ./funcs.nix { inherit lib; }) listVars;
 
   inherit (config.clan.core) machineName;
 
+  metaFile = sopsFile: dirOf sopsFile + "/meta.json";
+
+  metaData = sopsFile: if pathExists (metaFile sopsFile) then importJSON (metaFile sopsFile) else { };
+
+  toDeploy = secret: (metaData secret.sopsFile).deploy or true;
+
   varsDirMachines = config.clan.core.clanDir + "/sops/vars/per-machine/${machineName}";
   varsDirShared = config.clan.core.clanDir + "/sops/vars/shared";
 
-  vars = (listVars varsDirMachines) ++ (listVars varsDirShared);
+  vars' = (listVars varsDirMachines) ++ (listVars varsDirShared);
 
+  vars = lib.filter (secret: toDeploy secret) vars';
 in
 {
   config.clan.core.vars.settings = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") {

pkgs/clan-cli/clan_cli/secrets/secrets.py

@@ -87,9 +87,10 @@ def encrypt_secret(
     add_users: list[str] = [],
     add_machines: list[str] = [],
     add_groups: list[str] = [],
+    meta: dict = {},
 ) -> None:
     key = ensure_sops_key(flake_dir)
-    keys = set([])
+    recipient_keys = set([])
 
     files_to_commit = []
     for user in add_users:
@@ -122,10 +123,10 @@ def encrypt_secret(
             )
         )
 
-    keys = collect_keys_for_path(secret_path)
+    recipient_keys = collect_keys_for_path(secret_path)
 
-    if key.pubkey not in keys:
+    if key.pubkey not in recipient_keys:
-        keys.add(key.pubkey)
+        recipient_keys.add(key.pubkey)
         files_to_commit.extend(
             allow_member(
                 users_folder(secret_path),
@@ -136,7 +137,7 @@ def encrypt_secret(
         )
 
     secret_path = secret_path / "secret"
-    encrypt_file(secret_path, value, list(sorted(keys)))
+    encrypt_file(secret_path, value, list(sorted(recipient_keys)), meta)
     files_to_commit.append(secret_path)
     commit_files(
         files_to_commit,

pkgs/clan-cli/clan_cli/secrets/sops.py

@@ -143,12 +143,15 @@ def update_keys(secret_path: Path, keys: list[str]) -> list[Path]:
 
 
 def encrypt_file(
-    secret_path: Path, content: IO[str] | str | bytes | None, keys: list[str]
+    secret_path: Path,
+    content: IO[str] | str | bytes | None,
+    pubkeys: list[str],
+    meta: dict = {},
 ) -> None:
     folder = secret_path.parent
     folder.mkdir(parents=True, exist_ok=True)
 
-    with sops_manifest(keys) as manifest:
+    with sops_manifest(pubkeys) as manifest:
         if not content:
             args = ["sops", "--config", str(manifest)]
             args.extend([str(secret_path)])
@@ -186,6 +189,9 @@ def encrypt_file(
                 with NamedTemporaryFile(dir=folder, delete=False) as f2:
                     shutil.copyfile(f.name, f2.name)
                     os.rename(f2.name, secret_path)
+                meta_path = secret_path.parent / "meta.json"
+                with open(meta_path, "w") as f_meta:
+                    json.dump(meta, f_meta, indent=2)
             finally:
                 try:
                     os.remove(f.name)
@@ -203,6 +209,14 @@ def decrypt_file(secret_path: Path) -> str:
     return res.stdout
 
 
+def get_meta(secret_path: Path) -> dict:
+    meta_path = secret_path.parent / "meta.json"
+    if not meta_path.exists():
+        return {}
+    with open(meta_path) as f:
+        return json.load(f)
+
+
 def write_key(path: Path, publickey: str, overwrite: bool) -> None:
     path.mkdir(parents=True, exist_ok=True)
     try:

pkgs/clan-cli/clan_cli/vars/generate.py

@@ -152,6 +152,7 @@ def execute_generator(
         # store secrets
         files = machine.vars_generators[generator_name]["files"]
         for file_name, file in files.items():
+            is_deployed = file["deploy"]
             groups = machine.deployment["sops"]["defaultGroups"]
 
             secret_file = tmpdir_out / file_name
@@ -166,6 +167,7 @@ def execute_generator(
                     secret_file.read_bytes(),
                     groups,
                     shared=is_shared,
+                    deployed=is_deployed,
                 )
             else:
                 file_path = public_vars_store.set(

pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py

@@ -17,6 +17,7 @@ class SecretStoreBase(ABC):
         value: bytes,
         groups: list[str],
         shared: bool = False,
+        deployed: bool = True,
     ) -> Path | None:
         pass
 

pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py

@@ -31,6 +31,7 @@ class SecretStore(SecretStoreBase):
         value: bytes,
         groups: list[str],
         shared: bool = False,
+        deployed: bool = True,
     ) -> Path | None:
         subprocess.run(
             nix_shell(

pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py

@@ -58,6 +58,7 @@ class SecretStore(SecretStoreBase):
         value: bytes,
         groups: list[str],
         shared: bool = False,
+        deployed: bool = True,
     ) -> Path | None:
         path = self.secret_path(generator_name, name, shared)
         encrypt_secret(
@@ -66,6 +67,9 @@ class SecretStore(SecretStoreBase):
             value,
             add_machines=[self.machine.name],
             add_groups=groups,
+            meta=dict(
+                deploy=deployed,
+            ),
         )
         return path
 

pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py

@@ -21,6 +21,7 @@ class SecretStore(SecretStoreBase):
         value: bytes,
         groups: list[str],
         shared: bool = False,
+        deployed: bool = True,
     ) -> Path | None:
         secret_file = self.dir / service / name
         secret_file.parent.mkdir(parents=True, exist_ok=True)

pkgs/clan-cli/tests/test_vars_deployment.py

@@ -25,20 +25,19 @@ def test_vm_deployment(
     config["networking"]["firewall"]["enable"] = False
     my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
     my_generator["files"]["my_secret"]["secret"] = True
-    my_generator["files"]["my_value"]["secret"] = False
     my_generator["script"] = """
         echo hello > $out/my_secret
-        echo hello > $out/my_value
     """
     my_shared_generator = config["clan"]["core"]["vars"]["generators"][
         "my_shared_generator"
     ]
     my_shared_generator["share"] = True
-    my_shared_generator["files"]["my_shared_secret"]["secret"] = True
+    my_shared_generator["files"]["shared_secret"]["secret"] = True
-    my_shared_generator["files"]["my_shared_value"]["secret"] = False
+    my_shared_generator["files"]["no_deploy_secret"]["secret"] = True
+    my_shared_generator["files"]["no_deploy_secret"]["deploy"] = False
     my_shared_generator["script"] = """
-        echo hello > $out/my_shared_secret
+        echo hello > $out/shared_secret
-        echo hello > $out/my_shared_value
+        echo hello > $out/no_deploy_secret
     """
     flake = generate_flake(
         temporary_home,
@@ -69,11 +68,18 @@ def test_vm_deployment(
     assert "no-such-path" not in my_secret_path
     run_vm_in_thread("my_machine")
     qga = qga_connect("my_machine")
+    # check my_secret is deployed
     _, out, _ = qga.run("cat /run/secrets/vars/my_generator/my_secret", check=True)
     assert out == "hello\n"
+    # check shared_secret is deployed
     _, out, _ = qga.run(
-        "cat /run/secrets/vars/my_shared_generator/my_shared_secret", check=True
+        "cat /run/secrets/vars/my_shared_generator/shared_secret", check=True
     )
     assert out == "hello\n"
+    # check no_deploy_secret is not deployed
+    returncode, out, _ = qga.run(
+        "test -e /run/secrets/vars/my_shared_generator/no_deploy_secret", check=False
+    )
+    assert returncode != 0
     qga.exec_cmd("poweroff")
     wait_vm_down("my_machine")