Merge pull request 'fix vars migration prompts. add secretsForUsers to vars interface and implement that for pass' (#2551) from lassulus/clan-core:vars-stuff into main

nixosModules/clanCore/vars/default.nix

@@ -49,7 +49,12 @@ in
             ;
           files = lib.flip lib.mapAttrs generator.files (
             _name: file: {
-              inherit (file) name deploy secret;
+              inherit (file)
+                name
+                deploy
+                secret
+                neededForUsers
+                ;
             }
           );
         }

nixosModules/clanCore/vars/interface.nix

@@ -196,6 +196,15 @@ in
                       '';
                       type = str;
                     };
+                    neededForUsers = lib.mkOption {
+                      description = ''
+                        Enabling this option causes the secret to be decrypted/installed before users and groups are created.
+                        This can be used to retrieve user's passwords.
+                        Setting this option moves the secret to /run/secrets-for-users and disallows setting owner and group to anything else than root.
+                      '';
+                      type = bool;
+                      default = false;
+                    };
                     owner = lib.mkOption {
                       description = "The user name or id that will own the secret file.";
                       default = "root";

nixosModules/clanCore/vars/secret/password-store.nix

@@ -17,26 +17,37 @@ let
       set -efu -o pipefail
 
       src=$1
-      mkdir -p /run/secrets.tmp /run/secrets
+      target=$2
-      if mountpoint -q /run/secrets; then
+
-        mount -t tmpfs -o noswap -o private tmpfs /run/secrets.tmp
+      echo "installing secrets from $src to $target" >&2
-        chmod 511 /run/secrets.tmp
+
-        mount --bind --make-private /run/secrets.tmp /run/secrets.tmp
+      mkdir -p "$target".tmp "$target"
-        mount --bind --make-private /run/secrets /run/secrets
+      if mountpoint -q "$target"; then
-        tar -xf "$src" -C /run/secrets.tmp
+        mount -t tmpfs -o noswap -o private tmpfs "$target".tmp
-        move-mount --beneath --move /run/secrets.tmp /run/secrets >/dev/null
+        chmod 511 "$target".tmp
-        umount -R /run/secrets.tmp
+        mount --bind --make-private "$target".tmp "$target".tmp
-        rmdir /run/secrets.tmp
+        mount --bind --make-private "$target" "$target"
-        umount --lazy /run/secrets
+        tar -xf "$src" -C "$target".tmp
+        move-mount --beneath --move "$target".tmp "$target" 2>/dev/null
+        umount -R "$target".tmp
+        rmdir "$target".tmp
+        umount --lazy "$target"
       else
-        mount -t tmpfs -o noswap tmpfs /run/secrets
+        mount -t tmpfs -o noswap tmpfs "$target"
-        tar -xf "$src" -C /run/secrets
+        tar -xf "$src" -C "$target"
       fi
     '';
   };
   useSystemdActivation =
     (options.systemd ? sysusers && config.systemd.sysusers.enable)
     || (options.services ? userborn && config.services.userborn.enable);
+  normalSecrets = lib.any (gen: lib.any (file: !file.neededForUsers) (lib.attrValues gen.files)) (
+    lib.attrValues config.clan.core.vars.generators
+  );
+  userSecrets = lib.any (gen: lib.any (file: file.neededForUsers) (lib.attrValues gen.files)) (
+    lib.attrValues config.clan.core.vars.generators
+  );
+
 in
 {
   options.clan.vars.password-store = {
@@ -57,48 +68,75 @@ in
           fileModule =
             file:
             lib.mkIf file.config.secret {
-              path = "/run/secrets/${file.config.generatorName}/${file.config.name}";
+              path =
+                if file.config.neededForUsers then
+                  "/run/user-secrets/${file.config.generatorName}/${file.config.name}"
+                else
+                  "/run/secrets/${file.config.generatorName}/${file.config.name}";
             };
           secretModule = "clan_cli.vars.secret_modules.password_store";
         };
-    system.activationScripts.setupSecrets =
+    system.activationScripts =
-      lib.mkIf
+      lib.mkIf ((config.clan.core.vars.settings.secretStore == "password-store") && !useSystemdActivation)
-        (
-          (config.clan.core.vars.settings.secretStore == "password-store")
-          && (config.clan.core.vars.generators != { } && !useSystemdActivation)
-        )
-        (
-          lib.stringAfter
-            [
-              "specialfs"
-              "users"
-              "groups"
-            ]
-            ''
-              [ -e /run/current-system ] || echo setting up secrets...
-              ${installSecretTarball}/bin/install-secret-tarball ${config.clan.vars.password-store.secretLocation}/secrets.tar.gz
-            ''
-          // lib.optionalAttrs (config.system ? dryActivationScript) {
-            supportsDryActivation = true;
-          }
-        );
-    systemd.services.pass-install-secrets =
-      lib.mkIf
-        (
-          (config.clan.core.vars.settings.secretStore == "password-store")
-          && (config.clan.core.vars.generators != { } && useSystemdActivation)
-        )
         {
-          wantedBy = [ "sysinit.target" ];
+          setupUserSecrets = lib.mkIf userSecrets (
-          after = [ "systemd-sysusers.service" ];
+            lib.stringAfter
-          unitConfig.DefaultDependencies = "no";
+              [
+                "specialfs"
+              ]
+              ''
+                [ -e /run/current-system ] || echo setting up secrets...
+                ${installSecretTarball}/bin/install-secret-tarball ${config.clan.vars.password-store.secretLocation}/secrets_for_users.tar.gz /run/user-secrets
+              ''
+            // lib.optionalAttrs (config.system ? dryActivationScript) {
+              supportsDryActivation = true;
+            }
+          );
+          users.deps = lib.mkIf userSecrets [ "setupUserSecrets" ];
+          setupSecrets = lib.mkIf normalSecrets (
+            lib.stringAfter
+              [
+                "specialfs"
+                "users"
+                "groups"
+              ]
+              ''
+                [ -e /run/current-system ] || echo setting up secrets...
+                ${installSecretTarball}/bin/install-secret-tarball ${config.clan.vars.password-store.secretLocation}/secrets.tar.gz /run/secrets
+              ''
+            // lib.optionalAttrs (config.system ? dryActivationScript) {
+              supportsDryActivation = true;
+            }
+          );
+        };
+    systemd.services =
+      lib.mkIf ((config.clan.core.vars.settings.secretStore == "password-store") && useSystemdActivation)
+        {
+          pass-install-user-secrets = lib.mkIf userSecrets {
+            wantedBy = [ "systemd-sysusers.service" ];
+            before = [ "systemd-sysusers.service" ];
+            unitConfig.DefaultDependencies = "no";
 
-          serviceConfig = {
+            serviceConfig = {
-            Type = "oneshot";
+              Type = "oneshot";
-            ExecStart = [
+              ExecStart = [
-              "${installSecretTarball}/bin/install-secret-tarball ${config.clan.vars.password-store.secretLocation}/secrets.tar.gz"
+                "${installSecretTarball}/bin/install-secret-tarball ${config.clan.vars.password-store.secretLocation}/secrets_for_users.tar.gz /run/user-secrets"
-            ];
+              ];
-            RemainAfterExit = true;
+              RemainAfterExit = true;
+            };
+          };
+          pass-install-secrets = lib.mkIf normalSecrets {
+            wantedBy = [ "sysinit.target" ];
+            after = [ "systemd-sysusers.service" ];
+            unitConfig.DefaultDependencies = "no";
+
+            serviceConfig = {
+              Type = "oneshot";
+              ExecStart = [
+                "${installSecretTarball}/bin/install-secret-tarball ${config.clan.vars.password-store.secretLocation}/secrets.tar.gz /run/secrets"
+              ];
+              RemainAfterExit = true;
+            };
           };
         };
   };

nixosModules/clanCore/vars/secret/sops/default.nix

@@ -39,7 +39,7 @@ in
       flip map vars (secret: {
         name = "vars/${secret.generator}/${secret.name}";
         value = {
-          inherit (secret) owner group;
+          inherit (secret) owner group neededForUsers;
           sopsFile = secretPath secret;
           format = "binary";
         };

nixosModules/clanCore/vars/secret/sops/funcs.nix

@@ -25,7 +25,7 @@ in
               name = fname;
               generator = gen_name;
               inherit (generator) share;
-              inherit (file) owner group;
+              inherit (file) owner group neededForUsers;
             }
           )
         )

pkgs/clan-cli/clan_cli/vars/generate.py

@@ -233,15 +233,12 @@ def execute_generator(
 
 
 def _ask_prompts(
-    generators: list[Generator],
+    generator: Generator,
-) -> dict[str, dict[str, str]]:
+) -> dict[str, str]:
-    prompt_values: dict[str, dict[str, str]] = {}
+    prompt_values: dict[str, str] = {}
-    for generator in generators:
+    for prompt in generator.prompts:
-        for prompt in generator.prompts:
+        var_id = f"{generator.name}/{prompt.name}"
-            if generator.name not in prompt_values:
+        prompt_values[prompt.name] = ask(var_id, prompt.prompt_type)
-                prompt_values[generator.name] = {}
-            var_id = f"{generator.name}/{prompt.name}"
-            prompt_values[generator.name][prompt.name] = ask(var_id, prompt.prompt_type)
     return prompt_values
 
 
@@ -422,17 +419,16 @@ def generate_vars_for_machine(
     closure = get_closure(machine, generator_name, regenerate)
     if len(closure) == 0:
         return False
-    prompt_values = _ask_prompts(closure)
     for generator in closure:
         if _check_can_migrate(machine, generator):
             _migrate_files(machine, generator)
         else:
             execute_generator(
-                machine,
+                machine=machine,
-                generator,
+                generator=generator,
-                machine.secret_vars_store,
+                secret_vars_store=machine.secret_vars_store,
-                machine.public_vars_store,
+                public_vars_store=machine.public_vars_store,
-                prompt_values.get(generator.name, {}),
+                prompt_values=_ask_prompts(generator),
             )
     # flush caches to make sure the new secrets are available in evaluation
     machine.flush_caches()
@@ -464,7 +460,7 @@ def generate_vars(
             raise ClanError(msg) from errors[0][1]
 
     if not was_regenerated and len(machines) > 0:
-        machine.info("All vars are already up to date")
+        log.info("All vars are already up to date")
 
     return was_regenerated
 

pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py

@@ -150,7 +150,10 @@ class SecretStore(SecretStoreBase):
         return local_hash.decode() != remote_hash
 
     def populate_dir(self, output_dir: Path) -> None:
-        with tarfile.open(output_dir / "secrets.tar.gz", "w:gz") as tar:
+        with (
+            tarfile.open(output_dir / "secrets.tar.gz", "w:gz") as tar,
+            tarfile.open(output_dir / "secrets_for_users.tar.gz", "w:gz") as user_tar,
+        ):
             for generator in self.machine.vars_generators:
                 dir_exists = False
                 for file in generator.files:
@@ -170,7 +173,10 @@ class SecretStore(SecretStoreBase):
                     tar_file.mode = 0o440
                     tar_file.uname = file.owner
                     tar_file.gname = file.group
-                    tar.addfile(tarinfo=tar_file, fileobj=io.BytesIO(content))
+                    if file.needed_for_users:
+                        user_tar.addfile(tarinfo=tar_file, fileobj=io.BytesIO(content))
+                    else:
+                        tar.addfile(tarinfo=tar_file, fileobj=io.BytesIO(content))
         (output_dir / ".pass_info").write_bytes(self.generate_hash())
 
     def upload(self) -> None:
@@ -179,6 +185,7 @@ class SecretStore(SecretStoreBase):
             return
         with TemporaryDirectory(prefix="vars-upload-") as tempdir:
             pass_dir = Path(tempdir)
+            self.populate_dir(pass_dir)
             upload_dir = Path(
                 self.machine.deployment["password-store"]["secretLocation"]
             )

pkgs/clan-cli/clan_cli/vars/var.py

@@ -15,6 +15,7 @@ class Var:
     deploy: bool = False
     owner: str = "root"
     group: str = "root"
+    needed_for_users: bool = False
 
     # TODO: those shouldn't be set here
     _store: "StoreBase | None" = None
@@ -74,4 +75,5 @@ class Var:
             deploy=data["deploy"],
             owner=data.get("owner", "root"),
             group=data.get("group", "root"),
+            needed_for_users=data.get("neededForUsers", False),
         )