clan-cli secrets upload: secrets are populated into tmpdir

checks/impure/flake-module.nix

@@ -9,6 +9,7 @@
         export PATH="${lib.makeBinPath [
           pkgs.gitMinimal
           pkgs.nix
+          pkgs.rsync # needed to have rsync installed on the dummy ssh server
         ]}"
         ROOT=$(git rev-parse --show-toplevel)
         cd "$ROOT/pkgs/clan-cli"

nixosModules/clanCore/flake-module.nix

@@ -45,6 +45,7 @@
       system.clan.deployment.text = builtins.toJSON {
         inherit (config.system.clan) uploadSecrets generateSecrets;
         inherit (config.clan.networking) deploymentAddress;
+        inherit (config.clanCore) secretsUploadDirectory;
       };
       system.clan.deployment.file = pkgs.writeText "deployment.json" config.system.clan.deployment.text;
     };

nixosModules/clanCore/secrets/default.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, ... }:
 {
   options.clanCore.secretStore = lib.mkOption {
     type = lib.types.enum [ "sops" "password-store" "custom" ];
@@ -17,6 +17,13 @@
     '';
   };
 
+  options.clanCore.secretsUploadDirectory = lib.mkOption {
+    type = lib.types.path;
+    description = ''
+      The directory where secrets are uploaded into, This is backend specific.
+    '';
+  };
+
   options.clanCore.secretsPrefix = lib.mkOption {
     type = lib.types.str;
     default = "";
@@ -106,10 +113,6 @@
         };
       }));
   };
-  config.system.build.generateUploadSecrets = pkgs.writeScript "generate_upload_secrets" ''
-    ${config.system.clan.generateSecrets}
-    ${config.system.clan.uploadSecrets}
-  '';
   imports = [
     ./sops.nix
     ./password-store.nix

nixosModules/clanCore/secrets/password-store.nix

@@ -12,6 +12,7 @@ in
   };
   config = lib.mkIf (config.clanCore.secretStore == "password-store") {
     clanCore.secretsDirectory = config.clan.password-store.targetDirectory;
+    clanCore.secretsUploadDirectory = config.clan.password-store.targetDirectory;
     system.clan.generateSecrets = pkgs.writeScript "generate-secrets" ''
       #!/bin/sh
       set -efu
@@ -33,7 +34,7 @@ in
           trap "rm -rf $facts" EXIT
           secrets=$(mktemp -d)
           trap "rm -rf $secrets" EXIT
-          ${v.generator}
+          ( ${v.generator} )
 
           ${lib.concatMapStrings (fact: ''
             mkdir -p "$(dirname ${fact.path})"
@@ -50,8 +51,6 @@ in
       #!/bin/sh
       set -efu
 
-      target=$1
-
       umask 0077
 
       PATH=${lib.makeBinPath [
@@ -71,7 +70,7 @@ in
             sort |
             xargs -r -n 1 git -C ${passwordstoreDir} log -1 --format=%H
         )
-        remote_pass_info=$(ssh "$target" -- ${lib.escapeShellArg ''
+        remote_pass_info=$(ssh ${config.clan.networking.deploymentAddress} -- ${lib.escapeShellArg ''
           cat ${config.clan.password-store.targetDirectory}/.pass_info || :
         ''})
 
@@ -81,12 +80,6 @@ in
         fi
       fi
 
-      tmp_dir=$(mktemp -dt populate-pass.XXXXXXXX)
-      trap cleanup EXIT
-      cleanup() {
-        rm -fR "$tmp_dir"
-      }
-
       find ${passwordstoreDir}/machines/${config.clanCore.machineName} -type f -follow ! -name .gpg-id |
       while read -r gpg_path; do
 
@@ -99,7 +92,7 @@ in
           fi
         )
         pass_name=$rel_name
-        tmp_path=$tmp_dir/$(basename $rel_name)
+        tmp_path="$SECRETS_DIR"/$(basename $rel_name)
 
         mkdir -p "$(dirname "$tmp_path")"
         pass show "$pass_name" > "$tmp_path"
@@ -109,10 +102,8 @@ in
       done
 
       if test -n "''${local_pass_info-}"; then
-        echo "$local_pass_info" > "$tmp_dir"/.pass_info
+        echo "$local_pass_info" > "$SECRETS_DIR"/.pass_info
       fi
-
-      rsync --mkpath --delete -a "$tmp_dir"/ "$target":${config.clan.password-store.targetDirectory}/
     '';
   };
 }

nixosModules/clanCore/secrets/sops.nix

@@ -37,12 +37,10 @@ in
       uploadSecrets = pkgs.writeScript "upload-secrets" ''
         #!${pkgs.python3}/bin/python
         import json
-        import sys
         from clan_cli.secrets.sops_generate import upload_age_key_from_nix
         # the second toJSON is needed to escape the string for the python
-        deployment_address = sys.argv[1]
+        args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; })})
-        args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; age_key_file = config.sops.age.keyFile; })})
+        upload_age_key_from_nix(**args)
-        upload_age_key_from_nix(**args, deployment_address=deployment_address)
       '';
     };
     sops.secrets = builtins.mapAttrs
@@ -56,5 +54,6 @@ in
 
     sops.age.keyFile = lib.mkIf (builtins.pathExists (config.clanCore.clanDir + "/sops/secrets/${config.clanCore.machineName}-age.key/secret"))
       (lib.mkDefault "/var/lib/sops-nix/key.txt");
+    clanCore.secretsUploadDirectory = lib.mkDefault "/var/lib/sops-nix";
   };
 }

pkgs/clan-cli/clan_cli/machines/update.py

@@ -45,7 +45,7 @@ def deploy_nixos(hosts: HostGroup, clan_dir: Path) -> None:
             h.meta["uploadSecrets"],
             clan_dir,
             target=target,
-            target_directory=h.meta["targetDirectory"],
+            target_directory=h.meta["secretsUploadDirectory"],
         )
 
         target_host = h.meta.get("target_host")

pkgs/clan-cli/clan_cli/secrets/sops_generate.py

@@ -1,3 +1,4 @@
+import os
 import shlex
 import shutil
 import subprocess
@@ -10,7 +11,6 @@ from clan_cli.nix import nix_shell
 
 from ..dirs import get_clan_flake_toplevel
 from ..errors import ClanError
-from ..ssh import parse_deployment_address
 from .folders import sops_secrets_folder
 from .machines import add_machine, has_machine
 from .secrets import decrypt_secret, encrypt_secret, has_secret
@@ -102,27 +102,12 @@ def generate_secrets_from_nix(
 
 # this is called by the sops.nix clan core module
 def upload_age_key_from_nix(
-    machine_name: str, deployment_address: str, age_key_file: str
+    machine_name: str,
 ) -> None:
     secret_name = f"{machine_name}-age.key"
     if not has_secret(secret_name):  # skip uploading the secret, not managed by us
         return
     secret = decrypt_secret(secret_name)
 
-    h = parse_deployment_address(machine_name, deployment_address)
+    secrets_dir = Path(os.environ["SECRETS_DIR"])
-    path = Path(age_key_file)
+    (secrets_dir / "key.txt").write_text(secret)
-
-    proc = h.run(
-        [
-            "bash",
-            "-c",
-            'mkdir -p "$0" && echo -n "$1" > "$2"',
-            str(path.parent),
-            secret,
-            age_key_file,
-        ],
-        check=False,
-    )
-    if proc.returncode != 0:
-        print(f"failed to upload age key to {deployment_address}")
-        sys.exit(1)

pkgs/clan-cli/clan_cli/secrets/upload.py

@@ -4,10 +4,12 @@ import os
 import shlex
 import subprocess
 from pathlib import Path
+from tempfile import TemporaryDirectory
 
 from ..dirs import get_clan_flake_toplevel, module_root
 from ..errors import ClanError
-from ..nix import nix_build, nix_config, nix_eval
+from ..nix import nix_build, nix_config, nix_shell
+from ..ssh import parse_deployment_address
 
 
 def build_upload_script(machine: str, clan_dir: Path) -> str:
@@ -28,13 +30,13 @@ def build_upload_script(machine: str, clan_dir: Path) -> str:
     return proc.stdout.strip()
 
 
-def get_deployment_address(machine: str, clan_dir: Path) -> str:
+def get_deployment_info(machine: str, clan_dir: Path) -> dict:
     config = nix_config()
     system = config["system"]
 
-    cmd = nix_eval(
+    cmd = nix_build(
         [
-            f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.clan.networking.deploymentAddress'
+            f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.system.clan.deployment.file'
         ]
     )
     proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
@@ -43,29 +45,60 @@ def get_deployment_address(machine: str, clan_dir: Path) -> str:
             f"failed to get deploymentAddress:\n{shlex.join(cmd)}\nexited with {proc.returncode}"
         )
 
-    return json.loads(proc.stdout.strip())
+    return json.load(open(proc.stdout.strip()))
 
 
-def run_upload_secrets(flake_attr: str, clan_dir: Path, target: str) -> None:
+def run_upload_secrets(
+    flake_attr: str, clan_dir: Path, target: str, target_directory: str
+) -> None:
     env = os.environ.copy()
     env["CLAN_DIR"] = str(clan_dir)
     env["PYTHONPATH"] = str(module_root().parent)  # TODO do this in the clanCore module
     print(f"uploading secrets... {flake_attr}")
+    with TemporaryDirectory() as tempdir_:
+        tempdir = Path(tempdir_)
+        env["SECRETS_DIR"] = str(tempdir)
         proc = subprocess.run(
-        [flake_attr, target],
+            [flake_attr],
             env=env,
+            check=True,
+            stdout=subprocess.PIPE,
+            text=True,
         )
 
         if proc.returncode != 0:
             raise ClanError("failed to upload secrets")
-    else:
+
-        print("successfully uploaded secrets")
+        h = parse_deployment_address(flake_attr, target)
+        ssh_cmd = h.ssh_cmd()
+        subprocess.run(
+            nix_shell(
+                ["rsync"],
+                [
+                    "rsync",
+                    "-e",
+                    " ".join(["ssh"] + ssh_cmd[2:]),
+                    "-az",
+                    "--delete",
+                    f"{str(tempdir)}/",
+                    f"{h.user}@{h.host}:{target_directory}/",
+                ],
+            ),
+            check=True,
+        )
 
 
 def upload_secrets(machine: str) -> None:
     clan_dir = get_clan_flake_toplevel()
-    target = get_deployment_address(machine, clan_dir)
+    deployment_info = get_deployment_info(machine, clan_dir)
-    run_upload_secrets(build_upload_script(machine, clan_dir), clan_dir, target)
+    address = deployment_info.get("deploymentAddress", "")
+    secrets_upload_directory = deployment_info.get("secretsUploadDirectory", "")
+    run_upload_secrets(
+        build_upload_script(machine, clan_dir),
+        clan_dir,
+        address,
+        secrets_upload_directory,
+    )
 
 
 def upload_command(args: argparse.Namespace) -> None:

pkgs/clan-cli/clan_cli/ssh/__init__.py

@@ -373,7 +373,7 @@ class Host:
         Command to run locally for the host
 
         @cmd the commmand to run
-        @stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocss.PIPE
+        @stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocess.PIPE
         @stderr if not None stderr of the command will be redirected to this file i.e. stderr=subprocess.PIPE
         @extra_env environment variables to override whe running the command
         @cwd current working directory to run the process in
@@ -447,6 +447,33 @@ class Host:
             f"$ {displayed_cmd}", extra=dict(command_prefix=self.command_prefix)
         )
 
+        bash_cmd = export_cmd
+        bash_args = []
+        if isinstance(cmd, list):
+            bash_cmd += 'exec "$@"'
+            bash_args += cmd
+        else:
+            bash_cmd += cmd
+        # FIXME we assume bash to be present here? Should be documented...
+        ssh_cmd = self.ssh_cmd(verbose_ssh=verbose_ssh) + [
+            "--",
+            f"{sudo}bash -c {quote(bash_cmd)} -- {' '.join(map(quote, bash_args))}",
+        ]
+        return self._run(
+            ssh_cmd,
+            displayed_cmd,
+            shell=False,
+            stdout=stdout,
+            stderr=stderr,
+            cwd=cwd,
+            check=check,
+            timeout=timeout,
+        )
+
+    def ssh_cmd(
+        self,
+        verbose_ssh: bool = False,
+    ) -> List:
         if self.user is not None:
             ssh_target = f"{self.user}@{self.host}"
         else:
@@ -469,32 +496,7 @@ class Host:
         if verbose_ssh or self.verbose_ssh:
             ssh_opts.extend(["-v"])
 
-        bash_cmd = export_cmd
+        return ["ssh", ssh_target] + ssh_opts
-        bash_args = []
-        if isinstance(cmd, list):
-            bash_cmd += 'exec "$@"'
-            bash_args += cmd
-        else:
-            bash_cmd += cmd
-        # FIXME we assume bash to be present here? Should be documented...
-        ssh_cmd = (
-            ["ssh", ssh_target]
-            + ssh_opts
-            + [
-                "--",
-                f"{sudo}bash -c {quote(bash_cmd)} -- {' '.join(map(quote, bash_args))}",
-            ]
-        )
-        return self._run(
-            ssh_cmd,
-            displayed_cmd,
-            shell=False,
-            stdout=stdout,
-            stderr=stderr,
-            cwd=cwd,
-            check=check,
-            timeout=timeout,
-        )
 
 
 T = TypeVar("T")

pkgs/clan-cli/tests/test_flake.py

@@ -29,6 +29,7 @@ def create_flake(
             if clan_core_flake:
                 line = line.replace("__CLAN_CORE__", str(clan_core_flake))
             line = line.replace("__CLAN_SOPS_KEY_PATH__", sops_key)
+            line = line.replace("__CLAN_SOPS_KEY_DIR__", str(flake))
             print(line, end="")
         monkeypatch.chdir(flake)
         monkeypatch.setenv("HOME", str(home))

pkgs/clan-cli/tests/test_flake_with_core/flake.nix

@@ -14,6 +14,7 @@
             clan.networking.deploymentAddress = "__CLAN_DEPLOYMENT_ADDRESS__";
             system.stateVersion = lib.version;
             sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__";
+            clanCore.secretsUploadDirectory = "__CLAN_SOPS_KEY_DIR__";
 
             clan.networking.zerotier.controller.enable = true;
 

pkgs/clan-cli/tests/test_secrets_generate.py

@@ -13,7 +13,7 @@ if TYPE_CHECKING:
 
 
 @pytest.mark.impure
-def test_upload_secret(
+def test_generate_secret(
     monkeypatch: pytest.MonkeyPatch,
     test_flake_with_core: Path,
     age_keys: list["KeyPair"],

pkgs/clan-cli/tests/test_secrets_upload.py

@@ -36,6 +36,6 @@ def test_secrets_upload(
     cli.run(["secrets", "upload", "vm1"])
 
     # the flake defines this path as the location where the sops key should be installed
-    sops_key = test_flake_with_core.joinpath("sops.key")
+    sops_key = test_flake_with_core.joinpath("key.txt")
     assert sops_key.exists()
     assert sops_key.read_text() == age_keys[0].privkey