Merge pull request 'init diskLayouts' (#302) from lassulus-HEAD into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/302

.envrc

@@ -1 +1,5 @@
+if ! has nix_direnv_version || ! nix_direnv_version 2.3.0; then
+  source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/2.3.0/direnvrc" "sha256-Dmd+j63L84wuzgyjITIfSxSD57Tx7v51DMxVZOsiUD8="
+fi
+
 use flake

checks/impure/flake-module.nix

@@ -6,32 +6,20 @@
           #!${pkgs.bash}/bin/bash
           set -euo pipefail
 
-          export TMPDIR=$(${pkgs.coreutils}/bin/mktemp -d)
-          trap "${pkgs.coreutils}/bin/chmod -R +w '$TMPDIR'; ${pkgs.coreutils}/bin/rm -rf '$TMPDIR'" EXIT
-
           export PATH="${lib.makeBinPath [
-            pkgs.coreutils
             pkgs.gitMinimal
             pkgs.nix
-            self'.packages.clan-cli.checkPython
           ]}"
-
+          ROOT=$(git rev-parse --show-toplevel)
-          export CLAN_CORE=$TMPDIR/CLAN_CORE
+          cd "$ROOT/pkgs/clan-cli"
-          cp -r ${self} $CLAN_CORE
+          nix develop "$ROOT#clan-cli" -c bash -c 'TMPDIR=/tmp python -m pytest -m impure -s ./tests'
-          chmod +w -R $CLAN_CORE
-
-          cp -r ${self'.packages.clan-cli.src} $TMPDIR/src
-          chmod +w -R $TMPDIR/src
-          cd $TMPDIR/src
-
-          python -m pytest -m "impure" -s ./tests --workers "" "$@"
         '';
         check-clan-template = pkgs.writeShellScriptBin "check-clan-template" ''
           #!${pkgs.bash}/bin/bash
-          set -euo pipefail
+          set -euox pipefail
 
-          export TMPDIR=$(${pkgs.coreutils}/bin/mktemp -d)
+          export CLANTMP=$(${pkgs.coreutils}/bin/mktemp -d)
-          trap "${pkgs.coreutils}/bin/chmod -R +w '$TMPDIR'; ${pkgs.coreutils}/bin/rm -rf '$TMPDIR'" EXIT
+          trap "${pkgs.coreutils}/bin/chmod -R +w '$CLANTMP'; ${pkgs.coreutils}/bin/rm -rf '$CLANTMP'" EXIT
 
           export PATH="${lib.makeBinPath [
             pkgs.coreutils
@@ -44,7 +32,7 @@
             self'.packages.clan-cli
           ]}"
 
-          cd $TMPDIR
+          cd $CLANTMP
 
           echo initialize new clan
           nix flake init -t ${self}#new-clan

clanModules/diskLayouts/singleDiskExt4.nix

@@ -0,0 +1,44 @@
+{ config, lib, ... }:
+{
+  options.clan.diskLayouts.singleDiskExt4 = {
+    device = lib.mkOption {
+      type = lib.types.str;
+      example = "/dev/disk/by-id/ata-Samsung_SSD_850_EVO_250GB_S21PNXAGB12345";
+    };
+  };
+  config.disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        device = config.clan.diskLayouts.singleDiskExt4.device;
+        content = {
+          type = "gpt";
+          partitions = {
+            boot = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+            };
+            ESP = {
+              size = "512M";
+              type = "EF00";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+              };
+            };
+            root = {
+              size = "100%";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/";
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}
+

clanModules/flake-module.nix

@@ -0,0 +1,12 @@
+{ self, lib, ... }: {
+  flake.clanModules = {
+    diskLayouts = lib.mapAttrs'
+      (name: _: lib.nameValuePair (lib.removeSuffix ".nix" name) {
+        imports = [
+          self.inputs.disko.nixosModules.disko
+          ./diskLayouts/${name}
+        ];
+      })
+      (builtins.readDir ./diskLayouts);
+  };
+}

flake.nix

@@ -24,12 +24,12 @@
         "x86_64-linux"
         "aarch64-linux"
       ];
-      flake.clanModules = { };
       imports = [
         ./checks/flake-module.nix
         ./devShell.nix
         ./formatter.nix
         ./templates/flake-module.nix
+        ./clanModules/flake-module.nix
 
         ./pkgs/flake-module.nix
 

formatter.nix

@@ -21,7 +21,7 @@
 
     treefmt.programs.mypy.enable = true;
     treefmt.programs.mypy.directories = {
-      "pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.testDependencies;
+      "pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.pytestDependencies;
     };
 
     treefmt.settings.formatter.nix = {

lib/build-clan/default.nix

@@ -11,22 +11,44 @@ let
       (builtins.fromJSON
         (builtins.readFile (directory + /machines/${machineName}/settings.json)));
 
+  nixosConfiguration = { system ? "x86_64-linux", name }: nixpkgs.lib.nixosSystem {
+    modules = [
+      self.nixosModules.clanCore
+      (machineSettings name)
+      (machines.${name} or { })
+      {
+        clanCore.machineName = name;
+        clanCore.clanDir = directory;
+        # TODO: remove this once we have a hardware-config mechanism
+        nixpkgs.hostPlatform = lib.mkDefault system;
+      }
+    ];
+    inherit specialArgs;
+  };
+
   nixosConfigurations = lib.mapAttrs
     (name: _:
-      nixpkgs.lib.nixosSystem {
+      nixosConfiguration { inherit name; })
-        modules = [
-          self.nixosModules.clanCore
-          (machineSettings name)
-          (machines.${name} or { })
-          {
-            clanCore.machineName = name;
-            clanCore.clanDir = directory;
-            # TODO: remove this once we have a hardware-config mechanism
-            nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
-          }
-        ];
-        inherit specialArgs;
-      })
     (machinesDirs // machines);
+
+  systems = [
+    "x86_64-linux"
+    "aarch64-linux"
+    "riscv64-linux"
+    "x86_64-darwin"
+    "aarch64-darwin"
+  ];
+
+  clanInternals = {
+    machines = lib.mapAttrs
+      (name: _:
+        (builtins.listToAttrs (map
+          (system:
+            lib.nameValuePair system (nixosConfiguration { inherit name system; })
+          )
+          systems))
+      )
+      (machinesDirs // machines);
+  };
 in
-nixosConfigurations
+{ inherit nixosConfigurations clanInternals; }

nixosModules/clanCore/secrets/default.nix

@@ -18,14 +18,17 @@
             type = lib.types.str;
             default = secret.config._module.args.name;
             description = ''
-              namespace of the secret
+              Namespace of the secret
             '';
           };
           generator = lib.mkOption {
-            type = lib.types.nullOr lib.types.str;
+            type = lib.types.str;
             description = ''
-              script to generate the secret.
+              Script to generate the secret.
-              can be set to null. then the user has to provide the secret via the clan cli
+              The script will be called with the following variables:
+              - facts: path to a directory where facts can be stored
+              - secrets: path to a directory where secrets can be stored
+              The script is expected to generate all secrets and facts defined in the module.
             '';
           };
           secrets = lib.mkOption {
@@ -63,7 +66,11 @@
                 };
                 value = lib.mkOption {
                   defaultText = lib.literalExpression "\${config.clanCore.clanDir}/\${fact.config.path}";
-                  default = builtins.readFile "${config.clanCore.clanDir}/${fact.config.path}";
+                  default =
+                    if builtins.pathExists "${config.clanCore.clanDir}/${fact.config.path}" then
+                      builtins.readFile "${config.clanCore.clanDir}/${fact.config.path}"
+                    else
+                      "";
                 };
               };
             }));

nixosModules/clanCore/secrets/sops.nix

@@ -22,52 +22,23 @@ let
 in
 {
   config = lib.mkIf (config.clanCore.secretStore == "sops") {
-    system.clan.generateSecrets = pkgs.writeScript "generate-secrets" ''
+    system.clan = {
-      #!/bin/sh
+      generateSecrets = pkgs.writeScript "generate-secrets" ''
-      set -efu
+        #!${pkgs.python3}/bin/python
-
+        import json
-      test -d "$CLAN_DIR"
+        from clan_cli.secrets.sops_generate import generate_secrets_from_nix
-
+        args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; secret_submodules = config.clanCore.secrets; })})
-      PATH=$PATH:${lib.makeBinPath [
+        generate_secrets_from_nix(**args)
-        config.clanCore.clanPkgs.clan-cli
+      '';
-      ]}
+      uploadSecrets = pkgs.writeScript "upload-secrets" ''
-
+        #!${pkgs.python3}/bin/python
-      # initialize secret store
+        import json
-      if ! clan secrets machines list | grep -q ${config.clanCore.machineName}; then (
+        from clan_cli.secrets.sops_generate import upload_age_key_from_nix
-        INITTMP=$(mktemp -d)
+        # the second toJSON is needed to escape the string for the python
-        trap 'rm -rf "$INITTMP"' EXIT
+        args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; deployment_address = config.clan.networking.deploymentAddress; age_key_file = config.sops.age.keyFile; })})
-        ${pkgs.age}/bin/age-keygen -o "$INITTMP/secret" 2> "$INITTMP/public"
+        upload_age_key_from_nix(**args)
-        PUBKEY=$(cat "$INITTMP/public" | sed 's/.*: //')
+      '';
-        clan secrets machines add ${config.clanCore.machineName} "$PUBKEY"
+    };
-        tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-age.key
-      ) fi
-
-      ${lib.foldlAttrs (acc: n: v: ''
-        ${acc}
-        # ${n}
-        # if any of the secrets are missing, we regenerate all connected facts/secrets
-        (if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clanCore.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then
-
-          facts=$(mktemp -d)
-          trap "rm -rf $facts" EXIT
-          secrets=$(mktemp -d)
-          trap "rm -rf $secrets" EXIT
-          ${v.generator}
-
-          ${lib.concatMapStrings (fact: ''
-            mkdir -p "$(dirname ${fact.path})"
-            cp "$facts"/${fact.name} "$CLAN_DIR"/${fact.path}
-          '') (lib.attrValues v.facts)}
-
-          ${lib.concatMapStrings (secret: ''
-            cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-${secret.name}
-          '') (lib.attrValues v.secrets)}
-        fi)
-      '') "" config.clanCore.secrets}
-    '';
-    system.clan.uploadSecrets = pkgs.writeScript "upload-secrets" ''
-      echo upload is not needed for sops secret store, since the secrets are part of the flake
-    '';
     sops.secrets = builtins.mapAttrs
       (name: _: {
         sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret";
@@ -76,5 +47,8 @@ in
       secrets;
     # To get proper error messages about missing secrets we need a dummy secret file that is always present
     sops.defaultSopsFile = lib.mkIf config.sops.validateSopsFiles (lib.mkDefault (builtins.toString (pkgs.writeText "dummy.yaml" "")));
+
+    sops.age.keyFile = lib.mkIf (builtins.pathExists (config.clanCore.clanDir + "/sops/secrets/${config.clanCore.machineName}-age.key/secret"))
+      (lib.mkDefault "/var/lib/sops-nix/key.txt");
   };
 }

pkgs/clan-cli/clan_cli/machines/facts.py

@@ -0,0 +1,9 @@
+from .folders import machine_folder
+
+
+def machine_has_fact(machine: str, fact: str) -> bool:
+    return (machine_folder(machine) / "facts" / fact).exists()
+
+
+def machine_get_fact(machine: str, fact: str) -> str:
+    return (machine_folder(machine) / "facts" / fact).read_text()

pkgs/clan-cli/clan_cli/machines/update.py

@@ -2,13 +2,12 @@ import argparse
 import json
 import os
 import subprocess
-from typing import Optional
 
 from ..dirs import get_clan_flake_toplevel
 from ..nix import nix_command, nix_eval
 from ..secrets.generate import generate_secrets
 from ..secrets.upload import upload_secrets
-from ..ssh import Host, HostGroup, HostKeyCheck
+from ..ssh import Host, HostGroup, HostKeyCheck, parse_deployment_address
 
 
 def deploy_nixos(hosts: HostGroup) -> None:
@@ -78,11 +77,12 @@ def deploy_nixos(hosts: HostGroup) -> None:
 # FIXME: we want some kind of inventory here.
 def update(args: argparse.Namespace) -> None:
     clan_dir = get_clan_flake_toplevel().as_posix()
-    host = json.loads(
+    machine = args.machine
+    address = json.loads(
         subprocess.run(
             nix_eval(
                 [
-                    f'{clan_dir}#nixosConfigurations."{args.machine}".config.clan.networking.deploymentAddress'
+                    f'{clan_dir}#nixosConfigurations."{machine}".config.clan.networking.deploymentAddress'
                 ]
             ),
             stdout=subprocess.PIPE,
@@ -90,31 +90,9 @@ def update(args: argparse.Namespace) -> None:
             text=True,
         ).stdout
     )
-    parts = host.split("@")
+    host = parse_deployment_address(machine, address)
-    user: Optional[str] = None
+    print(f"deploying {machine}")
-    if len(parts) > 1:
+    deploy_nixos(HostGroup([host]))
-        user = parts[0]
-        hostname = parts[1]
-    else:
-        hostname = parts[0]
-    maybe_port = hostname.split(":")
-    port = None
-    if len(maybe_port) > 1:
-        hostname = maybe_port[0]
-        port = int(maybe_port[1])
-    print(f"deploying {host}")
-    deploy_nixos(
-        HostGroup(
-            [
-                Host(
-                    host=hostname,
-                    port=port,
-                    user=user,
-                    meta=dict(flake_attr=args.machine),
-                )
-            ]
-        )
-    )
 
 
 def register_update_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/nix.py

@@ -1,5 +1,8 @@
+import json
 import os
+import subprocess
 import tempfile
+from typing import Any
 
 from .dirs import nixpkgs_flake, nixpkgs_source, unfree_nixpkgs
 
@@ -25,6 +28,16 @@ def nix_build(
     )
 
 
+def nix_config() -> dict[str, Any]:
+    cmd = nix_command(["show-config", "--json"])
+    proc = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE)
+    data = json.loads(proc.stdout)
+    config = {}
+    for key, value in data.items():
+        config[key] = value["value"]
+    return config
+
+
 def nix_eval(flags: list[str]) -> list[str]:
     default_flags = nix_command(
         [

pkgs/clan-cli/clan_cli/secrets/generate.py

@@ -1,31 +1,32 @@
 import argparse
 import os
+import shlex
 import subprocess
-import sys
 
 from clan_cli.errors import ClanError
 
-from ..dirs import get_clan_flake_toplevel
+from ..dirs import get_clan_flake_toplevel, module_root
-from ..nix import nix_build
+from ..nix import nix_build, nix_config
 
 
 def generate_secrets(machine: str) -> None:
     clan_dir = get_clan_flake_toplevel().as_posix().strip()
     env = os.environ.copy()
     env["CLAN_DIR"] = clan_dir
+    env["PYTHONPATH"] = str(module_root().parent)  # TODO do this in the clanCore module
+    config = nix_config()
+    system = config["system"]
 
-    proc = subprocess.run(
+    cmd = nix_build(
-        nix_build(
+        [
-            [
+            f'path:{clan_dir}#clanInternals.machines."{machine}".{system}.config.system.clan.generateSecrets'
-                f'path:{clan_dir}#nixosConfigurations."{machine}".config.system.clan.generateSecrets'
+        ]
-            ]
-        ),
-        capture_output=True,
-        text=True,
     )
+    proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
     if proc.returncode != 0:
-        print(proc.stderr, file=sys.stderr)
+        raise ClanError(
-        raise ClanError(f"failed to generate secrets:\n{proc.stderr}")
+            f"failed to generate secrets:\n{shlex.join(cmd)}\nexited with {proc.returncode}"
+        )
 
     secret_generator_script = proc.stdout.strip()
     print(secret_generator_script)

pkgs/clan-cli/clan_cli/secrets/key.py

@@ -9,8 +9,8 @@ def generate_key() -> str:
     path = default_sops_key_path()
     if path.exists():
         raise ClanError(f"Key already exists at {path}")
-    generate_private_key(path)
+    priv_key, pub_key = generate_private_key()
-    pub_key = get_public_key(path.read_text())
+    path.write_text(priv_key)
     return pub_key
 
 

pkgs/clan-cli/clan_cli/secrets/machines.py

@@ -19,11 +19,15 @@ def get_machine(name: str) -> str:
     return read_key(sops_machines_folder() / name)
 
 
+def has_machine(name: str) -> bool:
+    return (sops_machines_folder() / name / "key.json").exists()
+
+
 def list_machines() -> list[str]:
     path = sops_machines_folder()
 
     def validate(name: str) -> bool:
-        return validate_hostname(name) and (path / name / "key.json").exists()
+        return validate_hostname(name) and has_machine(name)
 
     return list_objects(path, validate)
 

pkgs/clan-cli/clan_cli/secrets/secrets.py

@@ -171,14 +171,15 @@ def disallow_member(group_folder: Path, name: str) -> None:
     )
 
 
+def has_secret(secret: str) -> bool:
+    return (sops_secrets_folder() / secret / "secret").exists()
+
+
 def list_secrets() -> list[str]:
     path = sops_secrets_folder()
 
     def validate(name: str) -> bool:
-        return (
+        return VALID_SECRET_NAME.match(name) is not None and has_secret(name)
-            VALID_SECRET_NAME.match(name) is not None
-            and (path / name / "secret").exists()
-        )
 
     return list_objects(path, validate)
 

pkgs/clan-cli/clan_cli/secrets/sops.py

@@ -30,10 +30,25 @@ def get_public_key(privkey: str) -> str:
     return res.stdout.strip()
 
 
-def generate_private_key(path: Path) -> None:
+def generate_private_key() -> tuple[str, str]:
-    path.parent.mkdir(parents=True, exist_ok=True)
+    cmd = nix_shell(["age"], ["age-keygen"])
-    cmd = nix_shell(["age"], ["age-keygen", "-o", str(path)])
+    try:
-    subprocess.run(cmd, check=True)
+        proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, text=True)
+        res = proc.stdout.strip()
+        pubkey = None
+        private_key = None
+        for line in res.splitlines():
+            if line.startswith("# public key:"):
+                pubkey = line.split(":")[1].strip()
+            if not line.startswith("#"):
+                private_key = line
+        if not pubkey:
+            raise ClanError("Could not find public key in age-keygen output")
+        if not private_key:
+            raise ClanError("Could not find private key in age-keygen output")
+        return private_key, pubkey
+    except subprocess.CalledProcessError as e:
+        raise ClanError("Failed to generate private sops key") from e
 
 
 def get_user_name(user: str) -> str:

pkgs/clan-cli/clan_cli/secrets/sops_generate.py

@@ -0,0 +1,124 @@
+import shlex
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any
+
+from ..dirs import get_clan_flake_toplevel
+from ..errors import ClanError
+from ..ssh import parse_deployment_address
+from .folders import sops_secrets_folder
+from .machines import add_machine, has_machine
+from .secrets import decrypt_secret, encrypt_secret, has_secret
+from .sops import generate_private_key
+
+
+def generate_host_key(machine_name: str) -> None:
+    if has_machine(machine_name):
+        return
+    priv_key, pub_key = generate_private_key()
+    encrypt_secret(sops_secrets_folder() / f"{machine_name}-age.key", priv_key)
+    add_machine(machine_name, pub_key, False)
+
+
+def generate_secrets_group(
+    secret_group: str, machine_name: str, tempdir: Path, secret_options: dict[str, Any]
+) -> None:
+    clan_dir = get_clan_flake_toplevel()
+    secrets = secret_options["secrets"]
+    needs_regeneration = any(
+        not has_secret(f"{machine_name}-{secret['name']}")
+        for secret in secrets.values()
+    )
+    generator = secret_options["generator"]
+    subdir = tempdir / secret_group
+    if needs_regeneration:
+        facts_dir = subdir / "facts"
+        facts_dir.mkdir(parents=True)
+        secrets_dir = subdir / "secrets"
+        secrets_dir.mkdir(parents=True)
+
+        text = f"""\
+set -euo pipefail
+facts={shlex.quote(str(facts_dir))}
+secrets={shlex.quote(str(secrets_dir))}
+{generator}
+        """
+        try:
+            subprocess.run(["bash", "-c", text], check=True)
+        except subprocess.CalledProcessError:
+            msg = "failed to the following command:\n"
+            msg += text
+            raise ClanError(msg)
+        for secret in secrets.values():
+            secret_file = secrets_dir / secret["name"]
+            if not secret_file.is_file():
+                msg = f"did not generate a file for '{secret['name']}' when running the following command:\n"
+                msg += text
+                raise ClanError(msg)
+            encrypt_secret(
+                sops_secrets_folder() / f"{machine_name}-{secret['name']}",
+                secret_file.read_text(),
+            )
+        for fact in secret_options["facts"].values():
+            fact_file = facts_dir / fact["name"]
+            if not fact_file.is_file():
+                msg = f"did not generate a file for '{fact['name']}' when running the following command:\n"
+                msg += text
+                raise ClanError(msg)
+            fact_path = clan_dir.joinpath(fact["path"])
+            fact_path.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copyfile(fact_file, fact_path)
+
+
+# this is called by the sops.nix clan core module
+def generate_secrets_from_nix(
+    machine_name: str,
+    secret_submodules: dict[str, Any],
+) -> None:
+    generate_host_key(machine_name)
+    errors = {}
+    with TemporaryDirectory() as d:
+        # if any of the secrets are missing, we regenerate all connected facts/secrets
+        for secret_group, secret_options in secret_submodules.items():
+            try:
+                generate_secrets_group(
+                    secret_group, machine_name, Path(d), secret_options
+                )
+            except ClanError as e:
+                errors[secret_group] = e
+    for secret_group, error in errors.items():
+        print(f"failed to generate secrets for {machine_name}/{secret_group}:")
+        print(error, file=sys.stderr)
+    if len(errors) > 0:
+        sys.exit(1)
+
+
+# this is called by the sops.nix clan core module
+def upload_age_key_from_nix(
+    machine_name: str, deployment_address: str, age_key_file: str
+) -> None:
+    secret_name = f"{machine_name}-age.key"
+    if not has_secret(secret_name):  # skip uploading the secret, not managed by us
+        return
+    secret = decrypt_secret(secret_name)
+
+    h = parse_deployment_address(machine_name, deployment_address)
+    path = Path(age_key_file)
+
+    proc = h.run(
+        [
+            "bash",
+            "-c",
+            'mkdir -p "$0" && echo -n "$1" > "$2"',
+            str(path.parent),
+            secret,
+            age_key_file,
+        ],
+        check=False,
+    )
+    if proc.returncode != 0:
+        print(f"failed to upload age key to {deployment_address}")
+        sys.exit(1)

pkgs/clan-cli/clan_cli/secrets/upload.py

@@ -1,31 +1,36 @@
 import argparse
 import json
+import os
 import subprocess
 
-from clan_cli.errors import ClanError
+from ..dirs import get_clan_flake_toplevel, module_root
-
+from ..errors import ClanError
-from ..dirs import get_clan_flake_toplevel
+from ..nix import nix_build, nix_config, nix_eval
-from ..nix import nix_build, nix_eval
 
 
 def upload_secrets(machine: str) -> None:
     clan_dir = get_clan_flake_toplevel().as_posix()
+    config = nix_config()
+    system = config["system"]
 
     proc = subprocess.run(
         nix_build(
             [
-                f'{clan_dir}#nixosConfigurations."{machine}".config.system.clan.uploadSecrets'
+                f'{clan_dir}#clanInternals.machines."{machine}".{system}.config.system.clan.uploadSecrets'
             ]
         ),
         stdout=subprocess.PIPE,
         text=True,
         check=True,
     )
+
+    env = os.environ.copy()
+    env["PYTHONPATH"] = str(module_root().parent)  # TODO do this in the clanCore module
     host = json.loads(
         subprocess.run(
             nix_eval(
                 [
-                    f'{clan_dir}#nixosConfigurations."{machine}".config.clan.networking.deploymentAddress'
+                    f'{clan_dir}#clanInternals.machines."{machine}".{system}.config.clan.networking.deploymentAddress'
                 ]
             ),
             stdout=subprocess.PIPE,
@@ -40,6 +45,7 @@ def upload_secrets(machine: str) -> None:
             secret_upload_script,
             host,
         ],
+        env=env,
     )
 
     if secret_upload.returncode != 0:

pkgs/clan-cli/clan_cli/ssh/__init__.py

@@ -756,6 +756,36 @@ class HostGroup:
         return HostGroup(list(filter(pred, self.hosts)))
 
 
+def parse_deployment_address(machine_name: str, host: str) -> Host:
+    parts = host.split("@")
+    user: Optional[str] = None
+    if len(parts) > 1:
+        user = parts[0]
+        hostname = parts[1]
+    else:
+        hostname = parts[0]
+    maybe_options = hostname.split("?")
+    options: Dict[str, str] = {}
+    if len(maybe_options) > 1:
+        hostname = maybe_options[0]
+        for option in maybe_options[1].split("&"):
+            k, v = option.split("=")
+            options[k] = v
+    maybe_port = hostname.split(":")
+    port = None
+    if len(maybe_port) > 1:
+        hostname = maybe_port[0]
+        port = int(maybe_port[1])
+    return Host(
+        hostname,
+        user=user,
+        port=port,
+        command_prefix=machine_name,
+        meta=dict(flake_attr=machine_name),
+        ssh_options=options,
+    )
+
+
 @overload
 def run(
     cmd: Union[List[str], str],

pkgs/clan-cli/default.nix

@@ -1,4 +1,5 @@
 { age
+, lib
 , argcomplete
 , fastapi
 , uvicorn
@@ -20,7 +21,12 @@
 , rsync
 , pkgs
 , ui-assets
-, lib
+, bash
+, sshpass
+, zbar
+, tor
+, git
+, ipdb
 }:
 let
 
@@ -30,16 +36,36 @@ let
     uvicorn # optional dependencies: if not enabled, webui subcommand will not work
   ];
 
-  testDependencies = [
+  pytestDependencies = runtimeDependencies ++ dependencies ++ [
     pytest
     pytest-cov
     pytest-subprocess
     pytest-parallel
     openssh
+    git
     stdenv.cc
+    ipdb # used for debugging
   ];
 
-  checkPython = python3.withPackages (_ps: dependencies ++ testDependencies);
+  # Optional dependencies for clan cli, we re-expose them here to make sure they all build.
+  runtimeDependencies = [
+    bash
+    nix
+    zerotierone
+    bubblewrap
+    openssh
+    sshpass
+    zbar
+    tor
+    age
+    rsync
+    sops
+    git
+  ];
+
+  runtimeDependenciesAsSet = builtins.listToAttrs (builtins.map (p: lib.nameValuePair (lib.getName p.name) p) runtimeDependencies);
+
+  checkPython = python3.withPackages (_ps: pytestDependencies);
 
   # - vendor the jsonschema nix lib (copy instead of symlink).
   source = runCommand "clan-cli-source" { } ''
@@ -73,6 +99,7 @@ let
       --experimental-features 'nix-command flakes' \
       --override-input nixpkgs ${pkgs.path}
   '';
+
 in
 python3.pkgs.buildPythonPackage {
   name = "clan-cli";
@@ -85,25 +112,24 @@ python3.pkgs.buildPythonPackage {
   ];
   propagatedBuildInputs = dependencies;
 
-  passthru.tests.clan-pytest = runCommand "clan-pytest"
+  # also re-expose dependencies so we test them in CI
-    {
+  passthru.tests = (lib.mapAttrs' (n: lib.nameValuePair "package-${n}") runtimeDependenciesAsSet) // {
-      nativeBuildInputs = [ age zerotierone bubblewrap sops nix openssh rsync stdenv.cc ];
+    clan-pytest = runCommand "clan-pytest" { nativeBuildInputs = [ checkPython ] ++ pytestDependencies; } ''
-    } ''
+      cp -r ${source} ./src
-    cp -r ${source} ./src
+      chmod +w -R ./src
-    chmod +w -R ./src
+      cd ./src
-    cd ./src
 
-    # git is needed for test_git.py
+      export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1
-    export PATH="${lib.makeBinPath [pkgs.git]}:$PATH"
+      ${checkPython}/bin/python -m pytest -m "not impure" -s ./tests
-
+      touch $out
-    export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1
+    '';
-    ${checkPython}/bin/python -m pytest -m "not impure" -s ./tests
+  };
-    touch $out
-  '';
   passthru.clan-openapi = runCommand "clan-openapi" { } ''
     cp -r ${source} ./src
     chmod +w -R ./src
     cd ./src
+    export PATH=${checkPython}/bin:$PATH
+
     ${checkPython}/bin/python ./bin/gen-openapi --out $out/openapi.json --app-dir . clan_cli.webui.app:app
     touch $out
   '';
@@ -113,9 +139,10 @@ python3.pkgs.buildPythonPackage {
   passthru.devDependencies = [
     setuptools
     wheel
-  ] ++ testDependencies;
+  ] ++ pytestDependencies;
 
-  passthru.testDependencies = dependencies ++ testDependencies;
+  passthru.pytestDependencies = pytestDependencies;
+  passthru.runtimeDependencies = runtimeDependencies;
 
   postInstall = ''
     cp -r ${nixpkgs} $out/${python3.sitePackages}/clan_cli/nixpkgs

pkgs/clan-cli/flake-module.nix

@@ -1,3 +1,4 @@
+{ lib, ... }:
 {
   perSystem = { self', pkgs, ... }: {
     devShells.clan-cli = pkgs.callPackage ./shell.nix {
@@ -10,26 +11,13 @@
       clan-openapi = self'.packages.clan-cli.clan-openapi;
       default = self'.packages.clan-cli;
 
-      ## Optional dependencies for clan cli, we re-expose them here to make sure they all build.
-      inherit (pkgs)
-        age
-        bash
-        bubblewrap
-        git
-        openssh
-        rsync
-        sops
-        sshpass
-        tor
-        zbar
-        ;
       # Override license so that we can build zerotierone without
       # having to re-import nixpkgs.
       zerotierone = pkgs.zerotierone.overrideAttrs (_old: { meta = { }; });
       ## End optional dependencies
     };
 
-    checks = self'.packages.clan-cli.tests;
+    checks = lib.mkDefault self'.packages.clan-cli.tests;
   };
 
 }

pkgs/clan-cli/shell.nix

@@ -1,26 +1,16 @@
-{ nix-unit, clan-cli, ui-assets, python3, system, ruff, mkShell, writeScriptBin }:
+{ nix-unit, clan-cli, ui-assets, system, mkShell, writeScriptBin, openssh }:
 let
-  pythonWithDeps = python3.withPackages (
-    ps:
-    clan-cli.propagatedBuildInputs
-    ++ clan-cli.devDependencies
-    ++ [
-      ps.pip
-      ps.ipdb
-    ]
-  );
   checkScript = writeScriptBin "check" ''
     nix build .#checks.${system}.{treefmt,clan-pytest} -L "$@"
   '';
 in
 mkShell {
   packages = [
-    ruff
     nix-unit
-    pythonWithDeps
+    openssh
+    clan-cli.checkPython
   ];
   # sets up an editable install and add enty points to $PATH
-  PYTHONPATH = "${pythonWithDeps}/${pythonWithDeps.sitePackages}";
   PYTHONBREAKPOINT = "ipdb.set_trace";
 
   shellHook = ''

pkgs/clan-cli/tests/sshd.py

@@ -59,6 +59,7 @@ def sshd_config(project_root: Path, test_root: Path) -> Iterator[SshdConfig]:
         MaxStartups 64:30:256
         AuthorizedKeysFile {host_key}.pub
         AcceptEnv REALPATH
+        PasswordAuthentication no
         """
         )
         login_shell = dir / "shell"
@@ -109,7 +110,6 @@ def sshd(
 ) -> Iterator[Sshd]:
     import subprocess
 
-    subprocess.run(["echo", "hello"], check=True)
     port = unused_tcp_port()
     sshd = shutil.which("sshd")
     assert sshd is not None, "no sshd binary found"
@@ -123,6 +123,7 @@ def sshd(
     )
 
     while True:
+        print(sshd_config.path)
         if (
             subprocess.run(
                 [
@@ -137,7 +138,7 @@ def sshd(
                     "-p",
                     str(port),
                     "true",
-                ]
+                ],
             ).returncode
             == 0
         ):

pkgs/clan-cli/tests/test_flake_with_core/flake.nix

@@ -5,14 +5,29 @@
   # this placeholder is replaced by the path to nixpkgs
   inputs.clan-core.url = "__CLAN_CORE__";
 
-  outputs = { self, clan-core }: {
+  outputs = { self, clan-core }:
-    nixosConfigurations = clan-core.lib.buildClan {
+    let
-      directory = self;
+      clan = clan-core.lib.buildClan {
-      machines = {
+        directory = self;
-        vm1 = { modulesPath, ... }: {
+        machines = {
-          imports = [ "${toString modulesPath}/virtualisation/qemu-vm.nix" ];
+          vm1 = { modulesPath, ... }: {
+            imports = [ "${toString modulesPath}/virtualisation/qemu-vm.nix" ];
+            clan.networking.deploymentAddress = "__CLAN_DEPLOYMENT_ADDRESS__";
+            sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__";
+
+            clanCore.secrets.testpassword = {
+              generator = ''
+                echo "secret1" > "$secrets/secret1"
+                echo "fact1" > "$facts/fact1"
+              '';
+              secrets.secret1 = { };
+              facts.fact1 = { };
+            };
+          };
         };
       };
+    in
+    {
+      inherit (clan) nixosConfigurations clanInternals;
     };
-  };
 }

pkgs/clan-cli/tests/test_secrets_generate.py

@@ -0,0 +1,38 @@
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+from cli import Cli
+
+from clan_cli.machines.facts import machine_get_fact
+from clan_cli.secrets.folders import sops_secrets_folder
+from clan_cli.secrets.secrets import has_secret
+
+if TYPE_CHECKING:
+    from age_keys import KeyPair
+
+
+@pytest.mark.impure
+def test_upload_secret(
+    monkeypatch: pytest.MonkeyPatch,
+    test_flake_with_core: Path,
+    age_keys: list["KeyPair"],
+) -> None:
+    monkeypatch.chdir(test_flake_with_core)
+    monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
+    cli = Cli()
+    cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey])
+    cli.run(["secrets", "generate", "vm1"])
+    has_secret("vm1-age.key")
+    has_secret("vm1-secret1")
+    fact1 = machine_get_fact("vm1", "fact1")
+    assert fact1 == "fact1\n"
+    age_key = sops_secrets_folder().joinpath("vm1-age.key").joinpath("secret")
+    secret1 = sops_secrets_folder().joinpath("vm1-secret1").joinpath("secret")
+    age_key_mtime = age_key.lstat().st_mtime_ns
+    secret1_mtime = secret1.lstat().st_mtime_ns
+
+    # test idempotency
+    cli.run(["secrets", "generate", "vm1"])
+    assert age_key.lstat().st_mtime_ns == age_key_mtime
+    assert secret1.lstat().st_mtime_ns == secret1_mtime

pkgs/clan-cli/tests/test_secrets_upload.py

@@ -0,0 +1,40 @@
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+from cli import Cli
+
+from clan_cli.ssh import HostGroup
+
+if TYPE_CHECKING:
+    from age_keys import KeyPair
+
+
+@pytest.mark.impure
+def test_secrets_upload(
+    monkeypatch: pytest.MonkeyPatch,
+    test_flake_with_core: Path,
+    host_group: HostGroup,
+    age_keys: list["KeyPair"],
+) -> None:
+    monkeypatch.chdir(test_flake_with_core)
+    monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
+
+    cli = Cli()
+    cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey])
+
+    cli.run(["secrets", "machines", "add", "vm1", age_keys[1].pubkey])
+    monkeypatch.setenv("SOPS_NIX_SECRET", age_keys[0].privkey)
+    cli.run(["secrets", "set", "vm1-age.key"])
+
+    flake = test_flake_with_core.joinpath("flake.nix")
+    host = host_group.hosts[0]
+    addr = f"{host.user}@{host.host}:{host.port}?StrictHostKeyChecking=no&UserKnownHostsFile=/dev/null&IdentityFile={host.key}"
+    new_text = flake.read_text().replace("__CLAN_DEPLOYMENT_ADDRESS__", addr)
+    sops_key = test_flake_with_core.joinpath("sops.key")
+    new_text = new_text.replace("__CLAN_SOPS_KEY_PATH__", str(sops_key))
+
+    flake.write_text(new_text)
+    cli.run(["secrets", "upload", "vm1"])
+    assert sops_key.exists()
+    assert sops_key.read_text() == age_keys[0].privkey

templates/new-clan/flake.nix

@@ -7,12 +7,13 @@
     let
       system = "x86_64-linux";
       pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system};
+      clan = clan-core.lib.buildClan {
+        directory = self;
+      };
     in
     {
       # all machines managed by cLAN
-      nixosConfigurations = clan-core.lib.buildClan {
+      inherit (clan) nixosConfigurations clanInternals;
-        directory = self;
-      };
       # add the cLAN cli tool to the dev shell
       devShells.${system}.default = pkgs.mkShell {
         packages = [