pkgs/clan(templates): Add shell completions

NormalUsers get:
- Home directory
- Can login

This is expected for users created through this module. We can make it configurable if the use arises

.gitea/workflows/create-pr.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Shared script for creating pull requests in Gitea workflows
+set -euo pipefail
+
+# Required environment variables:
+# - CI_BOT_TOKEN: Gitea bot token for authentication
+# - PR_BRANCH: Branch name for the pull request
+# - PR_TITLE: Title of the pull request
+# - PR_BODY: Body/description of the pull request
+
+if [[ -z "${CI_BOT_TOKEN:-}" ]]; then
+  echo "Error: CI_BOT_TOKEN is not set" >&2
+  exit 1
+fi
+
+if [[ -z "${PR_BRANCH:-}" ]]; then
+  echo "Error: PR_BRANCH is not set" >&2
+  exit 1
+fi
+
+if [[ -z "${PR_TITLE:-}" ]]; then
+  echo "Error: PR_TITLE is not set" >&2
+  exit 1
+fi
+
+if [[ -z "${PR_BODY:-}" ]]; then
+  echo "Error: PR_BODY is not set" >&2
+  exit 1
+fi
+
+# Push the branch
+git push origin "+HEAD:${PR_BRANCH}"
+
+# Create pull request
+resp=$(nix run --inputs-from . nixpkgs#curl -- -X POST \
+  -H "Authorization: token $CI_BOT_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d "{
+    \"head\": \"${PR_BRANCH}\",
+    \"base\": \"main\",
+    \"title\": \"${PR_TITLE}\",
+    \"body\": \"${PR_BODY}\"
+  }" \
+  "https://git.clan.lol/api/v1/repos/clan/clan-core/pulls")
+
+pr_number=$(echo "$resp" | jq -r '.number')
+
+if [[ "$pr_number" == "null" ]]; then
+  echo "Error creating pull request:" >&2
+  echo "$resp" | jq . >&2
+  exit 1
+fi
+
+echo "Created pull request #$pr_number"
+
+# Merge when checks succeed
+while true; do
+  resp=$(nix run --inputs-from . nixpkgs#curl -- -X POST \
+    -H "Authorization: token $CI_BOT_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{
+      "Do": "merge",
+      "merge_when_checks_succeed": true,
+      "delete_branch_after_merge": true
+    }' \
+    "https://git.clan.lol/api/v1/repos/clan/clan-core/pulls/$pr_number/merge")
+  msg=$(echo "$resp" | jq -r '.message')
+  if [[ "$msg" != "Please try again later" ]]; then
+    break
+  fi
+  echo "Retrying in 2 seconds..."
+  sleep 2
+done
+
+echo "Pull request #$pr_number merge initiated"

.gitea/workflows/update-clan-core-for-checks.yml

@@ -19,35 +19,10 @@ jobs:
         run: |
           export GIT_AUTHOR_NAME=clan-bot GIT_AUTHOR_EMAIL=clan-bot@clan.lol GIT_COMMITTER_NAME=clan-bot GIT_COMMITTER_EMAIL=clan-bot@clan.lol
           git commit -am "Update pinned clan-core for checks"
-          git push origin +HEAD:update-clan-core-for-checks
-          set -x
-          resp=$(nix run --inputs-from . nixpkgs#curl -- -X POST \
-            -H "Authorization: token $CI_BOT_TOKEN" \
-            -H "Content-Type: application/json" \
-            -d '{
-              "head": "update-clan-core-for-checks",
-              "base": "main",
-              "title": "Update Clan Core for Checks",
-              "body": "This PR updates the pinned clan-core flake input that is used for checks."
-            }' \
-           "https://git.clan.lol/api/v1/repos/clan/clan-core/pulls")
-          pr_number=$(echo "$resp" | jq -r '.number')
 
-          # Merge when succeed
-          while true; do
-            resp=$(nix run --inputs-from . nixpkgs#curl -- -X POST \
-              -H "Authorization: token $CI_BOT_TOKEN" \
-              -H "Content-Type: application/json" \
-              -d '{
-                "Do": "merge",
-                "merge_when_checks_succeed": true,
-                "delete_branch_after_merge": true
-              }' \
-              "https://git.clan.lol/api/v1/repos/clan/clan-core/pulls/$pr_number/merge")
-            msg=$(echo $resp | jq -r '.message')
-            if [[ "$msg" != "Please try again later" ]]; then
-              break
-            fi
-            echo "Retrying in 2 seconds..."
-            sleep 2
-          done
+          # Use shared PR creation script
+          export PR_BRANCH="update-clan-core-for-checks"
+          export PR_TITLE="Update Clan Core for Checks"
+          export PR_BODY="This PR updates the pinned clan-core flake input that is used for checks."
+
+          ./.gitea/workflows/create-pr.sh

.gitea/workflows/update-private-flake-inputs.yml

@@ -0,0 +1,40 @@
+name: "Update private flake inputs"
+on:
+  repository_dispatch:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 3 * * *" # Run daily at 3 AM
+jobs:
+  update-private-flake:
+    runs-on: nix
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Update private flake inputs
+        run: |
+          # Update the private flake lock file
+          cd devFlake/private
+          nix flake update
+          cd ../..
+
+          # Update the narHash
+          bash ./devFlake/update-private-narhash
+      - name: Create pull request
+        env:
+          CI_BOT_TOKEN: ${{ secrets.CI_BOT_TOKEN }}
+        run: |
+          export GIT_AUTHOR_NAME=clan-bot GIT_AUTHOR_EMAIL=clan-bot@clan.lol GIT_COMMITTER_NAME=clan-bot GIT_COMMITTER_EMAIL=clan-bot@clan.lol
+
+          # Check if there are any changes
+          if ! git diff --quiet; then
+            git add devFlake/private/flake.lock devFlake/private.narHash
+            git commit -m "Update dev flake"
+            
+            # Use shared PR creation script
+            export PR_BRANCH="update-dev-flake"
+            export PR_TITLE="Update dev flake"
+            export PR_BODY="This PR updates the dev flake inputs and corresponding narHash."
+          else
+            echo "No changes detected in dev flake inputs"
+          fi

checks/backups/flake-module.nix

@@ -19,11 +19,11 @@
         ...
       }:
       let
-        dependencies = [
-          self
-          pkgs.stdenv.drvPath
-          self.clan.clanInternals.machines.${pkgs.hostPlatform.system}.test-backup.config.system.clan.deployment.file
-        ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
+        dependencies =
+          [
+            pkgs.stdenv.drvPath
+          ]
+          ++ builtins.map (i: i.outPath) (builtins.attrValues (builtins.removeAttrs self.inputs [ "self" ]));
         closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
       in
       {

checks/borgbackup/default.nix

@@ -47,14 +47,6 @@ nixosLib.runTest (
 
       clientone =
         { config, pkgs, ... }:
-        let
-          dependencies = [
-            clan-core
-            pkgs.stdenv.drvPath
-          ] ++ builtins.map (i: i.outPath) (builtins.attrValues clan-core.inputs);
-          closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
-
-        in
         {
 
           services.openssh.enable = true;
@@ -65,15 +57,6 @@ nixosLib.runTest (
 
           environment.systemPackages = [ clan-core.packages.${pkgs.system}.clan-cli ];
 
-          environment.etc.install-closure.source = "${closureInfo}/store-paths";
-          nix.settings = {
-            substituters = pkgs.lib.mkForce [ ];
-            hashed-mirrors = null;
-            connect-timeout = pkgs.lib.mkForce 3;
-            flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
-          };
-          system.extraDependencies = dependencies;
-
           clan.core.state.test-backups.folders = [ "/var/test-backups" ];
         };
 

checks/clan-core-for-checks.nix

@@ -1,6 +1,6 @@
 { fetchgit }:
 fetchgit {
   url = "https://git.clan.lol/clan/clan-core.git";
-  rev = "28131afbbcd379a8ff04c79c66c670ef655ed889";
-  sha256 = "1294cwjlnc341fl6zbggn4rgq8z33gqkcyggjfvk9cf7zdgygrf6";
+  rev = "eea93ea22c9818da67e148ba586277bab9e73cea";
+  sha256 = "sha256-PV0Z+97QuxQbkYSVuNIJwUNXMbHZG/vhsA9M4cDTCOE=";
 }

checks/data-mesher/default.nix

@@ -1,89 +0,0 @@
-{
-  pkgs,
-  nixosLib,
-  clan-core,
-  lib,
-  ...
-}:
-let
-  machines = [
-    "admin"
-    "peer"
-    "signer"
-  ];
-in
-nixosLib.runTest (
-  { ... }:
-  {
-    imports = [
-      clan-core.modules.nixosTest.clanTest
-    ];
-
-    hostPkgs = pkgs;
-    name = "service-data-mesher";
-
-    clan = {
-      directory = ./.;
-      inventory = {
-        machines = lib.genAttrs machines (_: { });
-        services = {
-          data-mesher.default = {
-            roles.peer.machines = [ "peer" ];
-            roles.admin.machines = [ "admin" ];
-            roles.signer.machines = [ "signer" ];
-          };
-        };
-      };
-    };
-
-    defaults =
-      { config, ... }:
-      {
-        environment.systemPackages = [
-          config.services.data-mesher.package
-        ];
-
-        clan.data-mesher.network.interface = "eth1";
-        clan.data-mesher.bootstrapNodes = [
-          "[2001:db8:1::1]:7946" # peer1
-          "[2001:db8:1::2]:7946" # peer2
-        ];
-
-        # speed up for testing
-        services.data-mesher.settings = {
-          cluster.join_interval = lib.mkForce "2s";
-          cluster.push_pull_interval = lib.mkForce "5s";
-        };
-      };
-
-    nodes = {
-      admin.clan.data-mesher.network.tld = "foo";
-    };
-
-    # TODO Add better test script.
-    testScript = ''
-
-      def resolve(node, success = {}, fail = [], timeout = 60):
-        for hostname, ips in success.items():
-            for ip in ips:
-                node.wait_until_succeeds(f"getent ahosts {hostname} | grep {ip}", timeout)
-
-        for hostname in fail:
-            node.wait_until_fails(f"getent ahosts {hostname}")
-
-      start_all()
-
-      admin.wait_for_unit("data-mesher")
-      signer.wait_for_unit("data-mesher")
-      peer.wait_for_unit("data-mesher")
-
-      # check dns resolution
-      for node in [admin, signer, peer]:
-        resolve(node, {
-            "admin.foo": ["2001:db8:1::1", "192.168.1.1"],
-            "peer.foo": ["2001:db8:1::2", "192.168.1.2"],
-            "signer.foo": ["2001:db8:1::3", "192.168.1.3"]
-        })
-    '';
-  }
-)

checks/data-mesher/sops/machines/admin/key.json

@@ -1,4 +0,0 @@
-{
-  "publickey": "age10zxkj45fah3qa8uyg3a36jsd06d839xfq64nrez9etrsf4km0gtsp45gsz",
-  "type": "age"
-}

checks/data-mesher/sops/machines/peer/key.json

@@ -1,4 +0,0 @@
-{
-  "publickey": "age1faqrml2ukc6unfm75d3v2vnaf62v92rdxaagg3ty3cfna7vt99gqlzs43l",
-  "type": "age"
-}

checks/data-mesher/sops/machines/signer/key.json

@@ -1,4 +0,0 @@
-{
-  "publickey": "age153mke8v2qksyqjc7vta7wglzdqr5epazt83nch0ur5v7kl87cfdsr07qld",
-  "type": "age"
-}

checks/data-mesher/sops/secrets/admin-age.key/secret

@@ -1,20 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:7xyb6WoaN7uRWEO8QRkBw7iytP5hFrA94VRi+sy/UhzqT9AyDPmxB/F8ASFsBbzJUwi0Oqd2E1CeIYRoDhG7JHnDyL2bYonz2RQ=,iv:slh3x774m6oTHAXFwcen1qF+jEchOKCyNsJMbNhqXHE=,tag:wtK8H8PZCESPA1vZCd7Ptw==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPTzZ4RTVNb2I1MTBRMEcy\neU1Eek9GakkydEJBVm9kR3AyY1pEYkorNUYwCkh2WHhNQmc1eWI2cCtEUFFWdzJq\nS0FvQWtoOFkzRVBxVzhuczc0aVprbkkKLS0tIFRLdmpnbzY1Uk9LdklEWnQzZHM2\nVEx3dzhMSnMwaWE0V0J6VTZ5ZVFYMjgKdaICa/hprHxhH89XD7ri0vyTT4rM+Si0\niHcQU4x64dgoJa4gKxgr4k9XncjoNEjJhxL7i/ZNZ5deaaLRn5rKMg==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:24:55Z",
-		"mac": "ENC[AES256_GCM,data:TJWDHGSRBfOCW8Q+t3YxG3vlpf9a5u7B27AamnOk95huqIv0htqWV3RuV7NoOZ5v2ijqSe/pLfpwrmtdhO2sUBEvhdhJm8UzLShP7AbH9lxV+icJOsY7VSrp+R5W526V46ONP6p47b7fOQBbp03BMz01G191N68WYOf6k2arGxU=,iv:nEyTBwJ2EA+OAl8Ulo5cvFX6Ow2FwzTWooF/rdkPiXg=,tag:oYcG16zR+Fb5XzVsHhq2Qw==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/sops/secrets/peer-age.key/secret

@@ -1,20 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:JOOhvl0clDD/b5YO45CXR3wVopBSNe9dYBG+p5iD+nniN2OgOwBgYPNSCVtc+NemqutD12hFUSfCzXidkv0ijhD1JZeLar9Ygxc=,iv:XctQwSYSvKhDRk/XMacC9uMydZ8e9hnhpoWTgyXiFI0=,tag:foAhBlg4DwpQU2G9DzTo5g==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBVWMvWkp5TnZQcGs5Ykhp\nWC91YkoyZERqdXpxQm5JVmRhaUhueEJETDJVCkM4V0hSYldkV1U2Q0d1TGh3eGNR\nVjJ1VFd6ZEN0SXZjSVEvcnV2WW0vbVUKLS0tIFRCNW9nWHdYaUxLSVVUSXM0OGtN\nVFMzRXExNkYxcFE3QWlxVUM3ay9INm8KV6r8ftpwarly3qXoU9y8KxKrUKLvP9KX\nGsP0pORsaM+qPMsdfEo35CqhAeQu0+6DWd7/67+fUMp6Jr0DthtTmg==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:28Z",
-		"mac": "ENC[AES256_GCM,data:scY9+/fcXhfHEdrsZJLOM6nfjpRaURgTVbCRepUjhUo24B4ByEsAo2B8psVAaGEHEsFRZuoiByqrGzKhyUASmUs+wn+ziOKBTLzu55fOakp8PWYtQ4miiz2TQffp80gCQRJpykcbUgqIKXNSNutt4tosTBL7osXwCEnEQWd+SaA=,iv:1VXNvLP6DUxZYEr1juOLJmZCGbLp33DlwhxHQV9AMD4=,tag:uFM1R8OmkFS74/zkUG0k8A==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/sops/secrets/signer-age.key/secret

@@ -1,20 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:i1YBJdK8XmWnVnZKBpmWggSN8JSOr8pm2Zx+CeE8qqeLZ7xwMO8SYCutM8l94M5vzmmX0CmwzeMZ/JVPbEwFd3ZAImUfh685HOY=,iv:N4rHNaX+WmoPb0EZPqMt+CT1BzaWO9LyoemBxKn+u/s=,tag:PnzSvdGwVnTMK8Do8VzFaQ==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4RXlmcVNGTnlkY2ZqZFlH\nVnh0eHhRNE5hRDNDVkt0TEE0bmRNN2JIVkN3CkxnaGM4Y3M3a0xoK2xMRzBLMHRV\nT1FzKzNRMFZOeWc2K3E5K2FzdUsvWmsKLS0tIENtVlFSWElHN3RtOUY2alhxajhs\naXI1MmR4WC9EVGVFK3dHM1gvVnlZMVUKCyLz0DkdbWfSfccShO1xjWfxhunEIbD0\n6imeIBhZHvVJmZLXnVl7B0pNXo6be7WSBMAUM9gUtCNh4zaChBNwGw==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:52Z",
-		"mac": "ENC[AES256_GCM,data:WFGysoXN95e/RxL094CoL4iueqEcSqCSQZLahwz9HMLi+8HWZIXr55a+jyK7piqR8nBS4BquU5fKhlC6BvEbZFt69t4onTA+LxS3D7A8/TO0CWS0RymUjW9omJUseRQWwAHtE7l0qI5hdOUKhQ+o5pU+2bc3PUlaONM0aOCCoFo=,iv:l1f4aVqLl5VAMfjNxDbxQEQp/qY/nxzgv2GTuPVBoBA=,tag:4PPDCmDrviqdn42RLHQYbA==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/per-machine/admin/data-mesher-host-key/private_key/secret

@@ -1,24 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:w3bU23Pfe8W89lF+tOmEYPU/A4FkY6n7rgQ6yo+eqCJFxTyHydV6Mg4/g4jaL+4wwIqNYRiMR8J8jLhSvw3Bc59u7Ul+RGwdpiKoBBJfsHjO8r6uOz2u9Raa+iUJH1EJWmGvsQXAILpliZ+klS96VWnGN3pYMEI=,iv:7QbUxta6NPQLZrh6AOcNe+0wkrADuTI9VKVp8q+XoZ8=,tag:ZH0t3RylfQk5U23ZHWaw0g==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age10zxkj45fah3qa8uyg3a36jsd06d839xfq64nrez9etrsf4km0gtsp45gsz",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKaTBoSFJVSTdZeW4wZG9p\nWFR1LzVmYS8xWmRqTlNtWFVkSW9jZXpVejJBCkpqZm12L1dDSmNhekVsK1JBOU9r\nZThScGdDakFlRzNsVXp1eE5yOStFSW8KLS0tIFRrTkZBQlRsR2VNcUJvNEkzS2pw\nNksvM296UkFWTkZDVVp1ZVZMNUs4cWsKWTteB1G9Oo38a81PeqKO09NUQetuqosC\nhrToQ6NMo5O7/StmVG228MHbJS3KLXsvh2AFOEPyZrbpB2Opd2wwoA==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6U2FWRThRNkVQdk9yZ0VE\nM09iSVhmeldMcDZVaFRDNGtjWTdBa0VIT2pJCkdtd04xSXdicDY3OHI1WXl5TndB\nemtQeW1SS2tVVllPUHhLUTRla3haZGMKLS0tIGN0NVNEN3RKeWM0azBBMnBpQU4r\nTFFzQ0lOcGt0ek9UZmZZRjhibTNTc0EKReUwYBVM1NKX0FD/ZeokFAAknwju5Azq\nGzl4UVJBi5Es0GWORdCGElPXMd7jMud1SwgY04AdZj/dzinCSW4CZw==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:10Z",
-		"mac": "ENC[AES256_GCM,data:0vl9Gt4QeH+GJcnl8FuWSaqQXC8S6Pe50NmeDg5Nl2NWagz8aLCvOFyTqX/Icp/bTi1XQ5icHHhF3YhM+QAvdUL3aO0WGbh92dPRnFuvlZsdtwCFhT+LyHyYHFf6yP+0h/uFpJv9fE6xY22CezA6ZVQ8ywi1epaC548Gr27uVe4=,iv:G4hZVCLkIpbg9uwB7Y8xtHLdnlmBvFrPjxSoqdyHNvM=,tag:uvKwakhUY2aa7v0tmR/o8A==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/per-machine/admin/data-mesher-host-key/public_key/value

@@ -1,3 +0,0 @@
------BEGIN PUBLIC KEY-----
-MCowBQYDK2VwAyEAm204bpSFi4jOjZuXDpIZ/rcJBrbG4zAc7OSA4rAVSYE=
------END PUBLIC KEY-----

checks/data-mesher/vars/per-machine/peer/data-mesher-host-key/private_key/secret

@@ -1,24 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:kERPY40pyvke0mRBnafa4zOaF46rbueRbhpUCXjYP5ORpC7zoOhbdlVBhOsPqE2vfEP4RWkH+ZPdDYXOKXwotBCmlq2i7TfZeoNXFkzWXc3GyM5mndnjCc8hvYEQF1w6xkkVSUt4n06BAw/gT0ppz+vo5dExIA8=,iv:JmYD2o4DGqds6DV7ucUmUD0BRB61exbRsNAtINOR8cQ=,tag:Z58gVnHD+4s21Z84IRw+Vw==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1faqrml2ukc6unfm75d3v2vnaf62v92rdxaagg3ty3cfna7vt99gqlzs43l",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4OFluVThBdUJSTmRVTk94\neFZnLytvcnNSdmQvR3ZkT2UvWFVieFV1SUFNCm9jWHlyZXRwaVdFaG9ocnd4S3FU\ndTZ2dklBbkFVL0hVT0Y2L1o5dnUyNG8KLS0tIGFvYlBJR3l2b3F6OU9uMTFkYjli\nNVFLOWQzOStpU2kzb0xyZUFCMnBmMVUK5Jzssf1XBX25bq0RKlJY8NwtKIytxL/c\nBPPFDZywJiUgw1izsdfGVkRhhSFCQIz+yWIJWzr01NU2jLyFjSfCNw==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzYW92c3Q4SktwSnJ1TkRJ\nZEJyZk96cG8ybkpPQzYzVk0xZGs0eCtISVR3CmhDaWxTem1FMjJKNmZNaTkxN01n\nenUvdFI1UkFmL1lzNlM5N0Ixd0dpc1EKLS0tIHpyS2VHaHRRdUovQVgvRmRHaXh3\naFpSNURjTWkxaW9TOXpKL2IvcUFEbmMKq4Ch7DIL34NetFV+xygTdcpQjjmV8v1n\nlvYcjUO/9c3nVkxNMJYGjuxFLuFc4Gw+AyawCjpsIYXRskYRW4UR1w==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:43Z",
-		"mac": "ENC[AES256_GCM,data:YhL2d6i0VpUd15B4ow2BgRpyEm0KEA8NSb7jZcjI58d7d4lAqBMcDQB+8a9e2NZbPk8p1EYl3q4VXbEnuwsJiPZI2kabRusy/IGoHzUTUMFfVaOuUcC0eyINNVSmzJxnCbLCAA1Aj1yXzgRQ0MWr7r0RHMKw0D1e0HxdEsuAPrA=,iv:yPlMmE6+NEEQ9uOZzD3lUTBcfUwGX/Ar+bCu0XKnjIg=,tag:eR22BCFVAlRHdggg9oCeaA==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/per-machine/peer/data-mesher-host-key/public_key/value

@@ -1,3 +0,0 @@
------BEGIN PUBLIC KEY-----
-MCowBQYDK2VwAyEAv5dICFue2fYO0Zi1IyfYjoNfR6713WpISo7+2bSjL18=
------END PUBLIC KEY-----

checks/data-mesher/vars/per-machine/signer/data-mesher-host-key/private_key/secret

@@ -1,24 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:U8F7clQ2Tuj8zy5EoEga/Mc9N3LLZrlFf5m7UJKrP5yybFRCJSBs05hOcNe+LQZdEAvvr0Qbkry1pQyE84gCVbxHvwkD+l3GbguBuLMsW96bHcmstb6AvZyhMDBpm73Azf4lXhNaiB8p2pDWdxV77E+PPw1MNYI=,iv:hQhN6Ak8tB6cXSCnTmmQqHEpXWpWck3uIVCk5pUqFqU=,tag:uC4ljcs92WPlUOfwSkrK9Q==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age153mke8v2qksyqjc7vta7wglzdqr5epazt83nch0ur5v7kl87cfdsr07qld",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvV05lejQrdUQvQjZPOG9v\nZ01naXlYZ1JxWHhDT1M1aUs1RWJDSU1acVFFCmdHY094aGRPYWxpdVVxSFVHRU9v\nNnVaeTlpSEdtSWRDMmVMSjdSOEQ4ZlEKLS0tIFo5NVk2bzBxYjZ5ZWpDWTMrQ2VF\nVThWUk0rVXpTY2svSCtiVDhTQ2kvbFkKEM2DBuFtdEj1G/vS1TsyIfQxSFFvPTDq\nCmO7L/J5lHdyfIXzp/FlhdKpjvmchb8gbfJn7IWpKopc7Zimy/JnGQ==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArNzVUaHkzUzVEMlh1Q3Qr\nOEo0aDJIMG91amJiZG50MEhqblRCTWxRRVVRCk4xZlp4SkJuUHc2UnFyU1prczkz\nNGtlQlRlNnBDRFFvUGhReTh6MTBZaXMKLS0tIGxtaXhUMDM0RU4yQytualdzdTFt\nWGRiVG54MnYrR2lqZVZoT0VkbmV5WUUKbzAnOkn8RYOo7z4RISQ0yN875vSEQMDa\nnnttzVrQuK0/iZvzJ0Zq8U9+JJJKvFB1tHqye6CN0zMbv55CLLnA0g==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:26:07Z",
-		"mac": "ENC[AES256_GCM,data:uMss4+BiVupFqX7nHnMo+0yZ8RPuFD8VHYK2EtJSqzgurQrZVT4tJwY50mz2gVmwbrm49QYKk5S+H29DU0cM0HiEOgB5P5ObpXTRJPagWQ48CEFrDpBzLplobxulwnN6jJ1dpL3JF3jfrzrnSDFXMvx+n5x/86/AYXYRsi/UeyY=,iv:mPT1svKrNGmYpbL9hh2Bxxakml69q+U6gQ0ZnEcbEyg=,tag:zcZx1lTw/bEsX/1g+6T04g==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/per-machine/signer/data-mesher-host-key/public_key/value

@@ -1,3 +0,0 @@
------BEGIN PUBLIC KEY-----
-MCowBQYDK2VwAyEAeUkW5UIwA1svbNY71ePyJKX68UhxrqIUGQ2jd06w5WM=
------END PUBLIC KEY-----

checks/data-mesher/vars/shared/data-mesher-network-key/private_key/secret

@@ -1,32 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:nRlCMF58cnkdUAE2aVHEG1+vAckKtVt48Jr21Bklfbsqe1yTiHPFAMLL1ywgWWWd7FjI/Z8WID9sWzh9J8Vmotw4aJWU/rIQSeF8cJHALvfOxarJIIyb7purAiPoPPs6ggGmSmVFGB1aw8kH1JMcppQN8OItdQM=,iv:qTwaL2mgw6g7heN/H5qcjei3oY+h46PdSe3v2hDlkTs=,tag:jYNULrOPl9mcQTTrx1SDeA==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age153mke8v2qksyqjc7vta7wglzdqr5epazt83nch0ur5v7kl87cfdsr07qld",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRcG44cGFBWXk2Z0pmNklv\nTnJ5b0svLytzZmNNRkxCVU1zaDVhNUs2cld3CklsenpWd0g2OEdKKzBMQlNEejRn\nTlEvY01HYjdvVExadnN3aXZIRTZ4YlEKLS0tIGRPUXdNSHZCRDBMbno2MjJqRHBl\nSzdiSURDYitQWFpaSElkdmdicDVjMWsKweQiRqyzXmzabmU2fmgwHtOa9uDmhx9O\ns9NfUhC3ifooQUSeYp58b1ZGJQx5O5bn9q/DaEoit5LTOUprt1pUPA==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiTEdlL29sVWFpSDNNaXRJ\ndTJDRkU4VzFPQ0M4MkFha2IxV2FXN2o3ZEFRCjF3UnZ5U1hTc3VvSTIzcWxOZjl0\ncHlLVEFqRk1UbGdxaUxEeDFqbFVYaU0KLS0tIFFyMnJkZnRHdWg4Z1IyRHFkY0I5\nQjdIMGtGLzRGMFM0ektDZ3hzZDdHSmMKvxOQuKgePom0QfPSvn+4vsGHhJ4BoOvW\nc27Vn4/i4hbjfJr4JpULAwyIwt3F0RaTA2M6EkFkY8otEi3vkcpWvA==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age10zxkj45fah3qa8uyg3a36jsd06d839xfq64nrez9etrsf4km0gtsp45gsz",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5ZzdsaVRnSmsrMGR1Ylg3\nZkpscTdwNUl5NUVXN3kvMU1icE0yZU1WSEJBClB6SlJYZUhDSElRREx5b0VueFUw\nNVFRU3BSU24yWEtpRnJoUC83SDVaUWsKLS0tIGVxNEo3TjlwakpDZlNsSkVCOXlz\nNDgwaE1xNjZkSnJBVlU5YXVHeGxVNFEKsXKyTzq9VsERpXzbFJGv/pbAghFAcXkf\nMmCgQHsfIMBJQUstcO8sAkxv3ced0dAEz8O6NUd0FS2zlhBzt29Rnw==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1faqrml2ukc6unfm75d3v2vnaf62v92rdxaagg3ty3cfna7vt99gqlzs43l",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkK1hDMGxCc1IvYXlJMnBF\nWncxaXBQa1RpTWdwUHc3Yk16My8rVHNJc2dFCkNlK2h0dy9oU3Z5ZGhwRWVLYVUz\ncVBKT2x5VnlhbXNmdHkwbmZzVG5sd0EKLS0tIHJaMzhDanF4Rkl3akN4MEIxOHFC\nYWRUZ08xb1UwOFNRaktkMjIzNXZmNkUK1rlbJ96oUNQZLmCmPNDOKxfDMMa+Bl2E\nJPxcNc7XY3WBHa3xFUbcqiPxWxDyaZjhq/LYQGpepiGonGMEzR5JOQ==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:20Z",
-		"mac": "ENC[AES256_GCM,data:za9ku+9lu1TTRjbPcd5LYDM4tJsAYF/yuWFCGkAhqcYguEducsIfoKBwL42ahAzqLjCZp91YJuINtw16mM+Hmlhi/BVwhnXNHqcfnKoAS/zg9KJvWcvXwKMmjEjaBovqaCWXWoKS7dn/wZ7nfGrlsiUilCDkW4BzTIzkqNkyREU=,iv:2X9apXMatwCPRBIRbPxz6PJQwGrlr7O+z+MrsnFq+sQ=,tag:IYvitoV4MhyJyRO1ySxbLQ==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/shared/data-mesher-network-key/public_key/value

@@ -1,3 +0,0 @@
------BEGIN PUBLIC KEY-----
-MCowBQYDK2VwAyEA/5j+Js7oxwWvZdfjfEO/3UuRqMxLKXsaNc3/5N2WSaw=
------END PUBLIC KEY-----

checks/flake-module.nix

@@ -22,6 +22,7 @@ in
   imports = filter pathExists [
     ./backups/flake-module.nix
     ../nixosModules/clanCore/machine-id/tests/flake-module.nix
+    ../nixosModules/clanCore/state-version/tests/flake-module.nix
     ./devshell/flake-module.nix
     ./flash/flake-module.nix
     ./impure/flake-module.nix
@@ -93,7 +94,6 @@ in
 
             service-dummy-test = import ./service-dummy-test nixosTestArgs;
             service-dummy-test-from-flake = import ./service-dummy-test-from-flake nixosTestArgs;
-            service-data-mesher = import ./data-mesher nixosTestArgs;
           };
 
           packagesToBuild = lib.removeAttrs self'.packages [

checks/flash/flake-module.nix

@@ -50,8 +50,6 @@
         self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.build.toplevel
         self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.build.diskoScript
         self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.build.diskoScript.drvPath
-        self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.clan.deployment.file
-
       ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
       closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
     in
@@ -61,7 +59,7 @@
           name = "flash";
           nodes.target = {
             virtualisation.emptyDiskImages = [ 4096 ];
-            virtualisation.memorySize = 3000;
+            virtualisation.memorySize = 4096;
             environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
             environment.etc."install-closure".source = "${closureInfo}/store-paths";
 

checks/installation/flake-module.nix

@@ -1,63 +1,9 @@
 {
   self,
   lib,
+
   ...
 }:
-let
-  installer =
-    { modulesPath, pkgs, ... }:
-    let
-      dependencies = [
-        self.clan.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.toplevel
-        self.clan.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.diskoScript
-        self.clan.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.clan.deployment.file
-        pkgs.stdenv.drvPath
-        pkgs.bash.drvPath
-        pkgs.nixos-anywhere
-        pkgs.bubblewrap
-        pkgs.buildPackages.xorg.lndir
-      ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
-      closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
-    in
-    {
-      imports = [
-        (modulesPath + "/../tests/common/auto-format-root-device.nix")
-      ];
-      networking.useNetworkd = true;
-      services.openssh.enable = true;
-      services.openssh.settings.UseDns = false;
-      services.openssh.settings.PasswordAuthentication = false;
-      system.nixos.variant_id = "installer";
-      environment.systemPackages = [
-        self.packages.${pkgs.system}.clan-cli-full
-        pkgs.nixos-facter
-      ];
-      environment.etc."install-closure".source = "${closureInfo}/store-paths";
-      virtualisation.emptyDiskImages = [ 512 ];
-      virtualisation.diskSize = 8 * 1024;
-      virtualisation.rootDevice = "/dev/vdb";
-      # both installer and target need to use the same diskImage
-      virtualisation.diskImage = "./target.qcow2";
-      virtualisation.memorySize = 3048;
-      nix.settings = {
-        substituters = lib.mkForce [ ];
-        hashed-mirrors = null;
-        connect-timeout = lib.mkForce 3;
-        flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
-        experimental-features = [
-          "nix-command"
-          "flakes"
-        ];
-      };
-      users.users.nonrootuser = {
-        isNormalUser = true;
-        openssh.authorizedKeys.keyFiles = [ ../assets/ssh/pubkey ];
-        extraGroups = [ "wheel" ];
-      };
-      security.sudo.wheelNeedsPassword = false;
-      system.extraDependencies = dependencies;
-    };
-in
 {
 
   # The purpose of this test is to ensure `clan machines install` works
@@ -106,6 +52,25 @@ in
 
         environment.etc."install-successful".text = "ok";
 
+        # Enable SSH and add authorized key for testing
+        services.openssh.enable = true;
+        services.openssh.settings.PasswordAuthentication = false;
+        users.users.nonrootuser = {
+          isNormalUser = true;
+          openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ];
+          extraGroups = [ "wheel" ];
+          home = "/home/nonrootuser";
+          createHome = true;
+        };
+        users.users.root.openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ];
+        # Allow users to manage their own SSH keys
+        services.openssh.authorizedKeysFiles = [
+          "/root/.ssh/authorized_keys"
+          "/home/%u/.ssh/authorized_keys"
+          "/etc/ssh/authorized_keys.d/%u"
+        ];
+        security.sudo.wheelNeedsPassword = false;
+
         boot.consoleLogLevel = lib.mkForce 100;
         boot.kernelParams = [ "boot.shell_on_fail" ];
 
@@ -182,55 +147,199 @@ in
       # vm-test-run-test-installation-> target: waiting for the VM to finish booting
       # vm-test-run-test-installation-> target: Guest root shell did not produce any data yet...
       # vm-test-run-test-installation-> target:   To debug, enter the VM and run 'systemctl status backdoor.service'.
-      checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux && !pkgs.stdenv.isAarch64) {
-        nixos-test-installation = self.clanLib.test.baseTest {
-          name = "installation";
-          nodes.target = {
-            services.openssh.enable = true;
-            virtualisation.diskImage = "./target.qcow2";
-            virtualisation.useBootLoader = true;
+      checks =
+        let
+          # Custom Python package for port management utilities
+          closureInfo = pkgs.closureInfo {
+            rootPaths = [
+              self.checks.x86_64-linux.clan-core-for-checks
+              self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.toplevel
+              self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.initialRamdisk
+              self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.diskoScript
+              pkgs.stdenv.drvPath
+              pkgs.bash.drvPath
+              pkgs.buildPackages.xorg.lndir
+            ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
           };
-          nodes.installer = installer;
+        in
+        pkgs.lib.mkIf (pkgs.stdenv.isLinux && !pkgs.stdenv.isAarch64) {
+          nixos-test-installation = self.clanLib.test.baseTest {
+            name = "installation";
+            nodes.target = (import ./test-helpers.nix { inherit lib pkgs self; }).target;
+            extraPythonPackages = _p: [
+              self.legacyPackages.${pkgs.system}.nixosTestLib
+            ];
 
-          testScript = ''
-            installer.start()
+            testScript = ''
+              import tempfile
+              import os
+              import subprocess
+              from nixos_test_lib.ssh import setup_ssh_connection # type: ignore[import-untyped]
+              from nixos_test_lib.nix_setup import prepare_test_flake # type: ignore[import-untyped]
 
-            installer.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../assets/ssh/privkey} /root/.ssh/id_ed25519")
+              def create_test_machine(oldmachine, qemu_test_bin: str, **kwargs):
+                  """Create a new test machine from an installed disk image"""
+                  start_command = [
+                      f"{qemu_test_bin}/bin/qemu-kvm",
+                      "-cpu",
+                      "max",
+                      "-m",
+                      "3048",
+                      "-virtfs",
+                      "local,path=/nix/store,security_model=none,mount_tag=nix-store",
+                      "-drive",
+                      f"file={oldmachine.state_dir}/target.qcow2,id=drive1,if=none,index=1,werror=report",
+                      "-device",
+                      "virtio-blk-pci,drive=drive1",
+                      "-netdev",
+                      "user,id=net0",
+                      "-device",
+                      "virtio-net-pci,netdev=net0",
+                  ]
+                  machine = create_machine(start_command=" ".join(start_command), **kwargs)
+                  driver.machines.append(machine)
+                  return machine
 
-            installer.wait_until_succeeds("timeout 2 ssh -o StrictHostKeyChecking=accept-new -v nonrootuser@localhost hostname")
-            installer.succeed("cp -r ${self.checks.x86_64-linux.clan-core-for-checks} test-flake && chmod -R +w test-flake")
+              target.start()
 
-            installer.succeed("clan machines install --no-reboot --debug --flake test-flake --yes test-install-machine-without-system --target-host nonrootuser@localhost --update-hardware-config nixos-facter >&2")
-            installer.shutdown()
+              # Set up test environment
+              with tempfile.TemporaryDirectory() as temp_dir:
+                  # Prepare test flake and Nix store
+                  flake_dir = prepare_test_flake(
+                      temp_dir,
+                      "${self.checks.x86_64-linux.clan-core-for-checks}",
+                      "${closureInfo}"
+                  )
 
-            # We are missing the test instrumentation somehow. Test this later.
-            target.state_dir = installer.state_dir
-            target.start()
-            target.wait_for_unit("multi-user.target")
-          '';
-        } { inherit pkgs self; };
+                  # Set up SSH connection
+                  ssh_conn = setup_ssh_connection(
+                      target,
+                      temp_dir,
+                      "${../assets/ssh/privkey}"
+                  )
 
-        nixos-test-update-hardware-configuration = self.clanLib.test.baseTest {
-          name = "update-hardware-configuration";
-          nodes.installer = installer;
+                  # Run clan install from host using port forwarding
+                  clan_cmd = [
+                      "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan",
+                      "machines",
+                      "install",
+                      "--phases", "disko,install",
+                      "--debug",
+                      "--flake", flake_dir,
+                      "--yes", "test-install-machine-without-system",
+                      "--target-host", f"nonrootuser@localhost:{ssh_conn.host_port}",
+                      "-i", ssh_conn.ssh_key,
+                      "--option", "store", os.environ['CLAN_TEST_STORE'],
+                      "--update-hardware-config", "nixos-facter",
+                  ]
 
-          testScript = ''
-            installer.start()
-            installer.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../assets/ssh/privkey} /root/.ssh/id_ed25519")
-            installer.wait_until_succeeds("timeout 2 ssh -o StrictHostKeyChecking=accept-new -v nonrootuser@localhost hostname")
-            installer.succeed("cp -r ${self.checks.x86_64-linux.clan-core-for-checks} test-flake && chmod -R +w test-flake")
-            installer.fail("test -f test-flake/machines/test-install-machine/hardware-configuration.nix")
-            installer.fail("test -f test-flake/machines/test-install-machine/facter.json")
+                  subprocess.run(clan_cmd, check=True)
 
-            installer.succeed("clan machines update-hardware-config --debug --flake test-flake test-install-machine-without-system nonrootuser@localhost >&2")
-            installer.succeed("test -f test-flake/machines/test-install-machine-without-system/facter.json")
-            installer.succeed("rm test-flake/machines/test-install-machine-without-system/facter.json")
+              # Shutdown the installer machine gracefully
+              try:
+                  target.shutdown()
+              except BrokenPipeError:
+                  # qemu has already exited
+                  pass
 
-            installer.succeed("clan machines update-hardware-config --debug --backend nixos-generate-config --flake test-flake test-install-machine-without-system nonrootuser@localhost >&2")
-            installer.succeed("test -f test-flake/machines/test-install-machine-without-system/hardware-configuration.nix")
-            installer.succeed("rm test-flake/machines/test-install-machine-without-system/hardware-configuration.nix")
-          '';
-        } { inherit pkgs self; };
-      };
+              # Create a new machine instance that boots from the installed system
+              installed_machine = create_test_machine(target, "${pkgs.qemu_test}", name="after_install")
+              installed_machine.start()
+              installed_machine.wait_for_unit("multi-user.target")
+              installed_machine.succeed("test -f /etc/install-successful")
+            '';
+          } { inherit pkgs self; };
+
+          nixos-test-update-hardware-configuration = self.clanLib.test.baseTest {
+            name = "update-hardware-configuration";
+            nodes.target = (import ./test-helpers.nix { inherit lib pkgs self; }).target;
+            extraPythonPackages = _p: [
+              self.legacyPackages.${pkgs.system}.nixosTestLib
+            ];
+
+            testScript = ''
+              import tempfile
+              import os
+              import subprocess
+              from nixos_test_lib.ssh import setup_ssh_connection # type: ignore[import-untyped]
+              from nixos_test_lib.nix_setup import prepare_test_flake # type: ignore[import-untyped]
+
+              target.start()
+
+              # Set up test environment
+              with tempfile.TemporaryDirectory() as temp_dir:
+                  # Prepare test flake and Nix store
+                  flake_dir = prepare_test_flake(
+                      temp_dir,
+                      "${self.checks.x86_64-linux.clan-core-for-checks}",
+                      "${closureInfo}"
+                  )
+                  
+                  # Set up SSH connection
+                  ssh_conn = setup_ssh_connection(
+                      target,
+                      temp_dir,
+                      "${../assets/ssh/privkey}"
+                  )
+
+                  # Verify files don't exist initially
+                  hw_config_file = os.path.join(flake_dir, "machines/test-install-machine/hardware-configuration.nix")
+                  facter_file = os.path.join(flake_dir, "machines/test-install-machine/facter.json")
+
+                  assert not os.path.exists(hw_config_file), "hardware-configuration.nix should not exist initially"
+                  assert not os.path.exists(facter_file), "facter.json should not exist initially"
+
+                  # Set CLAN_FLAKE for the commands
+                  os.environ["CLAN_FLAKE"] = flake_dir
+
+                  # Test facter backend
+                  clan_cmd = [
+                      "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan",
+                      "machines",
+                      "update-hardware-config",
+                      "--debug",
+                      "--flake", ".",
+                      "--host-key-check", "none",
+                      "test-install-machine-without-system",
+                      "-i", ssh_conn.ssh_key,
+                      "--option", "store", os.environ['CLAN_TEST_STORE'],
+                      f"nonrootuser@localhost:{ssh_conn.host_port}"
+                  ]
+
+                  result = subprocess.run(clan_cmd, capture_output=True, cwd=flake_dir)
+                  if result.returncode != 0:
+                      print(f"Clan update-hardware-config failed: {result.stderr.decode()}")
+                      raise Exception(f"Clan update-hardware-config failed with return code {result.returncode}")
+
+                  facter_without_system_file = os.path.join(flake_dir, "machines/test-install-machine-without-system/facter.json")
+                  assert os.path.exists(facter_without_system_file), "facter.json should exist after update"
+                  os.remove(facter_without_system_file)
+
+                  # Test nixos-generate-config backend
+                  clan_cmd = [
+                      "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan",
+                      "machines",
+                      "update-hardware-config",
+                      "--debug",
+                      "--backend", "nixos-generate-config",
+                      "--host-key-check", "none",
+                      "--flake", ".",
+                      "test-install-machine-without-system",
+                      "-i", ssh_conn.ssh_key,
+                      "--option", "store", os.environ['CLAN_TEST_STORE'],
+                      f"nonrootuser@localhost:{ssh_conn.host_port}"
+                  ]
+
+                  result = subprocess.run(clan_cmd, capture_output=True, cwd=flake_dir)
+                  if result.returncode != 0:
+                      print(f"Clan update-hardware-config (nixos-generate-config) failed: {result.stderr.decode()}")
+                      raise Exception(f"Clan update-hardware-config failed with return code {result.returncode}")
+
+                  hw_config_without_system_file = os.path.join(flake_dir, "machines/test-install-machine-without-system/hardware-configuration.nix")
+                  assert os.path.exists(hw_config_without_system_file), "hardware-configuration.nix should exist after update"
+            '';
+          } { inherit pkgs self; };
+
+        };
     };
 }

checks/installation/pyproject.toml

@@ -0,0 +1,44 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "nixos-test-lib"
+version = "1.0.0"
+description = "NixOS test utilities for clan VM testing"
+authors = [
+    {name = "Clan Core Team"}
+]
+dependencies = []
+
+[project.optional-dependencies]
+dev = [
+    "mypy",
+    "ruff"
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["nixos_test_lib*"]
+
+[tool.setuptools.package-data]
+"nixos_test_lib" = ["py.typed"]
+
+[tool.mypy]
+python_version = "3.12"
+strict = true
+warn_return_any = true
+warn_unused_configs = true
+
+[tool.ruff]
+target-version = "py312"
+line-length = 88
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "D", # docstrings
+    "ANN", # type annotations  
+    "COM812", # trailing comma
+    "ISC001", # string concatenation
+]

checks/installation/test-helpers.nix

@@ -0,0 +1,173 @@
+{
+  lib,
+  pkgs,
+  self,
+  ...
+}:
+let
+  # Common target VM configuration used by both installation and update tests
+  target =
+    { modulesPath, pkgs, ... }:
+    {
+      imports = [
+        (modulesPath + "/../tests/common/auto-format-root-device.nix")
+      ];
+      networking.useNetworkd = true;
+      services.openssh.enable = true;
+      services.openssh.settings.UseDns = false;
+      services.openssh.settings.PasswordAuthentication = false;
+      system.nixos.variant_id = "installer";
+      environment.systemPackages = [
+        pkgs.nixos-facter
+      ];
+      # Disable cache.nixos.org to speed up tests
+      nix.settings.substituters = [ ];
+      nix.settings.trusted-public-keys = [ ];
+      virtualisation.emptyDiskImages = [ 512 ];
+      virtualisation.diskSize = 8 * 1024;
+      virtualisation.rootDevice = "/dev/vdb";
+      # both installer and target need to use the same diskImage
+      virtualisation.diskImage = "./target.qcow2";
+      virtualisation.memorySize = 3048;
+      users.users.nonrootuser = {
+        isNormalUser = true;
+        openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ];
+        extraGroups = [ "wheel" ];
+      };
+      users.users.root.openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ];
+      # Allow users to manage their own SSH keys
+      services.openssh.authorizedKeysFiles = [
+        "/root/.ssh/authorized_keys"
+        "/home/%u/.ssh/authorized_keys"
+        "/etc/ssh/authorized_keys.d/%u"
+      ];
+      security.sudo.wheelNeedsPassword = false;
+    };
+
+  # Common base test machine configuration
+  baseTestMachine =
+    { lib, modulesPath, ... }:
+    {
+      imports = [
+        (modulesPath + "/testing/test-instrumentation.nix")
+        (modulesPath + "/profiles/qemu-guest.nix")
+        self.clanLib.test.minifyModule
+      ];
+
+      # Enable SSH and add authorized key for testing
+      services.openssh.enable = true;
+      services.openssh.settings.PasswordAuthentication = false;
+      users.users.nonrootuser = {
+        isNormalUser = true;
+        openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ];
+        extraGroups = [ "wheel" ];
+        home = "/home/nonrootuser";
+        createHome = true;
+      };
+      users.users.root.openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ];
+      # Allow users to manage their own SSH keys
+      services.openssh.authorizedKeysFiles = [
+        "/root/.ssh/authorized_keys"
+        "/home/%u/.ssh/authorized_keys"
+        "/etc/ssh/authorized_keys.d/%u"
+      ];
+      security.sudo.wheelNeedsPassword = false;
+
+      boot.consoleLogLevel = lib.mkForce 100;
+      boot.kernelParams = [ "boot.shell_on_fail" ];
+
+      # disko config
+      boot.loader.grub.efiSupport = lib.mkDefault true;
+      boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
+      clan.core.vars.settings.secretStore = "vm";
+      clan.core.vars.generators.test = {
+        files.test.neededFor = "partitioning";
+        script = ''
+          echo "notok" > "$out"/test
+        '';
+      };
+      disko.devices = {
+        disk = {
+          main = {
+            type = "disk";
+            device = "/dev/vda";
+
+            preCreateHook = ''
+              test -e /run/partitioning-secrets/test/test
+            '';
+
+            content = {
+              type = "gpt";
+              partitions = {
+                boot = {
+                  size = "1M";
+                  type = "EF02"; # for grub MBR
+                  priority = 1;
+                };
+                ESP = {
+                  size = "512M";
+                  type = "EF00";
+                  content = {
+                    type = "filesystem";
+                    format = "vfat";
+                    mountpoint = "/boot";
+                    mountOptions = [ "umask=0077" ];
+                  };
+                };
+                root = {
+                  size = "100%";
+                  content = {
+                    type = "filesystem";
+                    format = "ext4";
+                    mountpoint = "/";
+                  };
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+
+  # NixOS test library combining port utils and clan VM test utilities
+  nixosTestLib = pkgs.python3Packages.buildPythonPackage {
+    pname = "nixos-test-lib";
+    version = "1.0.0";
+    format = "pyproject";
+    src = lib.fileset.toSource {
+      root = ./.;
+      fileset = lib.fileset.unions [
+        ./pyproject.toml
+        ./nixos_test_lib
+      ];
+    };
+    nativeBuildInputs = with pkgs.python3Packages; [
+      setuptools
+      wheel
+    ];
+    doCheck = false;
+  };
+
+  # Common closure info
+  closureInfo = pkgs.closureInfo {
+    rootPaths = [
+      self.checks.x86_64-linux.clan-core-for-checks
+      self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.toplevel
+      self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.initialRamdisk
+      self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.diskoScript
+      self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.clan.deployment.file
+      pkgs.stdenv.drvPath
+      pkgs.bash.drvPath
+      pkgs.buildPackages.xorg.lndir
+    ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
+  };
+
+in
+{
+  inherit
+    target
+    baseTestMachine
+    nixosTestLib
+    closureInfo
+    ;
+}

checks/morph/flake-module.nix

@@ -35,7 +35,6 @@
                   pkgs.stdenv.drvPath
                   pkgs.stdenvNoCC
                   self.nixosConfigurations.test-morph-machine.config.system.build.toplevel
-                  self.nixosConfigurations.test-morph-machine.config.system.clan.deployment.file
                 ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
                 closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
               in

checks/service-dummy-test-from-flake/default.nix

@@ -23,14 +23,14 @@ nixosLib.runTest (
     clan.test.fromFlake = ./.;
 
     extraPythonPackages = _p: [
-      clan-core.legacyPackages.${hostPkgs.system}.setupNixInNixPythonPackage
+      clan-core.legacyPackages.${hostPkgs.system}.nixosTestLib
     ];
 
     testScript =
       { nodes, ... }:
       ''
-        from setup_nix_in_nix import setup_nix_in_nix # type: ignore[import-untyped]
-        setup_nix_in_nix()
+        from nixos_test_lib.nix_setup import setup_nix_in_nix # type: ignore[import-untyped]
+        setup_nix_in_nix(None)  # No closure info for this test
 
         def run_clan(cmd: list[str], **kwargs) -> str:
             import subprocess

clanModules/borgbackup/roles/client.nix

@@ -196,7 +196,7 @@ in
         pkgs.xkcdpass
       ];
       script = ''
-        ssh-keygen -t ed25519 -N "" -f "$out"/borgbackup.ssh
+        ssh-keygen -t ed25519 -N "" -C "" -f "$out"/borgbackup.ssh
         xkcdpass -n 4 -d - > "$out"/borgbackup.repokey
       '';
     };

clanModules/disk-id/roles/default.nix

@@ -1,5 +1,4 @@
 {
-  config,
   pkgs,
   ...
 }:
@@ -9,9 +8,14 @@
   config = {
 
     warnings = [
-      "The clan.disk-id module is deprecated and will be removed on 2025-07-15.
-      Please migrate to user-maintained configuration or the new equivalent clan services
-      (https://docs.clan.lol/reference/clanServices)."
+      ''
+        The clan.disk-id module is deprecated and will be removed on 2025-07-15.
+        For migration see: https://docs.clan.lol/guides/migrations/disk-id/
+
+        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+        !!! Please migrate. Otherwise you may not be able to boot your system after that date.  !!!
+        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+      ''
     ];
     clan.core.vars.generators.disk-id = {
       files.diskId.secret = false;

clanModules/importer/README.md

@@ -7,7 +7,7 @@ The importer module allows users to configure importing modules in a flexible an
 
 It exposes the `extraModules` functionality of the inventory, without any added configuration.
 
-## Usage:
+## Usage
 
 ```nix
 inventory.services = {

clanModules/sshd/roles/server.nix

@@ -54,7 +54,7 @@ in
         pkgs.openssh
       ];
       script = ''
-        ssh-keygen -t ed25519 -N "" -f "$out"/ssh.id_ed25519
+        ssh-keygen -t ed25519 -N "" -C "" -f "$out"/ssh.id_ed25519
       '';
     };
 
@@ -74,7 +74,7 @@ in
         pkgs.openssh
       ];
       script = ''
-        ssh-keygen -t rsa -b 4096 -N "" -f "$out"/ssh.id_rsa
+        ssh-keygen -t rsa -b 4096 -N "" -C "" -f "$out"/ssh.id_rsa
       '';
     };
 

clanModules/sshd/shared.nix

@@ -36,7 +36,7 @@
             pkgs.openssh
           ];
           script = ''
-            ssh-keygen -t ed25519 -N "" -f "$out"/id_ed25519
+            ssh-keygen -t ed25519 -N "" -C "" -f "$out"/id_ed25519
           '';
         };
 

clanModules/state-version/roles/default.nix

@@ -5,9 +5,12 @@ in
 {
 
   warnings = [
-    "The clan.state-version module is deprecated and will be removed on 2025-07-15.
-      Please migrate to user-maintained configuration or the new equivalent clan services
-      (https://docs.clan.lol/reference/clanServices)."
+    ''
+      The clan.state-version service is deprecated and will be
+      removed on 2025-07-15 in favor of a nix option.
+
+      Please migrate your configuration to use `clan.core.settings.state-version.enable = true` instead.
+    ''
   ];
 
   system.stateVersion = lib.mkDefault (lib.removeSuffix "\n" var.value);

clanServices/borgbackup/default.nix

@@ -256,7 +256,7 @@
                     pkgs.xkcdpass
                   ];
                   script = ''
-                    ssh-keygen -t ed25519 -N "" -f "$out"/borgbackup.ssh
+                    ssh-keygen -t ed25519 -N "" -C "" -f "$out"/borgbackup.ssh
                     xkcdpass -n 4 -d - > "$out"/borgbackup.repokey
                   '';
                 };

clanServices/borgbackup/tests/vm/default.nix

@@ -41,14 +41,6 @@
         clan-core,
         ...
       }:
-      let
-        dependencies = [
-          clan-core
-          pkgs.stdenv.drvPath
-        ] ++ builtins.map (i: i.outPath) (builtins.attrValues clan-core.inputs);
-        closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
-
-      in
       {
 
         services.openssh.enable = true;
@@ -59,15 +51,6 @@
 
         environment.systemPackages = [ clan-core.packages.${pkgs.system}.clan-cli ];
 
-        environment.etc.install-closure.source = "${closureInfo}/store-paths";
-        nix.settings = {
-          substituters = pkgs.lib.mkForce [ ];
-          hashed-mirrors = null;
-          connect-timeout = pkgs.lib.mkForce 3;
-          flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
-        };
-        system.extraDependencies = dependencies;
-
         clan.core.state.test-backups.folders = [ "/var/test-backups" ];
       };
 

clanServices/data-mesher/README.md

@@ -0,0 +1,29 @@
+This service will set up data-mesher.
+
+## Usage
+
+```nix
+inventory.instances = {
+  data-mesher = {
+    module = {
+      name = "data-mesher";
+      input = "clan-core";
+    };
+    roles.admin.machines.server0 = {
+      settings = {
+        bootstrapNodes = {
+          node1 = "192.168.1.1:7946";
+          node2 = "192.168.1.2:7946";
+        };
+
+        network = {
+          hostTTL = "24h";
+          interface = "tailscale0";
+        };
+      };
+    };
+    roles.peer.machines.server1 = { };
+    roles.signer.machines.server2 = { };
+  };
+}
+```

clanServices/data-mesher/admin.nix

@@ -0,0 +1,29 @@
+{
+  lib,
+  config,
+  settings,
+  ...
+}:
+{
+
+  services.data-mesher.initNetwork =
+    let
+      # for a given machine, read it's public key and remove any new lines
+      readHostKey =
+        machine:
+        let
+          path = "${config.clan.core.settings.directory}/vars/per-machine/${machine}/data-mesher-host-key/public_key/value";
+        in
+        builtins.elemAt (lib.splitString "\n" (builtins.readFile path)) 1;
+    in
+    {
+      enable = true;
+      keyPath = config.clan.core.vars.generators.data-mesher-network-key.files.private_key.path;
+
+      tld = settings.network.tld;
+      hostTTL = settings.network.hostTTL;
+
+      # admin and signer host public keys
+      signingKeys = builtins.map readHostKey (builtins.attrNames settings.bootstrapNodes);
+    };
+}

clanServices/data-mesher/default.nix

@@ -0,0 +1,142 @@
+{ ... }:
+let
+  sharedInterface =
+    { lib, ... }:
+    {
+      options = {
+        bootstrapNodes = lib.mkOption {
+          type = lib.types.nullOr (lib.types.attrsOf lib.types.str);
+          # the default bootstrap nodes are any machines with the admin or signers role
+          # we iterate through those machines, determining an IP address for them based on their VPN
+          # currently only supports zerotier
+          # default = builtins.foldl' (
+          #   urls: name:
+          #   let
+          #     ipPath = "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value";
+          #   in
+          #   if builtins.pathExists ipPath then
+          #     let
+          #       ip = builtins.readFile ipPath;
+          #     in
+          #     urls ++ [ "[${ip}]:${builtins.toString settings.network.port}" ]
+          #   else
+          #     urls
+          # ) [ ] (dmLib.machines config).bootstrap;
+          description = ''
+            A list of bootstrap nodes that act as an initial gateway when joining
+            the cluster.
+          '';
+          example = {
+            "node1" = "192.168.1.1:7946";
+            "node2" = "192.168.1.2:7946";
+          };
+        };
+
+        network = {
+          interface = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              The interface over which cluster communication should be performed.
+              All the ip addresses associate with this interface will be part of
+              our host claim, including both ipv4 and ipv6.
+
+              This should be set to an internal/VPN interface.
+            '';
+            example = "tailscale0";
+          };
+
+          port = lib.mkOption {
+            type = lib.types.port;
+            default = 7946;
+            description = ''
+              Port to listen on for cluster communication.
+            '';
+          };
+        };
+      };
+    };
+in
+{
+  _class = "clan.service";
+  manifest.name = "data-mesher";
+  manifest.description = "Set up data-mesher";
+  manifest.categories = [ "System" ];
+  manifest.readme = builtins.readFile ./README.md;
+
+  roles.admin = {
+    interface =
+      { lib, ... }:
+      {
+
+        imports = [ sharedInterface ];
+
+        options = {
+
+          network = {
+            tld = lib.mkOption {
+              type = lib.types.str;
+              default = "clan";
+              description = "Top level domain to use for the network";
+            };
+
+            hostTTL = lib.mkOption {
+              type = lib.types.str;
+              default = "${toString (24 * 28)}h";
+              example = "24h";
+              description = "The TTL for hosts in the network, in the form of a Go time.Duration";
+            };
+          };
+        };
+      };
+    perInstance =
+      { settings, roles, ... }:
+      {
+        nixosModule = {
+          imports = [
+            ./admin.nix
+            ./shared.nix
+          ];
+          _module.args = { inherit settings roles; };
+        };
+      };
+  };
+
+  roles.signer = {
+    interface =
+      { ... }:
+      {
+        imports = [ sharedInterface ];
+      };
+    perInstance =
+      { settings, roles, ... }:
+      {
+        nixosModule = {
+          imports = [
+            ./signer.nix
+            ./shared.nix
+          ];
+          _module.args = { inherit settings roles; };
+        };
+      };
+  };
+
+  roles.peer = {
+    interface =
+      { ... }:
+      {
+        imports = [ sharedInterface ];
+      };
+    perInstance =
+      { settings, roles, ... }:
+      {
+        nixosModule = {
+          imports = [
+            ./peer.nix
+            ./shared.nix
+          ];
+          _module.args = { inherit settings roles; };
+        };
+      };
+  };
+
+}

clanServices/data-mesher/flake-module.nix

@@ -0,0 +1,17 @@
+{ lib, ... }:
+let
+  module = lib.modules.importApply ./default.nix { };
+in
+{
+  clan.modules = {
+    data-mesher = module;
+  };
+  perSystem =
+    { ... }:
+    {
+      clan.nixosTests.service-data-mesher = {
+        imports = [ ./tests/vm/default.nix ];
+        clan.modules."@clan/data-mesher" = module;
+      };
+    };
+}

clanServices/data-mesher/peer.nix

@@ -0,0 +1,2 @@
+{
+}

clanServices/data-mesher/shared.nix

@@ -0,0 +1,86 @@
+{
+  config,
+  settings,
+  ...
+}:
+{
+
+  services.data-mesher = {
+    enable = true;
+    openFirewall = true;
+
+    settings = {
+      log_level = "warn";
+      state_dir = "/var/lib/data-mesher";
+
+      # read network id from vars
+      network.id = config.clan.core.vars.generators.data-mesher-network-key.files.public_key.value;
+
+      host = {
+        names = [ config.networking.hostName ];
+        key_path = config.clan.core.vars.generators.data-mesher-host-key.files.private_key.path;
+      };
+
+      cluster = {
+        port = settings.network.port;
+        join_interval = "30s";
+        push_pull_interval = "30s";
+        interface = settings.network.interface;
+        bootstrap_nodes = (builtins.attrValues settings.bootstrapNodes);
+      };
+
+      http.port = 7331;
+      http.interface = "lo";
+    };
+  };
+
+  # Generate host key.
+  clan.core.vars.generators.data-mesher-host-key = {
+    files =
+      let
+        owner = config.users.users.data-mesher.name;
+      in
+      {
+        private_key = {
+          inherit owner;
+        };
+        public_key.secret = false;
+      };
+
+    runtimeInputs = [
+      config.services.data-mesher.package
+    ];
+
+    script = ''
+      data-mesher generate keypair \
+          --public-key-path "$out"/public_key \
+          --private-key-path "$out"/private_key
+    '';
+  };
+
+  clan.core.vars.generators.data-mesher-network-key = {
+    # generated once per clan
+    share = true;
+
+    files =
+      let
+        owner = config.users.users.data-mesher.name;
+      in
+      {
+        private_key = {
+          inherit owner;
+        };
+        public_key.secret = false;
+      };
+
+    runtimeInputs = [
+      config.services.data-mesher.package
+    ];
+
+    script = ''
+      data-mesher generate keypair \
+          --public-key-path "$out"/public_key \
+          --private-key-path "$out"/private_key
+    '';
+  };
+}

clanServices/data-mesher/signer.nix

@@ -0,0 +1,2 @@
+{
+}

clanServices/data-mesher/tests/vm/default.nix

@@ -0,0 +1,90 @@
+{
+  ...
+}:
+{
+  name = "service-data-mesher";
+
+  clan = {
+    directory = ./.;
+    test.useContainers = true;
+    inventory = {
+
+      machines.peer = { };
+      machines.admin = { };
+      machines.signer = { };
+
+      instances = {
+        data-mesher =
+          let
+            bootstrapNodes = {
+              admin = "[2001:db8:1::1]:7946";
+              peer = "[2001:db8:1::2]:7946";
+              # signer = "2001:db8:1::3:7946";
+            };
+          in
+          {
+            roles.peer.machines.peer.settings = {
+              network.interface = "eth1";
+              inherit bootstrapNodes;
+            };
+            roles.signer.machines.signer.settings = {
+              network.interface = "eth1";
+              inherit bootstrapNodes;
+            };
+            roles.admin.machines.admin.settings = {
+              network.tld = "foo";
+              network.interface = "eth1";
+              inherit bootstrapNodes;
+            };
+          };
+      };
+    };
+  };
+
+  nodes =
+    let
+      commonConfig =
+        { lib, config, ... }:
+        {
+          environment.systemPackages = [
+            config.services.data-mesher.package
+          ];
+
+          # speed up for testing
+          services.data-mesher.settings = {
+            cluster.join_interval = lib.mkForce "2s";
+            cluster.push_pull_interval = lib.mkForce "5s";
+          };
+
+        };
+    in
+    {
+      peer = commonConfig;
+      admin = commonConfig;
+      signer = commonConfig;
+    };
+
+  testScript = ''
+    def resolve(node, success = {}, fail = [], timeout = 60):
+      for hostname, ips in success.items():
+          for ip in ips:
+              node.wait_until_succeeds(f"getent ahosts {hostname} | grep {ip}", timeout)
+
+      for hostname in fail:
+          node.wait_until_fails(f"getent ahosts {hostname}")
+
+    start_all()
+
+    admin.wait_for_unit("data-mesher")
+    signer.wait_for_unit("data-mesher")
+    peer.wait_for_unit("data-mesher")
+
+    # check dns resolution
+    for node in [admin, signer, peer]:
+      resolve(node, {
+          "admin.foo": ["2001:db8:1::1", "192.168.1.1"],
+          "peer.foo": ["2001:db8:1::2", "192.168.1.2"],
+          "signer.foo": ["2001:db8:1::3", "192.168.1.3"]
+      })
+  '';
+}

clanServices/data-mesher/tests/vm/sops/machines/admin/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1r99qtxl0v86wg8ndcem87yk5wag5xcsk98ngaumqzww6t7pyms0q5cyl80",
+    "type": "age"
+  }
+]

clanServices/data-mesher/tests/vm/sops/machines/peer/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1hgjs2yqxhcxfgtvhydnfe5wzlagxw2dw4hu658e8neduy0lkye0skmjfc7",
+    "type": "age"
+  }
+]

clanServices/data-mesher/tests/vm/sops/machines/signer/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1k6h9mespmnr9zhtwwqlhnla80x5jhpd4c2p7hp0nfanr5tspup0s0rld2f",
+    "type": "age"
+  }
+]

clanServices/data-mesher/tests/vm/sops/secrets/admin-age.key/secret

@@ -0,0 +1,15 @@
+{
+	"data": "ENC[AES256_GCM,data:87WFWukgpTGlH67MTkHxzTosABK/6flJObt+u9UrGSOzBr1lx4V5IsMQ9HAM4jvLpveBNH4hlFDCxbD5666n2oYylGoyBph2vAg=,iv:GKLcU7Xqmb0ImvY7M71NddkOlUDSPa/fcXrXny2iZ1o=,tag:589QMSZeXdmTxRFtMFasZg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFaXlqZEU0eHRZZjBncDE1\nV2hzTGZiVy9rM0NnWjc1NlpHVVZEUFd5S2pJCmo3Nm11bGQyWWt1R2tHS2pOYlpn\nY3lGa0w3UFpDT1RLSDU4cnJ2YVBkSU0KLS0tIEJjZVc1YXJqcHczYSt6WjV3ai93\nakdPd3VHWkVnWkdhNCtZakp4VXhBUG8Kg3xd9w5oW3/q+s59LkDy5N+xmvuvHRmh\njUv6KFLaB81yv3kb7bzj8E3aMzX0x2fMIDZ3EoPVggqA/sCWQu0p5Q==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:45Z",
+		"mac": "ENC[AES256_GCM,data:IWKfE1Y6SNg/SK+OOAmra5SwqAUfhepCNPClWPDWpOyJDwXSpk/OKl7hi3KFfIZOGupaC0xV2tTni0Uj6IBwf8zW2Mb/b1T+fWkGiyafoKlucfNPXPCob/fyf4Ju4iD/u1mD5BYYYqNTNqJWE+MCyQigL0MPE4tXGEPDa7htM6w=,iv:5RKArbEKnYjacopfL+4QhzGB8txqc3gnlwNPfRWQSlM=,tag:mdXf02nYiW7CexIbUUaMyw==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/sops/secrets/peer-age.key/secret

@@ -0,0 +1,15 @@
+{
+	"data": "ENC[AES256_GCM,data:C9evAr01JpYiMBwuy31h+G9phm+uOYoQu+PegPFAMRbjgkjh0R+uolKtweedtHumMhzEkvz7y+BlfrriVh16ceyMozfzDEkVSWM=,iv:jM4Qx4B/j5Mvc3ybOf+10hKU19l1fCc5KcKulKgMP3c=,tag:mz01kIv5kU6u3f2+FeItYA==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAydzZrdDVidGpyd1NXT0Fu\nUEtZV3I4S0p5Z095QjBGaXpwOExJSkxVclVJCm54Vk12czQ5dm5TUExNNzlEcFNp\nUWorcWc1c1pvL3pkUFlQY3BJUGhUS3MKLS0tIHd2a291M0xkcjJvTXNnelRNZXda\nQi93R3FQVm0xTXBGR3E3SVpIMzgvR3MKmps5ObV1nODBQ0TKgZ++RLkjCEQM6sMn\nzonKtBingYzfeq+0+cASVkHZJpt/t0G5wmTgivKfv0OIP5eNSgIWFw==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:57Z",
+		"mac": "ENC[AES256_GCM,data:Jk5eL2SmNpakrGF4N/31Q/PWShV5KYfA8NmlxEkD82UsIpPiIJ4Nec6NOoo7Y4bl/J53MLjK3u0/S6q7vv0Tih6+ze6hIddMJHTCp2qqclJvpH2xn6Ln+2ZK4okK2ZbWeSDF+LHc6nIpBak8JVjC/d8dQFT2L49Dkufc1nCD46w=,iv:oR0aQzjaEpFNrpWGc1TX6/zpg0WSfQjVG6VjAMwoLTI=,tag:pigUaCkVv91tynuaNoZenA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/sops/secrets/signer-age.key/secret

@@ -0,0 +1,15 @@
+{
+	"data": "ENC[AES256_GCM,data:bIx3chjDwy4epCyFuJoZlO7EglT/vEg6pdf6x+ISxqekGrrGNdiGtw3Z9foXWAPQrzngVztbwIlcEpUusKwoRPpdGIj5YzbGZbU=,iv:Gi1hjn6cL8z+LP5g6o3bUMsuIzoZRr8e3j3EBwG3p+Y=,tag:ttIfOLhDroV/WK57KBFd0w==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHNVh6OGE4aGJxbFd2Zks1\nL1ZoNkgrQjFSVFFUL2UzOGNqRXFkZURTMkJRCnZMWk00enRndzNXQmFvMG1UekI0\nUjhwZW9sQnFvb0FGbVE0N042UjF2OTAKLS0tIEdickxQdDdaZkVmN3RsemJzSElY\nWThGQVNMcnpxRlJ3bC9wVE56blljQUUK21wWOBiQc0Kyvl047nJ1N6QKR0/5Dd6r\nlqhhdFWninzqfVXJUk2pcMio8RVlvBujDsyjrPuhbRceSi+bUXIn+w==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:03:08Z",
+		"mac": "ENC[AES256_GCM,data:kA2KCDZkZuR5rD7uU4xn5sIkizcnpGcoa3PYMbl73eux7JJYuSpUojFBRcYo1WCwMeOQUGsqo8LVF/rYhH4BVJ9LERs5zTLBaUsTarY8r/UK0Q5lNYZqIrqcb5LgOf1uCvfdXg5yfaFgPFJrEqjeekb9bx8xvhDZXpsND93rrUI=,iv:B6JqWWcQV/MxP4ucAIe7EnLiq9c4pnAUj3dnEp9IXJU=,tag:1i0Fv2i7Lak5JzIbPa2/cw==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/per-machine/admin/data-mesher-host-key/private_key/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:2FgvnmawAdk+/k+RVWNsKQlUFUF+pZrrEBuupdG50uLNyxHd7Gi772gKNgHWyzZ/lpODg5mQi0rL+GmZYQwtZ7h76AGUEeQvuMMTzVUop69txxwhJD2dxZyhUAxZpibwo/St84ai+8+VksLkCSYfTXCulaeOVh4=,iv:YkPNq4zDj35PRNgt2kHEkHhbLcVc9dHP/zrAwdd94sM=,tag:KwW/74C7Z/+3dNoXB3NHwQ==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoaS94M0JsR2Q5N21DNnFB\nUHgvelRTK3FKZkNKcTJFbEJ1VGFIM256MVVRCmw5YjdyTVlXMlFpWnczV2dTSzhu\nSm5mMVRPeU1pYVFZNEN5MjJFZHVTejgKLS0tIDB0V2hSRkt5QzFYald0TWVza1lC\ncGNXemhGcklENTJiV1QvTFZxUDNRRlUK2dVEzSbdDNXZy7rQi5/Vq4KyHq5rMtEz\npTI8i1rFKIAy4TC7to03bOIudOIzKSCCzX31xARkM6qON0vEU9aHFg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1r99qtxl0v86wg8ndcem87yk5wag5xcsk98ngaumqzww6t7pyms0q5cyl80",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEOEMzcExzTTF3MmpaenRN\ncS9RbnM0aStZSjNqbjF4QisrRjhoaDg1T0ZBCmFVOWJYZkFaOXBOUGJTdytYWk52\nVXV1MDdmSWQ1OS9iODAvN2c2Q3VGYXMKLS0tIEQxeWR4bmRoOWJ2Z1FyUk1PUk1n\nM0c5Ri9FdG9FNE9CZ29VSmgvN2xDdjgKjfG38gVOXXN2ftGiCPxMFbnh7lKM1USl\nqf11k+rgvR8M9XsDy2SnirKAaNmpks1dR6Zs5ppQuYJDEYyQCrEO5g==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:45Z",
+		"mac": "ENC[AES256_GCM,data:TEH57vUZ/swTsWQPJ1X3J//xa1Q1LYPETZS7fuXCH1LCK51u88XGqVpNzSETREQ8LAOt34qN284b03UQIBGTeTr7I9cqt+/l8ew/0rFTiO3aiaT49q9aBkeFZlA+gy47r4hkhMmzGQJMUenvnzTHwT3Pw2RES5Vjs/2TSitpqlA=,iv:ffIotRGKU8y6j/VDLKbTmA8dZJVP5vafeG4F3wd60tc=,tag:q4xOwzLw5jxDR0pPIy2irA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/per-machine/admin/data-mesher-host-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEAi6qF8u2uvPXlSflB4fzJNlOhj5PgAmRiv+JyyYOOgg4=
+-----END PUBLIC KEY-----

clanServices/data-mesher/tests/vm/vars/per-machine/peer/data-mesher-host-key/private_key/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:LUNuEP/xSmzJ44sheoIYN6F24Qpr3svn6rTVUpr4KZA8uVJ9gPUd4ko4+pDisc9PyXCcxx+cYGRqr1cBp8Q3R+IyFFlR2HzuReQJaScvgjlntGtMJ2hin/aBp4pHS0F4nqPcKKROiZvIN4NHsxQ6XRVDOZbI3kE=,iv:BdRHjQXJL/OGgmqWaEDLit/zHgduNfPe3GUmYDrWLPw=,tag:N0n7CCiu+COgrfrwHUwQBQ==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hgjs2yqxhcxfgtvhydnfe5wzlagxw2dw4hu658e8neduy0lkye0skmjfc7",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCYlhrU2c1NnIyTzlVRHEx\nQTFxOUY1OWJXcHl1OHpPdWN1ZGpQV0UvZ1NzCnlKbmx0bllWMTd1ZnIxUHY0ZUU0\nVG9Jb3grSEdWeVpwaHoyQUxvNERqT00KLS0tIGtwZm5aMU1DOUhJbVVpVzIxZFow\nNVEvMy91SEg3M094MEFBSkVMRkhKZmMKuUzbEITGkYS39G14JXbKWLjiQFd4SVft\nWH34B97TFhOqusVF3zHsSCMxm/0BMeBvLxO/3RmzlwBtgNiKOqLwtQ==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2SUhJQW5EN0VKVHpQdlZC\nYTczdVJiRFdFNGtURFc2SmxKWFFycjZkQUgwCnRBVkJvUytuUDlhVlhFYno2cnBR\nRUdjL0lab1MwZzhGTklyVWZDVFJmN3cKLS0tIFRjOC9DS3llWGZWMGI2aThVYTRu\nVEFhK2Y2YkRTZHEyMWV0Q05ISHdhVVUKo9bPdV1dUeIkm4gI0r9V/s1dAfJC+H5Z\nEIUdYA7fl3jRZ01cSZ0iYWlvdl2jj0XzKafZsEQU7rL0jg9zbA2s2g==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:59Z",
+		"mac": "ENC[AES256_GCM,data:+JtuPacwUMHXtp93DZmkiVne7bQUP8J7VpoS8koM0oJWJqZoQRHd9qH/04lrpp8q/YoOXtqXwhViZvFLieJVRexiXf/AAHfAfMn0EI7ois9oHhscN88Ps9nY6JUxhNd0h0OrUA58KKhrkGoqreAKAPADtVhaVCmWbU7vMUu1StE=,iv:BmJnTsgMSbl4XsBUkhSLfKd0XjhrEQfurEkaRJ6uD/g=,tag:jg21c4y4bQp0RwWTXkxF1A==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/per-machine/peer/data-mesher-host-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEA7kRKjQpj+BXPe5buvDZtBAcU1HIcfGmbuHZqaVm3zCo=
+-----END PUBLIC KEY-----

clanServices/data-mesher/tests/vm/vars/per-machine/signer/data-mesher-host-key/private_key/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:armAfuTE0mkoy1fxAysCX/UPNM4/mt9P6/zEDwtagTSvQjMTwVzzsM+kRdLOUV4fbZ7HdqMceaZWzurAQJenXvWlBXgn87YFOFBSpf3OnpEwCTUs9H8dsVrdSUk4SrKjCjV33mybTrae/h9tMHdkRhKJzPD1+/8=,iv:x9KVGqT2Ug6B6PNwzL7NVDQqyOmFUptUsHAJEdn30dg=,tag:XSSO6JvXaXq8aezYvpF65Q==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1k6h9mespmnr9zhtwwqlhnla80x5jhpd4c2p7hp0nfanr5tspup0s0rld2f",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIMVUwMEFzVjJhYXg5MXR4\nMzZPZUFrUWdEU2hPWUVDNHpVVENpdEdYSWtnCnN0R2pVdEIxYWZXYWNBb3N5bGNK\naVpWOXp5aWVJWG9vUWtMUnhYSmMyV0UKLS0tIEtMdFAybk1PN0t2M2lkaEYzUTY3\nVzVOdTBFbnlNVTAvRU5kU0dReEZ6MlUKNHIkAUUAqnuMtXbvXqLxQwuFALsnD/i0\naBCiz6J4S18uqt3kFbXAEksbD7jCexI8m5SMp4iuumWJ/Bx1lL4TWg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkbzBFSGt1dXI2bDN5TmFU\nY3N6djNmMTh2ek4vUzdHbTF6Z1hDQ2t5WVNJClEzZDZiaVpBekFrYTYweDNsNmk5\nTlhYZGRNd0llMndyMkZWMyt5N3pwTE0KLS0tIGJJbU9vbnBhSE5vRW1pRG83cEFJ\nR2xDTHk3VkJaVUZSVThRV3Jldkp6cnMK1V37txaSFYfLQM0qqRWjojyTN4fTJkRm\nGO3yHX9uwo/4D2xI7LM48n4vnNhSF05bWpq0X4r13fI4DofCJeEo1g==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:03:11Z",
+		"mac": "ENC[AES256_GCM,data:qD1w+DO8cWFDQMBOrmO9FvxvJRn+mlUbh13exTGgmsdPn3uzTXknIDDHeWfkpF699nSzS6wRmgrB21e55rBU6iHMx1TW16S8wvCoYMFwib8zTrJzND7EJr/gRwQa0N080kBY3xBivKLUFlctgKtFUYZ9GQ6UTQeq18QKPoROjww=,iv:1mt8Er6YHxQ42F5Kb+xNtjbCAzokbeoNlHesC9Uzmhk=,tag:provO4tKDzoL5PHDg5EmhA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/per-machine/signer/data-mesher-host-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEAVA6c25s+yNe5225PnELDV9FwbWi9ppLoTfgmdY8kILo=
+-----END PUBLIC KEY-----

clanServices/data-mesher/tests/vm/vars/shared/data-mesher-network-key/private_key/secret

@@ -0,0 +1,27 @@
+{
+	"data": "ENC[AES256_GCM,data:VzcB/JABSPoFdKYhRSn+nKxasn9zO/9fyNMrg3XstBelQNPpbO8mhmcnSamc/7e5GkpoVWgLRSULvosv+o6sz9EHRZ3UpSLBBTkDGAJmoBnkR8DbstPA9EgScpQ9IGOUP5tQ0oEOcJC3FrivdbWIzeXjpWb9BrU=,iv:6BNUrubJ9aNCkgonDRNgdyckCTndkPVDLE4X3J5d2zA=,tag:YqHTiGslEkslzUk24bmPZg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hgjs2yqxhcxfgtvhydnfe5wzlagxw2dw4hu658e8neduy0lkye0skmjfc7",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwK2lMUTkrSmM4dHQxU0tI\nRVV6Wm4wWlJMYXBGbGdubExrMi8vRnJjdVd3CjI3aFVpdHRURHp6UEk3ZEZMcDZT\nZWZWaGFWYmY2Mk1iQ1BjalZkUnpUUm8KLS0tIEhFUVhBUjg1dC9LWHg2TytkRTlX\nNnlJZkJQc2ExK1BwaVVFcEw2b3BLZjQK8kqf3ZP9uLtbjCJLSEYpAqgq9zOS2HrY\n5MbPAKQI8iCUfnegti6hU+/MxjvPlaX1vT4V0Kd3gT4Khjl+OPw0Og==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1r99qtxl0v86wg8ndcem87yk5wag5xcsk98ngaumqzww6t7pyms0q5cyl80",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWeUk3ZW9rdnZBTk9vQlFZ\nTzFZVDAvcXZyQjdkcGNNbnA0T3UyM3lzVERvCjFreE9RdWxnb2xWWmI4amJVdHBv\nNE9JN2tFazRnSGhiM0FId2RCUHNKWVEKLS0tIGlmM3JNSVZtR21ndFliUVpLTzJO\ncHJ2SjI1OExQK2hEN01WdG9wZ3RmVTAKi0BXp9yV2/9a9NeT7aTSK2CfkQ5yColJ\nm0+uv5AJndZ9IsaZGJxNOdAOspYdvsW38hFdfjUtVuUCyIOPc20WUg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1k6h9mespmnr9zhtwwqlhnla80x5jhpd4c2p7hp0nfanr5tspup0s0rld2f",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkSUR1QVMvZ3F0NUxXd00z\nOWJGZFlsUy8vUmMxa1NoakZRVmJrSmd1RzBrCk1ZcDlBMFB0WVdWeFZaT3ZBTTh5\nS2RReWpUOGRBdGV6MDdjcEY5dFYrdjAKLS0tIG9oRWhUaWJZSElRdmlOZmRKSnNq\nUUNDZFdZbmM0c25MOGpvem1JSm9pVWsKxCLPivdHc6IN6Jbf9FujLGJaXP6ieO1S\nKsrs3Fe0RdYcEKI7P9EQNebQD2kKXficM0kKV5lRRVtW5024PftWoQ==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3amkyWWlxSTJkZEdMZFhL\nU2t5OGFIa25TRmdFM0ZNcUhFRHk0eDJQN2tjCm9UcUs2V0lEZ0hyNU9uaDVrckpj\nZ1JSQlhNeExjOER2aFJTM2NDS25PN2MKLS0tIFhmM21rT0Z4aUI5TUZyNnNBQ3Jy\nSDAxejhhZDZNQTVCNjNUSTBsZncra1kKFFQrFxNMyg0AEMb1wpKBc7LOVtEHyFZW\n/o7L52fTNa0GFJ3SVEdqg0PpnRzTyA8F5L77FBGKtx6auCVVHyZZ9g==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:48Z",
+		"mac": "ENC[AES256_GCM,data:HooesDb1S24Cfb7H0lVTA8fAjM2QAN9MaJFvOSHniR6ICJAX8t8X0xfWIFRFuwPjAxi4kpBYSjW0420Yz9lZ2m4Fxswo1TV3lzHDVN2u9hdrsfpKXg5fW+2oZihuvCRStDagT3l2fKv+C+gBnGs1qyCM60BStvrEiQxTxTTHfho=,iv:kL8N0qBj4q+ZJbNJ8Y8RcV1KpUUMvNCpdwKbTPGpG6k=,tag:o2PmRsSkqTP5Idq7veGDOw==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/shared/data-mesher-network-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEA/MuamRX6ZLcJunm7lZvlai0OZh++YuqMa56GiTwO68A=
+-----END PUBLIC KEY-----

clanServices/emergency-access/default.nix

@@ -4,6 +4,7 @@
   manifest.name = "clan-core/emergency-access";
   manifest.description = "Set recovery password for emergency access to machine";
   manifest.categories = [ "System" ];
+  manifest.readme = builtins.readFile ./README.md;
 
   roles.default.perInstance = {
     nixosModule =

clanServices/hello-world/flake-module.nix

@@ -23,7 +23,13 @@ in
       unit-test-module = (
         self.clanLib.test.flakeModules.makeEvalChecks {
           inherit module;
-          inherit self inputs;
+          inherit inputs;
+          fileset = lib.fileset.unions [
+            # The hello-world service being tested
+            ../../clanServices/hello-world
+            # Required modules
+            ../../nixosModules/clanCore
+          ];
           testName = "hello-world";
           tests = ./tests/eval-tests.nix;
           # Optional arguments passed to the test

clanServices/importer/README.md

@@ -1,7 +1,7 @@
 The importer module allows users to configure importing modules in a flexible and structured way.
 It exposes the `extraModules` functionality of the inventory, without any added configuration.
 
-## Usage:
+## Usage
 
 ```nix
 inventory.instances = {

clanServices/sshd/README.md

@@ -0,0 +1,36 @@
+The `sshd` Clan service manages SSH to make it easy to securely access your machines over the internet. The service uses `vars` to store the SSH host keys for each machine to ensure they remain stable across deployments.
+
+`sshd` also generates SSH certificates for both servers and clients allowing for certificate-based authentication for SSH.
+
+The service also disables password-based authentication over SSH, to access your machines you'll need to use public key authentication or certificate-based authentication.
+
+## Usage
+
+```nix
+{
+  inventory.instances = {
+    # By default this service only generates ed25519 host keys
+    sshd-basic = {
+      module = {
+        name = "sshd";
+        input = "clan-core";
+      };
+      roles.server.tags.all = { };
+      roles.client.tags.all = { };
+    };
+
+    # Also generate RSA host keys for all servers
+    sshd-with-rsa = {
+      module = {
+        name = "sshd";
+        input = "clan-core";
+      };
+      roles.server.tags.all = { };
+      roles.server.settings = {
+        hostKeys.rsa.enable = true;
+      };
+      roles.client.tags.all = { };
+    };
+  };
+}
+```

clanServices/sshd/default.nix

@@ -2,11 +2,12 @@
 {
   _class = "clan.service";
   manifest.name = "clan-core/sshd";
-  manifest.description = "Enables secure remote access to the machine over ssh.";
+  manifest.description = "Enables secure remote access to the machine over SSH";
   manifest.categories = [
     "System"
     "Network"
   ];
+  manifest.readme = builtins.readFile ./README.md;
 
   roles.client = {
     interface =
@@ -49,7 +50,7 @@
                 pkgs.openssh
               ];
               script = ''
-                ssh-keygen -t ed25519 -N "" -f "$out"/id_ed25519
+                ssh-keygen -t ed25519 -N "" -C "" -f "$out"/id_ed25519
               '';
             };
 
@@ -109,7 +110,7 @@
                   pkgs.openssh
                 ];
                 script = ''
-                  ssh-keygen -t ed25519 -N "" -f "$out"/id_ed25519
+                  ssh-keygen -t ed25519 -N "" -C "" -f "$out"/id_ed25519
                 '';
               };
 
@@ -151,7 +152,7 @@
                   pkgs.openssh
                 ];
                 script = ''
-                  ssh-keygen -t rsa -b 4096 -N "" -f "$out"/ssh.id_rsa
+                  ssh-keygen -t rsa -b 4096 -N "" -C "" -f "$out"/ssh.id_rsa
                 '';
               };
 
@@ -164,7 +165,7 @@
                   pkgs.openssh
                 ];
                 script = ''
-                  ssh-keygen -t ed25519 -N "" -f "$out"/ssh.id_ed25519
+                  ssh-keygen -t ed25519 -N "" -C "" -f "$out"/ssh.id_ed25519
                 '';
               };
             };

clanServices/state-version/default.nix

@@ -4,6 +4,7 @@
   manifest.name = "clan-core/state-version";
   manifest.description = "Automatically generate the state version of the nixos installation.";
   manifest.categories = [ "System" ];
+  manifest.readme = builtins.readFile ./README.md;
 
   roles.default = {
 
@@ -20,6 +21,16 @@
             var = config.clan.core.vars.generators.state-version.files.version or { };
           in
           {
+
+            warnings = [
+              ''
+                The clan.state-version service is deprecated and will be
+                removed on 2025-07-15 in favor of a nix option.
+
+                Please migrate your configuration to use `clan.core.settings.state-version.enable = true` instead.
+              ''
+            ];
+
             system.stateVersion = lib.mkDefault (lib.removeSuffix "\n" var.value);
 
             clan.core.vars.generators.state-version = {

clanServices/state-version/tests/vm/default.nix

@@ -1,5 +1,6 @@
+{ lib, ... }:
 {
-  name = "state-version";
+  name = "service-state-version";
 
   clan = {
     directory = ./.;
@@ -15,7 +16,7 @@
 
   nodes.server = { };
 
-  testScript = ''
+  testScript = lib.mkDefault ''
     start_all()
   '';
 }

clanServices/trusted-nix-caches/default.nix

@@ -4,6 +4,7 @@
   manifest.name = "clan-core/trusted-nix-caches";
   manifest.description = "This module sets the `clan.lol` and `nix-community` cache up as a trusted cache.";
   manifest.categories = [ "System" ];
+  manifest.readme = builtins.readFile ./README.md;
 
   roles.default = {
 

clanServices/users/README.md

@@ -1,30 +1,31 @@
 ## Usage
 
-```
-inventory.instances = {
-
-  # Deploy user alice on all machines. Don't prompt for password (will be
-  # auto-generated).
-
-  user-alice = {
-    module = {
-      name = "users";
-      input = "clan";
+```nix
+{
+  inventory.instances = {
+    # Deploy user alice on all machines. Don't prompt for password (will be
+    # auto-generated).
+    user-alice = {
+      module = {
+        name = "users";
+        input = "clan";
+      };
+      roles.default.tags.all = { };
+      roles.default.settings = {
+        user = "alice";
+        prompt = false;
+      };
     };
-    roles.default.tags.all = { };
-    roles.default.settings = {
-      user = "alice";
-      prompt = false;
+
+    # Deploy user bob only on his laptop. Prompt for a password.
+    user-bob = {
+      module = {
+        name = "users";
+        input = "clan";
+      };
+      roles.default.machines.bobs-laptop = { };
+      roles.default.settings.user = "bob";
     };
   };
-
-  # Deploy user bob only on his laptop. Prompt for a password.
-  user-bob = {
-    module = {
-      name = "users";
-      input = "clan";
-    };
-    roles.default.machines.bobs-laptop = { };
-    roles.default.settings.user = "bob";
-  };
+}
 ```

clanServices/users/default.nix

@@ -1,9 +1,13 @@
 { ... }:
 {
   _class = "clan.service";
-  manifest.name = "clan-core/users";
-  manifest.description = "Automatically generates and configures a password for the specified user account.";
+  manifest.name = "clan-core/user";
+  manifest.description = ''
+    An instance of this module will create a user account on the added machines,
+    along with a generated password that is constant across machines and user settings.
+  '';
   manifest.categories = [ "System" ];
+  manifest.readme = builtins.readFile ./README.md;
 
   roles.default = {
     interface =
@@ -19,7 +23,57 @@
             type = lib.types.bool;
             default = true;
             example = false;
-            description = "Whether the user should be prompted.";
+            description = ''
+              Whether the user should be prompted for a password.
+
+              Effects:
+
+              - *enabled* (`true`) - Prompt for a passwort during the machine installation or update workflow.
+              - *disabled* (`false`) - Generate a passwort during the machine installation or update workflow.
+
+              The password can be shown in two steps:
+
+              - `clan vars list <machine-name>`
+              - `clan vars get <machine-name> <name-of-password-variable>`
+            '';
+          };
+          regularUser = lib.mkOption {
+            type = lib.types.bool;
+            default = true;
+            example = false;
+            description = ''
+              Whether the user should be a regular user or a system user.
+
+              Regular users are normal users that can log in and have a home directory.
+
+              System users are used for system services and do not have a home directory.
+
+              !!! Warning
+                  `root` cannot be a regular user.
+                  You must set this to `false` for `root`
+            '';
+          };
+          groups = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [ ];
+            example = [
+              "wheel"
+              "networkmanager"
+              "video"
+              "input"
+            ];
+            description = ''
+              Additional groups the user should be added to.
+              You can add any group that exists on your system.
+              Make sure these group exists on all machines where the user is enabled.
+
+              Commonly used groups:
+
+              - "wheel" - Allows the user to run commands as root using `sudo`.
+              - "networkmanager" - Allows the user to manage network connections.
+              - "video" - Allows the user to access video devices.
+              - "input" - Allows the user to access input devices.
+            '';
           };
         };
       };
@@ -35,9 +89,13 @@
             ...
           }:
           {
-            users.mutableUsers = false;
-            users.users.${settings.user}.hashedPasswordFile =
-              config.clan.core.vars.generators."user-password-${settings.user}".files.user-password-hash.path;
+            users.users.${settings.user} = {
+              isNormalUser = settings.regularUser;
+              extraGroups = settings.groups;
+
+              hashedPasswordFile =
+                config.clan.core.vars.generators."user-password-${settings.user}".files.user-password-hash.path;
+            };
 
             clan.core.vars.generators."user-password-${settings.user}" = {
 
@@ -80,4 +138,11 @@
           };
       };
   };
+
+  perMachine = {
+    nixosModule = {
+      # Immutable users to ensure that this module has exclusive control over the users.
+      users.mutableUsers = false;
+    };
+  };
 }

clanServices/users/tests/vm/default.nix

@@ -13,6 +13,8 @@
           roles.default.machines."server".settings = {
             user = "root";
             prompt = false;
+            # Important: 'root' must not be a regular user. See: https://github.com/NixOS/nixpkgs/issues/424404
+            regularUser = false;
           };
         };
         user-password-test = {
@@ -31,7 +33,6 @@
     server = {
       users.users.testuser.group = "testuser";
       users.groups.testuser = { };
-      users.users.testuser.isNormalUser = true;
     };
   };
 

clanServices/wifi/default.nix

@@ -73,9 +73,10 @@ in
             ];
 
             networking.networkmanager.ensureProfiles.profiles = flip mapAttrs settings.networks (
-              name: _network: {
+              name: networkCfg: {
                 connection.id = "$ssid_${name}";
                 connection.type = "wifi";
+                connection.autoconnect = networkCfg.autoConnect;
                 wifi.mode = "infrastructure";
                 wifi.ssid = "$ssid_${name}";
                 wifi-security.psk = "$pw_${name}";
@@ -102,7 +103,7 @@ in
                   # Generate the secrets file
                   echo "Generating wifi secrets file: $env_file"
                   ${flip (concatMapAttrsStringSep "\n") settings.networks (
-                    name: _network: ''
+                    name: _networkCfg: ''
                       echo "ssid_${name}=\"$(cat "${ssid_path name}")\"" >> /run/secrets/NetworkManager/wifi-secrets
                       echo "pw_${name}=\"$(cat "${password_path name}")\"" >> /run/secrets/NetworkManager/wifi-secrets
                     ''

clanServices/zerotier/flake-module.nix

@@ -15,7 +15,15 @@ in
       unit-test-module = (
         self.clanLib.test.flakeModules.makeEvalChecks {
           inherit module;
-          inherit self inputs;
+          inherit inputs;
+          fileset = lib.fileset.unions [
+            # The zerotier service being tested
+            ../../clanServices/zerotier
+            # Required modules
+            ../../nixosModules/clanCore
+            # Dependencies like clan-cli
+            ../../pkgs/clan-cli
+          ];
           testName = "zerotier";
           tests = ./tests/eval-tests.nix;
           testArgs = { };

devFlake/private.narHash

@@ -0,0 +1 @@
+sha256-LdjcFZLL8WNldUO2LbdqFlss/ERiGeXVqMee0IxV2z0=

devFlake/private/flake.lock

@@ -0,0 +1,165 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": [
+          "systems"
+        ]
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "ixx": {
+      "inputs": {
+        "flake-utils": [
+          "nuschtos",
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "nuschtos",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1748294338,
+        "narHash": "sha256-FVO01jdmUNArzBS7NmaktLdGA5qA3lUMJ4B7a05Iynw=",
+        "owner": "NuschtOS",
+        "repo": "ixx",
+        "rev": "cc5f390f7caf265461d4aab37e98d2292ebbdb85",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NuschtOS",
+        "ref": "v0.0.8",
+        "repo": "ixx",
+        "type": "github"
+      }
+    },
+    "nixpkgs-dev": {
+      "locked": {
+        "lastModified": 1752039390,
+        "narHash": "sha256-DTHMN6kh1cGoc5hc9O0pYN+VAOnjsyy0wxq4YO5ZRvg=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "6ec4d5f023c3c000cda569255a3486e8710c39bf",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable-small",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nuschtos": {
+      "inputs": {
+        "flake-utils": "flake-utils_2",
+        "ixx": "ixx",
+        "nixpkgs": [
+          "nixpkgs-dev"
+        ]
+      },
+      "locked": {
+        "lastModified": 1749730855,
+        "narHash": "sha256-L3x2nSlFkXkM6tQPLJP3oCBMIsRifhIDPMQQdHO5xWo=",
+        "owner": "NuschtOS",
+        "repo": "search",
+        "rev": "8dfe5879dd009ff4742b668d9c699bc4b9761742",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NuschtOS",
+        "repo": "search",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs-dev": "nixpkgs-dev",
+        "nuschtos": "nuschtos",
+        "systems": "systems_2",
+        "treefmt-nix": "treefmt-nix"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": []
+      },
+      "locked": {
+        "lastModified": 1752055615,
+        "narHash": "sha256-19m7P4O/Aw/6+CzncWMAJu89JaKeMh3aMle1CNQSIwM=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "c9d477b5d5bd7f26adddd3f96cfd6a904768d4f9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

devFlake/private/flake.nix

@@ -0,0 +1,19 @@
+{
+  description = "private dev inputs";
+
+  # Dev dependencies
+  inputs.nixpkgs-dev.url = "github:NixOS/nixpkgs/nixos-unstable-small";
+
+  inputs.flake-utils.url = "github:numtide/flake-utils";
+  inputs.flake-utils.inputs.systems.follows = "systems";
+
+  inputs.nuschtos.url = "github:NuschtOS/search";
+  inputs.nuschtos.inputs.nixpkgs.follows = "nixpkgs-dev";
+
+  inputs.treefmt-nix.url = "github:numtide/treefmt-nix";
+  inputs.treefmt-nix.inputs.nixpkgs.follows = "";
+
+  inputs.systems.url = "github:nix-systems/default";
+
+  outputs = _: { };
+}

devFlake/update-private-narhash

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Used to update the private dev flake hash reference.
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+echo "Updating $PWD/private.narHash" >&2
+
+nix --extra-experimental-features 'flakes nix-command' flake lock ./private
+nix --extra-experimental-features 'flakes nix-command' hash path ./private >./private.narHash
+
+echo OK

docs/.gitignore

@@ -1,4 +1,5 @@
 /site/reference
 /site/static
 /site/options-page
+/site/openapi.json
 !/site/static/extra.css

docs/mkdocs.yml

@@ -48,13 +48,13 @@ nav:
   - Home: index.md
   - Guides:
       - Getting Started:
-          - Creating Your First Clan: guides/getting-started/index.md
-          - Create USB Installer (optional): guides/getting-started/installer.md
-          - Add Machines: guides/getting-started/add-machines.md
-          - Add Services: guides/getting-started/add-services.md
-          - Secrets & Facts: guides/getting-started/secrets.md
-          - Deploy Machine: guides/getting-started/deploy.md
-          - Continuous Integration: guides/getting-started/check.md
+          - 🚀 Creating Your First Clan: guides/getting-started/index.md
+          - 📀 Create USB Installer (optional): guides/getting-started/installer.md
+          - ⚙️ Add Machines: guides/getting-started/add-machines.md
+          - ⚙️ Add Services: guides/getting-started/add-services.md
+          - 🔐 Secrets & Facts: guides/getting-started/secrets.md
+          - 🚢 Deploy Machine: guides/getting-started/deploy.md
+          - 🧪 Continuous Integration: guides/getting-started/check.md
       - clanServices: guides/clanServices.md
       - Disk Encryption: guides/disk-encryption.md
       - Mesh VPN: guides/mesh-vpn.md
@@ -62,6 +62,7 @@ nav:
       - Vars Backend: guides/vars-backend.md
       - Facts Backend: guides/secrets.md
       - Adding more machines: guides/more-machines.md
+      - Target Host: guides/target-host.md
       - Inventory:
           - Inventory: guides/inventory.md
       - Secure Boot: guides/secure-boot.md
@@ -78,6 +79,7 @@ nav:
           - Migrate existing Flakes: guides/migrations/migration-guide.md
           - Migrate inventory Services: guides/migrations/migrate-inventory-services.md
           - Facts Vars Migration: guides/migrations/migration-facts-vars.md
+          - Disk id: guides/migrations/disk-id.md
       - macOS: guides/macos.md
   - Reference:
       - Overview: reference/index.md
@@ -85,6 +87,7 @@ nav:
           - Overview: reference/clanServices/index.md
           - reference/clanServices/admin.md
           - reference/clanServices/borgbackup.md
+          - reference/clanServices/data-mesher.md
           - reference/clanServices/emergency-access.md
           - reference/clanServices/garage.md
           - reference/clanServices/hello-world.md
@@ -154,6 +157,7 @@ nav:
           - reference/cli/show.md
           - reference/cli/ssh.md
           - reference/cli/state.md
+          - reference/cli/templates.md
           - reference/cli/vars.md
           - reference/cli/vms.md
       - NixOS Modules:
@@ -181,6 +185,9 @@ nav:
           - 05-deployment-parameters: decisions/05-deployment-parameters.md
           - Template: decisions/_template.md
   - Options: options.md
+  - Developer:
+      - Introduction: intern/index.md
+      - API: intern/api.md
 
 docs_dir: site
 site_dir: out
@@ -194,7 +201,6 @@ theme:
     - navigation.instant
     - navigation.tabs
     - navigation.tabs.sticky
-    - navigation.footer
     - content.code.annotate
     - content.code.copy
     - content.tabs.link
@@ -238,3 +244,4 @@ extra:
 plugins:
   - search
   - macros
+  - redoc-tag

docs/nix/default.nix

@@ -1,8 +1,8 @@
 {
-  clan-core,
   pkgs,
   module-docs,
   clan-cli-docs,
+  clan-lib-openapi,
   asciinema-player-js,
   asciinema-player-css,
   roboto,
@@ -18,7 +18,17 @@ pkgs.stdenv.mkDerivation {
 
   # Points to repository root.
   # so that we can access directories outside of docs to include code snippets
-  src = clan-core;
+  src = pkgs.lib.fileset.toSource {
+    root = ../..;
+    fileset = pkgs.lib.fileset.unions [
+      # Docs directory
+      ../../docs
+      # Icons needed for the build
+      ../../pkgs/clan-app/ui/icons
+      # Any other directories that might be referenced for code snippets
+      # Add them here as needed based on what mkdocs actually uses
+    ];
+  };
 
   nativeBuildInputs =
     [
@@ -29,6 +39,7 @@ pkgs.stdenv.mkDerivation {
       mkdocs
       mkdocs-material
       mkdocs-macros
+      mkdocs-redoc-tag
     ]);
   configurePhase = ''
     pushd docs
@@ -36,6 +47,10 @@ pkgs.stdenv.mkDerivation {
     mkdir -p ./site/reference/cli
     cp -af ${module-docs}/* ./site/reference/
     cp -af ${clan-cli-docs}/* ./site/reference/cli/
+
+    mkdir -p ./site/reference/internal
+    cp -af ${clan-lib-openapi} ./site/openapi.json
+
     chmod -R +w ./site/reference
     echo "Generated API documentation in './site/reference/' "
 

docs/nix/flake-module.nix

@@ -29,7 +29,10 @@
       # Frontmatter for clanModules
       clanModulesFrontmatter =
         let
-          docs = pkgs.nixosOptionsDoc { options = self.clanLib.modules.frontmatterOptions; };
+          docs = pkgs.nixosOptionsDoc {
+            options = self.clanLib.modules.frontmatterOptions;
+            transformOptions = self.clanLib.docs.stripStorePathsFromDeclarations;
+          };
         in
         docs.optionsJSON;
 
@@ -82,10 +85,9 @@
           }
           ''
             export CLAN_CORE_PATH=${
-              self.filter {
-                include = [
-                  "clanModules"
-                ];
+              inputs.nixpkgs.lib.fileset.toSource {
+                root = ../..;
+                fileset = ../../clanModules;
               }
             }
             export CLAN_CORE_DOCS=${jsonDocs.clanCore}/share/doc/nixos/options.json
@@ -126,8 +128,12 @@
       });
       packages = {
         docs = pkgs.python3.pkgs.callPackage ./default.nix {
-          clan-core = self;
-          inherit (self'.packages) clan-cli-docs docs-options inventory-api-docs;
+          inherit (self'.packages)
+            clan-cli-docs
+            docs-options
+            inventory-api-docs
+            clan-lib-openapi
+            ;
           inherit (inputs) nixpkgs;
           inherit module-docs;
           inherit asciinema-player-js;

docs/nix/get-module-docs.nix

@@ -7,6 +7,10 @@
   pkgs,
   clan-core,
 }:
+let
+  inherit (clan-core.clanLib.docs) stripStorePathsFromDeclarations;
+  transformOptions = stripStorePathsFromDeclarations;
+in
 {
   # clanModules docs
   clanModulesViaNix = lib.mapAttrs (
@@ -20,6 +24,7 @@
           }).options
           ).clan.${name} or { };
         warningsAreErrors = true;
+        inherit transformOptions;
       }).optionsJSON
     else
       { }
@@ -32,6 +37,7 @@
       (nixosOptionsDoc {
         inherit options;
         warningsAreErrors = true;
+        inherit transformOptions;
       }).optionsJSON
     ) rolesOptions
   ) modulesRolesOptions;
@@ -52,7 +58,15 @@
 
         (nixosOptionsDoc {
           transformOptions =
-            opt: if lib.strings.hasPrefix "_" opt.name then opt // { visible = false; } else opt;
+            opt:
+            let
+              # Apply store path stripping first
+              transformed = transformOptions opt;
+            in
+            if lib.strings.hasPrefix "_" transformed.name then
+              transformed // { visible = false; }
+            else
+              transformed;
           options = (lib.evalModules { modules = [ role.interface ]; }).options;
           warningsAreErrors = true;
         }).optionsJSON
@@ -72,5 +86,6 @@
         }).options
         ).clan.core or { };
       warningsAreErrors = true;
+      inherit transformOptions;
     }).optionsJSON;
 }

docs/nix/options/flake-module.nix

@@ -1,9 +1,15 @@
-{ self, config, ... }:
+{
+  self,
+  config,
+  inputs,
+  privateInputs ? { },
+  ...
+}:
 {
   perSystem =
     {
-      inputs',
       lib,
+      pkgs,
       ...
     }:
     let
@@ -157,11 +163,16 @@
       };
     in
     {
-      packages.docs-options = inputs'.nuschtos.packages.mkMultiSearch {
-        inherit baseHref;
-        title = "Clan Options";
-        # scopes = mapAttrsToList mkScope serviceModules;
-        scopes = [ (mkScope "Clan Inventory" serviceModules) ];
+      packages = lib.optionalAttrs ((privateInputs ? nuschtos) || (inputs ? nuschtos)) {
+        docs-options =
+          (privateInputs.nuschtos or inputs.nuschtos)
+          .packages.${pkgs.stdenv.hostPlatform.system}.mkMultiSearch
+            {
+              inherit baseHref;
+              title = "Clan Options";
+              # scopes = mapAttrsToList mkScope serviceModules;
+              scopes = [ (mkScope "Clan Inventory" serviceModules) ];
+            };
       };
     };
 }

docs/nix/render_options/__init__.py

@@ -62,14 +62,11 @@ def sanitize(text: str) -> str:
     return text.replace(">", "\\>")
 
 
-def replace_store_path(text: str) -> tuple[str, str]:
+def replace_git_url(text: str) -> tuple[str, str]:
     res = text
-    if text.startswith("/nix/store/"):
-        res = "https://git.clan.lol/clan/clan-core/src/branch/main/" + str(
-            Path(*Path(text).parts[4:])
-        )
-    # name = Path(res).name
-    name = str(Path(*Path(text).parts[4:]))
+    name = Path(res).name
+    if text.startswith("https://git.clan.lol/clan/clan-core/src/branch/main/"):
+        name = str(Path(*Path(text).parts[7:]))
     return (res, name)
 
 
@@ -159,7 +156,7 @@ def render_option(
 
     decls = option.get("declarations", [])
     if decls:
-        source_path, name = replace_store_path(decls[0])
+        source_path, name = replace_git_url(decls[0])
 
         name = name.split(",")[0]
         source_path = source_path.split(",")[0]
@@ -452,7 +449,6 @@ Each `clanService`:
 * Is a module of class **`clan.service`**
 * Can define **roles** (e.g., `client`, `server`)
 * Uses **`inventory.instances`** to configure where and how it is deployed
-* Replaces the legacy `clanModules` and `inventory.services` system altogether
 
 !!! Note
     `clanServices` are part of Clan's next-generation service model and are intended to replace `clanModules`.

docs/site/decisions/04-fetching-nix-from-python.md

@@ -28,7 +28,7 @@ Benefits:
 * Caching mechanism is very simple.
 
 
-### Method 2: Direct access:
+### Method 2: Direct access
 
 Directly calling the evaluator / build sandbox via `nix build` and `nix eval`within the Python code
 

docs/site/guides/authoring/clanModules/index.md

@@ -52,6 +52,7 @@ clanModules/borgbackup
 
     ```nix title="flake.nix"
     # ...
+    # Sometimes this attribute set is defined in clan.nix
     clan-core.lib.clan {
         # 1. Add the module to the available clanModules with inventory support
         inventory.modules = {
@@ -175,6 +176,7 @@ The following shows how to add options to your module.
     Configuration can be set as follows.
 
     ```nix title="flake.nix"
+    # Sometimes this attribute set is defined in clan.nix
     clan-core.lib.clan {
         inventory.services = {
             custom-module.instance_1 = {

docs/site/guides/authoring/clanServices/index.md

@@ -27,6 +27,7 @@ i.e. `@hsjobeki/customNetworking`
 outputs = inputs: inputs.flake-parts.lib.mkFlake { inherit inputs; } ({
     imports = [ inputs.clan-core.flakeModules.default ];
     # ...
+    # Sometimes this attribute set is defined in clan.nix
     clan = {
         # If needed: Exporting the module for other people
         modules."@hsjobeki/customNetworking" = import ./service-modules/networking.nix;
@@ -218,6 +219,7 @@ To import the module use `importApply`
 outputs = inputs: flake-parts.lib.mkFlake { inherit inputs; } ({self, lib, ...}: {
     imports = [ inputs.clan-core.flakeModules.default ];
     # ...
+    # Sometimes this attribute set is defined in clan.nix
     clan = {
         # Register the module
         modules."@hsjobeki/messaging" = lib.importApply ./service-modules/messaging.nix { inherit self; };
@@ -244,6 +246,7 @@ Then wrap the module and forward the variable `self` from the outer context into
 outputs = inputs: flake-parts.lib.mkFlake { inherit inputs; } ({self, lib, ...}: {
     imports = [ inputs.clan-core.flakeModules.default ];
     # ...
+    # Sometimes this attribute set is defined in clan.nix
     clan = {
         # Register the module
         modules."@hsjobeki/messaging" = {

docs/site/guides/clanServices.md

@@ -17,8 +17,10 @@ For example:
 ```nix
 inventory.instances = {
   borgbackup = {
-    roles.client.machines = [ "laptop" "server1" ];
-    roles.server.machines = [ "backup-box" ];
+    roles.client.machines."laptop" = {};
+    roles.client.machines."server1" = {};
+
+    roles.server.machines."backup-box" = {};
   };
 }
 ```
@@ -40,7 +42,8 @@ Example of instantiating a `borgbackup` service using `clan-core`:
 ```nix
 inventory.instances = {
     # Instance Name: Different name for this 'borgbackup' instance
-    borgbackup-example = {
+    borgbackup = {
+        # Since this is instances."borgbackup" the whole `module = { ... }` below is equivalent and optional.
         module =  {
             name = "borgbackup";  # <-- Name of the module (optional)
             input = "clan-core"; # <-- The flake input where the service is defined (optional)