pkgs/clan(templates): Add shell completions

NormalUsers get:
- Home directory
- Can login

This is expected for users created through this module. We can make it configurable if the use arises

checks/data-mesher/default.nix

@@ -1,89 +0,0 @@
-{
-  pkgs,
-  nixosLib,
-  clan-core,
-  lib,
-  ...
-}:
-let
-  machines = [
-    "admin"
-    "peer"
-    "signer"
-  ];
-in
-nixosLib.runTest (
-  { ... }:
-  {
-    imports = [
-      clan-core.modules.nixosTest.clanTest
-    ];
-
-    hostPkgs = pkgs;
-    name = "service-data-mesher";
-
-    clan = {
-      directory = ./.;
-      inventory = {
-        machines = lib.genAttrs machines (_: { });
-        services = {
-          data-mesher.default = {
-            roles.peer.machines = [ "peer" ];
-            roles.admin.machines = [ "admin" ];
-            roles.signer.machines = [ "signer" ];
-          };
-        };
-      };
-    };
-
-    defaults =
-      { config, ... }:
-      {
-        environment.systemPackages = [
-          config.services.data-mesher.package
-        ];
-
-        clan.data-mesher.network.interface = "eth1";
-        clan.data-mesher.bootstrapNodes = [
-          "[2001:db8:1::1]:7946" # peer1
-          "[2001:db8:1::2]:7946" # peer2
-        ];
-
-        # speed up for testing
-        services.data-mesher.settings = {
-          cluster.join_interval = lib.mkForce "2s";
-          cluster.push_pull_interval = lib.mkForce "5s";
-        };
-      };
-
-    nodes = {
-      admin.clan.data-mesher.network.tld = "foo";
-    };
-
-    # TODO Add better test script.
-    testScript = ''
-
-      def resolve(node, success = {}, fail = [], timeout = 60):
-        for hostname, ips in success.items():
-            for ip in ips:
-                node.wait_until_succeeds(f"getent ahosts {hostname} | grep {ip}", timeout)
-
-        for hostname in fail:
-            node.wait_until_fails(f"getent ahosts {hostname}")
-
-      start_all()
-
-      admin.wait_for_unit("data-mesher")
-      signer.wait_for_unit("data-mesher")
-      peer.wait_for_unit("data-mesher")
-
-      # check dns resolution
-      for node in [admin, signer, peer]:
-        resolve(node, {
-            "admin.foo": ["2001:db8:1::1", "192.168.1.1"],
-            "peer.foo": ["2001:db8:1::2", "192.168.1.2"],
-            "signer.foo": ["2001:db8:1::3", "192.168.1.3"]
-        })
-    '';
-  }
-)

checks/data-mesher/sops/machines/admin/key.json

@@ -1,4 +0,0 @@
-{
-  "publickey": "age10zxkj45fah3qa8uyg3a36jsd06d839xfq64nrez9etrsf4km0gtsp45gsz",
-  "type": "age"
-}

checks/data-mesher/sops/machines/peer/key.json

@@ -1,4 +0,0 @@
-{
-  "publickey": "age1faqrml2ukc6unfm75d3v2vnaf62v92rdxaagg3ty3cfna7vt99gqlzs43l",
-  "type": "age"
-}

checks/data-mesher/sops/machines/signer/key.json

@@ -1,4 +0,0 @@
-{
-  "publickey": "age153mke8v2qksyqjc7vta7wglzdqr5epazt83nch0ur5v7kl87cfdsr07qld",
-  "type": "age"
-}

checks/data-mesher/sops/secrets/admin-age.key/secret

@@ -1,20 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:7xyb6WoaN7uRWEO8QRkBw7iytP5hFrA94VRi+sy/UhzqT9AyDPmxB/F8ASFsBbzJUwi0Oqd2E1CeIYRoDhG7JHnDyL2bYonz2RQ=,iv:slh3x774m6oTHAXFwcen1qF+jEchOKCyNsJMbNhqXHE=,tag:wtK8H8PZCESPA1vZCd7Ptw==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPTzZ4RTVNb2I1MTBRMEcy\neU1Eek9GakkydEJBVm9kR3AyY1pEYkorNUYwCkh2WHhNQmc1eWI2cCtEUFFWdzJq\nS0FvQWtoOFkzRVBxVzhuczc0aVprbkkKLS0tIFRLdmpnbzY1Uk9LdklEWnQzZHM2\nVEx3dzhMSnMwaWE0V0J6VTZ5ZVFYMjgKdaICa/hprHxhH89XD7ri0vyTT4rM+Si0\niHcQU4x64dgoJa4gKxgr4k9XncjoNEjJhxL7i/ZNZ5deaaLRn5rKMg==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:24:55Z",
-		"mac": "ENC[AES256_GCM,data:TJWDHGSRBfOCW8Q+t3YxG3vlpf9a5u7B27AamnOk95huqIv0htqWV3RuV7NoOZ5v2ijqSe/pLfpwrmtdhO2sUBEvhdhJm8UzLShP7AbH9lxV+icJOsY7VSrp+R5W526V46ONP6p47b7fOQBbp03BMz01G191N68WYOf6k2arGxU=,iv:nEyTBwJ2EA+OAl8Ulo5cvFX6Ow2FwzTWooF/rdkPiXg=,tag:oYcG16zR+Fb5XzVsHhq2Qw==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/sops/secrets/peer-age.key/secret

@@ -1,20 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:JOOhvl0clDD/b5YO45CXR3wVopBSNe9dYBG+p5iD+nniN2OgOwBgYPNSCVtc+NemqutD12hFUSfCzXidkv0ijhD1JZeLar9Ygxc=,iv:XctQwSYSvKhDRk/XMacC9uMydZ8e9hnhpoWTgyXiFI0=,tag:foAhBlg4DwpQU2G9DzTo5g==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBVWMvWkp5TnZQcGs5Ykhp\nWC91YkoyZERqdXpxQm5JVmRhaUhueEJETDJVCkM4V0hSYldkV1U2Q0d1TGh3eGNR\nVjJ1VFd6ZEN0SXZjSVEvcnV2WW0vbVUKLS0tIFRCNW9nWHdYaUxLSVVUSXM0OGtN\nVFMzRXExNkYxcFE3QWlxVUM3ay9INm8KV6r8ftpwarly3qXoU9y8KxKrUKLvP9KX\nGsP0pORsaM+qPMsdfEo35CqhAeQu0+6DWd7/67+fUMp6Jr0DthtTmg==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:28Z",
-		"mac": "ENC[AES256_GCM,data:scY9+/fcXhfHEdrsZJLOM6nfjpRaURgTVbCRepUjhUo24B4ByEsAo2B8psVAaGEHEsFRZuoiByqrGzKhyUASmUs+wn+ziOKBTLzu55fOakp8PWYtQ4miiz2TQffp80gCQRJpykcbUgqIKXNSNutt4tosTBL7osXwCEnEQWd+SaA=,iv:1VXNvLP6DUxZYEr1juOLJmZCGbLp33DlwhxHQV9AMD4=,tag:uFM1R8OmkFS74/zkUG0k8A==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/sops/secrets/signer-age.key/secret

@@ -1,20 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:i1YBJdK8XmWnVnZKBpmWggSN8JSOr8pm2Zx+CeE8qqeLZ7xwMO8SYCutM8l94M5vzmmX0CmwzeMZ/JVPbEwFd3ZAImUfh685HOY=,iv:N4rHNaX+WmoPb0EZPqMt+CT1BzaWO9LyoemBxKn+u/s=,tag:PnzSvdGwVnTMK8Do8VzFaQ==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4RXlmcVNGTnlkY2ZqZFlH\nVnh0eHhRNE5hRDNDVkt0TEE0bmRNN2JIVkN3CkxnaGM4Y3M3a0xoK2xMRzBLMHRV\nT1FzKzNRMFZOeWc2K3E5K2FzdUsvWmsKLS0tIENtVlFSWElHN3RtOUY2alhxajhs\naXI1MmR4WC9EVGVFK3dHM1gvVnlZMVUKCyLz0DkdbWfSfccShO1xjWfxhunEIbD0\n6imeIBhZHvVJmZLXnVl7B0pNXo6be7WSBMAUM9gUtCNh4zaChBNwGw==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:52Z",
-		"mac": "ENC[AES256_GCM,data:WFGysoXN95e/RxL094CoL4iueqEcSqCSQZLahwz9HMLi+8HWZIXr55a+jyK7piqR8nBS4BquU5fKhlC6BvEbZFt69t4onTA+LxS3D7A8/TO0CWS0RymUjW9omJUseRQWwAHtE7l0qI5hdOUKhQ+o5pU+2bc3PUlaONM0aOCCoFo=,iv:l1f4aVqLl5VAMfjNxDbxQEQp/qY/nxzgv2GTuPVBoBA=,tag:4PPDCmDrviqdn42RLHQYbA==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/per-machine/admin/data-mesher-host-key/private_key/secret

@@ -1,24 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:w3bU23Pfe8W89lF+tOmEYPU/A4FkY6n7rgQ6yo+eqCJFxTyHydV6Mg4/g4jaL+4wwIqNYRiMR8J8jLhSvw3Bc59u7Ul+RGwdpiKoBBJfsHjO8r6uOz2u9Raa+iUJH1EJWmGvsQXAILpliZ+klS96VWnGN3pYMEI=,iv:7QbUxta6NPQLZrh6AOcNe+0wkrADuTI9VKVp8q+XoZ8=,tag:ZH0t3RylfQk5U23ZHWaw0g==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age10zxkj45fah3qa8uyg3a36jsd06d839xfq64nrez9etrsf4km0gtsp45gsz",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKaTBoSFJVSTdZeW4wZG9p\nWFR1LzVmYS8xWmRqTlNtWFVkSW9jZXpVejJBCkpqZm12L1dDSmNhekVsK1JBOU9r\nZThScGdDakFlRzNsVXp1eE5yOStFSW8KLS0tIFRrTkZBQlRsR2VNcUJvNEkzS2pw\nNksvM296UkFWTkZDVVp1ZVZMNUs4cWsKWTteB1G9Oo38a81PeqKO09NUQetuqosC\nhrToQ6NMo5O7/StmVG228MHbJS3KLXsvh2AFOEPyZrbpB2Opd2wwoA==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6U2FWRThRNkVQdk9yZ0VE\nM09iSVhmeldMcDZVaFRDNGtjWTdBa0VIT2pJCkdtd04xSXdicDY3OHI1WXl5TndB\nemtQeW1SS2tVVllPUHhLUTRla3haZGMKLS0tIGN0NVNEN3RKeWM0azBBMnBpQU4r\nTFFzQ0lOcGt0ek9UZmZZRjhibTNTc0EKReUwYBVM1NKX0FD/ZeokFAAknwju5Azq\nGzl4UVJBi5Es0GWORdCGElPXMd7jMud1SwgY04AdZj/dzinCSW4CZw==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:10Z",
-		"mac": "ENC[AES256_GCM,data:0vl9Gt4QeH+GJcnl8FuWSaqQXC8S6Pe50NmeDg5Nl2NWagz8aLCvOFyTqX/Icp/bTi1XQ5icHHhF3YhM+QAvdUL3aO0WGbh92dPRnFuvlZsdtwCFhT+LyHyYHFf6yP+0h/uFpJv9fE6xY22CezA6ZVQ8ywi1epaC548Gr27uVe4=,iv:G4hZVCLkIpbg9uwB7Y8xtHLdnlmBvFrPjxSoqdyHNvM=,tag:uvKwakhUY2aa7v0tmR/o8A==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/per-machine/admin/data-mesher-host-key/public_key/value

@@ -1,3 +0,0 @@
------BEGIN PUBLIC KEY-----
-MCowBQYDK2VwAyEAm204bpSFi4jOjZuXDpIZ/rcJBrbG4zAc7OSA4rAVSYE=
------END PUBLIC KEY-----

checks/data-mesher/vars/per-machine/peer/data-mesher-host-key/private_key/secret

@@ -1,24 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:kERPY40pyvke0mRBnafa4zOaF46rbueRbhpUCXjYP5ORpC7zoOhbdlVBhOsPqE2vfEP4RWkH+ZPdDYXOKXwotBCmlq2i7TfZeoNXFkzWXc3GyM5mndnjCc8hvYEQF1w6xkkVSUt4n06BAw/gT0ppz+vo5dExIA8=,iv:JmYD2o4DGqds6DV7ucUmUD0BRB61exbRsNAtINOR8cQ=,tag:Z58gVnHD+4s21Z84IRw+Vw==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1faqrml2ukc6unfm75d3v2vnaf62v92rdxaagg3ty3cfna7vt99gqlzs43l",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4OFluVThBdUJSTmRVTk94\neFZnLytvcnNSdmQvR3ZkT2UvWFVieFV1SUFNCm9jWHlyZXRwaVdFaG9ocnd4S3FU\ndTZ2dklBbkFVL0hVT0Y2L1o5dnUyNG8KLS0tIGFvYlBJR3l2b3F6OU9uMTFkYjli\nNVFLOWQzOStpU2kzb0xyZUFCMnBmMVUK5Jzssf1XBX25bq0RKlJY8NwtKIytxL/c\nBPPFDZywJiUgw1izsdfGVkRhhSFCQIz+yWIJWzr01NU2jLyFjSfCNw==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzYW92c3Q4SktwSnJ1TkRJ\nZEJyZk96cG8ybkpPQzYzVk0xZGs0eCtISVR3CmhDaWxTem1FMjJKNmZNaTkxN01n\nenUvdFI1UkFmL1lzNlM5N0Ixd0dpc1EKLS0tIHpyS2VHaHRRdUovQVgvRmRHaXh3\naFpSNURjTWkxaW9TOXpKL2IvcUFEbmMKq4Ch7DIL34NetFV+xygTdcpQjjmV8v1n\nlvYcjUO/9c3nVkxNMJYGjuxFLuFc4Gw+AyawCjpsIYXRskYRW4UR1w==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:43Z",
-		"mac": "ENC[AES256_GCM,data:YhL2d6i0VpUd15B4ow2BgRpyEm0KEA8NSb7jZcjI58d7d4lAqBMcDQB+8a9e2NZbPk8p1EYl3q4VXbEnuwsJiPZI2kabRusy/IGoHzUTUMFfVaOuUcC0eyINNVSmzJxnCbLCAA1Aj1yXzgRQ0MWr7r0RHMKw0D1e0HxdEsuAPrA=,iv:yPlMmE6+NEEQ9uOZzD3lUTBcfUwGX/Ar+bCu0XKnjIg=,tag:eR22BCFVAlRHdggg9oCeaA==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/per-machine/peer/data-mesher-host-key/public_key/value

@@ -1,3 +0,0 @@
------BEGIN PUBLIC KEY-----
-MCowBQYDK2VwAyEAv5dICFue2fYO0Zi1IyfYjoNfR6713WpISo7+2bSjL18=
------END PUBLIC KEY-----

checks/data-mesher/vars/per-machine/signer/data-mesher-host-key/private_key/secret

@@ -1,24 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:U8F7clQ2Tuj8zy5EoEga/Mc9N3LLZrlFf5m7UJKrP5yybFRCJSBs05hOcNe+LQZdEAvvr0Qbkry1pQyE84gCVbxHvwkD+l3GbguBuLMsW96bHcmstb6AvZyhMDBpm73Azf4lXhNaiB8p2pDWdxV77E+PPw1MNYI=,iv:hQhN6Ak8tB6cXSCnTmmQqHEpXWpWck3uIVCk5pUqFqU=,tag:uC4ljcs92WPlUOfwSkrK9Q==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age153mke8v2qksyqjc7vta7wglzdqr5epazt83nch0ur5v7kl87cfdsr07qld",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvV05lejQrdUQvQjZPOG9v\nZ01naXlYZ1JxWHhDT1M1aUs1RWJDSU1acVFFCmdHY094aGRPYWxpdVVxSFVHRU9v\nNnVaeTlpSEdtSWRDMmVMSjdSOEQ4ZlEKLS0tIFo5NVk2bzBxYjZ5ZWpDWTMrQ2VF\nVThWUk0rVXpTY2svSCtiVDhTQ2kvbFkKEM2DBuFtdEj1G/vS1TsyIfQxSFFvPTDq\nCmO7L/J5lHdyfIXzp/FlhdKpjvmchb8gbfJn7IWpKopc7Zimy/JnGQ==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArNzVUaHkzUzVEMlh1Q3Qr\nOEo0aDJIMG91amJiZG50MEhqblRCTWxRRVVRCk4xZlp4SkJuUHc2UnFyU1prczkz\nNGtlQlRlNnBDRFFvUGhReTh6MTBZaXMKLS0tIGxtaXhUMDM0RU4yQytualdzdTFt\nWGRiVG54MnYrR2lqZVZoT0VkbmV5WUUKbzAnOkn8RYOo7z4RISQ0yN875vSEQMDa\nnnttzVrQuK0/iZvzJ0Zq8U9+JJJKvFB1tHqye6CN0zMbv55CLLnA0g==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:26:07Z",
-		"mac": "ENC[AES256_GCM,data:uMss4+BiVupFqX7nHnMo+0yZ8RPuFD8VHYK2EtJSqzgurQrZVT4tJwY50mz2gVmwbrm49QYKk5S+H29DU0cM0HiEOgB5P5ObpXTRJPagWQ48CEFrDpBzLplobxulwnN6jJ1dpL3JF3jfrzrnSDFXMvx+n5x/86/AYXYRsi/UeyY=,iv:mPT1svKrNGmYpbL9hh2Bxxakml69q+U6gQ0ZnEcbEyg=,tag:zcZx1lTw/bEsX/1g+6T04g==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/per-machine/signer/data-mesher-host-key/public_key/value

@@ -1,3 +0,0 @@
------BEGIN PUBLIC KEY-----
-MCowBQYDK2VwAyEAeUkW5UIwA1svbNY71ePyJKX68UhxrqIUGQ2jd06w5WM=
------END PUBLIC KEY-----

checks/data-mesher/vars/shared/data-mesher-network-key/private_key/secret

@@ -1,32 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:nRlCMF58cnkdUAE2aVHEG1+vAckKtVt48Jr21Bklfbsqe1yTiHPFAMLL1ywgWWWd7FjI/Z8WID9sWzh9J8Vmotw4aJWU/rIQSeF8cJHALvfOxarJIIyb7purAiPoPPs6ggGmSmVFGB1aw8kH1JMcppQN8OItdQM=,iv:qTwaL2mgw6g7heN/H5qcjei3oY+h46PdSe3v2hDlkTs=,tag:jYNULrOPl9mcQTTrx1SDeA==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age153mke8v2qksyqjc7vta7wglzdqr5epazt83nch0ur5v7kl87cfdsr07qld",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRcG44cGFBWXk2Z0pmNklv\nTnJ5b0svLytzZmNNRkxCVU1zaDVhNUs2cld3CklsenpWd0g2OEdKKzBMQlNEejRn\nTlEvY01HYjdvVExadnN3aXZIRTZ4YlEKLS0tIGRPUXdNSHZCRDBMbno2MjJqRHBl\nSzdiSURDYitQWFpaSElkdmdicDVjMWsKweQiRqyzXmzabmU2fmgwHtOa9uDmhx9O\ns9NfUhC3ifooQUSeYp58b1ZGJQx5O5bn9q/DaEoit5LTOUprt1pUPA==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiTEdlL29sVWFpSDNNaXRJ\ndTJDRkU4VzFPQ0M4MkFha2IxV2FXN2o3ZEFRCjF3UnZ5U1hTc3VvSTIzcWxOZjl0\ncHlLVEFqRk1UbGdxaUxEeDFqbFVYaU0KLS0tIFFyMnJkZnRHdWg4Z1IyRHFkY0I5\nQjdIMGtGLzRGMFM0ektDZ3hzZDdHSmMKvxOQuKgePom0QfPSvn+4vsGHhJ4BoOvW\nc27Vn4/i4hbjfJr4JpULAwyIwt3F0RaTA2M6EkFkY8otEi3vkcpWvA==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age10zxkj45fah3qa8uyg3a36jsd06d839xfq64nrez9etrsf4km0gtsp45gsz",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5ZzdsaVRnSmsrMGR1Ylg3\nZkpscTdwNUl5NUVXN3kvMU1icE0yZU1WSEJBClB6SlJYZUhDSElRREx5b0VueFUw\nNVFRU3BSU24yWEtpRnJoUC83SDVaUWsKLS0tIGVxNEo3TjlwakpDZlNsSkVCOXlz\nNDgwaE1xNjZkSnJBVlU5YXVHeGxVNFEKsXKyTzq9VsERpXzbFJGv/pbAghFAcXkf\nMmCgQHsfIMBJQUstcO8sAkxv3ced0dAEz8O6NUd0FS2zlhBzt29Rnw==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1faqrml2ukc6unfm75d3v2vnaf62v92rdxaagg3ty3cfna7vt99gqlzs43l",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkK1hDMGxCc1IvYXlJMnBF\nWncxaXBQa1RpTWdwUHc3Yk16My8rVHNJc2dFCkNlK2h0dy9oU3Z5ZGhwRWVLYVUz\ncVBKT2x5VnlhbXNmdHkwbmZzVG5sd0EKLS0tIHJaMzhDanF4Rkl3akN4MEIxOHFC\nYWRUZ08xb1UwOFNRaktkMjIzNXZmNkUK1rlbJ96oUNQZLmCmPNDOKxfDMMa+Bl2E\nJPxcNc7XY3WBHa3xFUbcqiPxWxDyaZjhq/LYQGpepiGonGMEzR5JOQ==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:20Z",
-		"mac": "ENC[AES256_GCM,data:za9ku+9lu1TTRjbPcd5LYDM4tJsAYF/yuWFCGkAhqcYguEducsIfoKBwL42ahAzqLjCZp91YJuINtw16mM+Hmlhi/BVwhnXNHqcfnKoAS/zg9KJvWcvXwKMmjEjaBovqaCWXWoKS7dn/wZ7nfGrlsiUilCDkW4BzTIzkqNkyREU=,iv:2X9apXMatwCPRBIRbPxz6PJQwGrlr7O+z+MrsnFq+sQ=,tag:IYvitoV4MhyJyRO1ySxbLQ==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/shared/data-mesher-network-key/public_key/value

@@ -1,3 +0,0 @@
------BEGIN PUBLIC KEY-----
-MCowBQYDK2VwAyEA/5j+Js7oxwWvZdfjfEO/3UuRqMxLKXsaNc3/5N2WSaw=
------END PUBLIC KEY-----

checks/flake-module.nix

@@ -94,7 +94,6 @@ in
 
             service-dummy-test = import ./service-dummy-test nixosTestArgs;
             service-dummy-test-from-flake = import ./service-dummy-test-from-flake nixosTestArgs;
-            service-data-mesher = import ./data-mesher nixosTestArgs;
           };
 
           packagesToBuild = lib.removeAttrs self'.packages [

checks/flash/flake-module.nix

@@ -59,7 +59,7 @@
           name = "flash";
           nodes.target = {
             virtualisation.emptyDiskImages = [ 4096 ];
-            virtualisation.memorySize = 3000;
+            virtualisation.memorySize = 4096;
             environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
             environment.etc."install-closure".source = "${closureInfo}/store-paths";
 

clanModules/disk-id/roles/default.nix

@@ -1,5 +1,4 @@
 {
-  config,
   pkgs,
   ...
 }:
@@ -9,9 +8,14 @@
   config = {
 
     warnings = [
-      "The clan.disk-id module is deprecated and will be removed on 2025-07-15.
-      Please migrate to user-maintained configuration or the new equivalent clan services
-      (https://docs.clan.lol/reference/clanServices)."
+      ''
+        The clan.disk-id module is deprecated and will be removed on 2025-07-15.
+        For migration see: https://docs.clan.lol/guides/migrations/disk-id/
+
+        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+        !!! Please migrate. Otherwise you may not be able to boot your system after that date.  !!!
+        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+      ''
     ];
     clan.core.vars.generators.disk-id = {
       files.diskId.secret = false;

clanServices/data-mesher/README.md

@@ -0,0 +1,29 @@
+This service will set up data-mesher.
+
+## Usage
+
+```nix
+inventory.instances = {
+  data-mesher = {
+    module = {
+      name = "data-mesher";
+      input = "clan-core";
+    };
+    roles.admin.machines.server0 = {
+      settings = {
+        bootstrapNodes = {
+          node1 = "192.168.1.1:7946";
+          node2 = "192.168.1.2:7946";
+        };
+
+        network = {
+          hostTTL = "24h";
+          interface = "tailscale0";
+        };
+      };
+    };
+    roles.peer.machines.server1 = { };
+    roles.signer.machines.server2 = { };
+  };
+}
+```

clanServices/data-mesher/admin.nix

@@ -0,0 +1,29 @@
+{
+  lib,
+  config,
+  settings,
+  ...
+}:
+{
+
+  services.data-mesher.initNetwork =
+    let
+      # for a given machine, read it's public key and remove any new lines
+      readHostKey =
+        machine:
+        let
+          path = "${config.clan.core.settings.directory}/vars/per-machine/${machine}/data-mesher-host-key/public_key/value";
+        in
+        builtins.elemAt (lib.splitString "\n" (builtins.readFile path)) 1;
+    in
+    {
+      enable = true;
+      keyPath = config.clan.core.vars.generators.data-mesher-network-key.files.private_key.path;
+
+      tld = settings.network.tld;
+      hostTTL = settings.network.hostTTL;
+
+      # admin and signer host public keys
+      signingKeys = builtins.map readHostKey (builtins.attrNames settings.bootstrapNodes);
+    };
+}

clanServices/data-mesher/default.nix

@@ -0,0 +1,142 @@
+{ ... }:
+let
+  sharedInterface =
+    { lib, ... }:
+    {
+      options = {
+        bootstrapNodes = lib.mkOption {
+          type = lib.types.nullOr (lib.types.attrsOf lib.types.str);
+          # the default bootstrap nodes are any machines with the admin or signers role
+          # we iterate through those machines, determining an IP address for them based on their VPN
+          # currently only supports zerotier
+          # default = builtins.foldl' (
+          #   urls: name:
+          #   let
+          #     ipPath = "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value";
+          #   in
+          #   if builtins.pathExists ipPath then
+          #     let
+          #       ip = builtins.readFile ipPath;
+          #     in
+          #     urls ++ [ "[${ip}]:${builtins.toString settings.network.port}" ]
+          #   else
+          #     urls
+          # ) [ ] (dmLib.machines config).bootstrap;
+          description = ''
+            A list of bootstrap nodes that act as an initial gateway when joining
+            the cluster.
+          '';
+          example = {
+            "node1" = "192.168.1.1:7946";
+            "node2" = "192.168.1.2:7946";
+          };
+        };
+
+        network = {
+          interface = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              The interface over which cluster communication should be performed.
+              All the ip addresses associate with this interface will be part of
+              our host claim, including both ipv4 and ipv6.
+
+              This should be set to an internal/VPN interface.
+            '';
+            example = "tailscale0";
+          };
+
+          port = lib.mkOption {
+            type = lib.types.port;
+            default = 7946;
+            description = ''
+              Port to listen on for cluster communication.
+            '';
+          };
+        };
+      };
+    };
+in
+{
+  _class = "clan.service";
+  manifest.name = "data-mesher";
+  manifest.description = "Set up data-mesher";
+  manifest.categories = [ "System" ];
+  manifest.readme = builtins.readFile ./README.md;
+
+  roles.admin = {
+    interface =
+      { lib, ... }:
+      {
+
+        imports = [ sharedInterface ];
+
+        options = {
+
+          network = {
+            tld = lib.mkOption {
+              type = lib.types.str;
+              default = "clan";
+              description = "Top level domain to use for the network";
+            };
+
+            hostTTL = lib.mkOption {
+              type = lib.types.str;
+              default = "${toString (24 * 28)}h";
+              example = "24h";
+              description = "The TTL for hosts in the network, in the form of a Go time.Duration";
+            };
+          };
+        };
+      };
+    perInstance =
+      { settings, roles, ... }:
+      {
+        nixosModule = {
+          imports = [
+            ./admin.nix
+            ./shared.nix
+          ];
+          _module.args = { inherit settings roles; };
+        };
+      };
+  };
+
+  roles.signer = {
+    interface =
+      { ... }:
+      {
+        imports = [ sharedInterface ];
+      };
+    perInstance =
+      { settings, roles, ... }:
+      {
+        nixosModule = {
+          imports = [
+            ./signer.nix
+            ./shared.nix
+          ];
+          _module.args = { inherit settings roles; };
+        };
+      };
+  };
+
+  roles.peer = {
+    interface =
+      { ... }:
+      {
+        imports = [ sharedInterface ];
+      };
+    perInstance =
+      { settings, roles, ... }:
+      {
+        nixosModule = {
+          imports = [
+            ./peer.nix
+            ./shared.nix
+          ];
+          _module.args = { inherit settings roles; };
+        };
+      };
+  };
+
+}

clanServices/data-mesher/flake-module.nix

@@ -0,0 +1,17 @@
+{ lib, ... }:
+let
+  module = lib.modules.importApply ./default.nix { };
+in
+{
+  clan.modules = {
+    data-mesher = module;
+  };
+  perSystem =
+    { ... }:
+    {
+      clan.nixosTests.service-data-mesher = {
+        imports = [ ./tests/vm/default.nix ];
+        clan.modules."@clan/data-mesher" = module;
+      };
+    };
+}

clanServices/data-mesher/peer.nix

@@ -0,0 +1,2 @@
+{
+}

clanServices/data-mesher/shared.nix

@@ -0,0 +1,86 @@
+{
+  config,
+  settings,
+  ...
+}:
+{
+
+  services.data-mesher = {
+    enable = true;
+    openFirewall = true;
+
+    settings = {
+      log_level = "warn";
+      state_dir = "/var/lib/data-mesher";
+
+      # read network id from vars
+      network.id = config.clan.core.vars.generators.data-mesher-network-key.files.public_key.value;
+
+      host = {
+        names = [ config.networking.hostName ];
+        key_path = config.clan.core.vars.generators.data-mesher-host-key.files.private_key.path;
+      };
+
+      cluster = {
+        port = settings.network.port;
+        join_interval = "30s";
+        push_pull_interval = "30s";
+        interface = settings.network.interface;
+        bootstrap_nodes = (builtins.attrValues settings.bootstrapNodes);
+      };
+
+      http.port = 7331;
+      http.interface = "lo";
+    };
+  };
+
+  # Generate host key.
+  clan.core.vars.generators.data-mesher-host-key = {
+    files =
+      let
+        owner = config.users.users.data-mesher.name;
+      in
+      {
+        private_key = {
+          inherit owner;
+        };
+        public_key.secret = false;
+      };
+
+    runtimeInputs = [
+      config.services.data-mesher.package
+    ];
+
+    script = ''
+      data-mesher generate keypair \
+          --public-key-path "$out"/public_key \
+          --private-key-path "$out"/private_key
+    '';
+  };
+
+  clan.core.vars.generators.data-mesher-network-key = {
+    # generated once per clan
+    share = true;
+
+    files =
+      let
+        owner = config.users.users.data-mesher.name;
+      in
+      {
+        private_key = {
+          inherit owner;
+        };
+        public_key.secret = false;
+      };
+
+    runtimeInputs = [
+      config.services.data-mesher.package
+    ];
+
+    script = ''
+      data-mesher generate keypair \
+          --public-key-path "$out"/public_key \
+          --private-key-path "$out"/private_key
+    '';
+  };
+}

clanServices/data-mesher/signer.nix

@@ -0,0 +1,2 @@
+{
+}

clanServices/data-mesher/tests/vm/default.nix

@@ -0,0 +1,90 @@
+{
+  ...
+}:
+{
+  name = "service-data-mesher";
+
+  clan = {
+    directory = ./.;
+    test.useContainers = true;
+    inventory = {
+
+      machines.peer = { };
+      machines.admin = { };
+      machines.signer = { };
+
+      instances = {
+        data-mesher =
+          let
+            bootstrapNodes = {
+              admin = "[2001:db8:1::1]:7946";
+              peer = "[2001:db8:1::2]:7946";
+              # signer = "2001:db8:1::3:7946";
+            };
+          in
+          {
+            roles.peer.machines.peer.settings = {
+              network.interface = "eth1";
+              inherit bootstrapNodes;
+            };
+            roles.signer.machines.signer.settings = {
+              network.interface = "eth1";
+              inherit bootstrapNodes;
+            };
+            roles.admin.machines.admin.settings = {
+              network.tld = "foo";
+              network.interface = "eth1";
+              inherit bootstrapNodes;
+            };
+          };
+      };
+    };
+  };
+
+  nodes =
+    let
+      commonConfig =
+        { lib, config, ... }:
+        {
+          environment.systemPackages = [
+            config.services.data-mesher.package
+          ];
+
+          # speed up for testing
+          services.data-mesher.settings = {
+            cluster.join_interval = lib.mkForce "2s";
+            cluster.push_pull_interval = lib.mkForce "5s";
+          };
+
+        };
+    in
+    {
+      peer = commonConfig;
+      admin = commonConfig;
+      signer = commonConfig;
+    };
+
+  testScript = ''
+    def resolve(node, success = {}, fail = [], timeout = 60):
+      for hostname, ips in success.items():
+          for ip in ips:
+              node.wait_until_succeeds(f"getent ahosts {hostname} | grep {ip}", timeout)
+
+      for hostname in fail:
+          node.wait_until_fails(f"getent ahosts {hostname}")
+
+    start_all()
+
+    admin.wait_for_unit("data-mesher")
+    signer.wait_for_unit("data-mesher")
+    peer.wait_for_unit("data-mesher")
+
+    # check dns resolution
+    for node in [admin, signer, peer]:
+      resolve(node, {
+          "admin.foo": ["2001:db8:1::1", "192.168.1.1"],
+          "peer.foo": ["2001:db8:1::2", "192.168.1.2"],
+          "signer.foo": ["2001:db8:1::3", "192.168.1.3"]
+      })
+  '';
+}

clanServices/data-mesher/tests/vm/sops/machines/admin/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1r99qtxl0v86wg8ndcem87yk5wag5xcsk98ngaumqzww6t7pyms0q5cyl80",
+    "type": "age"
+  }
+]

clanServices/data-mesher/tests/vm/sops/machines/peer/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1hgjs2yqxhcxfgtvhydnfe5wzlagxw2dw4hu658e8neduy0lkye0skmjfc7",
+    "type": "age"
+  }
+]

clanServices/data-mesher/tests/vm/sops/machines/signer/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1k6h9mespmnr9zhtwwqlhnla80x5jhpd4c2p7hp0nfanr5tspup0s0rld2f",
+    "type": "age"
+  }
+]

clanServices/data-mesher/tests/vm/sops/secrets/admin-age.key/secret

@@ -0,0 +1,15 @@
+{
+	"data": "ENC[AES256_GCM,data:87WFWukgpTGlH67MTkHxzTosABK/6flJObt+u9UrGSOzBr1lx4V5IsMQ9HAM4jvLpveBNH4hlFDCxbD5666n2oYylGoyBph2vAg=,iv:GKLcU7Xqmb0ImvY7M71NddkOlUDSPa/fcXrXny2iZ1o=,tag:589QMSZeXdmTxRFtMFasZg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFaXlqZEU0eHRZZjBncDE1\nV2hzTGZiVy9rM0NnWjc1NlpHVVZEUFd5S2pJCmo3Nm11bGQyWWt1R2tHS2pOYlpn\nY3lGa0w3UFpDT1RLSDU4cnJ2YVBkSU0KLS0tIEJjZVc1YXJqcHczYSt6WjV3ai93\nakdPd3VHWkVnWkdhNCtZakp4VXhBUG8Kg3xd9w5oW3/q+s59LkDy5N+xmvuvHRmh\njUv6KFLaB81yv3kb7bzj8E3aMzX0x2fMIDZ3EoPVggqA/sCWQu0p5Q==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:45Z",
+		"mac": "ENC[AES256_GCM,data:IWKfE1Y6SNg/SK+OOAmra5SwqAUfhepCNPClWPDWpOyJDwXSpk/OKl7hi3KFfIZOGupaC0xV2tTni0Uj6IBwf8zW2Mb/b1T+fWkGiyafoKlucfNPXPCob/fyf4Ju4iD/u1mD5BYYYqNTNqJWE+MCyQigL0MPE4tXGEPDa7htM6w=,iv:5RKArbEKnYjacopfL+4QhzGB8txqc3gnlwNPfRWQSlM=,tag:mdXf02nYiW7CexIbUUaMyw==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/sops/secrets/peer-age.key/secret

@@ -0,0 +1,15 @@
+{
+	"data": "ENC[AES256_GCM,data:C9evAr01JpYiMBwuy31h+G9phm+uOYoQu+PegPFAMRbjgkjh0R+uolKtweedtHumMhzEkvz7y+BlfrriVh16ceyMozfzDEkVSWM=,iv:jM4Qx4B/j5Mvc3ybOf+10hKU19l1fCc5KcKulKgMP3c=,tag:mz01kIv5kU6u3f2+FeItYA==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAydzZrdDVidGpyd1NXT0Fu\nUEtZV3I4S0p5Z095QjBGaXpwOExJSkxVclVJCm54Vk12czQ5dm5TUExNNzlEcFNp\nUWorcWc1c1pvL3pkUFlQY3BJUGhUS3MKLS0tIHd2a291M0xkcjJvTXNnelRNZXda\nQi93R3FQVm0xTXBGR3E3SVpIMzgvR3MKmps5ObV1nODBQ0TKgZ++RLkjCEQM6sMn\nzonKtBingYzfeq+0+cASVkHZJpt/t0G5wmTgivKfv0OIP5eNSgIWFw==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:57Z",
+		"mac": "ENC[AES256_GCM,data:Jk5eL2SmNpakrGF4N/31Q/PWShV5KYfA8NmlxEkD82UsIpPiIJ4Nec6NOoo7Y4bl/J53MLjK3u0/S6q7vv0Tih6+ze6hIddMJHTCp2qqclJvpH2xn6Ln+2ZK4okK2ZbWeSDF+LHc6nIpBak8JVjC/d8dQFT2L49Dkufc1nCD46w=,iv:oR0aQzjaEpFNrpWGc1TX6/zpg0WSfQjVG6VjAMwoLTI=,tag:pigUaCkVv91tynuaNoZenA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/sops/secrets/signer-age.key/secret

@@ -0,0 +1,15 @@
+{
+	"data": "ENC[AES256_GCM,data:bIx3chjDwy4epCyFuJoZlO7EglT/vEg6pdf6x+ISxqekGrrGNdiGtw3Z9foXWAPQrzngVztbwIlcEpUusKwoRPpdGIj5YzbGZbU=,iv:Gi1hjn6cL8z+LP5g6o3bUMsuIzoZRr8e3j3EBwG3p+Y=,tag:ttIfOLhDroV/WK57KBFd0w==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHNVh6OGE4aGJxbFd2Zks1\nL1ZoNkgrQjFSVFFUL2UzOGNqRXFkZURTMkJRCnZMWk00enRndzNXQmFvMG1UekI0\nUjhwZW9sQnFvb0FGbVE0N042UjF2OTAKLS0tIEdickxQdDdaZkVmN3RsemJzSElY\nWThGQVNMcnpxRlJ3bC9wVE56blljQUUK21wWOBiQc0Kyvl047nJ1N6QKR0/5Dd6r\nlqhhdFWninzqfVXJUk2pcMio8RVlvBujDsyjrPuhbRceSi+bUXIn+w==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:03:08Z",
+		"mac": "ENC[AES256_GCM,data:kA2KCDZkZuR5rD7uU4xn5sIkizcnpGcoa3PYMbl73eux7JJYuSpUojFBRcYo1WCwMeOQUGsqo8LVF/rYhH4BVJ9LERs5zTLBaUsTarY8r/UK0Q5lNYZqIrqcb5LgOf1uCvfdXg5yfaFgPFJrEqjeekb9bx8xvhDZXpsND93rrUI=,iv:B6JqWWcQV/MxP4ucAIe7EnLiq9c4pnAUj3dnEp9IXJU=,tag:1i0Fv2i7Lak5JzIbPa2/cw==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/per-machine/admin/data-mesher-host-key/private_key/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:2FgvnmawAdk+/k+RVWNsKQlUFUF+pZrrEBuupdG50uLNyxHd7Gi772gKNgHWyzZ/lpODg5mQi0rL+GmZYQwtZ7h76AGUEeQvuMMTzVUop69txxwhJD2dxZyhUAxZpibwo/St84ai+8+VksLkCSYfTXCulaeOVh4=,iv:YkPNq4zDj35PRNgt2kHEkHhbLcVc9dHP/zrAwdd94sM=,tag:KwW/74C7Z/+3dNoXB3NHwQ==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoaS94M0JsR2Q5N21DNnFB\nUHgvelRTK3FKZkNKcTJFbEJ1VGFIM256MVVRCmw5YjdyTVlXMlFpWnczV2dTSzhu\nSm5mMVRPeU1pYVFZNEN5MjJFZHVTejgKLS0tIDB0V2hSRkt5QzFYald0TWVza1lC\ncGNXemhGcklENTJiV1QvTFZxUDNRRlUK2dVEzSbdDNXZy7rQi5/Vq4KyHq5rMtEz\npTI8i1rFKIAy4TC7to03bOIudOIzKSCCzX31xARkM6qON0vEU9aHFg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1r99qtxl0v86wg8ndcem87yk5wag5xcsk98ngaumqzww6t7pyms0q5cyl80",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEOEMzcExzTTF3MmpaenRN\ncS9RbnM0aStZSjNqbjF4QisrRjhoaDg1T0ZBCmFVOWJYZkFaOXBOUGJTdytYWk52\nVXV1MDdmSWQ1OS9iODAvN2c2Q3VGYXMKLS0tIEQxeWR4bmRoOWJ2Z1FyUk1PUk1n\nM0c5Ri9FdG9FNE9CZ29VSmgvN2xDdjgKjfG38gVOXXN2ftGiCPxMFbnh7lKM1USl\nqf11k+rgvR8M9XsDy2SnirKAaNmpks1dR6Zs5ppQuYJDEYyQCrEO5g==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:45Z",
+		"mac": "ENC[AES256_GCM,data:TEH57vUZ/swTsWQPJ1X3J//xa1Q1LYPETZS7fuXCH1LCK51u88XGqVpNzSETREQ8LAOt34qN284b03UQIBGTeTr7I9cqt+/l8ew/0rFTiO3aiaT49q9aBkeFZlA+gy47r4hkhMmzGQJMUenvnzTHwT3Pw2RES5Vjs/2TSitpqlA=,iv:ffIotRGKU8y6j/VDLKbTmA8dZJVP5vafeG4F3wd60tc=,tag:q4xOwzLw5jxDR0pPIy2irA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/per-machine/admin/data-mesher-host-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEAi6qF8u2uvPXlSflB4fzJNlOhj5PgAmRiv+JyyYOOgg4=
+-----END PUBLIC KEY-----

clanServices/data-mesher/tests/vm/vars/per-machine/peer/data-mesher-host-key/private_key/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:LUNuEP/xSmzJ44sheoIYN6F24Qpr3svn6rTVUpr4KZA8uVJ9gPUd4ko4+pDisc9PyXCcxx+cYGRqr1cBp8Q3R+IyFFlR2HzuReQJaScvgjlntGtMJ2hin/aBp4pHS0F4nqPcKKROiZvIN4NHsxQ6XRVDOZbI3kE=,iv:BdRHjQXJL/OGgmqWaEDLit/zHgduNfPe3GUmYDrWLPw=,tag:N0n7CCiu+COgrfrwHUwQBQ==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hgjs2yqxhcxfgtvhydnfe5wzlagxw2dw4hu658e8neduy0lkye0skmjfc7",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCYlhrU2c1NnIyTzlVRHEx\nQTFxOUY1OWJXcHl1OHpPdWN1ZGpQV0UvZ1NzCnlKbmx0bllWMTd1ZnIxUHY0ZUU0\nVG9Jb3grSEdWeVpwaHoyQUxvNERqT00KLS0tIGtwZm5aMU1DOUhJbVVpVzIxZFow\nNVEvMy91SEg3M094MEFBSkVMRkhKZmMKuUzbEITGkYS39G14JXbKWLjiQFd4SVft\nWH34B97TFhOqusVF3zHsSCMxm/0BMeBvLxO/3RmzlwBtgNiKOqLwtQ==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2SUhJQW5EN0VKVHpQdlZC\nYTczdVJiRFdFNGtURFc2SmxKWFFycjZkQUgwCnRBVkJvUytuUDlhVlhFYno2cnBR\nRUdjL0lab1MwZzhGTklyVWZDVFJmN3cKLS0tIFRjOC9DS3llWGZWMGI2aThVYTRu\nVEFhK2Y2YkRTZHEyMWV0Q05ISHdhVVUKo9bPdV1dUeIkm4gI0r9V/s1dAfJC+H5Z\nEIUdYA7fl3jRZ01cSZ0iYWlvdl2jj0XzKafZsEQU7rL0jg9zbA2s2g==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:59Z",
+		"mac": "ENC[AES256_GCM,data:+JtuPacwUMHXtp93DZmkiVne7bQUP8J7VpoS8koM0oJWJqZoQRHd9qH/04lrpp8q/YoOXtqXwhViZvFLieJVRexiXf/AAHfAfMn0EI7ois9oHhscN88Ps9nY6JUxhNd0h0OrUA58KKhrkGoqreAKAPADtVhaVCmWbU7vMUu1StE=,iv:BmJnTsgMSbl4XsBUkhSLfKd0XjhrEQfurEkaRJ6uD/g=,tag:jg21c4y4bQp0RwWTXkxF1A==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/per-machine/peer/data-mesher-host-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEA7kRKjQpj+BXPe5buvDZtBAcU1HIcfGmbuHZqaVm3zCo=
+-----END PUBLIC KEY-----

clanServices/data-mesher/tests/vm/vars/per-machine/signer/data-mesher-host-key/private_key/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:armAfuTE0mkoy1fxAysCX/UPNM4/mt9P6/zEDwtagTSvQjMTwVzzsM+kRdLOUV4fbZ7HdqMceaZWzurAQJenXvWlBXgn87YFOFBSpf3OnpEwCTUs9H8dsVrdSUk4SrKjCjV33mybTrae/h9tMHdkRhKJzPD1+/8=,iv:x9KVGqT2Ug6B6PNwzL7NVDQqyOmFUptUsHAJEdn30dg=,tag:XSSO6JvXaXq8aezYvpF65Q==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1k6h9mespmnr9zhtwwqlhnla80x5jhpd4c2p7hp0nfanr5tspup0s0rld2f",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIMVUwMEFzVjJhYXg5MXR4\nMzZPZUFrUWdEU2hPWUVDNHpVVENpdEdYSWtnCnN0R2pVdEIxYWZXYWNBb3N5bGNK\naVpWOXp5aWVJWG9vUWtMUnhYSmMyV0UKLS0tIEtMdFAybk1PN0t2M2lkaEYzUTY3\nVzVOdTBFbnlNVTAvRU5kU0dReEZ6MlUKNHIkAUUAqnuMtXbvXqLxQwuFALsnD/i0\naBCiz6J4S18uqt3kFbXAEksbD7jCexI8m5SMp4iuumWJ/Bx1lL4TWg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkbzBFSGt1dXI2bDN5TmFU\nY3N6djNmMTh2ek4vUzdHbTF6Z1hDQ2t5WVNJClEzZDZiaVpBekFrYTYweDNsNmk5\nTlhYZGRNd0llMndyMkZWMyt5N3pwTE0KLS0tIGJJbU9vbnBhSE5vRW1pRG83cEFJ\nR2xDTHk3VkJaVUZSVThRV3Jldkp6cnMK1V37txaSFYfLQM0qqRWjojyTN4fTJkRm\nGO3yHX9uwo/4D2xI7LM48n4vnNhSF05bWpq0X4r13fI4DofCJeEo1g==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:03:11Z",
+		"mac": "ENC[AES256_GCM,data:qD1w+DO8cWFDQMBOrmO9FvxvJRn+mlUbh13exTGgmsdPn3uzTXknIDDHeWfkpF699nSzS6wRmgrB21e55rBU6iHMx1TW16S8wvCoYMFwib8zTrJzND7EJr/gRwQa0N080kBY3xBivKLUFlctgKtFUYZ9GQ6UTQeq18QKPoROjww=,iv:1mt8Er6YHxQ42F5Kb+xNtjbCAzokbeoNlHesC9Uzmhk=,tag:provO4tKDzoL5PHDg5EmhA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/per-machine/signer/data-mesher-host-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEAVA6c25s+yNe5225PnELDV9FwbWi9ppLoTfgmdY8kILo=
+-----END PUBLIC KEY-----

clanServices/data-mesher/tests/vm/vars/shared/data-mesher-network-key/private_key/secret

@@ -0,0 +1,27 @@
+{
+	"data": "ENC[AES256_GCM,data:VzcB/JABSPoFdKYhRSn+nKxasn9zO/9fyNMrg3XstBelQNPpbO8mhmcnSamc/7e5GkpoVWgLRSULvosv+o6sz9EHRZ3UpSLBBTkDGAJmoBnkR8DbstPA9EgScpQ9IGOUP5tQ0oEOcJC3FrivdbWIzeXjpWb9BrU=,iv:6BNUrubJ9aNCkgonDRNgdyckCTndkPVDLE4X3J5d2zA=,tag:YqHTiGslEkslzUk24bmPZg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hgjs2yqxhcxfgtvhydnfe5wzlagxw2dw4hu658e8neduy0lkye0skmjfc7",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwK2lMUTkrSmM4dHQxU0tI\nRVV6Wm4wWlJMYXBGbGdubExrMi8vRnJjdVd3CjI3aFVpdHRURHp6UEk3ZEZMcDZT\nZWZWaGFWYmY2Mk1iQ1BjalZkUnpUUm8KLS0tIEhFUVhBUjg1dC9LWHg2TytkRTlX\nNnlJZkJQc2ExK1BwaVVFcEw2b3BLZjQK8kqf3ZP9uLtbjCJLSEYpAqgq9zOS2HrY\n5MbPAKQI8iCUfnegti6hU+/MxjvPlaX1vT4V0Kd3gT4Khjl+OPw0Og==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1r99qtxl0v86wg8ndcem87yk5wag5xcsk98ngaumqzww6t7pyms0q5cyl80",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWeUk3ZW9rdnZBTk9vQlFZ\nTzFZVDAvcXZyQjdkcGNNbnA0T3UyM3lzVERvCjFreE9RdWxnb2xWWmI4amJVdHBv\nNE9JN2tFazRnSGhiM0FId2RCUHNKWVEKLS0tIGlmM3JNSVZtR21ndFliUVpLTzJO\ncHJ2SjI1OExQK2hEN01WdG9wZ3RmVTAKi0BXp9yV2/9a9NeT7aTSK2CfkQ5yColJ\nm0+uv5AJndZ9IsaZGJxNOdAOspYdvsW38hFdfjUtVuUCyIOPc20WUg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1k6h9mespmnr9zhtwwqlhnla80x5jhpd4c2p7hp0nfanr5tspup0s0rld2f",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkSUR1QVMvZ3F0NUxXd00z\nOWJGZFlsUy8vUmMxa1NoakZRVmJrSmd1RzBrCk1ZcDlBMFB0WVdWeFZaT3ZBTTh5\nS2RReWpUOGRBdGV6MDdjcEY5dFYrdjAKLS0tIG9oRWhUaWJZSElRdmlOZmRKSnNq\nUUNDZFdZbmM0c25MOGpvem1JSm9pVWsKxCLPivdHc6IN6Jbf9FujLGJaXP6ieO1S\nKsrs3Fe0RdYcEKI7P9EQNebQD2kKXficM0kKV5lRRVtW5024PftWoQ==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3amkyWWlxSTJkZEdMZFhL\nU2t5OGFIa25TRmdFM0ZNcUhFRHk0eDJQN2tjCm9UcUs2V0lEZ0hyNU9uaDVrckpj\nZ1JSQlhNeExjOER2aFJTM2NDS25PN2MKLS0tIFhmM21rT0Z4aUI5TUZyNnNBQ3Jy\nSDAxejhhZDZNQTVCNjNUSTBsZncra1kKFFQrFxNMyg0AEMb1wpKBc7LOVtEHyFZW\n/o7L52fTNa0GFJ3SVEdqg0PpnRzTyA8F5L77FBGKtx6auCVVHyZZ9g==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:48Z",
+		"mac": "ENC[AES256_GCM,data:HooesDb1S24Cfb7H0lVTA8fAjM2QAN9MaJFvOSHniR6ICJAX8t8X0xfWIFRFuwPjAxi4kpBYSjW0420Yz9lZ2m4Fxswo1TV3lzHDVN2u9hdrsfpKXg5fW+2oZihuvCRStDagT3l2fKv+C+gBnGs1qyCM60BStvrEiQxTxTTHfho=,iv:kL8N0qBj4q+ZJbNJ8Y8RcV1KpUUMvNCpdwKbTPGpG6k=,tag:o2PmRsSkqTP5Idq7veGDOw==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/shared/data-mesher-network-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEA/MuamRX6ZLcJunm7lZvlai0OZh++YuqMa56GiTwO68A=
+-----END PUBLIC KEY-----

clanServices/emergency-access/default.nix

@@ -4,6 +4,7 @@
   manifest.name = "clan-core/emergency-access";
   manifest.description = "Set recovery password for emergency access to machine";
   manifest.categories = [ "System" ];
+  manifest.readme = builtins.readFile ./README.md;
 
   roles.default.perInstance = {
     nixosModule =

clanServices/sshd/default.nix

@@ -7,6 +7,7 @@
     "System"
     "Network"
   ];
+  manifest.readme = builtins.readFile ./README.md;
 
   roles.client = {
     interface =

clanServices/state-version/default.nix

@@ -4,6 +4,7 @@
   manifest.name = "clan-core/state-version";
   manifest.description = "Automatically generate the state version of the nixos installation.";
   manifest.categories = [ "System" ];
+  manifest.readme = builtins.readFile ./README.md;
 
   roles.default = {
 
@@ -24,7 +25,7 @@
             warnings = [
               ''
                 The clan.state-version service is deprecated and will be
-                removed on 2025-07-15 in favor of a nix option. 
+                removed on 2025-07-15 in favor of a nix option.
 
                 Please migrate your configuration to use `clan.core.settings.state-version.enable = true` instead.
               ''

clanServices/trusted-nix-caches/default.nix

@@ -4,6 +4,7 @@
   manifest.name = "clan-core/trusted-nix-caches";
   manifest.description = "This module sets the `clan.lol` and `nix-community` cache up as a trusted cache.";
   manifest.categories = [ "System" ];
+  manifest.readme = builtins.readFile ./README.md;
 
   roles.default = {
 

clanServices/users/default.nix

@@ -1,9 +1,13 @@
 { ... }:
 {
   _class = "clan.service";
-  manifest.name = "clan-core/users";
-  manifest.description = "Automatically generates and configures a password for the specified user account.";
+  manifest.name = "clan-core/user";
+  manifest.description = ''
+    An instance of this module will create a user account on the added machines,
+    along with a generated password that is constant across machines and user settings.
+  '';
   manifest.categories = [ "System" ];
+  manifest.readme = builtins.readFile ./README.md;
 
   roles.default = {
     interface =
@@ -19,7 +23,57 @@
             type = lib.types.bool;
             default = true;
             example = false;
-            description = "Whether the user should be prompted.";
+            description = ''
+              Whether the user should be prompted for a password.
+
+              Effects:
+
+              - *enabled* (`true`) - Prompt for a passwort during the machine installation or update workflow.
+              - *disabled* (`false`) - Generate a passwort during the machine installation or update workflow.
+
+              The password can be shown in two steps:
+
+              - `clan vars list <machine-name>`
+              - `clan vars get <machine-name> <name-of-password-variable>`
+            '';
+          };
+          regularUser = lib.mkOption {
+            type = lib.types.bool;
+            default = true;
+            example = false;
+            description = ''
+              Whether the user should be a regular user or a system user.
+
+              Regular users are normal users that can log in and have a home directory.
+
+              System users are used for system services and do not have a home directory.
+
+              !!! Warning
+                  `root` cannot be a regular user.
+                  You must set this to `false` for `root`
+            '';
+          };
+          groups = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [ ];
+            example = [
+              "wheel"
+              "networkmanager"
+              "video"
+              "input"
+            ];
+            description = ''
+              Additional groups the user should be added to.
+              You can add any group that exists on your system.
+              Make sure these group exists on all machines where the user is enabled.
+
+              Commonly used groups:
+
+              - "wheel" - Allows the user to run commands as root using `sudo`.
+              - "networkmanager" - Allows the user to manage network connections.
+              - "video" - Allows the user to access video devices.
+              - "input" - Allows the user to access input devices.
+            '';
           };
         };
       };
@@ -35,9 +89,13 @@
             ...
           }:
           {
-            users.mutableUsers = false;
-            users.users.${settings.user}.hashedPasswordFile =
-              config.clan.core.vars.generators."user-password-${settings.user}".files.user-password-hash.path;
+            users.users.${settings.user} = {
+              isNormalUser = settings.regularUser;
+              extraGroups = settings.groups;
+
+              hashedPasswordFile =
+                config.clan.core.vars.generators."user-password-${settings.user}".files.user-password-hash.path;
+            };
 
             clan.core.vars.generators."user-password-${settings.user}" = {
 
@@ -80,4 +138,11 @@
           };
       };
   };
+
+  perMachine = {
+    nixosModule = {
+      # Immutable users to ensure that this module has exclusive control over the users.
+      users.mutableUsers = false;
+    };
+  };
 }

clanServices/users/tests/vm/default.nix

@@ -13,6 +13,8 @@
           roles.default.machines."server".settings = {
             user = "root";
             prompt = false;
+            # Important: 'root' must not be a regular user. See: https://github.com/NixOS/nixpkgs/issues/424404
+            regularUser = false;
           };
         };
         user-password-test = {
@@ -31,7 +33,6 @@
     server = {
       users.users.testuser.group = "testuser";
       users.groups.testuser = { };
-      users.users.testuser.isNormalUser = true;
     };
   };
 

devFlake/private.narHash

@@ -1 +1 @@
-sha256-pFUj3KhQ4FkzZT19t+FHBru8u8Lspax0rS2cv7nXIgM=
+sha256-LdjcFZLL8WNldUO2LbdqFlss/ERiGeXVqMee0IxV2z0=

devFlake/private/flake.lock

@@ -66,11 +66,11 @@
     },
     "nixpkgs-dev": {
       "locked": {
-        "lastModified": 1751867001,
-        "narHash": "sha256-3I49W0s3WVEDBO5S1RxYr74E2LLG7X8Wuvj9AmU0RDk=",
+        "lastModified": 1752039390,
+        "narHash": "sha256-DTHMN6kh1cGoc5hc9O0pYN+VAOnjsyy0wxq4YO5ZRvg=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "73feb5e20ec7259e280ca6f424ba165059b3bb6b",
+        "rev": "6ec4d5f023c3c000cda569255a3486e8710c39bf",
         "type": "github"
       },
       "original": {
@@ -146,11 +146,11 @@
         "nixpkgs": []
       },
       "locked": {
-        "lastModified": 1750931469,
-        "narHash": "sha256-0IEdQB1nS+uViQw4k3VGUXntjkDp7aAlqcxdewb/hAc=",
+        "lastModified": 1752055615,
+        "narHash": "sha256-19m7P4O/Aw/6+CzncWMAJu89JaKeMh3aMle1CNQSIwM=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "ac8e6f32e11e9c7f153823abc3ab007f2a65d3e1",
+        "rev": "c9d477b5d5bd7f26adddd3f96cfd6a904768d4f9",
         "type": "github"
       },
       "original": {

docs/mkdocs.yml

@@ -79,6 +79,7 @@ nav:
           - Migrate existing Flakes: guides/migrations/migration-guide.md
           - Migrate inventory Services: guides/migrations/migrate-inventory-services.md
           - Facts Vars Migration: guides/migrations/migration-facts-vars.md
+          - Disk id: guides/migrations/disk-id.md
       - macOS: guides/macos.md
   - Reference:
       - Overview: reference/index.md
@@ -86,6 +87,7 @@ nav:
           - Overview: reference/clanServices/index.md
           - reference/clanServices/admin.md
           - reference/clanServices/borgbackup.md
+          - reference/clanServices/data-mesher.md
           - reference/clanServices/emergency-access.md
           - reference/clanServices/garage.md
           - reference/clanServices/hello-world.md
@@ -199,7 +201,6 @@ theme:
     - navigation.instant
     - navigation.tabs
     - navigation.tabs.sticky
-    - navigation.footer
     - content.code.annotate
     - content.code.copy
     - content.tabs.link

docs/nix/flake-module.nix

@@ -29,7 +29,10 @@
       # Frontmatter for clanModules
       clanModulesFrontmatter =
         let
-          docs = pkgs.nixosOptionsDoc { options = self.clanLib.modules.frontmatterOptions; };
+          docs = pkgs.nixosOptionsDoc {
+            options = self.clanLib.modules.frontmatterOptions;
+            transformOptions = self.clanLib.docs.stripStorePathsFromDeclarations;
+          };
         in
         docs.optionsJSON;
 

docs/nix/get-module-docs.nix

@@ -7,6 +7,10 @@
   pkgs,
   clan-core,
 }:
+let
+  inherit (clan-core.clanLib.docs) stripStorePathsFromDeclarations;
+  transformOptions = stripStorePathsFromDeclarations;
+in
 {
   # clanModules docs
   clanModulesViaNix = lib.mapAttrs (
@@ -20,6 +24,7 @@
           }).options
           ).clan.${name} or { };
         warningsAreErrors = true;
+        inherit transformOptions;
       }).optionsJSON
     else
       { }
@@ -32,6 +37,7 @@
       (nixosOptionsDoc {
         inherit options;
         warningsAreErrors = true;
+        inherit transformOptions;
       }).optionsJSON
     ) rolesOptions
   ) modulesRolesOptions;
@@ -52,7 +58,15 @@
 
         (nixosOptionsDoc {
           transformOptions =
-            opt: if lib.strings.hasPrefix "_" opt.name then opt // { visible = false; } else opt;
+            opt:
+            let
+              # Apply store path stripping first
+              transformed = transformOptions opt;
+            in
+            if lib.strings.hasPrefix "_" transformed.name then
+              transformed // { visible = false; }
+            else
+              transformed;
           options = (lib.evalModules { modules = [ role.interface ]; }).options;
           warningsAreErrors = true;
         }).optionsJSON
@@ -72,5 +86,6 @@
         }).options
         ).clan.core or { };
       warningsAreErrors = true;
+      inherit transformOptions;
     }).optionsJSON;
 }

docs/nix/render_options/__init__.py

@@ -62,14 +62,11 @@ def sanitize(text: str) -> str:
     return text.replace(">", "\\>")
 
 
-def replace_store_path(text: str) -> tuple[str, str]:
+def replace_git_url(text: str) -> tuple[str, str]:
     res = text
-    if text.startswith("/nix/store/"):
-        res = "https://git.clan.lol/clan/clan-core/src/branch/main/" + str(
-            Path(*Path(text).parts[4:])
-        )
-    # name = Path(res).name
-    name = str(Path(*Path(text).parts[4:]))
+    name = Path(res).name
+    if text.startswith("https://git.clan.lol/clan/clan-core/src/branch/main/"):
+        name = str(Path(*Path(text).parts[7:]))
     return (res, name)
 
 
@@ -159,7 +156,7 @@ def render_option(
 
     decls = option.get("declarations", [])
     if decls:
-        source_path, name = replace_store_path(decls[0])
+        source_path, name = replace_git_url(decls[0])
 
         name = name.split(",")[0]
         source_path = source_path.split(",")[0]
@@ -452,7 +449,6 @@ Each `clanService`:
 * Is a module of class **`clan.service`**
 * Can define **roles** (e.g., `client`, `server`)
 * Uses **`inventory.instances`** to configure where and how it is deployed
-* Replaces the legacy `clanModules` and `inventory.services` system altogether
 
 !!! Note
     `clanServices` are part of Clan's next-generation service model and are intended to replace `clanModules`.

docs/site/guides/authoring/clanModules/index.md

@@ -52,6 +52,7 @@ clanModules/borgbackup
 
     ```nix title="flake.nix"
     # ...
+    # Sometimes this attribute set is defined in clan.nix
     clan-core.lib.clan {
         # 1. Add the module to the available clanModules with inventory support
         inventory.modules = {
@@ -175,6 +176,7 @@ The following shows how to add options to your module.
     Configuration can be set as follows.
 
     ```nix title="flake.nix"
+    # Sometimes this attribute set is defined in clan.nix
     clan-core.lib.clan {
         inventory.services = {
             custom-module.instance_1 = {

docs/site/guides/authoring/clanServices/index.md

@@ -27,6 +27,7 @@ i.e. `@hsjobeki/customNetworking`
 outputs = inputs: inputs.flake-parts.lib.mkFlake { inherit inputs; } ({
     imports = [ inputs.clan-core.flakeModules.default ];
     # ...
+    # Sometimes this attribute set is defined in clan.nix
     clan = {
         # If needed: Exporting the module for other people
         modules."@hsjobeki/customNetworking" = import ./service-modules/networking.nix;
@@ -218,6 +219,7 @@ To import the module use `importApply`
 outputs = inputs: flake-parts.lib.mkFlake { inherit inputs; } ({self, lib, ...}: {
     imports = [ inputs.clan-core.flakeModules.default ];
     # ...
+    # Sometimes this attribute set is defined in clan.nix
     clan = {
         # Register the module
         modules."@hsjobeki/messaging" = lib.importApply ./service-modules/messaging.nix { inherit self; };
@@ -244,6 +246,7 @@ Then wrap the module and forward the variable `self` from the outer context into
 outputs = inputs: flake-parts.lib.mkFlake { inherit inputs; } ({self, lib, ...}: {
     imports = [ inputs.clan-core.flakeModules.default ];
     # ...
+    # Sometimes this attribute set is defined in clan.nix
     clan = {
         # Register the module
         modules."@hsjobeki/messaging" = {

docs/site/guides/getting-started/add-machines.md

@@ -90,6 +90,7 @@ See the complete [list](../../guides/more-machines.md#automatic-registration) of
     The option: `machines.<name>` is used to add extra *nixosConfiguration* to a machine
 
 ```{.nix .annotate title="flake.nix" hl_lines="3-13 18-22"}
+# Sometimes this attribute set is defined in clan.nix
 clan = {
     inventory.machines = {
         jon = {

docs/site/guides/getting-started/add-services.md

@@ -28,6 +28,7 @@ To learn more: [Guide about clanService](../clanServices.md)
         inputs@{ flake-parts, ... }:
         flake-parts.lib.mkFlake { inherit inputs; } {
             imports = [ inputs.clan-core.flakeModules.default ];
+            # Sometimes this attribute set is defined in clan.nix
             clan = {
                 inventory.machines = {
                     jon = {
@@ -76,6 +77,7 @@ Adding the following services is recommended for most users:
         inputs@{ flake-parts, ... }:
         flake-parts.lib.mkFlake { inherit inputs; } {
             imports = [ inputs.clan-core.flakeModules.default ];
+            # Sometimes this attribute set is defined in clan.nix
             clan = {
                 inventory.machines = {
                     jon = {

docs/site/guides/getting-started/index.md

@@ -48,7 +48,7 @@ nix shell git+https://git.clan.lol/clan/clan-core#clan-cli --refresh
 clan --help
 ```
 
-Should print the avilable commands.
+Should print the available commands.
 
 Also checkout the [cli-reference documentation](../../reference/cli/index.md).
 
@@ -103,10 +103,15 @@ Don’t worry if your output looks different—the template evolves over time.
     run `nix develop` every time, we recommend setting up [direnv](https://direnv.net/).
 
 ```
-clan machines list
+clan show
 ```
 
-If you see no output yet, that’s expected — [add machines](./add-machines.md) to populate it.
+You should see something like this:
+
+```terminal-session
+Name: my-clan
+Description: None
+```
 
 ---
 

docs/site/guides/getting-started/secrets.md

@@ -152,6 +152,7 @@ are loaded when using Clan:
   outputs =
     { self, clan-core, ... }:
     let
+      # Sometimes this attribute set is defined in clan.nix
       clan = clan-core.lib.clan {
         inherit self;
 

docs/site/guides/mesh-vpn.md

@@ -39,6 +39,7 @@ For the purpose of this guide we have two machines:
   outputs =
     { self, clan-core, ... }:
     let
+      # Sometimes this attribute set is defined in clan.nix
       clan = clan-core.lib.clan {
         inherit self;
 

docs/site/guides/migrations/disk-id.md

@@ -0,0 +1,98 @@
+# Migrate disko config from `clanModules.disk-id`
+
+If you previously bootstrapped a machine's disk using `clanModules.disk-id`, you should now migrate to a standalone, self-contained disko configuration. This ensures long-term stability and avoids reliance on dynamic values from Clan.
+
+If your `disko.nix` currently looks something like this:
+
+```nix title="disko.nix"
+{
+  lib,
+  clan-core,
+  config,
+  ...
+}:
+
+let
+  suffix = config.clan.core.vars.generators.disk-id.files.diskId.value;
+in
+{
+  imports = [
+    clan-core.clanModules.disk-id
+  ];
+
+  # DO NOT EDIT THIS FILE AFTER INSTALLATION of a machine
+  # Otherwise your system might not boot because of missing partitions / filesystems
+  boot.loader.grub.efiSupport = lib.mkDefault true;
+  boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
+  disko.devices = {
+    disk = {
+      "main" = {
+        # suffix is to prevent disk name collisions
+        name = "main-" + suffix;
+        type = "disk";
+        # Set the following in flake.nix for each maschine:
+        # device = <uuid>;
+        content = {
+          # edlied
+        };
+      };
+    };
+  };
+}
+```
+
+## Step 1: Retrieve your `disk-id`
+
+Run the following command to retrieve the generated disk ID for your machine:
+
+```bash
+clan vars list <machineName>
+```
+
+Which should print the generated `disk-id/diskId` value in clear text
+You should see output like:
+
+```terminal-session
+disk-id/diskId: fcef30a749f8451d8f60c46e1ead726f
+# ...
+# elided
+```
+
+Copy this value — you'll need it in the next step.
+
+## ✍️ Step 2: Replace Dynamic Configuration with Static Values
+
+✅ Goal: Make your disko.nix file standalone.
+
+We are going to make three changes:
+
+- Remove `let in, imports, {lib,clan-core,config, ...}:` to isolate the file.
+- Replace `suffix` with the actual disk-id
+- Move `disko.devices.disk.main.device` from `flake.nix` or `configuration.nix` into this file.
+
+```{.nix title="disko.nix" hl_lines="7-9 11-14"}
+{
+  boot.loader.grub.efiSupport = lib.mkDefault true;
+  boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
+  disko.devices = {
+    disk = {
+      "main" = {
+        #       ↓ Copy the disk-id into place
+        name = "main-fcef30a749f8451d8f60c46e1ead726f";
+        type = "disk";
+
+        # Some earlier guides had this line in a flake.nix
+        # disko.devices.disk.main.device = "/dev/disk/by-id/__CHANGE_ME__";
+        #        ↓ Copy the '/dev/disk/by-id' into here instead
+        device = "/dev/disk/by-id/nvme-eui.e8238fa6bf530001001b448b4aec2929";
+
+        # edlied;
+      };
+    };
+  };
+}
+```
+
+These steps are only needed for existing configurations that depend on the `diskId` module.
+
+For newer machines clan offers simple *disk templates* via its [templates cli](../../reference/cli/templates.md)

docs/site/guides/target-host.md

@@ -14,6 +14,7 @@ If the hostname is **static**, like `server.example.com`, set it in the **invent
   outputs =
     { self, clan-core, ... }:
     let
+      # Sometimes this attribute set is defined in clan.nix
       clan = clan-core.lib.clan {
         inventory.machines.jon = {
             deploy.targetHost = "root@server.example.com";
@@ -41,6 +42,7 @@ If your target host depends on a **dynamic expression** (like using the machine
   outputs =
     { self, clan-core, ... }:
     let
+      # Sometimes this attribute set is defined in clan.nix
       clan = clan-core.lib.clan {
         machines.jon = {config, ...}: {
             clan.core.networking.targetHost = "jon@${config.networking.fqdn}";

flake.lock

@@ -34,11 +34,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1751854533,
-        "narHash": "sha256-U/OQFplExOR1jazZY4KkaQkJqOl59xlh21HP9mI79Vc=",
+        "lastModified": 1752113600,
+        "narHash": "sha256-7LYDxKxZgBQ8LZUuolAQ8UkIB+jb4A2UmiR+kzY9CLI=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "16b74a1e304197248a1bc663280f2548dbfcae3c",
+        "rev": "79264292b7e3482e5702932949de9cbb69fedf6d",
         "type": "github"
       },
       "original": {
@@ -184,11 +184,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1750931469,
-        "narHash": "sha256-0IEdQB1nS+uViQw4k3VGUXntjkDp7aAlqcxdewb/hAc=",
+        "lastModified": 1752055615,
+        "narHash": "sha256-19m7P4O/Aw/6+CzncWMAJu89JaKeMh3aMle1CNQSIwM=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "ac8e6f32e11e9c7f153823abc3ab007f2a65d3e1",
+        "rev": "c9d477b5d5bd7f26adddd3f96cfd6a904768d4f9",
         "type": "github"
       },
       "original": {

lib/default.nix

@@ -45,6 +45,7 @@ lib.fix (
     introspection = import ./introspection { inherit lib; };
     jsonschema = import ./jsonschema { inherit lib; };
     facts = import ./facts.nix { inherit lib; };
+    docs = import ./docs.nix { inherit lib; };
 
     # flakes
     flakes = clanLib.callLib ./flakes.nix { };

lib/docs.nix

@@ -0,0 +1,25 @@
+{ lib, ... }:
+rec {
+  prefix = "https://git.clan.lol/clan/clan-core/src/branch/main/";
+  # Strip store paths from option declarations to make docs more stable
+  # This prevents documentation from rebuilding when store paths change
+  # but the actual content remains the same
+  stripStorePathsFromDeclarations =
+    opt:
+    opt
+    // {
+      declarations = map (
+        decl:
+        if lib.isString decl && lib.hasPrefix "/nix/store/" decl then
+          let
+            parts = lib.splitString "/" decl;
+          in
+          if builtins.length parts > 4 then
+            (prefix + "/" + lib.concatStringsSep "/" (lib.drop 4 parts))
+          else
+            decl
+        else
+          decl
+      ) opt.declarations;
+    };
+}

lib/modules/eval-docs.nix

@@ -9,9 +9,11 @@ let
       clan-core.modules.clan.default
     ];
   };
+
   evalDocs = pkgs.nixosOptionsDoc {
     options = eval.options;
     warningsAreErrors = false;
+    transformOptions = clan-core.clanLib.docs.stripStorePathsFromDeclarations;
   };
 in
 {

lib/modules/inventory/distributed-service/all-services-wrapper.nix

@@ -22,6 +22,7 @@ in
       type = attrsWith {
         placeholder = "mappedServiceName";
         elemType = submoduleWith {
+          class = "clan.service";
           modules = [
             (
               { name, ... }:

lib/modules/inventory/distributed-service/inventory-adapter.nix

@@ -122,6 +122,7 @@ in
       evalServices =
         { modules, prefix }:
         lib.evalModules {
+          class = "clan";
           specialArgs = {
             inherit clanLib;
             _ctx = prefix;

lib/modules/inventory/flake-module.nix

@@ -49,6 +49,7 @@ in
               prefix = [ ];
             }).options;
           warningsAreErrors = true;
+          transformOptions = self.clanLib.docs.stripStorePathsFromDeclarations;
         }).optionsJSON;
 
       # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests

nixosModules/clanCore/vars/secret/sops/collectFiles.nix

@@ -0,0 +1,37 @@
+# collectFiles helper function
+{
+  lib ? import <nixpkgs/lib>,
+}:
+let
+  inherit (lib)
+    filterAttrs
+    flatten
+    mapAttrsToList
+    ;
+in
+generators:
+let
+  relevantFiles =
+    generator:
+    filterAttrs (
+      _name: f: f.secret && f.deploy && (f.neededFor == "users" || f.neededFor == "services")
+    ) generator.files;
+  allFiles = flatten (
+    mapAttrsToList (
+      gen_name: generator:
+      mapAttrsToList (fname: file: {
+        name = fname;
+        generator = gen_name;
+        neededForUsers = file.neededFor == "users";
+        inherit (generator) share;
+        inherit (file)
+          owner
+          group
+          mode
+          restartUnits
+          ;
+      }) (relevantFiles generator)
+    ) generators
+  );
+in
+allFiles

nixosModules/clanCore/vars/secret/sops/default.nix

@@ -7,7 +7,7 @@
 }:
 let
 
-  inherit (import ./funcs.nix { inherit lib; }) collectFiles;
+  collectFiles = import ./collectFiles.nix { inherit lib; };
 
   machineName = config.clan.core.settings.machine.name;
 

nixosModules/clanCore/vars/secret/sops/funcs.nix

@@ -1,42 +0,0 @@
-{
-  lib ? import <nixpkgs/lib>,
-  ...
-}:
-let
-
-  inherit (lib)
-    filterAttrs
-    flatten
-    mapAttrsToList
-    ;
-in
-{
-
-  collectFiles =
-    generators:
-    let
-      relevantFiles =
-        generator:
-        filterAttrs (
-          _name: f: f.secret && f.deploy && (f.neededFor == "users" || f.neededFor == "services")
-        ) generator.files;
-      allFiles = flatten (
-        mapAttrsToList (
-          gen_name: generator:
-          mapAttrsToList (fname: file: {
-            name = fname;
-            generator = gen_name;
-            neededForUsers = file.neededFor == "users";
-            inherit (generator) share;
-            inherit (file)
-              owner
-              group
-              mode
-              restartUnits
-              ;
-          }) (relevantFiles generator)
-        ) generators
-      );
-    in
-    allFiles;
-}

pkgs/clan-app/clan_app/__init__.py

@@ -4,10 +4,10 @@ import sys
 
 from clan_cli.profiler import profile
 
-log = logging.getLogger(__name__)
-
 from clan_app.app import ClanAppOptions, app_run
 
+log = logging.getLogger(__name__)
+
 
 @profile
 def main(argv: list[str] = sys.argv) -> int:
@@ -16,9 +16,32 @@ def main(argv: list[str] = sys.argv) -> int:
         "--content-uri", type=str, help="The URI of the content to display"
     )
     parser.add_argument("--debug", action="store_true", help="Enable debug mode")
+    parser.add_argument(
+        "--http-api",
+        action="store_true",
+        help="Enable HTTP API mode (default: False)",
+    )
+    parser.add_argument(
+        "--http-host",
+        type=str,
+        default="localhost",
+        help="The host for the HTTP API server (default: localhost)",
+    )
+    parser.add_argument(
+        "--http-port",
+        type=int,
+        default=8080,
+        help="The host and port for the HTTP API server (default: 8080)",
+    )
     args = parser.parse_args(argv[1:])
 
-    app_opts = ClanAppOptions(content_uri=args.content_uri, debug=args.debug)
+    app_opts = ClanAppOptions(
+        content_uri=args.content_uri,
+        http_api=args.http_api,
+        http_host=args.http_host,
+        http_port=args.http_port,
+        debug=args.debug,
+    )
     try:
         app_run(app_opts)
     except KeyboardInterrupt:

pkgs/clan-app/clan_app/__main__.py

@@ -1,6 +1,6 @@
 import sys
 
-from . import main
+from clan_app import main
 
 if __name__ == "__main__":
     sys.exit(main())

pkgs/clan-app/clan_app/api/api_bridge.py

@@ -0,0 +1,141 @@
+import logging
+import threading
+from abc import ABC, abstractmethod
+from contextlib import ExitStack
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+from clan_lib.api import ApiResponse
+from clan_lib.api.tasks import WebThread
+from clan_lib.async_run import set_should_cancel
+
+if TYPE_CHECKING:
+    from .middleware import Middleware
+
+log = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class BackendRequest:
+    method_name: str
+    args: dict[str, Any]
+    header: dict[str, Any]
+    op_key: str | None
+
+
+@dataclass(frozen=True)
+class BackendResponse:
+    body: ApiResponse
+    header: dict[str, Any]
+    _op_key: str
+
+
+@dataclass
+class ApiBridge(ABC):
+    """Generic interface for API bridges that can handle method calls from different sources."""
+
+    middleware_chain: tuple["Middleware", ...]
+    threads: dict[str, WebThread] = field(default_factory=dict)
+
+    @abstractmethod
+    def send_api_response(self, response: BackendResponse) -> None:
+        """Send response back to the client."""
+
+    def process_request(self, request: BackendRequest) -> None:
+        """Process an API request through the middleware chain."""
+        from .middleware import MiddlewareContext
+
+        with ExitStack() as stack:
+            context = MiddlewareContext(
+                request=request,
+                bridge=self,
+                exit_stack=stack,
+            )
+
+            # Process through middleware chain
+            for middleware in self.middleware_chain:
+                try:
+                    log.debug(
+                        f"{middleware.__class__.__name__} => {request.method_name}"
+                    )
+                    middleware.process(context)
+                except Exception as e:
+                    # If middleware fails, handle error
+                    self.send_api_error_response(
+                        request.op_key or "unknown", str(e), ["middleware_error"]
+                    )
+                    return
+
+    def send_api_error_response(
+        self, op_key: str, error_message: str, location: list[str]
+    ) -> None:
+        """Send an error response."""
+        from clan_lib.api import ApiError, ErrorDataClass
+
+        error_data = ErrorDataClass(
+            op_key=op_key,
+            status="error",
+            errors=[
+                ApiError(
+                    message="An internal error occured",
+                    description=error_message,
+                    location=location,
+                )
+            ],
+        )
+
+        response = BackendResponse(
+            body=error_data,
+            header={},
+            _op_key=op_key,
+        )
+
+        self.send_api_response(response)
+
+    def process_request_in_thread(
+        self,
+        request: BackendRequest,
+        *,
+        thread_name: str = "ApiBridgeThread",
+        wait_for_completion: bool = False,
+        timeout: float = 60.0,
+    ) -> None:
+        """Process an API request in a separate thread with cancellation support.
+
+        Args:
+            request: The API request to process
+            thread_name: Name for the thread (for debugging)
+            wait_for_completion: Whether to wait for the thread to complete
+            timeout: Timeout in seconds when waiting for completion
+        """
+        op_key = request.op_key or "unknown"
+
+        def thread_task(stop_event: threading.Event) -> None:
+            set_should_cancel(lambda: stop_event.is_set())
+            try:
+                log.debug(
+                    f"Processing {request.method_name} with args {request.args} "
+                    f"and header {request.header} in thread {thread_name}"
+                )
+                self.process_request(request)
+            finally:
+                self.threads.pop(op_key, None)
+
+        stop_event = threading.Event()
+        thread = threading.Thread(
+            target=thread_task, args=(stop_event,), name=thread_name
+        )
+        thread.start()
+
+        self.threads[op_key] = WebThread(thread=thread, stop_event=stop_event)
+
+        if wait_for_completion:
+            # Wait for the thread to complete (this blocks until response is sent)
+            thread.join(timeout=timeout)
+
+            # Handle timeout
+            if thread.is_alive():
+                stop_event.set()  # Cancel the thread
+                self.send_api_error_response(
+                    op_key, "Request timeout", ["api_bridge", request.method_name]
+                )

pkgs/clan-app/clan_app/api/file_gtk.py

@@ -1,16 +1,19 @@
-import gi
-
-gi.require_version("Gtk", "4.0")
-
 import logging
 import time
 from pathlib import Path
 from typing import Any
 
+import gi
+
+gi.require_version("Gtk", "4.0")
+
 from clan_lib.api import ApiError, ErrorDataClass, SuccessDataClass
 from clan_lib.api.directory import FileRequest
+from clan_lib.clan.check import check_clan_valid
+from clan_lib.flake import Flake
 from gi.repository import Gio, GLib, Gtk
 
+gi.require_version("Gtk", "4.0")
 log = logging.getLogger(__name__)
 
 
@@ -21,13 +24,58 @@ def remove_none(_list: list) -> list:
 RESULT: dict[str, SuccessDataClass[list[str] | None] | ErrorDataClass] = {}
 
 
-def open_file(
+def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass:
+    """
+    Opens the clan folder using the GTK file dialog.
+    Returns the path to the clan folder or an error if it fails.
+    """
+    file_request = FileRequest(
+        mode="select_folder",
+        title="Select Clan Folder",
+        initial_folder=str(Path.home()),
+    )
+    response = get_system_file(file_request, op_key=op_key)
+
+    if isinstance(response, ErrorDataClass):
+        return response
+
+    if not response.data or len(response.data) == 0:
+        return ErrorDataClass(
+            op_key=op_key,
+            status="error",
+            errors=[
+                ApiError(
+                    message="No folder selected",
+                    description="You must select a folder to open.",
+                    location=["get_clan_folder"],
+                )
+            ],
+        )
+
+    clan_folder = Flake(response.data[0])
+    if not check_clan_valid(clan_folder):
+        return ErrorDataClass(
+            op_key=op_key,
+            status="error",
+            errors=[
+                ApiError(
+                    message="Invalid clan folder",
+                    description=f"The selected folder '{clan_folder}' is not a valid clan folder.",
+                    location=["get_clan_folder"],
+                )
+            ],
+        )
+
+    return SuccessDataClass(op_key=op_key, data=clan_folder, status="success")
+
+
+def get_system_file(
     file_request: FileRequest, *, op_key: str
 ) -> SuccessDataClass[list[str] | None] | ErrorDataClass:
     GLib.idle_add(gtk_open_file, file_request, op_key)
 
     while RESULT.get(op_key) is None:
-        time.sleep(0.2)
+        time.sleep(0.1)
     response = RESULT[op_key]
     del RESULT[op_key]
     return response
@@ -58,7 +106,7 @@ def gtk_open_file(file_request: FileRequest, op_key: str) -> bool:
                         ApiError(
                             message=e.__class__.__name__,
                             description=str(e),
-                            location=["open_file"],
+                            location=["get_system_file"],
                         )
                     ],
                 )
@@ -86,7 +134,7 @@ def gtk_open_file(file_request: FileRequest, op_key: str) -> bool:
                         ApiError(
                             message=e.__class__.__name__,
                             description=str(e),
-                            location=["open_file"],
+                            location=["get_system_file"],
                         )
                     ],
                 )
@@ -114,7 +162,7 @@ def gtk_open_file(file_request: FileRequest, op_key: str) -> bool:
                         ApiError(
                             message=e.__class__.__name__,
                             description=str(e),
-                            location=["open_file"],
+                            location=["get_system_file"],
                         )
                     ],
                 )
@@ -142,7 +190,7 @@ def gtk_open_file(file_request: FileRequest, op_key: str) -> bool:
                         ApiError(
                             message=e.__class__.__name__,
                             description=str(e),
-                            location=["open_file"],
+                            location=["get_system_file"],
                         )
                     ],
                 )
@@ -191,7 +239,7 @@ def gtk_open_file(file_request: FileRequest, op_key: str) -> bool:
         dialog.select_folder(callback=on_folder_select)
     if file_request.mode == "open_multiple_files":
         dialog.open_multiple(callback=on_file_select_multiple)
-    elif file_request.mode == "open_file":
+    elif file_request.mode == "get_system_file":
         dialog.open(callback=on_file_select)
     elif file_request.mode == "save":
         dialog.save(callback=on_save_finish)

pkgs/clan-app/clan_app/api/middleware/__init__.py

@@ -0,0 +1,14 @@
+"""Middleware components for the webview API bridge."""
+
+from .argument_parsing import ArgumentParsingMiddleware
+from .base import Middleware, MiddlewareContext
+from .logging import LoggingMiddleware
+from .method_execution import MethodExecutionMiddleware
+
+__all__ = [
+    "ArgumentParsingMiddleware",
+    "LoggingMiddleware",
+    "MethodExecutionMiddleware",
+    "Middleware",
+    "MiddlewareContext",
+]

pkgs/clan-app/clan_app/api/middleware/argument_parsing.py

@@ -0,0 +1,55 @@
+import logging
+from dataclasses import dataclass
+
+from clan_lib.api import MethodRegistry, from_dict
+
+from clan_app.api.api_bridge import BackendRequest
+
+from .base import Middleware, MiddlewareContext
+
+log = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class ArgumentParsingMiddleware(Middleware):
+    """Middleware that handles argument parsing and dataclass construction."""
+
+    api: MethodRegistry
+
+    def process(self, context: MiddlewareContext) -> None:
+        try:
+            # Convert dictionary arguments to dataclass instances
+            reconciled_arguments = {}
+            for k, v in context.request.args.items():
+                if k == "op_key":
+                    continue
+
+                # Get the expected argument type from the API
+                arg_class = self.api.get_method_argtype(context.request.method_name, k)
+
+                # Convert dictionary to dataclass instance
+                reconciled_arguments[k] = from_dict(arg_class, v)
+
+            # Add op_key to arguments
+            reconciled_arguments["op_key"] = context.request.op_key
+
+            # Create a new request with reconciled arguments
+
+            updated_request = BackendRequest(
+                method_name=context.request.method_name,
+                args=reconciled_arguments,
+                header=context.request.header,
+                op_key=context.request.op_key,
+            )
+            context.request = updated_request
+
+        except Exception as e:
+            log.exception(
+                f"Error while parsing arguments for {context.request.method_name}"
+            )
+            context.bridge.send_api_error_response(
+                context.request.op_key or "unknown",
+                str(e),
+                ["argument_parsing", context.request.method_name],
+            )
+            raise

pkgs/clan-app/clan_app/api/middleware/base.py

@@ -0,0 +1,29 @@
+from abc import ABC, abstractmethod
+from contextlib import AbstractContextManager, ExitStack
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from clan_app.api.api_bridge import ApiBridge, BackendRequest
+
+
+@dataclass
+class MiddlewareContext:
+    request: "BackendRequest"
+    bridge: "ApiBridge"
+    exit_stack: ExitStack
+
+
+@dataclass(frozen=True)
+class Middleware(ABC):
+    """Abstract base class for middleware components."""
+
+    @abstractmethod
+    def process(self, context: MiddlewareContext) -> None:
+        """Process the request through this middleware."""
+
+    def register_context_manager(
+        self, context: MiddlewareContext, cm: AbstractContextManager[Any]
+    ) -> Any:
+        """Register a context manager with the exit stack."""
+        return context.exit_stack.enter_context(cm)

pkgs/clan-app/clan_app/api/middleware/logging.py

@@ -0,0 +1,99 @@
+import io
+import logging
+import types
+from dataclasses import dataclass
+from typing import Any
+
+from clan_lib.async_run import AsyncContext, get_async_ctx, set_async_ctx
+from clan_lib.custom_logger import RegisteredHandler, setup_logging
+from clan_lib.log_manager import LogManager
+
+from .base import Middleware, MiddlewareContext
+
+log = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class LoggingMiddleware(Middleware):
+    """Middleware that sets up logging context without executing methods."""
+
+    log_manager: LogManager
+
+    def process(self, context: MiddlewareContext) -> None:
+        method = context.request.method_name
+
+        try:
+            # Handle log group configuration
+            log_group: list[str] | None = context.request.header.get("logging", {}).get(
+                "group_path", None
+            )
+            if log_group is not None:
+                if not isinstance(log_group, list):
+                    msg = f"Expected log_group to be a list, got {type(log_group)}"
+                    raise TypeError(msg)  # noqa: TRY301
+                log.warning(
+                    f"Using log group {log_group} for {context.request.method_name} with op_key {context.request.op_key}"
+                )
+            # Create log file
+            log_file = self.log_manager.create_log_file(
+                method, op_key=context.request.op_key or "unknown", group_path=log_group
+            ).get_file_path()
+
+        except Exception as e:
+            log.exception(
+                f"Error while handling request header of {context.request.method_name}"
+            )
+            context.bridge.send_api_error_response(
+                context.request.op_key or "unknown",
+                str(e),
+                ["header_middleware", context.request.method_name],
+            )
+            return
+
+        # Register logging context manager
+        class LoggingContextManager:
+            def __init__(self, log_file: Any) -> None:
+                self.log_file = log_file
+                self.log_f: Any = None
+                self.handler: RegisteredHandler | None = None
+                self.original_ctx: AsyncContext | None = None
+
+            def __enter__(self) -> "LoggingContextManager":
+                self.log_f = self.log_file.open("ab")
+                self.original_ctx = get_async_ctx()
+
+                # Set up async context for logging
+                ctx = AsyncContext(**self.original_ctx.__dict__)
+                ctx.stderr = self.log_f
+                ctx.stdout = self.log_f
+                set_async_ctx(ctx)
+
+                # Set up logging handler
+                handler_stream = io.TextIOWrapper(
+                    self.log_f,  # type: ignore[arg-type]
+                    encoding="utf-8",
+                    write_through=True,
+                    line_buffering=True,
+                )
+                self.handler = setup_logging(
+                    log.getEffectiveLevel(), log_file=handler_stream
+                )
+
+                return self
+
+            def __exit__(
+                self,
+                exc_type: type[BaseException] | None,
+                exc_val: BaseException | None,
+                exc_tb: types.TracebackType | None,
+            ) -> None:
+                if self.handler:
+                    self.handler.root_logger.removeHandler(self.handler.new_handler)
+                    self.handler.new_handler.close()
+                if self.log_f:
+                    self.log_f.close()
+                if self.original_ctx:
+                    set_async_ctx(self.original_ctx)
+
+        # Register the logging context manager
+        self.register_context_manager(context, LoggingContextManager(log_file))

pkgs/clan-app/clan_app/api/middleware/method_execution.py

@@ -0,0 +1,41 @@
+import logging
+from dataclasses import dataclass
+
+from clan_lib.api import MethodRegistry
+
+from clan_app.api.api_bridge import BackendResponse
+
+from .base import Middleware, MiddlewareContext
+
+log = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class MethodExecutionMiddleware(Middleware):
+    """Middleware that handles actual method execution."""
+
+    api: MethodRegistry
+
+    def process(self, context: MiddlewareContext) -> None:
+        method = self.api.functions[context.request.method_name]
+
+        try:
+            # Execute the actual method
+            result = method(**context.request.args)
+
+            response = BackendResponse(
+                body=result,
+                header={},
+                _op_key=context.request.op_key or "unknown",
+            )
+            context.bridge.send_api_response(response)
+
+        except Exception as e:
+            log.exception(
+                f"Error while handling result of {context.request.method_name}"
+            )
+            context.bridge.send_api_error_response(
+                context.request.op_key or "unknown",
+                str(e),
+                ["method_execution", context.request.method_name],
+            )

pkgs/clan-app/clan_app/app.py

@@ -1,27 +1,33 @@
 import logging
-
-from clan_cli.profiler import profile
-
-log = logging.getLogger(__name__)
 import os
 from dataclasses import dataclass
 from pathlib import Path
 
-import clan_lib.machines.actions  # noqa: F401
+from clan_cli.profiler import profile
 from clan_lib.api import API, load_in_all_api_functions, tasks
 from clan_lib.custom_logger import setup_logging
 from clan_lib.dirs import user_data_dir
 from clan_lib.log_manager import LogGroupConfig, LogManager
 from clan_lib.log_manager import api as log_manager_api
 
-from clan_app.api.file_gtk import open_file
+from clan_app.api.file_gtk import get_clan_folder, get_system_file
+from clan_app.api.middleware import (
+    ArgumentParsingMiddleware,
+    LoggingMiddleware,
+    MethodExecutionMiddleware,
+)
 from clan_app.deps.webview.webview import Size, SizeHint, Webview
 
+log = logging.getLogger(__name__)
+
 
 @dataclass
 class ClanAppOptions:
     content_uri: str
     debug: bool
+    http_api: bool = False
+    http_host: str = "127.0.0.1"
+    http_port: int = 8080
 
 
 @profile
@@ -39,9 +45,6 @@ def app_run(app_opts: ClanAppOptions) -> int:
         site_index: Path = Path(os.getenv("WEBUI_PATH", ".")).resolve() / "index.html"
         content_uri = f"file://{site_index}"
 
-    webview = Webview(debug=app_opts.debug)
-    webview.title = "Clan App"
-
     # Add a log group ["clans", <dynamic_name>, "machines", <dynamic_name>]
     log_manager = LogManager(base_dir=user_data_dir() / "clan-app" / "logs")
     clan_log_group = LogGroupConfig("clans", "Clans").add_child(
@@ -51,15 +54,75 @@ def app_run(app_opts: ClanAppOptions) -> int:
     # Init LogManager global in log_manager_api module
     log_manager_api.LOG_MANAGER_INSTANCE = log_manager
 
-    # Init BAKEND_THREADS global in tasks module
-    tasks.BAKEND_THREADS = webview.threads
-
     # Populate the API global with all functions
     load_in_all_api_functions()
 
-    API.overwrite_fn(open_file)
-    webview.bind_jsonschema_api(API, log_manager=log_manager_api.LOG_MANAGER_INSTANCE)
-    webview.size = Size(1280, 1024, SizeHint.NONE)
-    webview.navigate(content_uri)
-    webview.run()
+    # Create a shared threads dictionary for both HTTP and Webview modes
+    shared_threads: dict[str, tasks.WebThread] = {}
+    tasks.BAKEND_THREADS = shared_threads
+
+    # Start HTTP API server if requested
+    http_server = None
+    if app_opts.http_api:
+        from clan_app.deps.http.http_server import HttpApiServer
+
+        openapi_file = os.getenv("OPENAPI_FILE", None)
+        swagger_dist = os.getenv("SWAGGER_UI_DIST", None)
+
+        http_server = HttpApiServer(
+            api=API,
+            openapi_file=Path(openapi_file) if openapi_file else None,
+            swagger_dist=Path(swagger_dist) if swagger_dist else None,
+            host=app_opts.http_host,
+            port=app_opts.http_port,
+            shared_threads=shared_threads,
+        )
+
+        # Add middleware to HTTP server
+        http_server.add_middleware(ArgumentParsingMiddleware(api=API))
+        http_server.add_middleware(LoggingMiddleware(log_manager=log_manager))
+        http_server.add_middleware(MethodExecutionMiddleware(api=API))
+
+        # Start the server (bridge will be created automatically)
+        http_server.start()
+
+        # HTTP-only mode - keep the server running
+        log.info("HTTP API server running...")
+        log.info(
+            f"Swagger: http://{app_opts.http_host}:{app_opts.http_port}/api/swagger"
+        )
+
+        log.info("Press Ctrl+C to stop the server")
+        try:
+            # Keep the main thread alive
+            import time
+
+            while True:
+                time.sleep(1)
+        except KeyboardInterrupt:
+            log.info("Shutting down HTTP API server...")
+            if http_server:
+                http_server.stop()
+
+    # Create webview if not running in HTTP-only mode
+    if not app_opts.http_api:
+        webview = Webview(
+            debug=app_opts.debug,
+            title="Clan App",
+            size=Size(1280, 1024, SizeHint.NONE),
+            shared_threads=shared_threads,
+        )
+
+        API.overwrite_fn(get_system_file)
+        API.overwrite_fn(get_clan_folder)
+
+        # Add middleware to the webview
+        webview.add_middleware(ArgumentParsingMiddleware(api=API))
+        webview.add_middleware(LoggingMiddleware(log_manager=log_manager))
+        webview.add_middleware(MethodExecutionMiddleware(api=API))
+
+        webview.bind_jsonschema_api(API, log_manager=log_manager)
+        webview.navigate(content_uri)
+        webview.run()
+
     return 0

pkgs/clan-app/clan_app/deps/http/http_bridge.py

@@ -0,0 +1,341 @@
+import json
+import logging
+import uuid
+from http.server import BaseHTTPRequestHandler
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+from urllib.parse import urlparse
+
+from clan_lib.api import MethodRegistry, SuccessDataClass, dataclass_to_dict
+from clan_lib.api.tasks import WebThread
+
+from clan_app.api.api_bridge import ApiBridge, BackendRequest, BackendResponse
+
+if TYPE_CHECKING:
+    from clan_app.api.middleware import Middleware
+
+log = logging.getLogger(__name__)
+
+
+class HttpBridge(ApiBridge, BaseHTTPRequestHandler):
+    """HTTP-specific implementation of the API bridge that handles HTTP requests directly.
+
+    This bridge combines the API bridge functionality with HTTP request handling.
+    """
+
+    def __init__(
+        self,
+        api: MethodRegistry,
+        middleware_chain: tuple["Middleware", ...],
+        request: Any,
+        client_address: Any,
+        server: Any,
+        *,
+        openapi_file: Path | None = None,
+        swagger_dist: Path | None = None,
+        shared_threads: dict[str, WebThread] | None = None,
+    ) -> None:
+        # Initialize API bridge fields
+        self.api = api
+        self.middleware_chain = middleware_chain
+        self.threads = shared_threads if shared_threads is not None else {}
+
+        # Initialize OpenAPI/Swagger fields
+        self.openapi_file = openapi_file
+        self.swagger_dist = swagger_dist
+
+        # Initialize HTTP handler
+        super(BaseHTTPRequestHandler, self).__init__(request, client_address, server)
+
+    def _send_cors_headers(self) -> None:
+        """Send CORS headers for cross-origin requests."""
+        self.send_header("Access-Control-Allow-Origin", "*")
+        self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
+        self.send_header("Access-Control-Allow-Headers", "Content-Type")
+
+    def _send_json_response_with_status(
+        self, data: dict[str, Any], status_code: int = 200
+    ) -> None:
+        """Send a JSON response with the given status code."""
+        try:
+            self.send_response_only(status_code)
+            self.send_header("Content-Type", "application/json")
+            self._send_cors_headers()
+            self.end_headers()
+
+            response_data = json.dumps(data, indent=2, ensure_ascii=False)
+            self.wfile.write(response_data.encode("utf-8"))
+        except BrokenPipeError as e:
+            log.warning(f"Client disconnected before we could send a response: {e!s}")
+
+    def send_api_response(self, response: BackendResponse) -> None:
+        """Send HTTP response directly to the client."""
+        response_dict = dataclass_to_dict(response)
+        self._send_json_response_with_status(response_dict, 200)
+        log.debug(
+            f"HTTP response for {response._op_key}: {json.dumps(response_dict, indent=2)}"  # noqa: SLF001
+        )
+
+    def _create_success_response(
+        self, op_key: str, data: dict[str, Any]
+    ) -> BackendResponse:
+        """Create a successful API response."""
+        return BackendResponse(
+            body=SuccessDataClass(op_key=op_key, status="success", data=data),
+            header={},
+            _op_key=op_key,
+        )
+
+    def _send_info_response(self) -> None:
+        """Send server information response."""
+        response = self._create_success_response(
+            "info", {"message": "Clan API Server", "version": "1.0.0"}
+        )
+        self.send_api_response(response)
+
+    def _send_methods_response(self) -> None:
+        """Send available API methods response."""
+        response = self._create_success_response(
+            "methods", {"methods": list(self.api.functions.keys())}
+        )
+        self.send_api_response(response)
+
+    def _handle_swagger_request(self, parsed_url: Any) -> None:
+        """Handle Swagger UI related requests."""
+        if not self.swagger_dist or not self.swagger_dist.exists():
+            self.send_error(404, "Swagger file not found")
+            return
+
+        rel_path = parsed_url.path[len("/api/swagger") :].lstrip("/")
+
+        # Redirect /api/swagger to /api/swagger/index.html
+        if rel_path == "":
+            self.send_response(302)
+            self.send_header("Location", "/api/swagger/index.html")
+            self.end_headers()
+            return
+
+        self._serve_swagger_file(rel_path)
+
+    def _serve_swagger_file(self, rel_path: str) -> None:
+        """Serve a specific Swagger UI file."""
+        file_path = self._get_swagger_file_path(rel_path)
+
+        if not file_path.exists() or not file_path.is_file():
+            self.send_error(404, "Swagger file not found")
+            return
+
+        try:
+            content_type = self._get_content_type(file_path)
+            file_data = self._read_and_process_file(file_path, rel_path)
+
+            self.send_response(200)
+            self.send_header("Content-Type", content_type)
+            self.end_headers()
+            self.wfile.write(file_data)
+        except Exception as e:
+            log.error(f"Error reading Swagger file: {e!s}")
+            self.send_error(500, "Internal Server Error")
+
+    def _get_swagger_file_path(self, rel_path: str) -> Path:
+        """Get the file path for a Swagger resource."""
+        if rel_path == "index.html":
+            return Path(__file__).parent / "swagger.html"
+        if rel_path == "openapi.json":
+            if not self.openapi_file:
+                return Path("/nonexistent")  # Will fail exists() check
+            return self.openapi_file
+        return (
+            self.swagger_dist / rel_path if self.swagger_dist else Path("/nonexistent")
+        )
+
+    def _get_content_type(self, file_path: Path) -> str:
+        """Get the content type for a file based on its extension."""
+        content_types = {
+            ".html": "text/html",
+            ".js": "application/javascript",
+            ".css": "text/css",
+            ".json": "application/json",
+            ".png": "image/png",
+            ".svg": "image/svg+xml",
+        }
+        return content_types.get(file_path.suffix, "application/octet-stream")
+
+    def _read_and_process_file(self, file_path: Path, rel_path: str) -> bytes:
+        """Read and optionally process a file (e.g., inject server URL into openapi.json)."""
+        with file_path.open("rb") as f:
+            file_data = f.read()
+
+        if rel_path == "openapi.json":
+            json_data = json.loads(file_data.decode("utf-8"))
+            server_address = getattr(self.server, "server_address", ("localhost", 80))
+            json_data["servers"] = [
+                {"url": f"http://{server_address[0]}:{server_address[1]}/api/v1/"}
+            ]
+            file_data = json.dumps(json_data, indent=2).encode("utf-8")
+
+        return file_data
+
+    def do_OPTIONS(self) -> None:  # noqa: N802
+        """Handle CORS preflight requests."""
+        self.send_response_only(200)
+        self._send_cors_headers()
+        self.end_headers()
+
+    def do_GET(self) -> None:  # noqa: N802
+        """Handle GET requests."""
+        parsed_url = urlparse(self.path)
+        path = parsed_url.path
+
+        if path == "/":
+            self._send_info_response()
+        elif path.startswith("/api/swagger"):
+            self._handle_swagger_request(parsed_url)
+        elif path == "/api/methods":
+            self._send_methods_response()
+        else:
+            self.send_api_error_response("info", "Not Found", ["http_bridge", "GET"])
+
+    def do_POST(self) -> None:  # noqa: N802
+        """Handle POST requests."""
+        parsed_url = urlparse(self.path)
+        path = parsed_url.path
+
+        # Validate API path
+        if not path.startswith("/api/v1/"):
+            self.send_api_error_response(
+                "post", f"Path not found: {path}", ["http_bridge", "POST"]
+            )
+            return
+
+        # Extract and validate method name
+        method_name = path[len("/api/v1/") :]
+        if not method_name:
+            self.send_api_error_response(
+                "post", "Method name required", ["http_bridge", "POST"]
+            )
+            return
+
+        if method_name not in self.api.functions:
+            self.send_api_error_response(
+                "post",
+                f"Method '{method_name}' not found",
+                ["http_bridge", "POST", method_name],
+            )
+            return
+
+        # Read and parse request body
+        request_data = self._read_request_body(method_name)
+        if request_data is None:
+            return  # Error already sent
+
+        # Generate operation key and handle request
+        gen_op_key = str(uuid.uuid4())
+        try:
+            self._handle_api_request(method_name, request_data, gen_op_key)
+        except Exception as e:
+            log.exception(f"Error processing API request {method_name}")
+            self.send_api_error_response(
+                gen_op_key,
+                f"Internal server error: {e!s}",
+                ["http_bridge", "POST", method_name],
+            )
+
+    def _read_request_body(self, method_name: str) -> dict[str, Any] | None:
+        """Read and parse the request body. Returns None if there was an error."""
+        try:
+            content_length = int(self.headers.get("Content-Length", 0))
+            if content_length > 0:
+                body = self.rfile.read(content_length)
+                return json.loads(body.decode("utf-8"))
+            return {}
+        except json.JSONDecodeError:
+            self.send_api_error_response(
+                "post",
+                "Invalid JSON in request body",
+                ["http_bridge", "POST", method_name],
+            )
+            return None
+        except Exception as e:
+            self.send_api_error_response(
+                "post",
+                f"Error reading request: {e!s}",
+                ["http_bridge", "POST", method_name],
+            )
+            return None
+
+    def _handle_api_request(
+        self,
+        method_name: str,
+        request_data: dict[str, Any],
+        gen_op_key: str,
+    ) -> None:
+        """Handle an API request by processing it through middleware."""
+        try:
+            # Validate and parse request data
+            header, body, op_key = self._parse_request_data(request_data, gen_op_key)
+
+            # Validate operation key
+            self._validate_operation_key(op_key)
+
+            # Create API request
+            api_request = BackendRequest(
+                method_name=method_name, args=body, header=header, op_key=op_key
+            )
+
+        except Exception as e:
+            self.send_api_error_response(
+                gen_op_key, str(e), ["http_bridge", method_name]
+            )
+            return
+
+        self._process_api_request_in_thread(api_request, method_name)
+
+    def _parse_request_data(
+        self, request_data: dict[str, Any], gen_op_key: str
+    ) -> tuple[dict[str, Any], dict[str, Any], str]:
+        """Parse and validate request data components."""
+        header = request_data.get("header", {})
+        if not isinstance(header, dict):
+            msg = f"Expected header to be a dict, got {type(header)}"
+            raise TypeError(msg)
+
+        body = request_data.get("body", {})
+        if not isinstance(body, dict):
+            msg = f"Expected body to be a dict, got {type(body)}"
+            raise TypeError(msg)
+
+        op_key = header.get("op_key", gen_op_key)
+        if not isinstance(op_key, str):
+            msg = f"Expected op_key to be a string, got {type(op_key)}"
+            raise TypeError(msg)
+
+        return header, body, op_key
+
+    def _validate_operation_key(self, op_key: str) -> None:
+        """Validate that the operation key is valid and not in use."""
+        try:
+            uuid.UUID(op_key)
+        except ValueError as e:
+            msg = f"op_key '{op_key}' is not a valid UUID"
+            raise TypeError(msg) from e
+
+        if op_key in self.threads:
+            msg = f"Operation key '{op_key}' is already in use. Please try again."
+            raise ValueError(msg)
+
+    def _process_api_request_in_thread(
+        self, api_request: BackendRequest, method_name: str
+    ) -> None:
+        """Process the API request in a separate thread."""
+        # Use the inherited thread processing method
+        self.process_request_in_thread(
+            api_request,
+            thread_name="HttpThread",
+            wait_for_completion=True,
+            timeout=60.0,
+        )
+
+    def log_message(self, format: str, *args: Any) -> None:  # noqa: A002
+        """Override default logging to use our logger."""
+        log.info(f"{self.address_string()} - {format % args}")

pkgs/clan-app/clan_app/deps/http/http_server.py

@@ -0,0 +1,114 @@
+import logging
+import threading
+from http.server import HTTPServer, ThreadingHTTPServer
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from clan_lib.api import MethodRegistry
+from clan_lib.api.tasks import WebThread
+
+if TYPE_CHECKING:
+    from clan_app.api.middleware import Middleware
+
+from .http_bridge import HttpBridge
+
+log = logging.getLogger(__name__)
+
+
+class HttpApiServer:
+    """HTTP server for the Clan API using Python's built-in HTTP server."""
+
+    def __init__(
+        self,
+        api: MethodRegistry,
+        host: str = "127.0.0.1",
+        port: int = 8080,
+        openapi_file: Path | None = None,
+        swagger_dist: Path | None = None,
+        shared_threads: dict[str, WebThread] | None = None,
+    ) -> None:
+        self.api = api
+        self.openapi = openapi_file
+        self.swagger_dist = swagger_dist
+        self.host = host
+        self.port = port
+        self._server: HTTPServer | None = None
+        self._server_thread: threading.Thread | None = None
+        # Bridge is now the request handler itself, no separate instance needed
+        self._middleware: list[Middleware] = []
+        self.shared_threads = shared_threads if shared_threads is not None else {}
+
+    def add_middleware(self, middleware: "Middleware") -> None:
+        """Add middleware to the middleware chain."""
+        if self._server is not None:
+            msg = "Cannot add middleware after server is started"
+            raise RuntimeError(msg)
+        self._middleware.append(middleware)
+
+    @property
+    def server(self) -> HTTPServer | None:
+        """Get the HTTP server instance."""
+        return self._server
+
+    @property
+    def server_thread(self) -> threading.Thread | None:
+        """Get the server thread."""
+        return self._server_thread
+
+    def _create_request_handler(self) -> type[HttpBridge]:
+        """Create a request handler class with injected dependencies."""
+        api = self.api
+        middleware_chain = tuple(self._middleware)
+        openapi_file = self.openapi
+        swagger_dist = self.swagger_dist
+        shared_threads = self.shared_threads
+
+        class RequestHandler(HttpBridge):
+            def __init__(self, request: Any, client_address: Any, server: Any) -> None:
+                super().__init__(
+                    api=api,
+                    middleware_chain=middleware_chain,
+                    request=request,
+                    client_address=client_address,
+                    server=server,
+                    openapi_file=openapi_file,
+                    swagger_dist=swagger_dist,
+                    shared_threads=shared_threads,
+                )
+
+        return RequestHandler
+
+    def start(self) -> None:
+        """Start the HTTP server in a separate thread."""
+        if self._server_thread is not None:
+            log.warning("HTTP server is already running")
+            return
+
+        # Create the server using ThreadingHTTPServer for concurrent request handling
+        handler_class = self._create_request_handler()
+        self._server = ThreadingHTTPServer((self.host, self.port), handler_class)
+
+        def run_server() -> None:
+            if self._server:
+                log.info(f"HTTP API server started on http://{self.host}:{self.port}")
+                self._server.serve_forever()
+
+        self._server_thread = threading.Thread(target=run_server, daemon=True)
+        self._server_thread.start()
+
+    def stop(self) -> None:
+        """Stop the HTTP server."""
+        if self._server:
+            self._server.shutdown()
+            self._server.server_close()
+            self._server = None
+
+        if self._server_thread:
+            self._server_thread.join(timeout=5)
+            self._server_thread = None
+
+        log.info("HTTP API server stopped")
+
+    def is_running(self) -> bool:
+        """Check if the server is running."""
+        return self._server_thread is not None and self._server_thread.is_alive()

pkgs/clan-app/clan_app/deps/http/swagger.html

@@ -0,0 +1,125 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>Swagger UI with Interceptors</title>
+    <!-- Assuming these files are in the same directory -->
+    <link rel="stylesheet" type="text/css" href="./swagger-ui.css" />
+    <link rel="stylesheet" type="text/css" href="index.css" />
+    <link
+      rel="icon"
+      type="image/png"
+      href="./favicon-32x32.png"
+      sizes="32x32"
+    />
+    <link
+      rel="icon"
+      type="image/png"
+      href="./favicon-16x16.png"
+      sizes="16x16"
+    />
+  </head>
+
+  <body>
+    <div id="swagger-ui"></div>
+    <script src="./swagger-ui-bundle.js" charset="UTF-8"></script>
+    <script src="./swagger-ui-standalone-preset.js" charset="UTF-8"></script>
+    <!-- Your swagger-initializer.js is not needed if you configure directly in the HTML -->
+    <script>
+      window.onload = () => {
+        SwaggerUIBundle({
+          url: "./openapi.json", // Path to your OpenAPI 3 spec
+          dom_id: "#swagger-ui",
+          presets: [SwaggerUIBundle.presets.apis, SwaggerUIStandalonePreset],
+          layout: "StandaloneLayout",
+          tryItOutEnabled: true,
+          deepLinking: true,
+          displayOperationId: true,
+
+          // --- INTERCEPTORS START HERE ---
+
+          /**
+           * requestInterceptor
+           * This function is called before a request is sent.
+           * It takes the request object and must return a modified request object.
+           * We will use it to wrap the user's input.
+           */
+          requestInterceptor: (request) => {
+            console.log("Intercepting request:", request);
+
+            // Only modify requests that have a body (like POST, PUT)
+            if (request.body) {
+              try {
+                // The body from the UI is a string, so we parse it to an object.
+                const originalBody = JSON.parse(request.body);
+
+                // Create the new, nested structure.
+                const newBody = {
+                  body: originalBody,
+                  header: {}, // Add an empty header object as per your example
+                };
+
+                // Replace the original body with the new, stringified, nested structure.
+                request.body = JSON.stringify(newBody);
+
+                // Update the 'Content-Length' header to match the new body size.
+                request.headers["Content-Length"] = new Blob([
+                  request.body,
+                ]).size;
+
+                console.log("Modified request body:", request.body);
+              } catch (e) {
+                // If the user's input isn't valid JSON, don't modify the request.
+                console.error(
+                  "Request Interceptor: Could not parse body as JSON.",
+                  e,
+                );
+              }
+            }
+            return request; // Always return the request object
+          },
+
+          /**
+           * responseInterceptor
+           * This function is called after a response is received, but before it's displayed.
+           * It takes the response object and must return a modified response object.
+           * We will use it to un-nest the data for display.
+           */
+          responseInterceptor: (response) => {
+            console.log("Intercepting response:", response);
+
+            // Check if the response was successful and has data to process.
+            if (response.ok && response.data) {
+              try {
+                // The response data is a string, so we parse it into an object.
+                const fullResponse = JSON.parse(response.data);
+
+                // Check if the expected 'body' property exists.
+                if (fullResponse && typeof fullResponse.body !== "undefined") {
+                  console.log(
+                    "Found nested 'body' property. Un-nesting for display.",
+                  );
+
+                  // Replace the response's data with JUST the nested 'body' object.
+                  // We stringify it with pretty-printing (2-space indentation) for readability in the UI.
+                  response.data = JSON.stringify(fullResponse.body, null, 2);
+                  response.text = response.data; // Also update the 'text' property
+                }
+              } catch (e) {
+                // If the response isn't the expected JSON structure, do nothing.
+                // This prevents errors on other endpoints that have a normal response.
+                console.error(
+                  "Response Interceptor: Could not parse response or un-nest data.",
+                  e,
+                );
+              }
+            }
+            return response; // Always return the response object
+          },
+
+          // --- INTERCEPTORS END HERE ---
+        });
+      };
+    </script>
+  </body>
+</html>

pkgs/clan-app/clan_app/deps/http/test_http_api.py

@@ -0,0 +1,372 @@
+"""Tests for HTTP API components."""
+
+import json
+import logging
+import threading
+import time
+from unittest.mock import Mock
+from urllib.request import Request, urlopen
+
+import pytest
+from clan_lib.api import MethodRegistry, tasks
+from clan_lib.async_run import is_async_cancelled
+from clan_lib.log_manager import LogManager
+
+from clan_app.api.middleware import (
+    ArgumentParsingMiddleware,
+    MethodExecutionMiddleware,
+)
+from clan_app.deps.http.http_server import HttpApiServer
+
+log = logging.getLogger(__name__)
+
+
+@pytest.fixture
+def mock_api() -> MethodRegistry:
+    """Create a mock API with test methods."""
+    api = MethodRegistry()
+
+    api.register(tasks.delete_task)
+
+    @api.register
+    def test_method(message: str) -> dict[str, str]:
+        return {"response": f"Hello {message}!"}
+
+    @api.register
+    def test_method_with_error() -> dict[str, str]:
+        msg = "Test error"
+        raise ValueError(msg)
+
+    @api.register
+    def run_task_blocking(wtime: int) -> str:
+        """A long blocking task that simulates a long-running operation."""
+        time.sleep(1)
+
+        for i in range(wtime):
+            if is_async_cancelled():
+                log.debug("Task was cancelled")
+                return "Task was cancelled"
+            log.debug(f"Processing {i} for {wtime}")
+            time.sleep(1)
+        return f"Task completed with wtime: {wtime}"
+
+    return api
+
+
+@pytest.fixture
+def mock_log_manager() -> Mock:
+    """Create a mock log manager."""
+    log_manager = Mock(spec=LogManager)
+    log_manager.create_log_file.return_value.get_file_path.return_value = Mock()
+    log_manager.create_log_file.return_value.get_file_path.return_value.open.return_value = Mock()
+    return log_manager
+
+
+@pytest.fixture
+def http_bridge(
+    mock_api: MethodRegistry, mock_log_manager: Mock
+) -> tuple[MethodRegistry, tuple]:
+    """Create HTTP bridge dependencies for testing."""
+    middleware_chain = (
+        ArgumentParsingMiddleware(api=mock_api),
+        # LoggingMiddleware(log_manager=mock_log_manager),
+        MethodExecutionMiddleware(api=mock_api),
+    )
+    return mock_api, middleware_chain
+
+
+@pytest.fixture
+def http_server(mock_api: MethodRegistry, mock_log_manager: Mock) -> HttpApiServer:
+    """Create HTTP server with mock dependencies."""
+    server = HttpApiServer(
+        api=mock_api,
+        host="127.0.0.1",
+        port=8081,  # Use different port for tests
+    )
+
+    # Add middleware
+    server.add_middleware(ArgumentParsingMiddleware(api=mock_api))
+    # server.add_middleware(LoggingMiddleware(log_manager=mock_log_manager))
+    server.add_middleware(MethodExecutionMiddleware(api=mock_api))
+
+    # Bridge will be created automatically when accessed
+
+    return server
+
+
+class TestHttpBridge:
+    """Tests for HttpBridge class."""
+
+    def test_http_bridge_initialization(self, http_bridge: tuple) -> None:
+        """Test HTTP bridge initialization."""
+        # Since HttpBridge is now a request handler, we can't instantiate it directly
+        # We'll test initialization through the server
+        api, middleware_chain = http_bridge
+        assert api is not None
+        assert len(middleware_chain) == 2
+
+    def test_http_bridge_middleware_setup(self, http_bridge: tuple) -> None:
+        """Test that middleware is properly set up."""
+        api, middleware_chain = http_bridge
+
+        # Test that we can create the bridge with middleware
+        # The actual HTTP handling will be tested through the server integration tests
+        assert len(middleware_chain) == 2
+        assert isinstance(middleware_chain[0], ArgumentParsingMiddleware)
+        # assert isinstance(middleware_chain[1], LoggingMiddleware)
+        assert isinstance(middleware_chain[1], MethodExecutionMiddleware)
+
+
+class TestHttpApiServer:
+    """Tests for HttpApiServer class."""
+
+    def test_server_initialization(self, http_server: HttpApiServer) -> None:
+        """Test HTTP server initialization."""
+        assert http_server.host == "127.0.0.1"
+        assert http_server.port == 8081
+        assert http_server.server is None
+        assert http_server.server_thread is None
+        assert not http_server.is_running()
+
+    def test_server_start_stop(self, http_server: HttpApiServer) -> None:
+        """Test starting and stopping the server."""
+        # Start server
+        http_server.start()
+        time.sleep(0.1)  # Give server time to start
+
+        assert http_server.is_running()
+
+        # Stop server
+        http_server.stop()
+        time.sleep(0.1)  # Give server time to stop
+
+        assert not http_server.is_running()
+
+    def test_server_endpoints(self, http_server: HttpApiServer) -> None:
+        """Test server endpoints."""
+        # Start server
+        http_server.start()
+        time.sleep(0.1)  # Give server time to start
+
+        try:
+            # Test root endpoint
+            response = urlopen("http://127.0.0.1:8081/")
+            data: dict = json.loads(response.read().decode())
+            assert data["body"]["status"] == "success"
+            assert data["body"]["data"]["message"] == "Clan API Server"
+            assert data["body"]["data"]["version"] == "1.0.0"
+
+            # Test methods endpoint
+            response = urlopen("http://127.0.0.1:8081/api/methods")
+            data = json.loads(response.read().decode())
+            assert data["body"]["status"] == "success"
+            assert "test_method" in data["body"]["data"]["methods"]
+            assert "test_method_with_error" in data["body"]["data"]["methods"]
+
+            # Test API call endpoint
+            request_data: dict = {"header": {}, "body": {"message": "World"}}
+            req: Request = Request(
+                "http://127.0.0.1:8081/api/v1/test_method",
+                data=json.dumps(request_data).encode(),
+                headers={"Content-Type": "application/json"},
+            )
+            response = urlopen(req)
+            data = json.loads(response.read().decode())
+
+            # Response should be BackendResponse format
+            assert "body" in data
+            assert "header" in data
+
+            assert data["body"]["status"] == "success"
+            assert data["body"]["data"] == {"response": "Hello World!"}
+
+        finally:
+            # Always stop server
+            http_server.stop()
+
+    def test_server_error_handling(self, http_server: HttpApiServer) -> None:
+        """Test server error handling."""
+        # Start server
+        http_server.start()
+        time.sleep(0.1)  # Give server time to start
+
+        try:
+            # Test 404 error
+
+            res = urlopen("http://127.0.0.1:8081/nonexistent")
+            assert res.status == 200
+            body = json.loads(res.read().decode())["body"]
+            assert body["status"] == "error"
+
+            # Test method not found
+            request_data: dict = {"header": {}, "body": {}}
+            req: Request = Request(
+                "http://127.0.0.1:8081/api/v1/nonexistent_method",
+                data=json.dumps(request_data).encode(),
+                headers={"Content-Type": "application/json"},
+            )
+
+            res = urlopen(req)
+            assert res.status == 200
+            body = json.loads(res.read().decode())["body"]
+            assert body["status"] == "error"
+
+            # Test invalid JSON
+            req = Request(
+                "http://127.0.0.1:8081/api/v1/test_method",
+                data=b"invalid json",
+                headers={"Content-Type": "application/json"},
+            )
+
+            res = urlopen(req)
+            assert res.status == 200
+            body = json.loads(res.read().decode())["body"]
+            assert body["status"] == "error"
+        finally:
+            # Always stop server
+            http_server.stop()
+
+    def test_server_cors_headers(self, http_server: HttpApiServer) -> None:
+        """Test CORS headers are properly set."""
+        # Start server
+        http_server.start()
+        time.sleep(0.1)  # Give server time to start
+
+        try:
+            # Test OPTIONS request
+            class OptionsRequest(Request):
+                def get_method(self) -> str:
+                    return "OPTIONS"
+
+            req: Request = OptionsRequest("http://127.0.0.1:8081/api/call/test_method")
+            response = urlopen(req)
+
+            # Check CORS headers
+            headers = response.info()
+            assert headers.get("Access-Control-Allow-Origin") == "*"
+            assert "GET" in headers.get("Access-Control-Allow-Methods", "")
+            assert "POST" in headers.get("Access-Control-Allow-Methods", "")
+
+        finally:
+            # Always stop server
+            http_server.stop()
+
+
+class TestIntegration:
+    """Integration tests for HTTP API components."""
+
+    def test_full_request_flow(
+        self, mock_api: MethodRegistry, mock_log_manager: Mock
+    ) -> None:
+        """Test complete request flow from server to bridge to middleware."""
+        server: HttpApiServer = HttpApiServer(
+            api=mock_api,
+            host="127.0.0.1",
+            port=8082,
+        )
+
+        # Add middleware
+        server.add_middleware(ArgumentParsingMiddleware(api=mock_api))
+        # server.add_middleware(LoggingMiddleware(log_manager=mock_log_manager))
+        server.add_middleware(MethodExecutionMiddleware(api=mock_api))
+
+        # Bridge will be created automatically when accessed
+
+        # Start server
+        server.start()
+        time.sleep(0.1)  # Give server time to start
+
+        try:
+            # Make API call
+            request_data: dict = {
+                "header": {"logging": {"group_path": ["test", "group"]}},
+                "body": {"message": "Integration"},
+            }
+            req: Request = Request(
+                "http://127.0.0.1:8082/api/v1/test_method",
+                data=json.dumps(request_data).encode(),
+                headers={"Content-Type": "application/json"},
+            )
+            response = urlopen(req)
+            data: dict = json.loads(response.read().decode())
+
+            # Verify response in BackendResponse format
+            assert "body" in data
+            assert "header" in data
+            assert data["body"]["status"] == "success"
+            assert data["body"]["data"] == {"response": "Hello Integration!"}
+
+        finally:
+            # Always stop server
+            server.stop()
+
+    def test_blocking_task(
+        self, mock_api: MethodRegistry, mock_log_manager: Mock
+    ) -> None:
+        shared_threads: dict[str, tasks.WebThread] = {}
+        tasks.BAKEND_THREADS = shared_threads
+
+        """Test a long-running blocking task."""
+        server: HttpApiServer = HttpApiServer(
+            api=mock_api,
+            host="127.0.0.1",
+            port=8083,
+            shared_threads=shared_threads,
+        )
+
+        # Add middleware
+        server.add_middleware(ArgumentParsingMiddleware(api=mock_api))
+        # server.add_middleware(LoggingMiddleware(log_manager=mock_log_manager))
+        server.add_middleware(MethodExecutionMiddleware(api=mock_api))
+
+        # Start server
+        server.start()
+        time.sleep(0.1)  # Give server time to start
+
+        blocking_op_key = "b37f920f-ce8c-4c8d-b595-28ca983d265e"  # str(uuid.uuid4())
+
+        def parallel_task() -> None:
+            # Make API call
+            request_data: dict = {
+                "body": {"wtime": 60},
+                "header": {"op_key": blocking_op_key},
+            }
+            req: Request = Request(
+                "http://127.0.0.1:8083/api/v1/run_task_blocking",
+                data=json.dumps(request_data).encode(),
+                headers={"Content-Type": "application/json"},
+            )
+            response = urlopen(req)
+            data: dict = json.loads(response.read().decode())
+
+            # thread.join()
+            assert "body" in data
+            assert data["body"]["status"] == "success"
+            assert data["body"]["data"] == "Task was cancelled"
+
+        thread = threading.Thread(
+            target=parallel_task,
+            name="ParallelTaskThread",
+            daemon=True,
+        )
+        thread.start()
+
+        time.sleep(1)
+        request_data: dict = {
+            "body": {"task_id": blocking_op_key},
+        }
+        req: Request = Request(
+            "http://127.0.0.1:8083/api/v1/delete_task",
+            data=json.dumps(request_data).encode(),
+            headers={"Content-Type": "application/json"},
+        )
+        response = urlopen(req)
+        data: dict = json.loads(response.read().decode())
+
+        assert "body" in data
+        assert "header" in data
+        assert data["body"]["status"] == "success"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])