Merge branch 'main' into update-templates-services-2

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4237

checks/data-mesher/default.nix

@@ -1,89 +0,0 @@
-{
-  pkgs,
-  nixosLib,
-  clan-core,
-  lib,
-  ...
-}:
-let
-  machines = [
-    "admin"
-    "peer"
-    "signer"
-  ];
-in
-nixosLib.runTest (
-  { ... }:
-  {
-    imports = [
-      clan-core.modules.nixosTest.clanTest
-    ];
-
-    hostPkgs = pkgs;
-    name = "service-data-mesher";
-
-    clan = {
-      directory = ./.;
-      inventory = {
-        machines = lib.genAttrs machines (_: { });
-        services = {
-          data-mesher.default = {
-            roles.peer.machines = [ "peer" ];
-            roles.admin.machines = [ "admin" ];
-            roles.signer.machines = [ "signer" ];
-          };
-        };
-      };
-    };
-
-    defaults =
-      { config, ... }:
-      {
-        environment.systemPackages = [
-          config.services.data-mesher.package
-        ];
-
-        clan.data-mesher.network.interface = "eth1";
-        clan.data-mesher.bootstrapNodes = [
-          "[2001:db8:1::1]:7946" # peer1
-          "[2001:db8:1::2]:7946" # peer2
-        ];
-
-        # speed up for testing
-        services.data-mesher.settings = {
-          cluster.join_interval = lib.mkForce "2s";
-          cluster.push_pull_interval = lib.mkForce "5s";
-        };
-      };
-
-    nodes = {
-      admin.clan.data-mesher.network.tld = "foo";
-    };
-
-    # TODO Add better test script.
-    testScript = ''
-
-      def resolve(node, success = {}, fail = [], timeout = 60):
-        for hostname, ips in success.items():
-            for ip in ips:
-                node.wait_until_succeeds(f"getent ahosts {hostname} | grep {ip}", timeout)
-
-        for hostname in fail:
-            node.wait_until_fails(f"getent ahosts {hostname}")
-
-      start_all()
-
-      admin.wait_for_unit("data-mesher")
-      signer.wait_for_unit("data-mesher")
-      peer.wait_for_unit("data-mesher")
-
-      # check dns resolution
-      for node in [admin, signer, peer]:
-        resolve(node, {
-            "admin.foo": ["2001:db8:1::1", "192.168.1.1"],
-            "peer.foo": ["2001:db8:1::2", "192.168.1.2"],
-            "signer.foo": ["2001:db8:1::3", "192.168.1.3"]
-        })
-    '';
-  }
-)

checks/data-mesher/sops/machines/admin/key.json

@@ -1,4 +0,0 @@
-{
-  "publickey": "age10zxkj45fah3qa8uyg3a36jsd06d839xfq64nrez9etrsf4km0gtsp45gsz",
-  "type": "age"
-}

checks/data-mesher/sops/machines/peer/key.json

@@ -1,4 +0,0 @@
-{
-  "publickey": "age1faqrml2ukc6unfm75d3v2vnaf62v92rdxaagg3ty3cfna7vt99gqlzs43l",
-  "type": "age"
-}

checks/data-mesher/sops/machines/signer/key.json

@@ -1,4 +0,0 @@
-{
-  "publickey": "age153mke8v2qksyqjc7vta7wglzdqr5epazt83nch0ur5v7kl87cfdsr07qld",
-  "type": "age"
-}

checks/data-mesher/sops/secrets/admin-age.key/secret

@@ -1,20 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:7xyb6WoaN7uRWEO8QRkBw7iytP5hFrA94VRi+sy/UhzqT9AyDPmxB/F8ASFsBbzJUwi0Oqd2E1CeIYRoDhG7JHnDyL2bYonz2RQ=,iv:slh3x774m6oTHAXFwcen1qF+jEchOKCyNsJMbNhqXHE=,tag:wtK8H8PZCESPA1vZCd7Ptw==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPTzZ4RTVNb2I1MTBRMEcy\neU1Eek9GakkydEJBVm9kR3AyY1pEYkorNUYwCkh2WHhNQmc1eWI2cCtEUFFWdzJq\nS0FvQWtoOFkzRVBxVzhuczc0aVprbkkKLS0tIFRLdmpnbzY1Uk9LdklEWnQzZHM2\nVEx3dzhMSnMwaWE0V0J6VTZ5ZVFYMjgKdaICa/hprHxhH89XD7ri0vyTT4rM+Si0\niHcQU4x64dgoJa4gKxgr4k9XncjoNEjJhxL7i/ZNZ5deaaLRn5rKMg==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:24:55Z",
-		"mac": "ENC[AES256_GCM,data:TJWDHGSRBfOCW8Q+t3YxG3vlpf9a5u7B27AamnOk95huqIv0htqWV3RuV7NoOZ5v2ijqSe/pLfpwrmtdhO2sUBEvhdhJm8UzLShP7AbH9lxV+icJOsY7VSrp+R5W526V46ONP6p47b7fOQBbp03BMz01G191N68WYOf6k2arGxU=,iv:nEyTBwJ2EA+OAl8Ulo5cvFX6Ow2FwzTWooF/rdkPiXg=,tag:oYcG16zR+Fb5XzVsHhq2Qw==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/sops/secrets/peer-age.key/secret

@@ -1,20 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:JOOhvl0clDD/b5YO45CXR3wVopBSNe9dYBG+p5iD+nniN2OgOwBgYPNSCVtc+NemqutD12hFUSfCzXidkv0ijhD1JZeLar9Ygxc=,iv:XctQwSYSvKhDRk/XMacC9uMydZ8e9hnhpoWTgyXiFI0=,tag:foAhBlg4DwpQU2G9DzTo5g==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBVWMvWkp5TnZQcGs5Ykhp\nWC91YkoyZERqdXpxQm5JVmRhaUhueEJETDJVCkM4V0hSYldkV1U2Q0d1TGh3eGNR\nVjJ1VFd6ZEN0SXZjSVEvcnV2WW0vbVUKLS0tIFRCNW9nWHdYaUxLSVVUSXM0OGtN\nVFMzRXExNkYxcFE3QWlxVUM3ay9INm8KV6r8ftpwarly3qXoU9y8KxKrUKLvP9KX\nGsP0pORsaM+qPMsdfEo35CqhAeQu0+6DWd7/67+fUMp6Jr0DthtTmg==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:28Z",
-		"mac": "ENC[AES256_GCM,data:scY9+/fcXhfHEdrsZJLOM6nfjpRaURgTVbCRepUjhUo24B4ByEsAo2B8psVAaGEHEsFRZuoiByqrGzKhyUASmUs+wn+ziOKBTLzu55fOakp8PWYtQ4miiz2TQffp80gCQRJpykcbUgqIKXNSNutt4tosTBL7osXwCEnEQWd+SaA=,iv:1VXNvLP6DUxZYEr1juOLJmZCGbLp33DlwhxHQV9AMD4=,tag:uFM1R8OmkFS74/zkUG0k8A==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/sops/secrets/signer-age.key/secret

@@ -1,20 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:i1YBJdK8XmWnVnZKBpmWggSN8JSOr8pm2Zx+CeE8qqeLZ7xwMO8SYCutM8l94M5vzmmX0CmwzeMZ/JVPbEwFd3ZAImUfh685HOY=,iv:N4rHNaX+WmoPb0EZPqMt+CT1BzaWO9LyoemBxKn+u/s=,tag:PnzSvdGwVnTMK8Do8VzFaQ==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4RXlmcVNGTnlkY2ZqZFlH\nVnh0eHhRNE5hRDNDVkt0TEE0bmRNN2JIVkN3CkxnaGM4Y3M3a0xoK2xMRzBLMHRV\nT1FzKzNRMFZOeWc2K3E5K2FzdUsvWmsKLS0tIENtVlFSWElHN3RtOUY2alhxajhs\naXI1MmR4WC9EVGVFK3dHM1gvVnlZMVUKCyLz0DkdbWfSfccShO1xjWfxhunEIbD0\n6imeIBhZHvVJmZLXnVl7B0pNXo6be7WSBMAUM9gUtCNh4zaChBNwGw==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:52Z",
-		"mac": "ENC[AES256_GCM,data:WFGysoXN95e/RxL094CoL4iueqEcSqCSQZLahwz9HMLi+8HWZIXr55a+jyK7piqR8nBS4BquU5fKhlC6BvEbZFt69t4onTA+LxS3D7A8/TO0CWS0RymUjW9omJUseRQWwAHtE7l0qI5hdOUKhQ+o5pU+2bc3PUlaONM0aOCCoFo=,iv:l1f4aVqLl5VAMfjNxDbxQEQp/qY/nxzgv2GTuPVBoBA=,tag:4PPDCmDrviqdn42RLHQYbA==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/per-machine/admin/data-mesher-host-key/private_key/secret

@@ -1,24 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:w3bU23Pfe8W89lF+tOmEYPU/A4FkY6n7rgQ6yo+eqCJFxTyHydV6Mg4/g4jaL+4wwIqNYRiMR8J8jLhSvw3Bc59u7Ul+RGwdpiKoBBJfsHjO8r6uOz2u9Raa+iUJH1EJWmGvsQXAILpliZ+klS96VWnGN3pYMEI=,iv:7QbUxta6NPQLZrh6AOcNe+0wkrADuTI9VKVp8q+XoZ8=,tag:ZH0t3RylfQk5U23ZHWaw0g==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age10zxkj45fah3qa8uyg3a36jsd06d839xfq64nrez9etrsf4km0gtsp45gsz",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKaTBoSFJVSTdZeW4wZG9p\nWFR1LzVmYS8xWmRqTlNtWFVkSW9jZXpVejJBCkpqZm12L1dDSmNhekVsK1JBOU9r\nZThScGdDakFlRzNsVXp1eE5yOStFSW8KLS0tIFRrTkZBQlRsR2VNcUJvNEkzS2pw\nNksvM296UkFWTkZDVVp1ZVZMNUs4cWsKWTteB1G9Oo38a81PeqKO09NUQetuqosC\nhrToQ6NMo5O7/StmVG228MHbJS3KLXsvh2AFOEPyZrbpB2Opd2wwoA==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6U2FWRThRNkVQdk9yZ0VE\nM09iSVhmeldMcDZVaFRDNGtjWTdBa0VIT2pJCkdtd04xSXdicDY3OHI1WXl5TndB\nemtQeW1SS2tVVllPUHhLUTRla3haZGMKLS0tIGN0NVNEN3RKeWM0azBBMnBpQU4r\nTFFzQ0lOcGt0ek9UZmZZRjhibTNTc0EKReUwYBVM1NKX0FD/ZeokFAAknwju5Azq\nGzl4UVJBi5Es0GWORdCGElPXMd7jMud1SwgY04AdZj/dzinCSW4CZw==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:10Z",
-		"mac": "ENC[AES256_GCM,data:0vl9Gt4QeH+GJcnl8FuWSaqQXC8S6Pe50NmeDg5Nl2NWagz8aLCvOFyTqX/Icp/bTi1XQ5icHHhF3YhM+QAvdUL3aO0WGbh92dPRnFuvlZsdtwCFhT+LyHyYHFf6yP+0h/uFpJv9fE6xY22CezA6ZVQ8ywi1epaC548Gr27uVe4=,iv:G4hZVCLkIpbg9uwB7Y8xtHLdnlmBvFrPjxSoqdyHNvM=,tag:uvKwakhUY2aa7v0tmR/o8A==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/per-machine/admin/data-mesher-host-key/public_key/value

@@ -1,3 +0,0 @@
------BEGIN PUBLIC KEY-----
-MCowBQYDK2VwAyEAm204bpSFi4jOjZuXDpIZ/rcJBrbG4zAc7OSA4rAVSYE=
------END PUBLIC KEY-----

checks/data-mesher/vars/per-machine/peer/data-mesher-host-key/private_key/secret

@@ -1,24 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:kERPY40pyvke0mRBnafa4zOaF46rbueRbhpUCXjYP5ORpC7zoOhbdlVBhOsPqE2vfEP4RWkH+ZPdDYXOKXwotBCmlq2i7TfZeoNXFkzWXc3GyM5mndnjCc8hvYEQF1w6xkkVSUt4n06BAw/gT0ppz+vo5dExIA8=,iv:JmYD2o4DGqds6DV7ucUmUD0BRB61exbRsNAtINOR8cQ=,tag:Z58gVnHD+4s21Z84IRw+Vw==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1faqrml2ukc6unfm75d3v2vnaf62v92rdxaagg3ty3cfna7vt99gqlzs43l",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4OFluVThBdUJSTmRVTk94\neFZnLytvcnNSdmQvR3ZkT2UvWFVieFV1SUFNCm9jWHlyZXRwaVdFaG9ocnd4S3FU\ndTZ2dklBbkFVL0hVT0Y2L1o5dnUyNG8KLS0tIGFvYlBJR3l2b3F6OU9uMTFkYjli\nNVFLOWQzOStpU2kzb0xyZUFCMnBmMVUK5Jzssf1XBX25bq0RKlJY8NwtKIytxL/c\nBPPFDZywJiUgw1izsdfGVkRhhSFCQIz+yWIJWzr01NU2jLyFjSfCNw==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzYW92c3Q4SktwSnJ1TkRJ\nZEJyZk96cG8ybkpPQzYzVk0xZGs0eCtISVR3CmhDaWxTem1FMjJKNmZNaTkxN01n\nenUvdFI1UkFmL1lzNlM5N0Ixd0dpc1EKLS0tIHpyS2VHaHRRdUovQVgvRmRHaXh3\naFpSNURjTWkxaW9TOXpKL2IvcUFEbmMKq4Ch7DIL34NetFV+xygTdcpQjjmV8v1n\nlvYcjUO/9c3nVkxNMJYGjuxFLuFc4Gw+AyawCjpsIYXRskYRW4UR1w==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:43Z",
-		"mac": "ENC[AES256_GCM,data:YhL2d6i0VpUd15B4ow2BgRpyEm0KEA8NSb7jZcjI58d7d4lAqBMcDQB+8a9e2NZbPk8p1EYl3q4VXbEnuwsJiPZI2kabRusy/IGoHzUTUMFfVaOuUcC0eyINNVSmzJxnCbLCAA1Aj1yXzgRQ0MWr7r0RHMKw0D1e0HxdEsuAPrA=,iv:yPlMmE6+NEEQ9uOZzD3lUTBcfUwGX/Ar+bCu0XKnjIg=,tag:eR22BCFVAlRHdggg9oCeaA==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/per-machine/peer/data-mesher-host-key/public_key/value

@@ -1,3 +0,0 @@
------BEGIN PUBLIC KEY-----
-MCowBQYDK2VwAyEAv5dICFue2fYO0Zi1IyfYjoNfR6713WpISo7+2bSjL18=
------END PUBLIC KEY-----

checks/data-mesher/vars/per-machine/signer/data-mesher-host-key/private_key/secret

@@ -1,24 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:U8F7clQ2Tuj8zy5EoEga/Mc9N3LLZrlFf5m7UJKrP5yybFRCJSBs05hOcNe+LQZdEAvvr0Qbkry1pQyE84gCVbxHvwkD+l3GbguBuLMsW96bHcmstb6AvZyhMDBpm73Azf4lXhNaiB8p2pDWdxV77E+PPw1MNYI=,iv:hQhN6Ak8tB6cXSCnTmmQqHEpXWpWck3uIVCk5pUqFqU=,tag:uC4ljcs92WPlUOfwSkrK9Q==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age153mke8v2qksyqjc7vta7wglzdqr5epazt83nch0ur5v7kl87cfdsr07qld",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvV05lejQrdUQvQjZPOG9v\nZ01naXlYZ1JxWHhDT1M1aUs1RWJDSU1acVFFCmdHY094aGRPYWxpdVVxSFVHRU9v\nNnVaeTlpSEdtSWRDMmVMSjdSOEQ4ZlEKLS0tIFo5NVk2bzBxYjZ5ZWpDWTMrQ2VF\nVThWUk0rVXpTY2svSCtiVDhTQ2kvbFkKEM2DBuFtdEj1G/vS1TsyIfQxSFFvPTDq\nCmO7L/J5lHdyfIXzp/FlhdKpjvmchb8gbfJn7IWpKopc7Zimy/JnGQ==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArNzVUaHkzUzVEMlh1Q3Qr\nOEo0aDJIMG91amJiZG50MEhqblRCTWxRRVVRCk4xZlp4SkJuUHc2UnFyU1prczkz\nNGtlQlRlNnBDRFFvUGhReTh6MTBZaXMKLS0tIGxtaXhUMDM0RU4yQytualdzdTFt\nWGRiVG54MnYrR2lqZVZoT0VkbmV5WUUKbzAnOkn8RYOo7z4RISQ0yN875vSEQMDa\nnnttzVrQuK0/iZvzJ0Zq8U9+JJJKvFB1tHqye6CN0zMbv55CLLnA0g==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:26:07Z",
-		"mac": "ENC[AES256_GCM,data:uMss4+BiVupFqX7nHnMo+0yZ8RPuFD8VHYK2EtJSqzgurQrZVT4tJwY50mz2gVmwbrm49QYKk5S+H29DU0cM0HiEOgB5P5ObpXTRJPagWQ48CEFrDpBzLplobxulwnN6jJ1dpL3JF3jfrzrnSDFXMvx+n5x/86/AYXYRsi/UeyY=,iv:mPT1svKrNGmYpbL9hh2Bxxakml69q+U6gQ0ZnEcbEyg=,tag:zcZx1lTw/bEsX/1g+6T04g==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/per-machine/signer/data-mesher-host-key/public_key/value

@@ -1,3 +0,0 @@
------BEGIN PUBLIC KEY-----
-MCowBQYDK2VwAyEAeUkW5UIwA1svbNY71ePyJKX68UhxrqIUGQ2jd06w5WM=
------END PUBLIC KEY-----

checks/data-mesher/vars/shared/data-mesher-network-key/private_key/secret

@@ -1,32 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:nRlCMF58cnkdUAE2aVHEG1+vAckKtVt48Jr21Bklfbsqe1yTiHPFAMLL1ywgWWWd7FjI/Z8WID9sWzh9J8Vmotw4aJWU/rIQSeF8cJHALvfOxarJIIyb7purAiPoPPs6ggGmSmVFGB1aw8kH1JMcppQN8OItdQM=,iv:qTwaL2mgw6g7heN/H5qcjei3oY+h46PdSe3v2hDlkTs=,tag:jYNULrOPl9mcQTTrx1SDeA==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age153mke8v2qksyqjc7vta7wglzdqr5epazt83nch0ur5v7kl87cfdsr07qld",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRcG44cGFBWXk2Z0pmNklv\nTnJ5b0svLytzZmNNRkxCVU1zaDVhNUs2cld3CklsenpWd0g2OEdKKzBMQlNEejRn\nTlEvY01HYjdvVExadnN3aXZIRTZ4YlEKLS0tIGRPUXdNSHZCRDBMbno2MjJqRHBl\nSzdiSURDYitQWFpaSElkdmdicDVjMWsKweQiRqyzXmzabmU2fmgwHtOa9uDmhx9O\ns9NfUhC3ifooQUSeYp58b1ZGJQx5O5bn9q/DaEoit5LTOUprt1pUPA==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiTEdlL29sVWFpSDNNaXRJ\ndTJDRkU4VzFPQ0M4MkFha2IxV2FXN2o3ZEFRCjF3UnZ5U1hTc3VvSTIzcWxOZjl0\ncHlLVEFqRk1UbGdxaUxEeDFqbFVYaU0KLS0tIFFyMnJkZnRHdWg4Z1IyRHFkY0I5\nQjdIMGtGLzRGMFM0ektDZ3hzZDdHSmMKvxOQuKgePom0QfPSvn+4vsGHhJ4BoOvW\nc27Vn4/i4hbjfJr4JpULAwyIwt3F0RaTA2M6EkFkY8otEi3vkcpWvA==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age10zxkj45fah3qa8uyg3a36jsd06d839xfq64nrez9etrsf4km0gtsp45gsz",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5ZzdsaVRnSmsrMGR1Ylg3\nZkpscTdwNUl5NUVXN3kvMU1icE0yZU1WSEJBClB6SlJYZUhDSElRREx5b0VueFUw\nNVFRU3BSU24yWEtpRnJoUC83SDVaUWsKLS0tIGVxNEo3TjlwakpDZlNsSkVCOXlz\nNDgwaE1xNjZkSnJBVlU5YXVHeGxVNFEKsXKyTzq9VsERpXzbFJGv/pbAghFAcXkf\nMmCgQHsfIMBJQUstcO8sAkxv3ced0dAEz8O6NUd0FS2zlhBzt29Rnw==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1faqrml2ukc6unfm75d3v2vnaf62v92rdxaagg3ty3cfna7vt99gqlzs43l",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkK1hDMGxCc1IvYXlJMnBF\nWncxaXBQa1RpTWdwUHc3Yk16My8rVHNJc2dFCkNlK2h0dy9oU3Z5ZGhwRWVLYVUz\ncVBKT2x5VnlhbXNmdHkwbmZzVG5sd0EKLS0tIHJaMzhDanF4Rkl3akN4MEIxOHFC\nYWRUZ08xb1UwOFNRaktkMjIzNXZmNkUK1rlbJ96oUNQZLmCmPNDOKxfDMMa+Bl2E\nJPxcNc7XY3WBHa3xFUbcqiPxWxDyaZjhq/LYQGpepiGonGMEzR5JOQ==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-04-08T13:25:20Z",
-		"mac": "ENC[AES256_GCM,data:za9ku+9lu1TTRjbPcd5LYDM4tJsAYF/yuWFCGkAhqcYguEducsIfoKBwL42ahAzqLjCZp91YJuINtw16mM+Hmlhi/BVwhnXNHqcfnKoAS/zg9KJvWcvXwKMmjEjaBovqaCWXWoKS7dn/wZ7nfGrlsiUilCDkW4BzTIzkqNkyREU=,iv:2X9apXMatwCPRBIRbPxz6PJQwGrlr7O+z+MrsnFq+sQ=,tag:IYvitoV4MhyJyRO1ySxbLQ==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.4"
-	}
-}

checks/data-mesher/vars/shared/data-mesher-network-key/public_key/value

@@ -1,3 +0,0 @@
------BEGIN PUBLIC KEY-----
-MCowBQYDK2VwAyEA/5j+Js7oxwWvZdfjfEO/3UuRqMxLKXsaNc3/5N2WSaw=
------END PUBLIC KEY-----

checks/flake-module.nix

@@ -94,7 +94,6 @@ in
 
             service-dummy-test = import ./service-dummy-test nixosTestArgs;
             service-dummy-test-from-flake = import ./service-dummy-test-from-flake nixosTestArgs;
-            service-data-mesher = import ./data-mesher nixosTestArgs;
           };
 
           packagesToBuild = lib.removeAttrs self'.packages [

clanModules/disk-id/roles/default.nix

@@ -1,5 +1,4 @@
 {
-  config,
   pkgs,
   ...
 }:
@@ -9,9 +8,14 @@
   config = {
 
     warnings = [
-      "The clan.disk-id module is deprecated and will be removed on 2025-07-15.
-      Please migrate to user-maintained configuration or the new equivalent clan services
-      (https://docs.clan.lol/reference/clanServices)."
+      ''
+        The clan.disk-id module is deprecated and will be removed on 2025-07-15.
+        For migration see: https://docs.clan.lol/guides/migrations/disk-id/
+
+        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+        !!! Please migrate. Otherwise you may not be able to boot your system after that date.  !!!
+        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+      ''
     ];
     clan.core.vars.generators.disk-id = {
       files.diskId.secret = false;

clanServices/admin/default.nix

@@ -9,15 +9,37 @@
     interface =
       { lib, ... }:
       {
-        options.allowedKeys = lib.mkOption {
-          default = { };
-          type = lib.types.attrsOf lib.types.str;
-          description = "The allowed public keys for ssh access to the admin user";
-          example = {
-            "key_1" = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD...";
+
+        options = {
+          allowedKeys = lib.mkOption {
+            default = { };
+            type = lib.types.attrsOf lib.types.str;
+            description = "The allowed public keys for ssh access to the admin user";
+            example = {
+              "key_1" = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD...";
+            };
+          };
+
+          rsaHostKey.enable = lib.mkEnableOption "Generate RSA host key";
+
+          # TODO: allow per-server domains that we than collect in the inventory
+          #certicficateDomains = lib.mkOption {
+          #  type = lib.types.listOf lib.types.str;
+          #  default = [ ];
+          #  example = [ "git.mydomain.com" ];
+          #  description = "List of domains to include in the certificate. This option will not prepend the machine name in front of each domain.";
+          #};
+
+          certificateSearchDomains = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [ ];
+            example = [ "mydomain.com" ];
+            description = ''
+              List of domains to include in the certificate.
+              This option will prepend the machine name in front of each domain before adding it to the certificate.
+            '';
           };
         };
-
       };
 
     perInstance =
@@ -27,10 +49,15 @@
           { ... }:
           {
             imports = [
-              ../../clanModules/sshd
-              ../../clanModules/root-password
+              # We don't have a good way to specify dependencies between
+              # clanServices for now. When it get's implemtende, we should just
+              # use the ssh and users modules here.
+              ./ssh.nix
+              ./root-password.nix
             ];
 
+            _module.args = { inherit settings; };
+
             users.users.root.openssh.authorizedKeys.keys = builtins.attrValues settings.allowedKeys;
           };
       };

clanServices/admin/root-password.nix

@@ -0,0 +1,39 @@
+# We don't have a way of specifying dependencies between clanServices for now.
+# When it get's added this file should be removed and the users module used instead.
+{
+  config,
+  pkgs,
+  ...
+}:
+{
+
+  users.mutableUsers = false;
+  users.users.root.hashedPasswordFile =
+    config.clan.core.vars.generators.root-password.files.password-hash.path;
+
+  clan.core.vars.generators.root-password = {
+    files.password-hash.neededFor = "users";
+
+    files.password.deploy = false;
+
+    runtimeInputs = [
+      pkgs.coreutils
+      pkgs.mkpasswd
+      pkgs.xkcdpass
+    ];
+
+    prompts.password.type = "hidden";
+    prompts.password.persist = true;
+    prompts.password.description = "You can autogenerate a password, if you leave this prompt blank.";
+
+    script = ''
+      prompt_value="$(cat "$prompts"/password)"
+      if [[ -n "''${prompt_value-}" ]]; then
+        echo "$prompt_value" | tr -d "\n" > "$out"/password
+      else
+        xkcdpass --numwords 5 --delimiter - --count 1 | tr -d "\n" > "$out"/password
+      fi
+      mkpasswd -s -m sha-512 < "$out"/password | tr -d "\n" > "$out"/password-hash
+    '';
+  };
+}

clanServices/admin/ssh.nix

@@ -0,0 +1,115 @@
+{
+  config,
+  pkgs,
+  lib,
+  settings,
+  ...
+}:
+let
+  stringSet = list: builtins.attrNames (builtins.groupBy lib.id list);
+
+  domains = stringSet settings.certificateSearchDomains;
+
+in
+{
+
+  services.openssh = {
+    enable = true;
+    settings.PasswordAuthentication = false;
+
+    settings.HostCertificate = lib.mkIf (
+      settings.certificateSearchDomains != [ ]
+    ) config.clan.core.vars.generators.openssh-cert.files."ssh.id_ed25519-cert.pub".path;
+
+    hostKeys =
+      [
+        {
+          path = config.clan.core.vars.generators.openssh.files."ssh.id_ed25519".path;
+          type = "ed25519";
+        }
+      ]
+      ++ lib.optional settings.rsaHostKey.enable {
+        path = config.clan.core.vars.generators.openssh-rsa.files."ssh.id_rsa".path;
+        type = "rsa";
+      };
+  };
+
+  clan.core.vars.generators.openssh = {
+    files."ssh.id_ed25519" = { };
+    files."ssh.id_ed25519.pub".secret = false;
+    migrateFact = "openssh";
+    runtimeInputs = [
+      pkgs.coreutils
+      pkgs.openssh
+    ];
+    script = ''
+      ssh-keygen -t ed25519 -N "" -C "" -f "$out"/ssh.id_ed25519
+    '';
+  };
+
+  programs.ssh.knownHosts.clan-sshd-self-ed25519 = {
+    hostNames = [
+      "localhost"
+      config.networking.hostName
+    ] ++ (lib.optional (config.networking.domain != null) config.networking.fqdn);
+    publicKey = config.clan.core.vars.generators.openssh.files."ssh.id_ed25519.pub".value;
+  };
+
+  clan.core.vars.generators.openssh-rsa = lib.mkIf settings.rsaHostKey.enable {
+    files."ssh.id_rsa" = { };
+    files."ssh.id_rsa.pub".secret = false;
+    runtimeInputs = [
+      pkgs.coreutils
+      pkgs.openssh
+    ];
+    script = ''
+      ssh-keygen -t rsa -b 4096 -N "" -C "" -f "$out"/ssh.id_rsa
+    '';
+  };
+
+  clan.core.vars.generators.openssh-cert = lib.mkIf (settings.certificateSearchDomains != [ ]) {
+    files."ssh.id_ed25519-cert.pub".secret = false;
+    dependencies = [
+      "openssh"
+      "openssh-ca"
+    ];
+    validation = {
+      name = config.clan.core.settings.machine.name;
+      domains = lib.genAttrs settings.certificateSearchDomains lib.id;
+    };
+    runtimeInputs = [
+      pkgs.openssh
+      pkgs.jq
+    ];
+    script = ''
+      ssh-keygen \
+        -s $in/openssh-ca/id_ed25519 \
+        -I ${config.clan.core.settings.machine.name} \
+        -h \
+        -n ${lib.concatMapStringsSep "," (d: "${config.clan.core.settings.machine.name}.${d}") domains} \
+        $in/openssh/ssh.id_ed25519.pub
+      mv $in/openssh/ssh.id_ed25519-cert.pub "$out"/ssh.id_ed25519-cert.pub
+    '';
+  };
+
+  clan.core.vars.generators.openssh-ca = lib.mkIf (settings.certificateSearchDomains != [ ]) {
+    share = true;
+    files.id_ed25519.deploy = false;
+    files."id_ed25519.pub" = {
+      deploy = false;
+      secret = false;
+    };
+    runtimeInputs = [
+      pkgs.openssh
+    ];
+    script = ''
+      ssh-keygen -t ed25519 -N "" -C "" -f "$out"/id_ed25519
+    '';
+  };
+
+  programs.ssh.knownHosts.ssh-ca = lib.mkIf (settings.certificateSearchDomains != [ ]) {
+    certAuthority = true;
+    extraHostNames = builtins.map (domain: "*.${domain}") settings.certificateSearchDomains;
+    publicKey = config.clan.core.vars.generators.openssh-ca.files."id_ed25519.pub".value;
+  };
+}

clanServices/data-mesher/README.md

@@ -0,0 +1,29 @@
+This service will set up data-mesher.
+
+## Usage
+
+```nix
+inventory.instances = {
+  data-mesher = {
+    module = {
+      name = "data-mesher";
+      input = "clan-core";
+    };
+    roles.admin.machines.server0 = {
+      settings = {
+        bootstrapNodes = {
+          node1 = "192.168.1.1:7946";
+          node2 = "192.168.1.2:7946";
+        };
+
+        network = {
+          hostTTL = "24h";
+          interface = "tailscale0";
+        };
+      };
+    };
+    roles.peer.machines.server1 = { };
+    roles.signer.machines.server2 = { };
+  };
+}
+```

clanServices/data-mesher/admin.nix

@@ -0,0 +1,29 @@
+{
+  lib,
+  config,
+  settings,
+  ...
+}:
+{
+
+  services.data-mesher.initNetwork =
+    let
+      # for a given machine, read it's public key and remove any new lines
+      readHostKey =
+        machine:
+        let
+          path = "${config.clan.core.settings.directory}/vars/per-machine/${machine}/data-mesher-host-key/public_key/value";
+        in
+        builtins.elemAt (lib.splitString "\n" (builtins.readFile path)) 1;
+    in
+    {
+      enable = true;
+      keyPath = config.clan.core.vars.generators.data-mesher-network-key.files.private_key.path;
+
+      tld = settings.network.tld;
+      hostTTL = settings.network.hostTTL;
+
+      # admin and signer host public keys
+      signingKeys = builtins.map readHostKey (builtins.attrNames settings.bootstrapNodes);
+    };
+}

clanServices/data-mesher/default.nix

@@ -0,0 +1,142 @@
+{ ... }:
+let
+  sharedInterface =
+    { lib, ... }:
+    {
+      options = {
+        bootstrapNodes = lib.mkOption {
+          type = lib.types.nullOr (lib.types.attrsOf lib.types.str);
+          # the default bootstrap nodes are any machines with the admin or signers role
+          # we iterate through those machines, determining an IP address for them based on their VPN
+          # currently only supports zerotier
+          # default = builtins.foldl' (
+          #   urls: name:
+          #   let
+          #     ipPath = "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value";
+          #   in
+          #   if builtins.pathExists ipPath then
+          #     let
+          #       ip = builtins.readFile ipPath;
+          #     in
+          #     urls ++ [ "[${ip}]:${builtins.toString settings.network.port}" ]
+          #   else
+          #     urls
+          # ) [ ] (dmLib.machines config).bootstrap;
+          description = ''
+            A list of bootstrap nodes that act as an initial gateway when joining
+            the cluster.
+          '';
+          example = {
+            "node1" = "192.168.1.1:7946";
+            "node2" = "192.168.1.2:7946";
+          };
+        };
+
+        network = {
+          interface = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              The interface over which cluster communication should be performed.
+              All the ip addresses associate with this interface will be part of
+              our host claim, including both ipv4 and ipv6.
+
+              This should be set to an internal/VPN interface.
+            '';
+            example = "tailscale0";
+          };
+
+          port = lib.mkOption {
+            type = lib.types.port;
+            default = 7946;
+            description = ''
+              Port to listen on for cluster communication.
+            '';
+          };
+        };
+      };
+    };
+in
+{
+  _class = "clan.service";
+  manifest.name = "data-mesher";
+  manifest.description = "Set up data-mesher";
+  manifest.categories = [ "System" ];
+  manifest.readme = builtins.readFile ./README.md;
+
+  roles.admin = {
+    interface =
+      { lib, ... }:
+      {
+
+        imports = [ sharedInterface ];
+
+        options = {
+
+          network = {
+            tld = lib.mkOption {
+              type = lib.types.str;
+              default = "clan";
+              description = "Top level domain to use for the network";
+            };
+
+            hostTTL = lib.mkOption {
+              type = lib.types.str;
+              default = "${toString (24 * 28)}h";
+              example = "24h";
+              description = "The TTL for hosts in the network, in the form of a Go time.Duration";
+            };
+          };
+        };
+      };
+    perInstance =
+      { settings, roles, ... }:
+      {
+        nixosModule = {
+          imports = [
+            ./admin.nix
+            ./shared.nix
+          ];
+          _module.args = { inherit settings roles; };
+        };
+      };
+  };
+
+  roles.signer = {
+    interface =
+      { ... }:
+      {
+        imports = [ sharedInterface ];
+      };
+    perInstance =
+      { settings, roles, ... }:
+      {
+        nixosModule = {
+          imports = [
+            ./signer.nix
+            ./shared.nix
+          ];
+          _module.args = { inherit settings roles; };
+        };
+      };
+  };
+
+  roles.peer = {
+    interface =
+      { ... }:
+      {
+        imports = [ sharedInterface ];
+      };
+    perInstance =
+      { settings, roles, ... }:
+      {
+        nixosModule = {
+          imports = [
+            ./peer.nix
+            ./shared.nix
+          ];
+          _module.args = { inherit settings roles; };
+        };
+      };
+  };
+
+}

clanServices/data-mesher/flake-module.nix

@@ -0,0 +1,17 @@
+{ lib, ... }:
+let
+  module = lib.modules.importApply ./default.nix { };
+in
+{
+  clan.modules = {
+    data-mesher = module;
+  };
+  perSystem =
+    { ... }:
+    {
+      clan.nixosTests.service-data-mesher = {
+        imports = [ ./tests/vm/default.nix ];
+        clan.modules."@clan/data-mesher" = module;
+      };
+    };
+}

clanServices/data-mesher/peer.nix

@@ -0,0 +1,2 @@
+{
+}

clanServices/data-mesher/shared.nix

@@ -0,0 +1,86 @@
+{
+  config,
+  settings,
+  ...
+}:
+{
+
+  services.data-mesher = {
+    enable = true;
+    openFirewall = true;
+
+    settings = {
+      log_level = "warn";
+      state_dir = "/var/lib/data-mesher";
+
+      # read network id from vars
+      network.id = config.clan.core.vars.generators.data-mesher-network-key.files.public_key.value;
+
+      host = {
+        names = [ config.networking.hostName ];
+        key_path = config.clan.core.vars.generators.data-mesher-host-key.files.private_key.path;
+      };
+
+      cluster = {
+        port = settings.network.port;
+        join_interval = "30s";
+        push_pull_interval = "30s";
+        interface = settings.network.interface;
+        bootstrap_nodes = (builtins.attrValues settings.bootstrapNodes);
+      };
+
+      http.port = 7331;
+      http.interface = "lo";
+    };
+  };
+
+  # Generate host key.
+  clan.core.vars.generators.data-mesher-host-key = {
+    files =
+      let
+        owner = config.users.users.data-mesher.name;
+      in
+      {
+        private_key = {
+          inherit owner;
+        };
+        public_key.secret = false;
+      };
+
+    runtimeInputs = [
+      config.services.data-mesher.package
+    ];
+
+    script = ''
+      data-mesher generate keypair \
+          --public-key-path "$out"/public_key \
+          --private-key-path "$out"/private_key
+    '';
+  };
+
+  clan.core.vars.generators.data-mesher-network-key = {
+    # generated once per clan
+    share = true;
+
+    files =
+      let
+        owner = config.users.users.data-mesher.name;
+      in
+      {
+        private_key = {
+          inherit owner;
+        };
+        public_key.secret = false;
+      };
+
+    runtimeInputs = [
+      config.services.data-mesher.package
+    ];
+
+    script = ''
+      data-mesher generate keypair \
+          --public-key-path "$out"/public_key \
+          --private-key-path "$out"/private_key
+    '';
+  };
+}

clanServices/data-mesher/signer.nix

@@ -0,0 +1,2 @@
+{
+}

clanServices/data-mesher/tests/vm/default.nix

@@ -0,0 +1,90 @@
+{
+  ...
+}:
+{
+  name = "service-data-mesher";
+
+  clan = {
+    directory = ./.;
+    test.useContainers = true;
+    inventory = {
+
+      machines.peer = { };
+      machines.admin = { };
+      machines.signer = { };
+
+      instances = {
+        data-mesher =
+          let
+            bootstrapNodes = {
+              admin = "[2001:db8:1::1]:7946";
+              peer = "[2001:db8:1::2]:7946";
+              # signer = "2001:db8:1::3:7946";
+            };
+          in
+          {
+            roles.peer.machines.peer.settings = {
+              network.interface = "eth1";
+              inherit bootstrapNodes;
+            };
+            roles.signer.machines.signer.settings = {
+              network.interface = "eth1";
+              inherit bootstrapNodes;
+            };
+            roles.admin.machines.admin.settings = {
+              network.tld = "foo";
+              network.interface = "eth1";
+              inherit bootstrapNodes;
+            };
+          };
+      };
+    };
+  };
+
+  nodes =
+    let
+      commonConfig =
+        { lib, config, ... }:
+        {
+          environment.systemPackages = [
+            config.services.data-mesher.package
+          ];
+
+          # speed up for testing
+          services.data-mesher.settings = {
+            cluster.join_interval = lib.mkForce "2s";
+            cluster.push_pull_interval = lib.mkForce "5s";
+          };
+
+        };
+    in
+    {
+      peer = commonConfig;
+      admin = commonConfig;
+      signer = commonConfig;
+    };
+
+  testScript = ''
+    def resolve(node, success = {}, fail = [], timeout = 60):
+      for hostname, ips in success.items():
+          for ip in ips:
+              node.wait_until_succeeds(f"getent ahosts {hostname} | grep {ip}", timeout)
+
+      for hostname in fail:
+          node.wait_until_fails(f"getent ahosts {hostname}")
+
+    start_all()
+
+    admin.wait_for_unit("data-mesher")
+    signer.wait_for_unit("data-mesher")
+    peer.wait_for_unit("data-mesher")
+
+    # check dns resolution
+    for node in [admin, signer, peer]:
+      resolve(node, {
+          "admin.foo": ["2001:db8:1::1", "192.168.1.1"],
+          "peer.foo": ["2001:db8:1::2", "192.168.1.2"],
+          "signer.foo": ["2001:db8:1::3", "192.168.1.3"]
+      })
+  '';
+}

clanServices/data-mesher/tests/vm/sops/machines/admin/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1r99qtxl0v86wg8ndcem87yk5wag5xcsk98ngaumqzww6t7pyms0q5cyl80",
+    "type": "age"
+  }
+]

clanServices/data-mesher/tests/vm/sops/machines/peer/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1hgjs2yqxhcxfgtvhydnfe5wzlagxw2dw4hu658e8neduy0lkye0skmjfc7",
+    "type": "age"
+  }
+]

clanServices/data-mesher/tests/vm/sops/machines/signer/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1k6h9mespmnr9zhtwwqlhnla80x5jhpd4c2p7hp0nfanr5tspup0s0rld2f",
+    "type": "age"
+  }
+]

clanServices/data-mesher/tests/vm/sops/secrets/admin-age.key/secret

@@ -0,0 +1,15 @@
+{
+	"data": "ENC[AES256_GCM,data:87WFWukgpTGlH67MTkHxzTosABK/6flJObt+u9UrGSOzBr1lx4V5IsMQ9HAM4jvLpveBNH4hlFDCxbD5666n2oYylGoyBph2vAg=,iv:GKLcU7Xqmb0ImvY7M71NddkOlUDSPa/fcXrXny2iZ1o=,tag:589QMSZeXdmTxRFtMFasZg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFaXlqZEU0eHRZZjBncDE1\nV2hzTGZiVy9rM0NnWjc1NlpHVVZEUFd5S2pJCmo3Nm11bGQyWWt1R2tHS2pOYlpn\nY3lGa0w3UFpDT1RLSDU4cnJ2YVBkSU0KLS0tIEJjZVc1YXJqcHczYSt6WjV3ai93\nakdPd3VHWkVnWkdhNCtZakp4VXhBUG8Kg3xd9w5oW3/q+s59LkDy5N+xmvuvHRmh\njUv6KFLaB81yv3kb7bzj8E3aMzX0x2fMIDZ3EoPVggqA/sCWQu0p5Q==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:45Z",
+		"mac": "ENC[AES256_GCM,data:IWKfE1Y6SNg/SK+OOAmra5SwqAUfhepCNPClWPDWpOyJDwXSpk/OKl7hi3KFfIZOGupaC0xV2tTni0Uj6IBwf8zW2Mb/b1T+fWkGiyafoKlucfNPXPCob/fyf4Ju4iD/u1mD5BYYYqNTNqJWE+MCyQigL0MPE4tXGEPDa7htM6w=,iv:5RKArbEKnYjacopfL+4QhzGB8txqc3gnlwNPfRWQSlM=,tag:mdXf02nYiW7CexIbUUaMyw==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/sops/secrets/peer-age.key/secret

@@ -0,0 +1,15 @@
+{
+	"data": "ENC[AES256_GCM,data:C9evAr01JpYiMBwuy31h+G9phm+uOYoQu+PegPFAMRbjgkjh0R+uolKtweedtHumMhzEkvz7y+BlfrriVh16ceyMozfzDEkVSWM=,iv:jM4Qx4B/j5Mvc3ybOf+10hKU19l1fCc5KcKulKgMP3c=,tag:mz01kIv5kU6u3f2+FeItYA==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAydzZrdDVidGpyd1NXT0Fu\nUEtZV3I4S0p5Z095QjBGaXpwOExJSkxVclVJCm54Vk12czQ5dm5TUExNNzlEcFNp\nUWorcWc1c1pvL3pkUFlQY3BJUGhUS3MKLS0tIHd2a291M0xkcjJvTXNnelRNZXda\nQi93R3FQVm0xTXBGR3E3SVpIMzgvR3MKmps5ObV1nODBQ0TKgZ++RLkjCEQM6sMn\nzonKtBingYzfeq+0+cASVkHZJpt/t0G5wmTgivKfv0OIP5eNSgIWFw==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:57Z",
+		"mac": "ENC[AES256_GCM,data:Jk5eL2SmNpakrGF4N/31Q/PWShV5KYfA8NmlxEkD82UsIpPiIJ4Nec6NOoo7Y4bl/J53MLjK3u0/S6q7vv0Tih6+ze6hIddMJHTCp2qqclJvpH2xn6Ln+2ZK4okK2ZbWeSDF+LHc6nIpBak8JVjC/d8dQFT2L49Dkufc1nCD46w=,iv:oR0aQzjaEpFNrpWGc1TX6/zpg0WSfQjVG6VjAMwoLTI=,tag:pigUaCkVv91tynuaNoZenA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/sops/secrets/signer-age.key/secret

@@ -0,0 +1,15 @@
+{
+	"data": "ENC[AES256_GCM,data:bIx3chjDwy4epCyFuJoZlO7EglT/vEg6pdf6x+ISxqekGrrGNdiGtw3Z9foXWAPQrzngVztbwIlcEpUusKwoRPpdGIj5YzbGZbU=,iv:Gi1hjn6cL8z+LP5g6o3bUMsuIzoZRr8e3j3EBwG3p+Y=,tag:ttIfOLhDroV/WK57KBFd0w==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHNVh6OGE4aGJxbFd2Zks1\nL1ZoNkgrQjFSVFFUL2UzOGNqRXFkZURTMkJRCnZMWk00enRndzNXQmFvMG1UekI0\nUjhwZW9sQnFvb0FGbVE0N042UjF2OTAKLS0tIEdickxQdDdaZkVmN3RsemJzSElY\nWThGQVNMcnpxRlJ3bC9wVE56blljQUUK21wWOBiQc0Kyvl047nJ1N6QKR0/5Dd6r\nlqhhdFWninzqfVXJUk2pcMio8RVlvBujDsyjrPuhbRceSi+bUXIn+w==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:03:08Z",
+		"mac": "ENC[AES256_GCM,data:kA2KCDZkZuR5rD7uU4xn5sIkizcnpGcoa3PYMbl73eux7JJYuSpUojFBRcYo1WCwMeOQUGsqo8LVF/rYhH4BVJ9LERs5zTLBaUsTarY8r/UK0Q5lNYZqIrqcb5LgOf1uCvfdXg5yfaFgPFJrEqjeekb9bx8xvhDZXpsND93rrUI=,iv:B6JqWWcQV/MxP4ucAIe7EnLiq9c4pnAUj3dnEp9IXJU=,tag:1i0Fv2i7Lak5JzIbPa2/cw==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/per-machine/admin/data-mesher-host-key/private_key/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:2FgvnmawAdk+/k+RVWNsKQlUFUF+pZrrEBuupdG50uLNyxHd7Gi772gKNgHWyzZ/lpODg5mQi0rL+GmZYQwtZ7h76AGUEeQvuMMTzVUop69txxwhJD2dxZyhUAxZpibwo/St84ai+8+VksLkCSYfTXCulaeOVh4=,iv:YkPNq4zDj35PRNgt2kHEkHhbLcVc9dHP/zrAwdd94sM=,tag:KwW/74C7Z/+3dNoXB3NHwQ==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoaS94M0JsR2Q5N21DNnFB\nUHgvelRTK3FKZkNKcTJFbEJ1VGFIM256MVVRCmw5YjdyTVlXMlFpWnczV2dTSzhu\nSm5mMVRPeU1pYVFZNEN5MjJFZHVTejgKLS0tIDB0V2hSRkt5QzFYald0TWVza1lC\ncGNXemhGcklENTJiV1QvTFZxUDNRRlUK2dVEzSbdDNXZy7rQi5/Vq4KyHq5rMtEz\npTI8i1rFKIAy4TC7to03bOIudOIzKSCCzX31xARkM6qON0vEU9aHFg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1r99qtxl0v86wg8ndcem87yk5wag5xcsk98ngaumqzww6t7pyms0q5cyl80",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEOEMzcExzTTF3MmpaenRN\ncS9RbnM0aStZSjNqbjF4QisrRjhoaDg1T0ZBCmFVOWJYZkFaOXBOUGJTdytYWk52\nVXV1MDdmSWQ1OS9iODAvN2c2Q3VGYXMKLS0tIEQxeWR4bmRoOWJ2Z1FyUk1PUk1n\nM0c5Ri9FdG9FNE9CZ29VSmgvN2xDdjgKjfG38gVOXXN2ftGiCPxMFbnh7lKM1USl\nqf11k+rgvR8M9XsDy2SnirKAaNmpks1dR6Zs5ppQuYJDEYyQCrEO5g==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:45Z",
+		"mac": "ENC[AES256_GCM,data:TEH57vUZ/swTsWQPJ1X3J//xa1Q1LYPETZS7fuXCH1LCK51u88XGqVpNzSETREQ8LAOt34qN284b03UQIBGTeTr7I9cqt+/l8ew/0rFTiO3aiaT49q9aBkeFZlA+gy47r4hkhMmzGQJMUenvnzTHwT3Pw2RES5Vjs/2TSitpqlA=,iv:ffIotRGKU8y6j/VDLKbTmA8dZJVP5vafeG4F3wd60tc=,tag:q4xOwzLw5jxDR0pPIy2irA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/per-machine/admin/data-mesher-host-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEAi6qF8u2uvPXlSflB4fzJNlOhj5PgAmRiv+JyyYOOgg4=
+-----END PUBLIC KEY-----

clanServices/data-mesher/tests/vm/vars/per-machine/peer/data-mesher-host-key/private_key/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:LUNuEP/xSmzJ44sheoIYN6F24Qpr3svn6rTVUpr4KZA8uVJ9gPUd4ko4+pDisc9PyXCcxx+cYGRqr1cBp8Q3R+IyFFlR2HzuReQJaScvgjlntGtMJ2hin/aBp4pHS0F4nqPcKKROiZvIN4NHsxQ6XRVDOZbI3kE=,iv:BdRHjQXJL/OGgmqWaEDLit/zHgduNfPe3GUmYDrWLPw=,tag:N0n7CCiu+COgrfrwHUwQBQ==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hgjs2yqxhcxfgtvhydnfe5wzlagxw2dw4hu658e8neduy0lkye0skmjfc7",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCYlhrU2c1NnIyTzlVRHEx\nQTFxOUY1OWJXcHl1OHpPdWN1ZGpQV0UvZ1NzCnlKbmx0bllWMTd1ZnIxUHY0ZUU0\nVG9Jb3grSEdWeVpwaHoyQUxvNERqT00KLS0tIGtwZm5aMU1DOUhJbVVpVzIxZFow\nNVEvMy91SEg3M094MEFBSkVMRkhKZmMKuUzbEITGkYS39G14JXbKWLjiQFd4SVft\nWH34B97TFhOqusVF3zHsSCMxm/0BMeBvLxO/3RmzlwBtgNiKOqLwtQ==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2SUhJQW5EN0VKVHpQdlZC\nYTczdVJiRFdFNGtURFc2SmxKWFFycjZkQUgwCnRBVkJvUytuUDlhVlhFYno2cnBR\nRUdjL0lab1MwZzhGTklyVWZDVFJmN3cKLS0tIFRjOC9DS3llWGZWMGI2aThVYTRu\nVEFhK2Y2YkRTZHEyMWV0Q05ISHdhVVUKo9bPdV1dUeIkm4gI0r9V/s1dAfJC+H5Z\nEIUdYA7fl3jRZ01cSZ0iYWlvdl2jj0XzKafZsEQU7rL0jg9zbA2s2g==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:59Z",
+		"mac": "ENC[AES256_GCM,data:+JtuPacwUMHXtp93DZmkiVne7bQUP8J7VpoS8koM0oJWJqZoQRHd9qH/04lrpp8q/YoOXtqXwhViZvFLieJVRexiXf/AAHfAfMn0EI7ois9oHhscN88Ps9nY6JUxhNd0h0OrUA58KKhrkGoqreAKAPADtVhaVCmWbU7vMUu1StE=,iv:BmJnTsgMSbl4XsBUkhSLfKd0XjhrEQfurEkaRJ6uD/g=,tag:jg21c4y4bQp0RwWTXkxF1A==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/per-machine/peer/data-mesher-host-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEA7kRKjQpj+BXPe5buvDZtBAcU1HIcfGmbuHZqaVm3zCo=
+-----END PUBLIC KEY-----

clanServices/data-mesher/tests/vm/vars/per-machine/signer/data-mesher-host-key/private_key/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:armAfuTE0mkoy1fxAysCX/UPNM4/mt9P6/zEDwtagTSvQjMTwVzzsM+kRdLOUV4fbZ7HdqMceaZWzurAQJenXvWlBXgn87YFOFBSpf3OnpEwCTUs9H8dsVrdSUk4SrKjCjV33mybTrae/h9tMHdkRhKJzPD1+/8=,iv:x9KVGqT2Ug6B6PNwzL7NVDQqyOmFUptUsHAJEdn30dg=,tag:XSSO6JvXaXq8aezYvpF65Q==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1k6h9mespmnr9zhtwwqlhnla80x5jhpd4c2p7hp0nfanr5tspup0s0rld2f",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIMVUwMEFzVjJhYXg5MXR4\nMzZPZUFrUWdEU2hPWUVDNHpVVENpdEdYSWtnCnN0R2pVdEIxYWZXYWNBb3N5bGNK\naVpWOXp5aWVJWG9vUWtMUnhYSmMyV0UKLS0tIEtMdFAybk1PN0t2M2lkaEYzUTY3\nVzVOdTBFbnlNVTAvRU5kU0dReEZ6MlUKNHIkAUUAqnuMtXbvXqLxQwuFALsnD/i0\naBCiz6J4S18uqt3kFbXAEksbD7jCexI8m5SMp4iuumWJ/Bx1lL4TWg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkbzBFSGt1dXI2bDN5TmFU\nY3N6djNmMTh2ek4vUzdHbTF6Z1hDQ2t5WVNJClEzZDZiaVpBekFrYTYweDNsNmk5\nTlhYZGRNd0llMndyMkZWMyt5N3pwTE0KLS0tIGJJbU9vbnBhSE5vRW1pRG83cEFJ\nR2xDTHk3VkJaVUZSVThRV3Jldkp6cnMK1V37txaSFYfLQM0qqRWjojyTN4fTJkRm\nGO3yHX9uwo/4D2xI7LM48n4vnNhSF05bWpq0X4r13fI4DofCJeEo1g==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:03:11Z",
+		"mac": "ENC[AES256_GCM,data:qD1w+DO8cWFDQMBOrmO9FvxvJRn+mlUbh13exTGgmsdPn3uzTXknIDDHeWfkpF699nSzS6wRmgrB21e55rBU6iHMx1TW16S8wvCoYMFwib8zTrJzND7EJr/gRwQa0N080kBY3xBivKLUFlctgKtFUYZ9GQ6UTQeq18QKPoROjww=,iv:1mt8Er6YHxQ42F5Kb+xNtjbCAzokbeoNlHesC9Uzmhk=,tag:provO4tKDzoL5PHDg5EmhA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/per-machine/signer/data-mesher-host-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEAVA6c25s+yNe5225PnELDV9FwbWi9ppLoTfgmdY8kILo=
+-----END PUBLIC KEY-----

clanServices/data-mesher/tests/vm/vars/shared/data-mesher-network-key/private_key/secret

@@ -0,0 +1,27 @@
+{
+	"data": "ENC[AES256_GCM,data:VzcB/JABSPoFdKYhRSn+nKxasn9zO/9fyNMrg3XstBelQNPpbO8mhmcnSamc/7e5GkpoVWgLRSULvosv+o6sz9EHRZ3UpSLBBTkDGAJmoBnkR8DbstPA9EgScpQ9IGOUP5tQ0oEOcJC3FrivdbWIzeXjpWb9BrU=,iv:6BNUrubJ9aNCkgonDRNgdyckCTndkPVDLE4X3J5d2zA=,tag:YqHTiGslEkslzUk24bmPZg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hgjs2yqxhcxfgtvhydnfe5wzlagxw2dw4hu658e8neduy0lkye0skmjfc7",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwK2lMUTkrSmM4dHQxU0tI\nRVV6Wm4wWlJMYXBGbGdubExrMi8vRnJjdVd3CjI3aFVpdHRURHp6UEk3ZEZMcDZT\nZWZWaGFWYmY2Mk1iQ1BjalZkUnpUUm8KLS0tIEhFUVhBUjg1dC9LWHg2TytkRTlX\nNnlJZkJQc2ExK1BwaVVFcEw2b3BLZjQK8kqf3ZP9uLtbjCJLSEYpAqgq9zOS2HrY\n5MbPAKQI8iCUfnegti6hU+/MxjvPlaX1vT4V0Kd3gT4Khjl+OPw0Og==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1r99qtxl0v86wg8ndcem87yk5wag5xcsk98ngaumqzww6t7pyms0q5cyl80",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWeUk3ZW9rdnZBTk9vQlFZ\nTzFZVDAvcXZyQjdkcGNNbnA0T3UyM3lzVERvCjFreE9RdWxnb2xWWmI4amJVdHBv\nNE9JN2tFazRnSGhiM0FId2RCUHNKWVEKLS0tIGlmM3JNSVZtR21ndFliUVpLTzJO\ncHJ2SjI1OExQK2hEN01WdG9wZ3RmVTAKi0BXp9yV2/9a9NeT7aTSK2CfkQ5yColJ\nm0+uv5AJndZ9IsaZGJxNOdAOspYdvsW38hFdfjUtVuUCyIOPc20WUg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1k6h9mespmnr9zhtwwqlhnla80x5jhpd4c2p7hp0nfanr5tspup0s0rld2f",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkSUR1QVMvZ3F0NUxXd00z\nOWJGZFlsUy8vUmMxa1NoakZRVmJrSmd1RzBrCk1ZcDlBMFB0WVdWeFZaT3ZBTTh5\nS2RReWpUOGRBdGV6MDdjcEY5dFYrdjAKLS0tIG9oRWhUaWJZSElRdmlOZmRKSnNq\nUUNDZFdZbmM0c25MOGpvem1JSm9pVWsKxCLPivdHc6IN6Jbf9FujLGJaXP6ieO1S\nKsrs3Fe0RdYcEKI7P9EQNebQD2kKXficM0kKV5lRRVtW5024PftWoQ==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3amkyWWlxSTJkZEdMZFhL\nU2t5OGFIa25TRmdFM0ZNcUhFRHk0eDJQN2tjCm9UcUs2V0lEZ0hyNU9uaDVrckpj\nZ1JSQlhNeExjOER2aFJTM2NDS25PN2MKLS0tIFhmM21rT0Z4aUI5TUZyNnNBQ3Jy\nSDAxejhhZDZNQTVCNjNUSTBsZncra1kKFFQrFxNMyg0AEMb1wpKBc7LOVtEHyFZW\n/o7L52fTNa0GFJ3SVEdqg0PpnRzTyA8F5L77FBGKtx6auCVVHyZZ9g==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-07-09T10:02:48Z",
+		"mac": "ENC[AES256_GCM,data:HooesDb1S24Cfb7H0lVTA8fAjM2QAN9MaJFvOSHniR6ICJAX8t8X0xfWIFRFuwPjAxi4kpBYSjW0420Yz9lZ2m4Fxswo1TV3lzHDVN2u9hdrsfpKXg5fW+2oZihuvCRStDagT3l2fKv+C+gBnGs1qyCM60BStvrEiQxTxTTHfho=,iv:kL8N0qBj4q+ZJbNJ8Y8RcV1KpUUMvNCpdwKbTPGpG6k=,tag:o2PmRsSkqTP5Idq7veGDOw==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

clanServices/data-mesher/tests/vm/vars/shared/data-mesher-network-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEA/MuamRX6ZLcJunm7lZvlai0OZh++YuqMa56GiTwO68A=
+-----END PUBLIC KEY-----

clanServices/users/default.nix

@@ -1,14 +1,17 @@
 { ... }:
 {
   _class = "clan.service";
-  manifest.name = "clan-core/users";
-  manifest.description = "Automatically generates and configures a password for the specified user account.";
+  manifest.name = "clan-core/user";
+  manifest.description = ''
+    An instance of this module will create a user account on the added machines,
+    along with a generated password that is constant across machines and user settings.
+  '';
   manifest.categories = [ "System" ];
   manifest.readme = builtins.readFile ./README.md;
 
   roles.default = {
     interface =
-      { lib, ... }:
+      { config, lib, ... }:
       {
         options = {
           user = lib.mkOption {
@@ -20,7 +23,58 @@
             type = lib.types.bool;
             default = true;
             example = false;
-            description = "Whether the user should be prompted.";
+            description = ''
+              Whether the user should be prompted for a password.
+
+              Effects:
+
+              - *enabled* (`true`) - Prompt for a password during the machine installation or update workflow.
+              - *disabled* (`false`) - Generate a password during the machine installation or update workflow.
+
+              The password can be shown in two steps:
+
+              - `clan vars list <machine-name>`
+              - `clan vars get <machine-name> <name-of-password-variable>`
+            '';
+          };
+          regularUser = lib.mkOption {
+            type = lib.types.bool;
+            default = config.user != "root";
+            defaultText = lib.literalExpression "config.user != \"root\"";
+            example = false;
+            description = ''
+              Whether the user should be a regular user or a system user.
+
+              Regular users are normal users that can log in and have a home directory.
+
+              System users are used for system services and do not have a home directory.
+
+              !!! Warning
+                  `root` cannot be a regular user.
+                  You must set this to `false` for `root`
+            '';
+          };
+          groups = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [ ];
+            example = [
+              "wheel"
+              "networkmanager"
+              "video"
+              "input"
+            ];
+            description = ''
+              Additional groups the user should be added to.
+              You can add any group that exists on your system.
+              Make sure these group exists on all machines where the user is enabled.
+
+              Commonly used groups:
+
+              - "wheel" - Allows the user to run commands as root using `sudo`.
+              - "networkmanager" - Allows the user to manage network connections.
+              - "video" - Allows the user to access video devices.
+              - "input" - Allows the user to access input devices.
+            '';
           };
         };
       };
@@ -36,9 +90,13 @@
             ...
           }:
           {
-            users.mutableUsers = false;
-            users.users.${settings.user}.hashedPasswordFile =
-              config.clan.core.vars.generators."user-password-${settings.user}".files.user-password-hash.path;
+            users.users.${settings.user} = {
+              isNormalUser = settings.regularUser;
+              extraGroups = settings.groups;
+
+              hashedPasswordFile =
+                config.clan.core.vars.generators."user-password-${settings.user}".files.user-password-hash.path;
+            };
 
             clan.core.vars.generators."user-password-${settings.user}" = {
 
@@ -81,4 +139,11 @@
           };
       };
   };
+
+  perMachine = {
+    nixosModule = {
+      # Immutable users to ensure that this module has exclusive control over the users.
+      users.mutableUsers = false;
+    };
+  };
 }

clanServices/users/tests/vm/default.nix

@@ -31,7 +31,6 @@
     server = {
       users.users.testuser.group = "testuser";
       users.groups.testuser = { };
-      users.users.testuser.isNormalUser = true;
     };
   };
 

docs/mkdocs.yml

@@ -51,6 +51,7 @@ nav:
           - 🚀 Creating Your First Clan: guides/getting-started/index.md
           - 📀 Create USB Installer (optional): guides/getting-started/installer.md
           - ⚙️ Add Machines: guides/getting-started/add-machines.md
+          - ⚙️ Add User: guides/getting-started/add-user.md
           - ⚙️ Add Services: guides/getting-started/add-services.md
           - 🔐 Secrets & Facts: guides/getting-started/secrets.md
           - 🚢 Deploy Machine: guides/getting-started/deploy.md
@@ -79,6 +80,7 @@ nav:
           - Migrate existing Flakes: guides/migrations/migration-guide.md
           - Migrate inventory Services: guides/migrations/migrate-inventory-services.md
           - Facts Vars Migration: guides/migrations/migration-facts-vars.md
+          - Disk id: guides/migrations/disk-id.md
       - macOS: guides/macos.md
   - Reference:
       - Overview: reference/index.md
@@ -86,6 +88,7 @@ nav:
           - Overview: reference/clanServices/index.md
           - reference/clanServices/admin.md
           - reference/clanServices/borgbackup.md
+          - reference/clanServices/data-mesher.md
           - reference/clanServices/emergency-access.md
           - reference/clanServices/garage.md
           - reference/clanServices/hello-world.md

docs/nix/render_options/__init__.py

@@ -29,13 +29,13 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
 
-from clan_lib.api.modules import (
+from clan_lib.errors import ClanError
+from clan_lib.services.modules import (
     CategoryInfo,
     Frontmatter,
     extract_frontmatter,
     get_roles,
 )
-from clan_lib.errors import ClanError
 
 # Get environment variables
 CLAN_CORE_PATH = Path(os.environ["CLAN_CORE_PATH"])

docs/site/guides/contributing/debugging.md

@@ -55,9 +55,37 @@ If you're using VSCode, it has a handy feature that makes paths to source code f
 
 ## Finding Print Messages
 
-To identify where a specific print message comes from, you can enable a helpful feature. Simply set the environment variable `export TRACE_PRINT=1`. When you run commands with `--debug` mode, each print message will include information about its source location.
+To trace the origin of print messages in `clan-cli`, you can enable special debugging features using environment variables:
+
+- Set `TRACE_PRINT=1` to include the source location with each print message:
+    ```bash
+    export TRACE_PRINT=1
+    ```
+    When running commands with `--debug`, every print will show where it was triggered in the code.
+
+- To see a deeper stack trace for each print, set `TRACE_DEPTH` to the desired number of stack frames (e.g., 3):
+    ```bash
+    export TRACE_DEPTH=3
+    ```
+
+### Additional Debug Logging
+
+You can enable more detailed logging for specific components by setting these environment variables:
+
+- `CLAN_DEBUG_NIX_SELECTORS=1` — verbose logs for flake.select operations
+- `CLAN_DEBUG_NIX_PREFETCH=1` — verbose logs for flake.prefetch operations
+- `CLAN_DEBUG_COMMANDS=1` — print the diffed environment of executed commands
+
+Example:
+```bash
+export CLAN_DEBUG_NIX_SELECTORS=1
+export CLAN_DEBUG_NIX_PREFETCH=1
+export CLAN_DEBUG_COMMANDS=1
+```
+
+These options help you pinpoint the source and context of print messages and debug logs during development.
+
 
-If you need more details, you can expand the stack trace information that appears with each print by setting the environment variable `export TRACE_DEPTH=3`.
 
 ## Analyzing Performance
 

docs/site/guides/getting-started/add-machines.md

@@ -10,64 +10,23 @@ See the complete [list](../../guides/more-machines.md#automatic-registration) of
 
 ## Create a machine
 
-=== "flake.nix (flake-parts)"
+=== "clan.nix (declarative)"
 
-    ```{.nix hl_lines=12-15}
+    ```{.nix hl_lines="3-4"}
     {
-        inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-        inputs.nixpkgs.follows = "clan-core/nixpkgs";
-        inputs.flake-parts.follows = "clan-core/flake-parts";
-        inputs.flake-parts.inputs.nixpkgs-lib.follows = "clan-core/nixpkgs";
-
-        outputs =
-            inputs@{ flake-parts, ... }:
-            flake-parts.lib.mkFlake { inherit inputs; } {
-                imports = [ inputs.clan-core.flakeModules.default ];
-                clan = {
-                    inventory.machines = {
-                        # Define a machine
-                        jon = { };
-                    };
-                };
-
-                systems = [
-                    "x86_64-linux"
-                    "aarch64-linux"
-                    "x86_64-darwin"
-                    "aarch64-darwin"
-                ];
+        inventory.machines = {
+            # Define a machine
+            jon = { };
         };
-    }
-    ```
 
-=== "flake.nix (classic)"
-
-    ```{.nix hl_lines=11-14}
-    {
-        inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-        inputs.nixpkgs.follows = "clan-core/nixpkgs";
-
-        outputs =
-            { self, clan-core, ... }:
-            let
-                clan = clan-core.lib.clan {
-                    inherit self;
-
-                    inventory.machines = {
-                        # Define a machine
-                        jon = { };
-                    };
-                };
-            in
-            {
-                inherit (clan.config)
-                    nixosConfigurations
-                    nixosModules
-                    clanInternals
-                    darwinConfigurations
-                    darwinModules
-                    ;
-            };
+        # Additional NixOS configuration can be added here.
+        # machines/jon/configuration.nix will be automatically imported.
+        # See: https://docs.clan.lol/guides/more-machines/#automatic-registration
+        machines = {
+            # jon = { config, ... }: {
+            #   environment.systemPackages = [ pkgs.asciinema ];
+            # };
+        };
     }
     ```
 
@@ -89,16 +48,15 @@ See the complete [list](../../guides/more-machines.md#automatic-registration) of
 
     The option: `machines.<name>` is used to add extra *nixosConfiguration* to a machine
 
-```{.nix .annotate title="flake.nix" hl_lines="3-13 18-22"}
-# Sometimes this attribute set is defined in clan.nix
-clan = {
+Add the following to your `clan.nix` file for each machine.
+This example demonstrates what is needed based on a machine called `jon`:
+
+```{.nix .annotate title="clan.nix" hl_lines="3-6 15-19"}
+{
     inventory.machines = {
         jon = {
-            # Define targetHost here
-            # Required before deployment
-            deploy.targetHost = "root@jon"; # (1)
-            # Define tags here
-            tags = [ ];
+            # Define tags here (optional)
+            tags = [ ]; # (1)
         };
         sara = {
             deploy.targetHost = "root@sara";
@@ -117,9 +75,24 @@ clan = {
 }
 ```
 
-1. It is required to define a *targetHost* for each machine before deploying. Best practice has been, to use the zerotier ip/hostname or the ip from the from overlay network you decided to use.
+1. Tags can be used to automatically add this machine to services later on. - You dont need to set this now.
 2. Add your *ssh key* here - That will ensure you can always login to your machine via *ssh* in case something goes wrong.
 
+### (Optional) Create a `configuration.nix`
+
+```nix title="./machines/jon/configuration.nix"
+{
+    imports = [
+        # enables GNOME desktop (optional)
+        ../../modules/gnome.nix
+    ];
+
+    # Set nixosOptions here
+    # Or import your own modules via 'imports'
+    # ...
+}
+```
+
 ### (Optional) Renaming a Machine
 
 Older templates included static machine folders like `jon` and `sara`.

docs/site/guides/getting-started/add-services.md

@@ -17,104 +17,61 @@ To learn more: [Guide about clanService](../clanServices.md)
 
 ## Configure a Zerotier Network (recommended)
 
-```{.nix title="flake.nix" hl_lines="20-28"}
+```{.nix title="clan.nix" hl_lines="8-16"}
 {
-    inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-    inputs.nixpkgs.follows = "clan-core/nixpkgs";
-    inputs.flake-parts.follows = "clan-core/flake-parts";
-    inputs.flake-parts.inputs.nixpkgs-lib.follows = "clan-core/nixpkgs";
-
-    outputs =
-        inputs@{ flake-parts, ... }:
-        flake-parts.lib.mkFlake { inherit inputs; } {
-            imports = [ inputs.clan-core.flakeModules.default ];
-            # Sometimes this attribute set is defined in clan.nix
-            clan = {
-                inventory.machines = {
-                    jon = {
-                        targetHost = "root@jon";
-                    };
-                    sara = {
-                        targetHost = "root@jon";
-                    };
-                };
-                inventory.instances = {
-                    zerotier = { # (1)
-                        # Defines 'jon' as the controller
-                        roles.controller.machines.jon = {};
-                        # Defines all machines as networking peer.
-                        # The 'all' tag is a clan builtin.
-                        roles.peer.tags.all = {};
-                    };
-                }
-            };
-
-            systems = [
-                "x86_64-linux"
-                "aarch64-linux"
-                "x86_64-darwin"
-                "aarch64-darwin"
-            ];
+    inventory.machines = {
+        jon = { };
+        sara = { };
     };
+
+    inventory.instances = {
+        zerotier = { # (1)
+            # Replace with the name (string) of your machine that you will use as zerotier-controller
+            # See: https://docs.zerotier.com/controller/
+            # Deploy this machine first to create the network secrets
+            roles.controller.machines."jon" = { }; # (2)
+            # Peers of the network
+            # this line means 'all' clan machines will be 'peers'
+            roles.peer.tags.all = { }; # (3)
+        };
+    };
+    # ...
+    # elided
 }
 ```
 
 1. See [reference/clanServices](../../reference/clanServices/index.md) for all available services and how to configure them.
    Or read [authoring/clanServices](../authoring/clanServices/index.md) if you want to bring your own
 
+2. Replace `__YOUR_CONTROLLER_` with the *name* of your machine.
+
+3. This line will add all machines of your clan as `peer` to zerotier
+
 ## Adding more recommended defaults
 
 Adding the following services is recommended for most users:
 
-```{.nix title="flake.nix" hl_lines="25-35"}
+```{.nix title="clan.nix" hl_lines="7-14"}
 {
-    inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-    inputs.nixpkgs.follows = "clan-core/nixpkgs";
-    inputs.flake-parts.follows = "clan-core/flake-parts";
-    inputs.flake-parts.inputs.nixpkgs-lib.follows = "clan-core/nixpkgs";
-
-    outputs =
-        inputs@{ flake-parts, ... }:
-        flake-parts.lib.mkFlake { inherit inputs; } {
-            imports = [ inputs.clan-core.flakeModules.default ];
-            # Sometimes this attribute set is defined in clan.nix
-            clan = {
-                inventory.machines = {
-                    jon = {
-                        targetHost = "root@jon";
-                    };
-                    sara = {
-                        targetHost = "root@jon";
-                    };
-                };
-                inventory.instances = {
-                    zerotier = {
-                        roles.controller.machines.jon = {};
-                        roles.peer.tags.all = {};
-                    };
-                    admin = { # (1)
-                        roles.default.tags.all = { };
-                        roles.default.settings = {
-                            allowedKeys = {
-                                "my-user" = "ssh-ed25519 AAAAC3N..."; # elided
-                            };
-                        };
-                    };
-                    state-version = { # (2)
-                        roles.default.tags.all = { };
-                    };
+    inventory.machines = {
+        jon = { };
+        sara = { };
+    };
+    inventory.instances = {
+        admin = { # (1)
+            roles.default.tags.all = { };
+            roles.default.settings = {
+                allowedKeys = {
+                    "my-user" = "ssh-ed25519 AAAAC3N..."; # (2)
                 };
             };
-            systems = [
-                "x86_64-linux"
-                "aarch64-linux"
-                "x86_64-darwin"
-                "aarch64-darwin"
-            ];
+        };
+        # ...
+        # elided
     };
 }
 ```
 
 1. The `admin` service will generate a **root-password** and **add your ssh-key** that allows for convienient administration.
-
-2. The `state-version` service will generate a [nixos state version](https://wiki.nixos.org/wiki/FAQ/When_do_I_update_stateVersion) for each system once it is deployed.
+2. Equivalent to directly setting `authorizedKeys` like in [configuring a machine](./add-machines.md#configuring-a-machine)
+3. Adds `user = jon` as a user on all machines. Will create a `home` directory, and prompt for a password before deployment.

docs/site/guides/getting-started/add-user.md

@@ -0,0 +1,127 @@
+# How to add users
+
+!!! Note "Under construction"
+
+The users concept of clan is not done yet. This guide outlines some solutions from our community.
+Defining users can be done in many different ways. We want to highlight two approaches:
+
+- Using clan's [users](../../reference/clanServices/users.md) service.
+- Using a custom approach.
+
+## Adding Users using the [users](../../reference/clanServices/users.md) service
+
+To add a first *user* this guide will be leveraging two things:
+
+- [clanServices](../../reference/clanServices/index.md): Allows to bind arbitrary logic to something we call an `ìnstance`.
+- [clanServices/users](../../reference/clanServices/users.md): Implements logic for adding a single user perInstance.
+
+The example shows how to add a user called `jon`:
+
+```{.nix title="clan.nix" hl_lines="7-21"}
+{
+    inventory.machines = {
+        jon = { };
+        sara = { };
+    };
+    inventory.instances = {
+        jon-user = { # (1)
+            module.name = "users";
+
+            roles.default.tags.all = { }; # (2)
+
+            roles.default.settings = {
+                user = "jon"; # (3)
+                groups = [
+                    "wheel" # Allow using 'sudo'
+                    "networkmanager" # Allows to manage network connections.
+                    "video" # Allows to access video devices.
+                    "input" # Allows to access input devices.
+                ];
+            };
+        };
+        # ...
+        # elided
+    };
+}
+```
+
+1. Add `user = jon` as a user on all machines. Will create a `home` directory, and prompt for a password before deployment.
+2. Add this user to `all` machines
+3. Define the `name` of the user to be `jon`
+
+The `users` service creates a `/home/jon` directory, allows `jon` to sign in and will take care of the users password as part of [deployment](./deploy.md).
+
+For more information see [clanService/users](../../reference/clanServices/users.md)
+
+## Using a custom approach
+
+Some people like to define a `users` folder in their repository root.
+That allows to bind all user specific logic to a single place (`default.nix`)
+Which can be imported into individual machines to make the user avilable on that machine.
+
+```bash
+.
+├── machines
+│   ├── jon
+# ......
+├── users
+│   ├── jon
+│   │   └── default.nix # <- a NixOS module; sets some options
+# ... ... ...
+```
+
+## using [home-manager](https://github.com/nix-community/home-manager)
+
+When using clan's `users` service it is possible to define extraModules.
+In fact this is always possible when using clan's services.
+
+We can use this property of clan services to bind a nixosModule to the user, which configures home-manager.
+
+```{.nix title="clan.nix" hl_lines="22"}
+{
+    inventory.machines = {
+        jon = { };
+        sara = { };
+    };
+    inventory.instances = {
+        jon-user = {
+            module.name = "users";
+
+            roles.default.tags.all = { };
+
+            roles.default.settings = {
+                user = "jon",
+                groups = [
+                    "wheel"
+                    "networkmanager"
+                    "video"
+                    "input"
+                ];
+            };
+
+            roles.default.extraModules = [ ./users/jon/home.nix ]; # (1)
+        };
+        # ...
+        # elided
+    };
+}
+```
+
+1. Type `path` or `string`: Must point to a seperate file. Inlining a module is not possible
+
+!!! Note "This is inspiration"
+    Our community might come up with better solutions soon.
+    We are seeking contributions to improve this pattern if you have a nicer solution in mind.
+
+```nix title="users/jon/home.nix"
+# NixOS module to import home-manager and the home-manager configuration of 'jon'
+{ self, ...}:
+{
+  imports = [ self.inputs.home-manager.nixosModules.default ];
+  home-manager.users.jon = {
+    imports = [
+      ./home-configuration.nix
+    ];
+  };
+}
+```

docs/site/guides/getting-started/deploy.md

@@ -1,6 +1,6 @@
 # Deploy a machine
 
-Now that you have created a new machine, we will walk through how to install it.
+Now that you have created a machines, added some services and setup secrets. This guide will walk through how to deploy it.
 
 ## Prerequisites
 
@@ -10,256 +10,212 @@ Now that you have created a new machine, we will walk through how to install it.
     - [x] **Machine configuration**: See our basic [adding and configuring machine guide](./add-machines.md)
     - [x] **Initialized secrets**: See [secrets](secrets.md) for how to initialize your secrets.
 
-=== "**Physical Hardware**"
+## Physical Hardware
 
-    - [x] **USB Flash Drive**: See [Clan Installer](installer.md)
+!!! note "skip this if using a cloud VM"
 
-    !!! Steps
+Steps:
 
-        1. Create a NixOS installer image and transfer it to a bootable USB drive as described in the [installer](./installer.md).
+- Create a NixOS installer image and transfer it to a bootable USB drive as described in the [installer](./installer.md).
+- Boot the target machine and connect it to a network that makes it reachable from your setup computer.
+- Note down a reachable ip adress (*ipv4*, *ipv6* or *tor*)
 
-        2. Boot the target machine and connect it to a network that makes it reachable from your setup computer.
+---
 
-=== "**Cloud VMs**"
+The installer will generate a password and local addresses on boot, then run ssh with these preconfigured.
+The installer shows it's deployment relevant information in two formats, a text form, as well as a QR code.
 
-    - [x] Any cloud machine if it is reachable via SSH and supports `kexec`.
+Sample boot screen shows:
 
-    !!! Warning "NixOS can cause strange issues when booting in certain cloud environments."
-        If on Linode: Make sure that the system uses Direct Disk boot kernel (found in the configuration pannel)
+- Root password
+- IP address
+- Optional Tor and mDNS details
+
+```{ .bash .annotate .no-copy .nohighlight}
+┌─────────────────────────────────────────────────────────────────────────────────────┐
+│ ┌───────────────────────────┐                                                       │
+│ │███████████████████████████│ # This is the QR Code (1)                             │
+│ │██ ▄▄▄▄▄ █▀▄█▀█▀▄█ ▄▄▄▄▄ ██│                                                       │
+│ │██ █   █ █▀▄▄▄█ ▀█ █   █ ██│                                                       │
+│ │██ █▄▄▄█ █▀▄ ▀▄▄▄█ █▄▄▄█ ██│                                                       │
+│ │██▄▄▄▄▄▄▄█▄▀ ▀▄▀▄█▄▄▄▄▄▄▄██│                                                       │
+│ │███▀▀▀ █▄▄█ ▀▄   ▄▀▄█   ███│                                                       │
+│ │██▄██▄▄█▄▄▀▀██▄▀ ▄▄▄ ▄▀█▀██│                                                       │
+│ │██ ▄▄▄▄▄ █▄▄▄▄ █ █▄█ █▀ ███│                                                       │
+│ │██ █   █ █ █  █ ▄▄▄  ▄▀▀ ██│                                                       │
+│ │██ █▄▄▄█ █ ▄ ▄    ▄ ▀█ ▄███│                                                       │
+│ │██▄▄▄▄▄▄▄█▄▄▄▄▄▄█▄▄▄▄▄█▄███│                                                       │
+│ │███████████████████████████│                                                       │
+│ └───────────────────────────┘                                                       │
+│ ┌─────────────────────────────────────────────────────────────────────────────────┐ │
+│ │Root password: cheesy-capital-unwell  # password (2)                             │ │
+│ │Local network addresses:                                                         │ │
+│ │enp1s0           UP    192.168.178.169/24 metric 1024 fe80::21e:6ff:fe45:3c92/64 │ │
+│ │enp2s0           DOWN                                                            │ │
+│ │wlan0            DOWN # connect to wlan (3)                                      │ │
+│ │Onion address: 6evxy5yhzytwpnhc2vpscrbti3iktxdhpnf6yim6bbs25p4v6beemzyd.onion    │ │
+│ │Multicast DNS: nixos-installer.local                                             │ │
+│ └─────────────────────────────────────────────────────────────────────────────────┘ │
+│ Press 'Ctrl-C' for console access                                                   │
+│                                                                                     │
+└─────────────────────────────────────────────────────────────────────────────────────┘
+```
+
+1.  This is not an actual QR code, because it is displayed rather poorly on text sites.
+    This would be the actual content of this specific QR code prettified:
+    ```json
+    {
+        "pass": "cheesy-capital-unwell",
+        "tor": "6evxy5yhzytwpnhc2vpscrbti3iktxdhpnf6yim6bbs25p4v6beemzyd.onion",
+        "addrs": [
+        "2001:9e8:347:ca00:21e:6ff:fe45:3c92"
+        ]
+    }
+    ```
+
+    To generate the actual QR code, that would be displayed use:
+    ```shellSession
+    echo '{"pass":"cheesy-capital-unwell","tor":"6evxy5yhzytwpnhc2vpscrbti3iktxdhpnf6yim6bbs25p4v6beemzyd.onion","addrs":["2001:9e8:347:ca00:21e:6ff:fe45:3c92"]}' | nix run nixpkgs#qrencode -- -s 2 -m 2 -t utf8
+    ```
+2.  The root password for the installer medium.
+    This password is autogenerated and meant to be easily typeable.
+3.  See [how to connect to wlan](./installer.md#optional-connect-to-wifi-manually).
+
+!!! tip
+    Use [KDE Connect](https://apps.kde.org/de/kdeconnect/) for easyily sharing QR codes from phone to desktop
+
+## Cloud VMs
+
+!!! note "skip this if using a physical machine"
+
+Clan supports any cloud machine if it is reachable via SSH and supports `kexec`.
+
+Steps:
+
+- Go to the configuration panel and note down how to connect to the machine via ssh.
+
+!!! tip "NixOS can cause strange issues when booting in certain cloud environments."
+    If on Linode: Make sure that the system uses "Direct Disk boot kernel" (found in the configuration panel)
 
 ## Setting `targetHost`
 
-=== "flake.nix (flake-parts)"
+In your nix files set the targetHost (reachable ip) that you retrieved in the previous step.
 
-    ```{.nix hl_lines="22"}
-    {
-        inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-        inputs.nixpkgs.follows = "clan-core/nixpkgs";
-        inputs.flake-parts.follows = "clan-core/flake-parts";
-        inputs.flake-parts.inputs.nixpkgs-lib.follows = "clan-core/nixpkgs";
+```{.nix title="clan.nix" hl_lines="9"}
+{
+  # Ensure this is unique among all clans you want to use.
+  meta.name = "my-clan";
 
-        outputs =
-            inputs@{ flake-parts, ... }:
-            flake-parts.lib.mkFlake { inherit inputs; } {
-            systems = [
-                "x86_64-linux"
-                "aarch64-linux"
-                "x86_64-darwin"
-                "aarch64-darwin"
-            ];
-            imports = [ inputs.clan-core.flakeModules.default ];
+  inventory.machines = {
+    # Define machines here.
+    # The machine name will be used as the hostname.
+    jon = {
+        deploy.targetHost = "root@192.168.192.4"; # (1)
+    };
+  };
+  # ...
+  # elided
+}
+```
 
-            clan = {
-                inventory.machines = {
-                    jon = {
-                        # targetHost will get picked up by cli commands
-                        deploy.targetHost = "root@jon";
-                    };
-                };
-            };
-        };
-    }
-    ```
-
-=== "flake.nix (classic)"
-
-    ```{.nix hl_lines="14"}
-    {
-        inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-        inputs.nixpkgs.follows = "clan-core/nixpkgs";
-
-        outputs =
-            { self, clan-core, ... }:
-            let
-                clan = clan-core.lib.clan {
-                    inherit self;
-
-                    inventory.machines = {
-                        jon = {
-                            # targetHost will get picked up by cli commands
-                            deploy.targetHost = "root@jon";
-                        };
-                    };
-                };
-            in
-            {
-                inherit (clan.config)
-                    nixosConfigurations
-                    nixosModules
-                    clanInternals
-                    darwinConfigurations
-                    darwinModules
-                    ;
-            };
-    }
-    ```
+1. Use the ip address of your targetMachine that you want to deploy. If using the [flash-installer](./installer.md) it should display its local ip-address when booted.
 
 !!! warning
     The use of `root@` in the target address implies SSH access as the `root` user.
     Ensure that the root login is secured and only used when necessary.
 
-## Identify the Target Disk
+See also [how to set TargetHost](../target-host.md) for other methods.
 
-On the setup computer, SSH into the target:
+## Retrieve the hardware report
 
-```bash title="setup computer"
-ssh root@<IP> lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT
+By default clan uses [nixos-facter](https://github.com/nix-community/nixos-facter) which captures detailed information about the machine or virtual environment.
+
+To generate the hardware-report (`facter.json`) run:
+
+```bash
+clan machines update-hardware-config <machineName>
 ```
 
-Replace `<IP>` with the machine's IP or hostname if mDNS (i.e. Avahi) is available.
+Example output:
 
-Which should show something like:
-
-```{.shellSession hl_lines="6" .no-copy}
-NAME        ID-LINK                                         FSTYPE   SIZE MOUNTPOINT
-sda         usb-ST_16GB_AA6271026J1000000509-0:0                    14.9G
-├─sda1      usb-ST_16GB_AA6271026J1000000509-0:0-part1                 1M
-├─sda2      usb-ST_16GB_AA6271026J1000000509-0:0-part2      vfat     100M /boot
-└─sda3      usb-ST_16GB_AA6271026J1000000509-0:0-part3      ext4     2.9G /
-nvme0n1     nvme-eui.e8238fa6bf530001001b448b4aec2929              476.9G
-├─nvme0n1p1 nvme-eui.e8238fa6bf530001001b448b4aec2929-part1 vfat     512M
-├─nvme0n1p2 nvme-eui.e8238fa6bf530001001b448b4aec2929-part2 ext4   459.6G
-└─nvme0n1p3 nvme-eui.e8238fa6bf530001001b448b4aec2929-part3 swap    16.8G
+```shell-session
+$ clan machines update-hardware-config jon
+[jon] $ nixos-facter
+Successfully generated: ./machines/jon/facter.json
 ```
 
-Look for the top-level disk device (e.g., nvme0n1 or sda) and copy its `ID-LINK`. Avoid using partition IDs like `nvme0n1p1`.
+See [update-hardware-config cli reference](../../reference/cli/machines.md#machines-update-hardware-config) for further configuration possibilities if needed.
 
-In this example we would copy `nvme-eui.e8238fa6bf530001001b448b4aec2929`
+## Configure your disk schema
+
+By default clan uses [disko](https://github.com/nix-community/disko) which allows for declarative disk partitioning.
+
+To setup a disk schema for a machine run
+
+```bash
+clan templates apply disk single-disk jon --set mainDisk ""
+```
+
+Which should fail and give the valid options for the specific hardware:
+
+```shellSession
+Invalid value  for placeholder mainDisk - Valid options:
+/dev/disk/by-id/nvme-WD_PC_SN740_SDDQNQD-512G-1201_232557804368
+```
+
+Re-run the command with the correct disk:
+
+```bash
+clan templates apply disk single-disk jon --set mainDisk "/dev/disk/by-id/nvme-WD_PC_SN740_SDDQNQD-512G-1201_232557804368"
+```
+
+Should now be succesfull
+
+```shellSession
+Applied disk template 'single-disk' to machine 'jon'
+```
+
+A disko.nix file should be created in `machines/jon`
+You can have a look and customize it if needed.
 
 !!! tip
     For advanced partitioning, see [Disko templates](https://github.com/nix-community/disko-templates) or [Disko examples](https://github.com/nix-community/disko/tree/master/example).
 
-## Fill in hardware specific machine configuration
+!!! Danger
+    Don't change the `disko.nix` after the machine is installed for the first time.
 
-Edit the following fields inside the `./machines/<machine_name>/configuration.nix`
+    Changing disko configuration requires wiping and reinstalling the machine.
 
-<!-- Note: Use "jon" instead of "<machine>" as "<" is not supported in title tag -->
-
-```nix title="./machines/jon/configuration.nix" hl_lines="12 15 19"
-{
-    imports = [
-    # contains your disk format and partitioning configuration.
-    ../../modules/disko.nix
-    # this file is shared among all machines
-    ../../modules/shared.nix
-    # enables GNOME desktop (optional)
-    ../../modules/gnome.nix
-    ];
-
-    # Put your username here for login
-    users.users.user.name = "__YOUR_USERNAME__";
-
-    # Replace this __CHANGE_ME__ with the copied result of the lsblk command
-    disko.devices.disk.main.device = "/dev/disk/by-id/__CHANGE_ME__";
-
-    # IMPORTANT! Add your SSH key here
-    # e.g. > cat ~/.ssh/id_ed25519.pub
-    users.users.root.openssh.authorizedKeys.keys = [ "__YOUR_SSH_KEY__" ];
-
-    # ...
-}
-```
-
-!!! Info "Replace `__YOUR_USERNAME__` with the ip of your machine, if you use avahi you can also use your hostname"
-!!! Info "Replace `__CHANGE_ME__` with the appropriate `ID-LINK` identifier, such as `nvme-eui.e8238fa6bf530001001b448b4aec2929`"
-!!! Info "Replace `__YOUR_SSH_KEY__` with your personal key, like `ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILoMI0NC5eT9pHlQExrvR5ASV3iW9+BXwhfchq0smXUJ jon@jon-desktop`"
+    Unless you really know what you are doing.
 
 ## Deploy the machine
 
-**Finally deployment time!** Use the following command to build and deploy the image via SSH onto your machine.
-
-=== "**Image Installer**"
-
-    The installer will generate a password and local addresses on boot, then run ssh with these preconfigured.
-    The installer shows it's deployment relevant information in two formats, a text form, as well as a QR code.
-
-    Sample boot screen shows:
-
-    - Root password
-    - IP address
-    - Optional Tor and mDNS details
-
-    ```{ .bash .annotate .no-copy .nohighlight}
-    ┌─────────────────────────────────────────────────────────────────────────────────────┐
-    │ ┌───────────────────────────┐                                                       │
-    │ │███████████████████████████│ # This is the QR Code (1)                             │
-    │ │██ ▄▄▄▄▄ █▀▄█▀█▀▄█ ▄▄▄▄▄ ██│                                                       │
-    │ │██ █   █ █▀▄▄▄█ ▀█ █   █ ██│                                                       │
-    │ │██ █▄▄▄█ █▀▄ ▀▄▄▄█ █▄▄▄█ ██│                                                       │
-    │ │██▄▄▄▄▄▄▄█▄▀ ▀▄▀▄█▄▄▄▄▄▄▄██│                                                       │
-    │ │███▀▀▀ █▄▄█ ▀▄   ▄▀▄█   ███│                                                       │
-    │ │██▄██▄▄█▄▄▀▀██▄▀ ▄▄▄ ▄▀█▀██│                                                       │
-    │ │██ ▄▄▄▄▄ █▄▄▄▄ █ █▄█ █▀ ███│                                                       │
-    │ │██ █   █ █ █  █ ▄▄▄  ▄▀▀ ██│                                                       │
-    │ │██ █▄▄▄█ █ ▄ ▄    ▄ ▀█ ▄███│                                                       │
-    │ │██▄▄▄▄▄▄▄█▄▄▄▄▄▄█▄▄▄▄▄█▄███│                                                       │
-    │ │███████████████████████████│                                                       │
-    │ └───────────────────────────┘                                                       │
-    │ ┌─────────────────────────────────────────────────────────────────────────────────┐ │
-    │ │Root password: cheesy-capital-unwell  # password (2)                             │ │
-    │ │Local network addresses:                                                         │ │
-    │ │enp1s0           UP    192.168.178.169/24 metric 1024 fe80::21e:6ff:fe45:3c92/64 │ │
-    │ │enp2s0           DOWN                                                            │ │
-    │ │wlan0            DOWN # connect to wlan (3)                                      │ │
-    │ │Onion address: 6evxy5yhzytwpnhc2vpscrbti3iktxdhpnf6yim6bbs25p4v6beemzyd.onion    │ │
-    │ │Multicast DNS: nixos-installer.local                                             │ │
-    │ └─────────────────────────────────────────────────────────────────────────────────┘ │
-    │ Press 'Ctrl-C' for console access                                                   │
-    │                                                                                     │
-    └─────────────────────────────────────────────────────────────────────────────────────┘
-    ```
-
-    1.  This is not an actual QR code, because it is displayed rather poorly on text sites.
-        This would be the actual content of this specific QR code prettified:
-        ```json
-        {
-            "pass": "cheesy-capital-unwell",
-            "tor": "6evxy5yhzytwpnhc2vpscrbti3iktxdhpnf6yim6bbs25p4v6beemzyd.onion",
-            "addrs": [
-            "2001:9e8:347:ca00:21e:6ff:fe45:3c92"
-            ]
-        }
-        ```
-
-        To generate the actual QR code, that would be displayed use:
-        ```shellSession
-        echo '{"pass":"cheesy-capital-unwell","tor":"6evxy5yhzytwpnhc2vpscrbti3iktxdhpnf6yim6bbs25p4v6beemzyd.onion","addrs":["2001:9e8:347:ca00:21e:6ff:fe45:3c92"]}' | nix run nixpkgs#qrencode -- -s 2 -m 2 -t utf8
-        ```
-    2.  The root password for the installer medium.
-        This password is autogenerated and meant to be easily typeable.
-    3.  See [how to connect to wlan](./installer.md#optional-connect-to-wifi-manually).
-
-    !!! tip
-        Use [KDE Connect](https://apps.kde.org/de/kdeconnect/) for easyily sharing QR codes from phone to desktop
-
-=== "**Cloud VM**"
-
-    Just run the command **Option B: Cloud VM** below
+**Finally deployment time!** Use one of the following commands to build and deploy the image via SSH onto your machine.
 
 ### Deployment Commands
 
 #### Using password auth
 
 ```bash
-clan machines install [MACHINE] --target-host <IP> --update-hardware-config nixos-facter
+clan machines install [MACHINE] --target-host <IP>
 ```
 
 #### Using QR JSON
 
 ```bash
-clan machines install [MACHINE] --json "[JSON]" --update-hardware-config nixos-facter
+clan machines install [MACHINE] --json "[JSON]"
 ```
 
 #### Using QR image file
 
 ```bash
-clan machines install [MACHINE] --png [PATH] --update-hardware-config nixos-facter
+clan machines install [MACHINE] --png [PATH]
 ```
 
 #### Option B: Cloud VM
 
 ```bash
-clan machines install [MACHINE] --target-host <IP> --update-hardware-config nixos-facter
+clan machines install [MACHINE] --target-host <IP>
 ```
 
 !!! success
@@ -318,4 +274,3 @@ clan {
 ```
 
 This is useful for machines that are not always online or are not part of the regular update cycle.
-

docs/site/guides/getting-started/index.md

@@ -38,31 +38,24 @@ By the end of this guide, you'll have a fresh NixOS configuration ready to push
 
 ## Add Clan CLI to Your Shell
 
-Add the Clan CLI into your environment:
+Create a new clan
 
 ```bash
-nix shell git+https://git.clan.lol/clan/clan-core#clan-cli --refresh
+nix run git+https://git.clan.lol/clan/clan-core#clan-cli --refresh -- flakes create
 ```
 
+This should prompt for a *name*:
+
 ```terminalSession
-clan --help
+Enter a name for the new clan: my-clan
 ```
 
-Should print the avilable commands.
+Enter a *name*, confirm with *enter*. A directory with that name will be created and initialized.
 
-Also checkout the [cli-reference documentation](../../reference/cli/index.md).
+!!! Note
+    This command uses the `default` template
 
-## Initialize Your Project
-
-If you want to migrate an existing project, follow this [guide](../migrations/migration-guide.md).
-
-Set the foundation of your Clan project by initializing it by running:
-
-```bash
-clan flakes create my-clan
-```
-
-This command creates a `flake.nix` and some other files for your project.
+    See `clan templates list` and the `--help` reference for how to use other templates.
 
 ## Explore the Project Structure
 
@@ -83,36 +76,48 @@ For example, you might see something like:
 └── README.md
 ```
 
-Don’t worry if your output looks different—the template evolves over time.
+Don’t worry if your output looks different — Clan templates evolve over time.
 
-??? info "Recommended way of sourcing the `clan` CLI tool"
+To interact with your newly created clan the you need to load the `clan` cli-package it into your environment by running:
 
-    The default template adds the `clan` CLI tool to the development shell.
-    This means that you can access the `clan` CLI tool directly from the folder
-    you are in right now.
+=== "Automatic (direnv, recommended)"
+    - prerequisite: [install nix-direnv](https://github.com/nix-community/nix-direnv)
 
-    In the `my-clan` directory, run the following command:
+    ```
+    direnv allow
+    ```
+
+=== "Manual (nix develop)"
 
     ```
     nix develop
     ```
 
-    This will ensure the `clan` CLI tool is available in your shell environment.
+verify that you can run `clan` commands:
 
-    To automatically add the `clan` CLI tool to your environment without having to
-    run `nix develop` every time, we recommend setting up [direnv](https://direnv.net/).
-
-```
+```bash
 clan show
 ```
 
 You should see something like this:
 
-```terminal-session
-Name: my-clan
+```shellSession
+Name: __CHANGE_ME__
 Description: None
 ```
 
+To change the name of your clan edit `meta.name` in the `clan.nix` or `flake.nix` file
+
+```{.nix title="clan.nix" hl_lines="3"}
+{
+  # Ensure this is unique among all clans you want to use.
+  meta.name = "__CHANGE_ME__";
+
+  # ...
+  # elided
+}
+```
+
 ---
 
 ## Next Steps
@@ -123,6 +128,7 @@ You can continue with **any** of the following steps at your own pace:
 - [x] [Initialize Clan](./index.md#initialize-your-project)
 - [ ] [Create USB Installer (optional)](./installer.md)
 - [ ] [Add Machines](./add-machines.md)
+- [ ] [Add a User](./add-user.md)
 - [ ] [Add Services](./add-services.md)
 - [ ] [Configure Secrets](./secrets.md)
 - [ ] [Deploy](./deploy.md) - Requires configured secrets

docs/site/guides/migrations/disk-id.md

@@ -0,0 +1,98 @@
+# Migrate disko config from `clanModules.disk-id`
+
+If you previously bootstrapped a machine's disk using `clanModules.disk-id`, you should now migrate to a standalone, self-contained disko configuration. This ensures long-term stability and avoids reliance on dynamic values from Clan.
+
+If your `disko.nix` currently looks something like this:
+
+```nix title="disko.nix"
+{
+  lib,
+  clan-core,
+  config,
+  ...
+}:
+
+let
+  suffix = config.clan.core.vars.generators.disk-id.files.diskId.value;
+in
+{
+  imports = [
+    clan-core.clanModules.disk-id
+  ];
+
+  # DO NOT EDIT THIS FILE AFTER INSTALLATION of a machine
+  # Otherwise your system might not boot because of missing partitions / filesystems
+  boot.loader.grub.efiSupport = lib.mkDefault true;
+  boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
+  disko.devices = {
+    disk = {
+      "main" = {
+        # suffix is to prevent disk name collisions
+        name = "main-" + suffix;
+        type = "disk";
+        # Set the following in flake.nix for each maschine:
+        # device = <uuid>;
+        content = {
+          # edlied
+        };
+      };
+    };
+  };
+}
+```
+
+## Step 1: Retrieve your `disk-id`
+
+Run the following command to retrieve the generated disk ID for your machine:
+
+```bash
+clan vars list <machineName>
+```
+
+Which should print the generated `disk-id/diskId` value in clear text
+You should see output like:
+
+```shellSession
+disk-id/diskId: fcef30a749f8451d8f60c46e1ead726f
+# ...
+# elided
+```
+
+Copy this value — you'll need it in the next step.
+
+## ✍️ Step 2: Replace Dynamic Configuration with Static Values
+
+✅ Goal: Make your disko.nix file standalone.
+
+We are going to make three changes:
+
+- Remove `let in, imports, {lib,clan-core,config, ...}:` to isolate the file.
+- Replace `suffix` with the actual disk-id
+- Move `disko.devices.disk.main.device` from `flake.nix` or `configuration.nix` into this file.
+
+```{.nix title="disko.nix" hl_lines="7-9 11-14"}
+{
+  boot.loader.grub.efiSupport = lib.mkDefault true;
+  boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
+  disko.devices = {
+    disk = {
+      "main" = {
+        #       ↓ Copy the disk-id into place
+        name = "main-fcef30a749f8451d8f60c46e1ead726f";
+        type = "disk";
+
+        # Some earlier guides had this line in a flake.nix
+        # disko.devices.disk.main.device = "/dev/disk/by-id/__CHANGE_ME__";
+        #        ↓ Copy the '/dev/disk/by-id' into here instead
+        device = "/dev/disk/by-id/nvme-eui.e8238fa6bf530001001b448b4aec2929";
+
+        # edlied;
+      };
+    };
+  };
+}
+```
+
+These steps are only needed for existing configurations that depend on the `diskId` module.
+
+For newer machines clan offers simple *disk templates* via its [templates cli](../../reference/cli/templates.md)

flake.lock

@@ -16,11 +16,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1751846468,
-        "narHash": "sha256-h0mpWZIOIAKj4fmLNyI2HDG+c0YOkbYmyJXSj/bQ9s0=",
-        "rev": "a2166c13b0cb3febdaf36391cd2019aa2ccf4366",
+        "lastModified": 1752451292,
+        "narHash": "sha256-jvLbfYFvcS5f0AEpUlFS2xZRnK770r9TRM2smpUFFaU=",
+        "rev": "309e06fbc9a6d133ab6dd1c7d8e4876526e058bb",
         "type": "tarball",
-        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/a2166c13b0cb3febdaf36391cd2019aa2ccf4366.tar.gz"
+        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/309e06fbc9a6d133ab6dd1c7d8e4876526e058bb.tar.gz"
       },
       "original": {
         "type": "tarball",

lib/modules/inventory/distributed-service/all-services-wrapper.nix

@@ -22,6 +22,7 @@ in
       type = attrsWith {
         placeholder = "mappedServiceName";
         elemType = submoduleWith {
+          class = "clan.service";
           modules = [
             (
               { name, ... }:

lib/modules/inventory/distributed-service/inventory-adapter.nix

@@ -122,6 +122,7 @@ in
       evalServices =
         { modules, prefix }:
         lib.evalModules {
+          class = "clan";
           specialArgs = {
             inherit clanLib;
             _ctx = prefix;

lib/modules/inventoryClass/service-list-from-inputs.nix

@@ -35,10 +35,20 @@ in
         inputName: v: lib.mapAttrs (inspectModule inputName) v.clan.modules
       ) inputsWithModules;
   };
-  options.localModules = lib.mkOption {
+  options.moduleSchemas = lib.mkOption {
+    # { sourceName :: { moduleName :: { roleName :: Schema }}}
     readOnly = true;
     type = lib.types.raw;
-    default = config.modulesPerSource.self;
+    default = lib.mapAttrs (
+      _inputName: moduleSet:
+      lib.mapAttrs (
+        _moduleName: module:
+        (clanLib.evalService {
+          modules = [ module ];
+          prefix = [ ];
+        }).config.result.api.schema
+      ) moduleSet
+    ) config.modulesPerSource;
   };
   options.templatesPerSource = lib.mkOption {
     # { sourceName :: { moduleName :: {} }}

nixosModules/clanCore/vars/secret/sops/collectFiles.nix

@@ -0,0 +1,37 @@
+# collectFiles helper function
+{
+  lib ? import <nixpkgs/lib>,
+}:
+let
+  inherit (lib)
+    filterAttrs
+    flatten
+    mapAttrsToList
+    ;
+in
+generators:
+let
+  relevantFiles =
+    generator:
+    filterAttrs (
+      _name: f: f.secret && f.deploy && (f.neededFor == "users" || f.neededFor == "services")
+    ) generator.files;
+  allFiles = flatten (
+    mapAttrsToList (
+      gen_name: generator:
+      mapAttrsToList (fname: file: {
+        name = fname;
+        generator = gen_name;
+        neededForUsers = file.neededFor == "users";
+        inherit (generator) share;
+        inherit (file)
+          owner
+          group
+          mode
+          restartUnits
+          ;
+      }) (relevantFiles generator)
+    ) generators
+  );
+in
+allFiles

nixosModules/clanCore/vars/secret/sops/default.nix

@@ -7,7 +7,7 @@
 }:
 let
 
-  inherit (import ./funcs.nix { inherit lib; }) collectFiles;
+  collectFiles = import ./collectFiles.nix { inherit lib; };
 
   machineName = config.clan.core.settings.machine.name;
 

nixosModules/clanCore/vars/secret/sops/funcs.nix

@@ -1,42 +0,0 @@
-{
-  lib ? import <nixpkgs/lib>,
-  ...
-}:
-let
-
-  inherit (lib)
-    filterAttrs
-    flatten
-    mapAttrsToList
-    ;
-in
-{
-
-  collectFiles =
-    generators:
-    let
-      relevantFiles =
-        generator:
-        filterAttrs (
-          _name: f: f.secret && f.deploy && (f.neededFor == "users" || f.neededFor == "services")
-        ) generator.files;
-      allFiles = flatten (
-        mapAttrsToList (
-          gen_name: generator:
-          mapAttrsToList (fname: file: {
-            name = fname;
-            generator = gen_name;
-            neededForUsers = file.neededFor == "users";
-            inherit (generator) share;
-            inherit (file)
-              owner
-              group
-              mode
-              restartUnits
-              ;
-          }) (relevantFiles generator)
-        ) generators
-      );
-    in
-    allFiles;
-}

pkgs/clan-app/clan-app.code-workspace

@@ -23,9 +23,6 @@
 		},
 		{
 			"path": "../clan-cli/clan_lib"
-		},
-		{
-			"path": "ui-2d"
 		}
 	],
 	"settings": {

pkgs/clan-app/clan_app/api/api_bridge.py

@@ -1,10 +1,13 @@
 import logging
+import threading
 from abc import ABC, abstractmethod
 from contextlib import ExitStack
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any
 
 from clan_lib.api import ApiResponse
+from clan_lib.api.tasks import WebThread
+from clan_lib.async_run import set_should_cancel
 
 if TYPE_CHECKING:
     from .middleware import Middleware
@@ -32,6 +35,7 @@ class ApiBridge(ABC):
     """Generic interface for API bridges that can handle method calls from different sources."""
 
     middleware_chain: tuple["Middleware", ...]
+    threads: dict[str, WebThread] = field(default_factory=dict)
 
     @abstractmethod
     def send_api_response(self, response: BackendResponse) -> None:
@@ -87,3 +91,51 @@ class ApiBridge(ABC):
         )
 
         self.send_api_response(response)
+
+    def process_request_in_thread(
+        self,
+        request: BackendRequest,
+        *,
+        thread_name: str = "ApiBridgeThread",
+        wait_for_completion: bool = False,
+        timeout: float = 60.0,
+    ) -> None:
+        """Process an API request in a separate thread with cancellation support.
+
+        Args:
+            request: The API request to process
+            thread_name: Name for the thread (for debugging)
+            wait_for_completion: Whether to wait for the thread to complete
+            timeout: Timeout in seconds when waiting for completion
+        """
+        op_key = request.op_key or "unknown"
+
+        def thread_task(stop_event: threading.Event) -> None:
+            set_should_cancel(lambda: stop_event.is_set())
+            try:
+                log.debug(
+                    f"Processing {request.method_name} with args {request.args} "
+                    f"and header {request.header} in thread {thread_name}"
+                )
+                self.process_request(request)
+            finally:
+                self.threads.pop(op_key, None)
+
+        stop_event = threading.Event()
+        thread = threading.Thread(
+            target=thread_task, args=(stop_event,), name=thread_name
+        )
+        thread.start()
+
+        self.threads[op_key] = WebThread(thread=thread, stop_event=stop_event)
+
+        if wait_for_completion:
+            # Wait for the thread to complete (this blocks until response is sent)
+            thread.join(timeout=timeout)
+
+            # Handle timeout
+            if thread.is_alive():
+                stop_event.set()  # Cancel the thread
+                self.send_api_error_response(
+                    op_key, "Request timeout", ["api_bridge", request.method_name]
+                )

pkgs/clan-app/clan_app/api/file_gtk.py

@@ -9,6 +9,8 @@ gi.require_version("Gtk", "4.0")
 
 from clan_lib.api import ApiError, ErrorDataClass, SuccessDataClass
 from clan_lib.api.directory import FileRequest
+from clan_lib.clan.check import check_clan_valid
+from clan_lib.flake import Flake
 from gi.repository import Gio, GLib, Gtk
 
 gi.require_version("Gtk", "4.0")
@@ -22,13 +24,58 @@ def remove_none(_list: list) -> list:
 RESULT: dict[str, SuccessDataClass[list[str] | None] | ErrorDataClass] = {}
 
 
-def open_file(
+def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass:
+    """
+    Opens the clan folder using the GTK file dialog.
+    Returns the path to the clan folder or an error if it fails.
+    """
+    file_request = FileRequest(
+        mode="select_folder",
+        title="Select Clan Folder",
+        initial_folder=str(Path.home()),
+    )
+    response = get_system_file(file_request, op_key=op_key)
+
+    if isinstance(response, ErrorDataClass):
+        return response
+
+    if not response.data or len(response.data) == 0:
+        return ErrorDataClass(
+            op_key=op_key,
+            status="error",
+            errors=[
+                ApiError(
+                    message="No folder selected",
+                    description="You must select a folder to open.",
+                    location=["get_clan_folder"],
+                )
+            ],
+        )
+
+    clan_folder = Flake(response.data[0])
+    if not check_clan_valid(clan_folder):
+        return ErrorDataClass(
+            op_key=op_key,
+            status="error",
+            errors=[
+                ApiError(
+                    message="Invalid clan folder",
+                    description=f"The selected folder '{clan_folder}' is not a valid clan folder.",
+                    location=["get_clan_folder"],
+                )
+            ],
+        )
+
+    return SuccessDataClass(op_key=op_key, data=clan_folder, status="success")
+
+
+def get_system_file(
     file_request: FileRequest, *, op_key: str
 ) -> SuccessDataClass[list[str] | None] | ErrorDataClass:
     GLib.idle_add(gtk_open_file, file_request, op_key)
 
     while RESULT.get(op_key) is None:
-        time.sleep(0.2)
+        time.sleep(0.1)
     response = RESULT[op_key]
     del RESULT[op_key]
     return response
@@ -59,7 +106,7 @@ def gtk_open_file(file_request: FileRequest, op_key: str) -> bool:
                         ApiError(
                             message=e.__class__.__name__,
                             description=str(e),
-                            location=["open_file"],
+                            location=["get_system_file"],
                         )
                     ],
                 )
@@ -87,7 +134,7 @@ def gtk_open_file(file_request: FileRequest, op_key: str) -> bool:
                         ApiError(
                             message=e.__class__.__name__,
                             description=str(e),
-                            location=["open_file"],
+                            location=["get_system_file"],
                         )
                     ],
                 )
@@ -115,7 +162,7 @@ def gtk_open_file(file_request: FileRequest, op_key: str) -> bool:
                         ApiError(
                             message=e.__class__.__name__,
                             description=str(e),
-                            location=["open_file"],
+                            location=["get_system_file"],
                         )
                     ],
                 )
@@ -143,7 +190,7 @@ def gtk_open_file(file_request: FileRequest, op_key: str) -> bool:
                         ApiError(
                             message=e.__class__.__name__,
                             description=str(e),
-                            location=["open_file"],
+                            location=["get_system_file"],
                         )
                     ],
                 )
@@ -192,7 +239,7 @@ def gtk_open_file(file_request: FileRequest, op_key: str) -> bool:
         dialog.select_folder(callback=on_folder_select)
     if file_request.mode == "open_multiple_files":
         dialog.open_multiple(callback=on_file_select_multiple)
-    elif file_request.mode == "open_file":
+    elif file_request.mode == "get_system_file":
         dialog.open(callback=on_file_select)
     elif file_request.mode == "save":
         dialog.save(callback=on_save_finish)

pkgs/clan-app/clan_app/app.py

@@ -10,7 +10,7 @@ from clan_lib.dirs import user_data_dir
 from clan_lib.log_manager import LogGroupConfig, LogManager
 from clan_lib.log_manager import api as log_manager_api
 
-from clan_app.api.file_gtk import open_file
+from clan_app.api.file_gtk import get_clan_folder, get_system_file
 from clan_app.api.middleware import (
     ArgumentParsingMiddleware,
     LoggingMiddleware,
@@ -56,7 +56,10 @@ def app_run(app_opts: ClanAppOptions) -> int:
 
     # Populate the API global with all functions
     load_in_all_api_functions()
-    API.overwrite_fn(open_file)
+
+    # Create a shared threads dictionary for both HTTP and Webview modes
+    shared_threads: dict[str, tasks.WebThread] = {}
+    tasks.BAKEND_THREADS = shared_threads
 
     # Start HTTP API server if requested
     http_server = None
@@ -72,6 +75,7 @@ def app_run(app_opts: ClanAppOptions) -> int:
             swagger_dist=Path(swagger_dist) if swagger_dist else None,
             host=app_opts.http_host,
             port=app_opts.http_port,
+            shared_threads=shared_threads,
         )
 
         # Add middleware to HTTP server
@@ -103,20 +107,20 @@ def app_run(app_opts: ClanAppOptions) -> int:
     # Create webview if not running in HTTP-only mode
     if not app_opts.http_api:
         webview = Webview(
-            debug=app_opts.debug, title="Clan App", size=Size(1280, 1024, SizeHint.NONE)
+            debug=app_opts.debug,
+            title="Clan App",
+            size=Size(1280, 1024, SizeHint.NONE),
+            shared_threads=shared_threads,
         )
 
+        API.overwrite_fn(get_system_file)
+        API.overwrite_fn(get_clan_folder)
+
         # Add middleware to the webview
         webview.add_middleware(ArgumentParsingMiddleware(api=API))
         webview.add_middleware(LoggingMiddleware(log_manager=log_manager))
         webview.add_middleware(MethodExecutionMiddleware(api=API))
 
-        # Create the bridge
-        webview.create_bridge()
-
-        # Init BAKEND_THREADS global in tasks module
-        tasks.BAKEND_THREADS = webview.threads
-
         webview.bind_jsonschema_api(API, log_manager=log_manager)
         webview.navigate(content_uri)
         webview.run()

pkgs/clan-app/clan_app/deps/http/http_bridge.py

@@ -1,6 +1,5 @@
 import json
 import logging
-import threading
 import uuid
 from http.server import BaseHTTPRequestHandler
 from pathlib import Path
@@ -9,7 +8,6 @@ from urllib.parse import urlparse
 
 from clan_lib.api import MethodRegistry, SuccessDataClass, dataclass_to_dict
 from clan_lib.api.tasks import WebThread
-from clan_lib.async_run import set_should_cancel
 
 from clan_app.api.api_bridge import ApiBridge, BackendRequest, BackendResponse
 
@@ -35,11 +33,12 @@ class HttpBridge(ApiBridge, BaseHTTPRequestHandler):
         *,
         openapi_file: Path | None = None,
         swagger_dist: Path | None = None,
+        shared_threads: dict[str, WebThread] | None = None,
     ) -> None:
         # Initialize API bridge fields
         self.api = api
         self.middleware_chain = middleware_chain
-        self.threads: dict[str, WebThread] = {}
+        self.threads = shared_threads if shared_threads is not None else {}
 
         # Initialize OpenAPI/Swagger fields
         self.openapi_file = openapi_file
@@ -329,31 +328,13 @@ class HttpBridge(ApiBridge, BaseHTTPRequestHandler):
         self, api_request: BackendRequest, method_name: str
     ) -> None:
         """Process the API request in a separate thread."""
-        op_key = api_request.op_key or "unknown"
-
-        def thread_task(stop_event: threading.Event) -> None:
-            set_should_cancel(lambda: stop_event.is_set())
-            try:
-                self.process_request(api_request)
-            finally:
-                self.threads.pop(op_key, None)
-
-        stop_event = threading.Event()
-        thread = threading.Thread(
-            target=thread_task, args=(stop_event,), name="HttpThread"
+        # Use the inherited thread processing method
+        self.process_request_in_thread(
+            api_request,
+            thread_name="HttpThread",
+            wait_for_completion=True,
+            timeout=60.0,
         )
-        thread.start()
-        self.threads[op_key] = WebThread(thread=thread, stop_event=stop_event)
-
-        # Wait for the thread to complete (this blocks until response is sent)
-        thread.join(timeout=60.0)
-
-        # Handle timeout
-        if thread.is_alive():
-            stop_event.set()  # Cancel the thread
-            self.send_api_error_response(
-                op_key, "Request timeout", ["http_bridge", method_name]
-            )
 
     def log_message(self, format: str, *args: Any) -> None:  # noqa: A002
         """Override default logging to use our logger."""

pkgs/clan-app/clan_app/deps/http/http_server.py

@@ -1,10 +1,11 @@
 import logging
 import threading
-from http.server import HTTPServer
+from http.server import HTTPServer, ThreadingHTTPServer
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 from clan_lib.api import MethodRegistry
+from clan_lib.api.tasks import WebThread
 
 if TYPE_CHECKING:
     from clan_app.api.middleware import Middleware
@@ -24,6 +25,7 @@ class HttpApiServer:
         port: int = 8080,
         openapi_file: Path | None = None,
         swagger_dist: Path | None = None,
+        shared_threads: dict[str, WebThread] | None = None,
     ) -> None:
         self.api = api
         self.openapi = openapi_file
@@ -34,6 +36,7 @@ class HttpApiServer:
         self._server_thread: threading.Thread | None = None
         # Bridge is now the request handler itself, no separate instance needed
         self._middleware: list[Middleware] = []
+        self.shared_threads = shared_threads if shared_threads is not None else {}
 
     def add_middleware(self, middleware: "Middleware") -> None:
         """Add middleware to the middleware chain."""
@@ -58,6 +61,7 @@ class HttpApiServer:
         middleware_chain = tuple(self._middleware)
         openapi_file = self.openapi
         swagger_dist = self.swagger_dist
+        shared_threads = self.shared_threads
 
         class RequestHandler(HttpBridge):
             def __init__(self, request: Any, client_address: Any, server: Any) -> None:
@@ -69,6 +73,7 @@ class HttpApiServer:
                     server=server,
                     openapi_file=openapi_file,
                     swagger_dist=swagger_dist,
+                    shared_threads=shared_threads,
                 )
 
         return RequestHandler
@@ -79,9 +84,9 @@ class HttpApiServer:
             log.warning("HTTP server is already running")
             return
 
-        # Create the server
+        # Create the server using ThreadingHTTPServer for concurrent request handling
         handler_class = self._create_request_handler()
-        self._server = HTTPServer((self.host, self.port), handler_class)
+        self._server = ThreadingHTTPServer((self.host, self.port), handler_class)
 
         def run_server() -> None:
             if self._server:

pkgs/clan-app/clan_app/deps/http/swagger.html

@@ -2,7 +2,8 @@
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
-    <title>Swagger UI</title>
+    <title>Swagger UI with Interceptors</title>
+    <!-- Assuming these files are in the same directory -->
     <link rel="stylesheet" type="text/css" href="./swagger-ui.css" />
     <link rel="stylesheet" type="text/css" href="index.css" />
     <link
@@ -23,14 +24,100 @@
     <div id="swagger-ui"></div>
     <script src="./swagger-ui-bundle.js" charset="UTF-8"></script>
     <script src="./swagger-ui-standalone-preset.js" charset="UTF-8"></script>
-    <script src="./swagger-initializer.js" charset="UTF-8"></script>
+    <!-- Your swagger-initializer.js is not needed if you configure directly in the HTML -->
     <script>
       window.onload = () => {
         SwaggerUIBundle({
-          url: "./openapi.json", // Path to your OpenAPI 3 spec (YAML or JSON)
+          url: "./openapi.json", // Path to your OpenAPI 3 spec
           dom_id: "#swagger-ui",
           presets: [SwaggerUIBundle.presets.apis, SwaggerUIStandalonePreset],
           layout: "StandaloneLayout",
+          tryItOutEnabled: true,
+          deepLinking: true,
+          displayOperationId: true,
+
+          // --- INTERCEPTORS START HERE ---
+
+          /**
+           * requestInterceptor
+           * This function is called before a request is sent.
+           * It takes the request object and must return a modified request object.
+           * We will use it to wrap the user's input.
+           */
+          requestInterceptor: (request) => {
+            console.log("Intercepting request:", request);
+
+            // Only modify requests that have a body (like POST, PUT)
+            if (request.body) {
+              try {
+                // The body from the UI is a string, so we parse it to an object.
+                const originalBody = JSON.parse(request.body);
+
+                // Create the new, nested structure.
+                const newBody = {
+                  body: originalBody,
+                  header: {}, // Add an empty header object as per your example
+                };
+
+                // Replace the original body with the new, stringified, nested structure.
+                request.body = JSON.stringify(newBody);
+
+                // Update the 'Content-Length' header to match the new body size.
+                request.headers["Content-Length"] = new Blob([
+                  request.body,
+                ]).size;
+
+                console.log("Modified request body:", request.body);
+              } catch (e) {
+                // If the user's input isn't valid JSON, don't modify the request.
+                console.error(
+                  "Request Interceptor: Could not parse body as JSON.",
+                  e,
+                );
+              }
+            }
+            return request; // Always return the request object
+          },
+
+          /**
+           * responseInterceptor
+           * This function is called after a response is received, but before it's displayed.
+           * It takes the response object and must return a modified response object.
+           * We will use it to un-nest the data for display.
+           */
+          responseInterceptor: (response) => {
+            console.log("Intercepting response:", response);
+
+            // Check if the response was successful and has data to process.
+            if (response.ok && response.data) {
+              try {
+                // The response data is a string, so we parse it into an object.
+                const fullResponse = JSON.parse(response.data);
+
+                // Check if the expected 'body' property exists.
+                if (fullResponse && typeof fullResponse.body !== "undefined") {
+                  console.log(
+                    "Found nested 'body' property. Un-nesting for display.",
+                  );
+
+                  // Replace the response's data with JUST the nested 'body' object.
+                  // We stringify it with pretty-printing (2-space indentation) for readability in the UI.
+                  response.data = JSON.stringify(fullResponse.body, null, 2);
+                  response.text = response.data; // Also update the 'text' property
+                }
+              } catch (e) {
+                // If the response isn't the expected JSON structure, do nothing.
+                // This prevents errors on other endpoints that have a normal response.
+                console.error(
+                  "Response Interceptor: Could not parse response or un-nest data.",
+                  e,
+                );
+              }
+            }
+            return response; // Always return the response object
+          },
+
+          // --- INTERCEPTORS END HERE ---
         });
       };
     </script>

pkgs/clan-app/clan_app/deps/http/test_http_api.py

@@ -1,27 +1,33 @@
 """Tests for HTTP API components."""
 
 import json
+import logging
+import threading
 import time
 from unittest.mock import Mock
 from urllib.request import Request, urlopen
 
 import pytest
-from clan_lib.api import MethodRegistry
+from clan_lib.api import MethodRegistry, tasks
+from clan_lib.async_run import is_async_cancelled
 from clan_lib.log_manager import LogManager
 
 from clan_app.api.middleware import (
     ArgumentParsingMiddleware,
-    LoggingMiddleware,
     MethodExecutionMiddleware,
 )
 from clan_app.deps.http.http_server import HttpApiServer
 
+log = logging.getLogger(__name__)
+
 
 @pytest.fixture
 def mock_api() -> MethodRegistry:
     """Create a mock API with test methods."""
     api = MethodRegistry()
 
+    api.register(tasks.delete_task)
+
     @api.register
     def test_method(message: str) -> dict[str, str]:
         return {"response": f"Hello {message}!"}
@@ -31,6 +37,19 @@ def mock_api() -> MethodRegistry:
         msg = "Test error"
         raise ValueError(msg)
 
+    @api.register
+    def run_task_blocking(wtime: int) -> str:
+        """A long blocking task that simulates a long-running operation."""
+        time.sleep(1)
+
+        for i in range(wtime):
+            if is_async_cancelled():
+                log.debug("Task was cancelled")
+                return "Task was cancelled"
+            log.debug(f"Processing {i} for {wtime}")
+            time.sleep(1)
+        return f"Task completed with wtime: {wtime}"
+
     return api
 
 
@@ -50,7 +69,7 @@ def http_bridge(
     """Create HTTP bridge dependencies for testing."""
     middleware_chain = (
         ArgumentParsingMiddleware(api=mock_api),
-        LoggingMiddleware(log_manager=mock_log_manager),
+        # LoggingMiddleware(log_manager=mock_log_manager),
         MethodExecutionMiddleware(api=mock_api),
     )
     return mock_api, middleware_chain
@@ -67,7 +86,7 @@ def http_server(mock_api: MethodRegistry, mock_log_manager: Mock) -> HttpApiServ
 
     # Add middleware
     server.add_middleware(ArgumentParsingMiddleware(api=mock_api))
-    server.add_middleware(LoggingMiddleware(log_manager=mock_log_manager))
+    # server.add_middleware(LoggingMiddleware(log_manager=mock_log_manager))
     server.add_middleware(MethodExecutionMiddleware(api=mock_api))
 
     # Bridge will be created automatically when accessed
@@ -84,7 +103,7 @@ class TestHttpBridge:
         # We'll test initialization through the server
         api, middleware_chain = http_bridge
         assert api is not None
-        assert len(middleware_chain) == 3
+        assert len(middleware_chain) == 2
 
     def test_http_bridge_middleware_setup(self, http_bridge: tuple) -> None:
         """Test that middleware is properly set up."""
@@ -92,10 +111,10 @@ class TestHttpBridge:
 
         # Test that we can create the bridge with middleware
         # The actual HTTP handling will be tested through the server integration tests
-        assert len(middleware_chain) == 3
+        assert len(middleware_chain) == 2
         assert isinstance(middleware_chain[0], ArgumentParsingMiddleware)
-        assert isinstance(middleware_chain[1], LoggingMiddleware)
-        assert isinstance(middleware_chain[2], MethodExecutionMiddleware)
+        # assert isinstance(middleware_chain[1], LoggingMiddleware)
+        assert isinstance(middleware_chain[1], MethodExecutionMiddleware)
 
 
 class TestHttpApiServer:
@@ -248,7 +267,7 @@ class TestIntegration:
 
         # Add middleware
         server.add_middleware(ArgumentParsingMiddleware(api=mock_api))
-        server.add_middleware(LoggingMiddleware(log_manager=mock_log_manager))
+        # server.add_middleware(LoggingMiddleware(log_manager=mock_log_manager))
         server.add_middleware(MethodExecutionMiddleware(api=mock_api))
 
         # Bridge will be created automatically when accessed
@@ -281,6 +300,73 @@ class TestIntegration:
             # Always stop server
             server.stop()
 
+    def test_blocking_task(
+        self, mock_api: MethodRegistry, mock_log_manager: Mock
+    ) -> None:
+        shared_threads: dict[str, tasks.WebThread] = {}
+        tasks.BAKEND_THREADS = shared_threads
+
+        """Test a long-running blocking task."""
+        server: HttpApiServer = HttpApiServer(
+            api=mock_api,
+            host="127.0.0.1",
+            port=8083,
+            shared_threads=shared_threads,
+        )
+
+        # Add middleware
+        server.add_middleware(ArgumentParsingMiddleware(api=mock_api))
+        # server.add_middleware(LoggingMiddleware(log_manager=mock_log_manager))
+        server.add_middleware(MethodExecutionMiddleware(api=mock_api))
+
+        # Start server
+        server.start()
+        time.sleep(0.1)  # Give server time to start
+
+        blocking_op_key = "b37f920f-ce8c-4c8d-b595-28ca983d265e"  # str(uuid.uuid4())
+
+        def parallel_task() -> None:
+            # Make API call
+            request_data: dict = {
+                "body": {"wtime": 60},
+                "header": {"op_key": blocking_op_key},
+            }
+            req: Request = Request(
+                "http://127.0.0.1:8083/api/v1/run_task_blocking",
+                data=json.dumps(request_data).encode(),
+                headers={"Content-Type": "application/json"},
+            )
+            response = urlopen(req)
+            data: dict = json.loads(response.read().decode())
+
+            # thread.join()
+            assert "body" in data
+            assert data["body"]["status"] == "success"
+            assert data["body"]["data"] == "Task was cancelled"
+
+        thread = threading.Thread(
+            target=parallel_task,
+            name="ParallelTaskThread",
+            daemon=True,
+        )
+        thread.start()
+
+        time.sleep(1)
+        request_data: dict = {
+            "body": {"task_id": blocking_op_key},
+        }
+        req: Request = Request(
+            "http://127.0.0.1:8083/api/v1/delete_task",
+            data=json.dumps(request_data).encode(),
+            headers={"Content-Type": "application/json"},
+        )
+        response = urlopen(req)
+        data: dict = json.loads(response.read().decode())
+
+        assert "body" in data
+        assert "header" in data
+        assert data["body"]["status"] == "success"
+
 
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])

pkgs/clan-app/clan_app/deps/webview/webview.py

@@ -45,6 +45,7 @@ class Webview:
     debug: bool = False
     size: Size | None = None
     window: int | None = None
+    shared_threads: dict[str, WebThread] | None = None
 
     # initialized later
     _bridge: "WebviewBridge | None" = None
@@ -116,7 +117,17 @@ class Webview:
         """Create and initialize the WebviewBridge with current middleware."""
         from .webview_bridge import WebviewBridge
 
-        bridge = WebviewBridge(webview=self, middleware_chain=tuple(self._middleware))
+        # Use shared_threads if provided, otherwise let WebviewBridge use its default
+        if self.shared_threads is not None:
+            bridge = WebviewBridge(
+                webview=self,
+                middleware_chain=tuple(self._middleware),
+                threads=self.shared_threads,
+            )
+        else:
+            bridge = WebviewBridge(
+                webview=self, middleware_chain=tuple(self._middleware), threads={}
+            )
         self._bridge = bridge
         return bridge
 

pkgs/clan-app/clan_app/deps/webview/webview_bridge.py

@@ -1,12 +1,10 @@
 import json
 import logging
-import threading
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
 from clan_lib.api import dataclass_to_dict
 from clan_lib.api.tasks import WebThread
-from clan_lib.async_run import set_should_cancel
 
 from clan_app.api.api_bridge import ApiBridge, BackendRequest, BackendResponse
 
@@ -23,7 +21,7 @@ class WebviewBridge(ApiBridge):
     """Webview-specific implementation of the API bridge."""
 
     webview: "Webview"
-    threads: dict[str, WebThread] = field(default_factory=dict)
+    threads: dict[str, WebThread]  # Inherited from ApiBridge
 
     def send_api_response(self, response: BackendResponse) -> None:
         """Send response back to the webview client."""
@@ -84,21 +82,9 @@ class WebviewBridge(ApiBridge):
             )
             return
 
-        # Process in a separate thread
-        def thread_task(stop_event: threading.Event) -> None:
-            set_should_cancel(lambda: stop_event.is_set())
-
-            try:
-                log.debug(
-                    f"Calling {method_name}({json.dumps(api_request.args, indent=4)}) with header {json.dumps(api_request.header, indent=4)} and op_key {op_key}"
-                )
-                self.process_request(api_request)
-            finally:
-                self.threads.pop(op_key, None)
-
-        stop_event = threading.Event()
-        thread = threading.Thread(
-            target=thread_task, args=(stop_event,), name="WebviewThread"
+        # Process in a separate thread using the inherited method
+        self.process_request_in_thread(
+            api_request,
+            thread_name="WebviewThread",
+            wait_for_completion=False,
         )
-        thread.start()
-        self.threads[op_key] = WebThread(thread=thread, stop_event=stop_event)

pkgs/clan-app/pyproject.toml

@@ -21,7 +21,7 @@ exclude = ["result", "**/__pycache__"]
 clan_app = ["**/assets/*"]
 
 [tool.pytest.ini_options]
-testpaths = "tests"
+testpaths = [ "tests", "clan_app" ]
 faulthandler_timeout = 60
 log_level = "DEBUG"
 log_format = "%(levelname)s: %(message)s\n       %(pathname)s:%(lineno)d::%(funcName)s"

pkgs/clan-app/shell.nix

@@ -58,6 +58,7 @@ mkShell {
       with ps;
       [
         mypy
+        pytest-cov
       ]
       ++ (clan-app.devshellPyDeps ps)
     ))

pkgs/clan-app/ui/src/index.tsx

@@ -3,7 +3,7 @@ import { render } from "solid-js/web";
 
 import "./index.css";
 import { QueryClient } from "@tanstack/solid-query";
-import { CubeScene } from "./scene/qubes";
+import { CubeScene } from "./scene/cubes";
 
 export const client = new QueryClient();
 

pkgs/clan-app/ui/src/scene/cubes.tsx

@@ -1,5 +1,11 @@
-// Working SolidJS + Three.js cube scene with grid arrangement
-import { createSignal, createEffect, onCleanup, onMount } from "solid-js";
+// Working SolidJS + Three.js cube scene with reactive positioning
+import {
+  createSignal,
+  createEffect,
+  onCleanup,
+  onMount,
+  createMemo,
+} from "solid-js";
 import * as THREE from "three";
 
 // Cube Data Model
@@ -29,27 +35,117 @@ export function CubeScene() {
   let isAnimating = false; // Flag to prevent multiple loops
   let frameCount = 0;
 
-  const [cubes, setCubes] = createSignal<CubeData[]>([]);
+  const [ids, setIds] = createSignal<string[]>([]);
   const [selectedIds, setSelectedIds] = createSignal<Set<string>>(new Set());
+  const [deletingIds, setDeletingIds] = createSignal<Set<string>>(new Set());
+  const [creatingIds, setCreatingIds] = createSignal<Set<string>>(new Set());
   const [cameraInfo, setCameraInfo] = createSignal({
     position: { x: 0, y: 0, z: 0 },
     spherical: { radius: 0, theta: 0, phi: 0 },
   });
 
+  // Animation configuration
+  const ANIMATION_DURATION = 800; // milliseconds
+  const DELETE_ANIMATION_DURATION = 400; // milliseconds
+  const CREATE_ANIMATION_DURATION = 600; // milliseconds
+
   // Grid configuration
-  const GRID_SIZE = 10;
+  const GRID_SIZE = 2;
   const CUBE_SPACING = 2;
 
   // Calculate grid position for a cube index with floating effect
-  function getGridPosition(index: number): [number, number, number] {
-    const x =
-      (index % GRID_SIZE) * CUBE_SPACING - (GRID_SIZE * CUBE_SPACING) / 2;
-    const z =
-      Math.floor(index / GRID_SIZE) * CUBE_SPACING -
-      (GRID_SIZE * CUBE_SPACING) / 2;
+  // function getGridPosition(index: number): [number, number, number] {
+  //   const x =
+  //     (index % GRID_SIZE) * CUBE_SPACING - (GRID_SIZE * CUBE_SPACING) / 2;
+  //   const z =
+  //     Math.floor(index / GRID_SIZE) * CUBE_SPACING -
+  //     (GRID_SIZE * CUBE_SPACING) / 2;
+  //   return [x, 0.5, z];
+  // }
+  // function getGridPosition(index: number): [number, number, number] {
+  //   if (index === 0) return [0, 0.5, 0];
+
+  //   let x = 0, z = 0;
+  //   let layer = 1;
+  //   let value = 1;
+
+  //   while (true) {
+  //     // right
+  //     for (let i = 0; i < layer; i++) {
+  //       x += 1;
+  //       if (value++ === index) return [x * CUBE_SPACING, 0.5, z * CUBE_SPACING];
+  //     }
+  //     // down
+  //     for (let i = 0; i < layer; i++) {
+  //       z += 1;
+  //       if (value++ === index) return [x * CUBE_SPACING, 0.5, z * CUBE_SPACING];
+  //     }
+  //     layer++;
+  //     // left
+  //     for (let i = 0; i < layer; i++) {
+  //       x -= 1;
+  //       if (value++ === index) return [x * CUBE_SPACING, 0.5, z * CUBE_SPACING];
+  //     }
+  //     // up
+  //     for (let i = 0; i < layer; i++) {
+  //       z -= 1;
+  //       if (value++ === index) return [x * CUBE_SPACING, 0.5, z * CUBE_SPACING];
+  //     }
+  //     layer++;
+
+  //     if (layer > 100) {
+  //       console.warn("Exceeded grid size, returning last position");
+  //       // If we exceed the index, return the last position
+  //       return [x * CUBE_SPACING, 0.5, z * CUBE_SPACING];
+  //     }
+  //   }
+  // }
+
+  // Circle IDEA:
+  // Need to talk with timo and W about this
+  function getCirclePosition(
+    index: number,
+    total: number,
+  ): [number, number, number] {
+    const r = Math.sqrt(total) * CUBE_SPACING; // Radius based on total cubes
+    const x = Math.cos((index / total) * 2 * Math.PI) * r;
+    const z = Math.sin((index / total) * 2 * Math.PI) * r;
+    // Position cubes at y = 0.5 to float above the ground
     return [x, 0.5, z];
   }
 
+  // Reactive cubes memo - this recalculates whenever ids() changes
+  const cubes = createMemo(() => {
+    const currentIds = ids();
+    const deleting = deletingIds();
+    const creating = creatingIds();
+
+    // Include both active and deleting cubes for smooth transitions
+    const allIds = [...new Set([...currentIds, ...Array.from(deleting)])];
+
+    return allIds.map((id, index) => {
+      const isDeleting = deleting.has(id);
+      const isCreating = creating.has(id);
+      const activeIndex = currentIds.indexOf(id);
+
+      return {
+        id,
+        position: getCirclePosition(
+          isDeleting ? -1 : activeIndex >= 0 ? activeIndex : index,
+          currentIds.length,
+        ),
+        // position: getGridPosition(isDeleting ? -1 : activeIndex >= 0 ? activeIndex : index),
+        isDeleting,
+        isCreating,
+        // targetPosition: activeIndex >= 0 ? getGridPosition(activeIndex) : getGridPosition(index),
+        targetPosition:
+          activeIndex >= 0
+            ? getCirclePosition(activeIndex, currentIds.length)
+            : getCirclePosition(index, currentIds.length),
+      };
+    });
+  });
+
   // Create multi-colored cube materials for different faces
   function createCubeMaterials() {
     const materials = [
@@ -62,6 +158,7 @@ export function CubeScene() {
     ];
     return materials;
   }
+
   function createBaseMaterials() {
     const materials = [
       new THREE.MeshBasicMaterial({ color: 0xdce4e5 }), // Right face - medium
@@ -74,7 +171,154 @@ export function CubeScene() {
     return materials;
   }
 
-  // Create white base for cube
+  // Animation helper function
+  function animateToPosition(
+    mesh: THREE.Mesh,
+    targetPosition: [number, number, number],
+    duration: number = ANIMATION_DURATION,
+  ) {
+    const startPosition = mesh.position.clone();
+    const endPosition = new THREE.Vector3(...targetPosition);
+    const startTime = Date.now();
+
+    function animate() {
+      const elapsed = Date.now() - startTime;
+      const progress = Math.min(elapsed / duration, 1);
+
+      // Smooth easing function
+      const easeProgress = 1 - Math.pow(1 - progress, 3);
+
+      mesh.position.lerpVectors(startPosition, endPosition, easeProgress);
+
+      if (progress < 1) {
+        requestAnimationFrame(animate);
+      }
+    }
+
+    animate();
+  }
+
+  // Create animation helper
+  function animateCreate(
+    mesh: THREE.Mesh,
+    baseMesh: THREE.Mesh,
+    onComplete: () => void,
+  ) {
+    const startTime = Date.now();
+
+    // Start with zero scale and full opacity
+    mesh.scale.setScalar(0);
+    baseMesh.scale.setScalar(0);
+
+    // Ensure materials are fully opaque
+    if (Array.isArray(mesh.material)) {
+      mesh.material.forEach((material) => {
+        (material as THREE.MeshBasicMaterial).opacity = 1;
+        material.transparent = false;
+      });
+    } else {
+      (mesh.material as THREE.MeshBasicMaterial).opacity = 1;
+      mesh.material.transparent = false;
+    }
+
+    if (Array.isArray(baseMesh.material)) {
+      baseMesh.material.forEach((material) => {
+        (material as THREE.MeshBasicMaterial).opacity = 1;
+        material.transparent = false;
+      });
+    } else {
+      (baseMesh.material as THREE.MeshBasicMaterial).opacity = 1;
+      baseMesh.material.transparent = false;
+    }
+
+    function animate() {
+      const elapsed = Date.now() - startTime;
+      const progress = Math.min(elapsed / CREATE_ANIMATION_DURATION, 1);
+
+      // Smooth easing function with slight overshoot effect
+      let easeProgress;
+      if (progress < 0.8) {
+        // First 80% - smooth scale up
+        easeProgress = 1 - Math.pow(1 - progress / 0.8, 3);
+      } else {
+        // Last 20% - slight overshoot and settle
+        const overshootProgress = (progress - 0.8) / 0.2;
+        const overshoot = Math.sin(overshootProgress * Math.PI) * 0.1;
+        easeProgress = 1 + overshoot;
+      }
+
+      const scale = easeProgress;
+      mesh.scale.setScalar(scale);
+      baseMesh.scale.setScalar(scale);
+
+      if (progress >= 1) {
+        // Ensure final scale is exactly 1
+        mesh.scale.setScalar(1);
+        baseMesh.scale.setScalar(1);
+        onComplete();
+      } else {
+        requestAnimationFrame(animate);
+      }
+    }
+
+    animate();
+  }
+
+  // Delete animation helper
+  function animateDelete(
+    mesh: THREE.Mesh,
+    baseMesh: THREE.Mesh,
+    onComplete: () => void,
+  ) {
+    const startTime = Date.now();
+    const startScale = mesh.scale.clone();
+    const startOpacity = Array.isArray(mesh.material)
+      ? (mesh.material[0] as THREE.MeshBasicMaterial).opacity
+      : (mesh.material as THREE.MeshBasicMaterial).opacity;
+
+    function animate() {
+      const elapsed = Date.now() - startTime;
+      const progress = Math.min(elapsed / DELETE_ANIMATION_DURATION, 1);
+
+      // Smooth easing function
+      const easeProgress = 1 - Math.pow(1 - progress, 3);
+      const scale = 1 - easeProgress;
+      const opacity = startOpacity * (1 - easeProgress);
+
+      mesh.scale.setScalar(scale);
+      baseMesh.scale.setScalar(scale);
+
+      // Update opacity for all materials
+      if (Array.isArray(mesh.material)) {
+        mesh.material.forEach((material) => {
+          (material as THREE.MeshBasicMaterial).opacity = opacity;
+          material.transparent = true;
+        });
+      } else {
+        (mesh.material as THREE.MeshBasicMaterial).opacity = opacity;
+        mesh.material.transparent = true;
+      }
+
+      if (Array.isArray(baseMesh.material)) {
+        baseMesh.material.forEach((material) => {
+          (material as THREE.MeshBasicMaterial).opacity = opacity;
+          material.transparent = true;
+        });
+      } else {
+        (baseMesh.material as THREE.MeshBasicMaterial).opacity = opacity;
+        baseMesh.material.transparent = true;
+      }
+
+      if (progress >= 1) {
+        onComplete();
+      } else {
+        requestAnimationFrame(animate);
+      }
+    }
+
+    animate();
+  }
+
   function createCubeBase(cube_pos: [number, number, number]) {
     const baseMaterials = createBaseMaterials();
     const base = new THREE.Mesh(sharedBaseGeometry, baseMaterials);
@@ -87,45 +331,55 @@ export function CubeScene() {
   // === Add/Delete Cube API ===
   function addCube() {
     const id = crypto.randomUUID();
-    const currentCount = cubes().length;
-    const cube: CubeData = {
-      id,
-      position: getGridPosition(currentCount),
-      color: "blue",
-    };
-    setCubes((prev) => [...prev, cube]);
+
+    // Add to creating set first
+    setCreatingIds((prev) => new Set([...prev, id]));
+
+    // Add to ids
+    setIds((prev) => [...prev, id]);
+
+    // Remove from creating set after animation completes
+    setTimeout(() => {
+      setCreatingIds((prev) => {
+        const next = new Set(prev);
+        next.delete(id);
+        return next;
+      });
+    }, CREATE_ANIMATION_DURATION);
+  }
+
+  function deleteSelectedCubes(selectedSet: Set<string>) {
+    if (selectedSet.size === 0) return;
+
+    // Add to deleting set to start animation
+    setDeletingIds(selectedSet);
+
+    // Start delete animations
+    selectedSet.forEach((id) => {
+      const mesh = meshMap.get(id);
+      const base = baseMap.get(id);
+
+      if (mesh && base) {
+        animateDelete(mesh, base, () => {
+          // Remove from deleting set when animation completes
+          setDeletingIds((prev) => {
+            const next = new Set(prev);
+            next.delete(id);
+            return next;
+          });
+        });
+      }
+    });
+
+    // Remove from ids after a short delay to allow animation to start
+    setTimeout(() => {
+      setIds((prev) => prev.filter((id) => !selectedSet.has(id)));
+      setSelectedIds(new Set<string>()); // Clear selection after deletion
+    }, 50);
   }
 
   function deleteCube(id: string) {
-    // Remove cube mesh
-    const mesh = meshMap.get(id);
-    if (mesh) {
-      scene.remove(mesh);
-      mesh.geometry.dispose();
-      // Dispose materials properly
-      if (Array.isArray(mesh.material)) {
-        mesh.material.forEach((material) => material.dispose());
-      } else {
-        mesh.material.dispose();
-      }
-      meshMap.delete(id);
-    }
-
-    // Remove base mesh - THIS WAS MISSING!
-    const base = baseMap.get(id);
-    if (base) {
-      scene.remove(base);
-      base.geometry.dispose();
-      // Dispose base materials properly
-      if (Array.isArray(base.material)) {
-        base.material.forEach((material) => material.dispose());
-      } else {
-        base.material.dispose();
-      }
-      baseMap.delete(id);
-    }
-
-    setCubes((prev) => prev.filter((c) => c.id !== id));
+    deleteSelectedCubes(new Set([id]));
   }
 
   function toggleSelection(id: string) {
@@ -148,24 +402,16 @@ export function CubeScene() {
       if (selected) {
         // When selected, make all faces red-ish but maintain the lighting difference
         materials.forEach((material, index) => {
-          (material as THREE.MeshBasicMaterial).color.set(
-            index === 2
-              ? 0xff6666 // Top face - lighter red
-              : index === 0 || index === 4
-                ? 0xdce4e5 // Front/right faces - keep
-                : 0xa4b3b5, // Shadow faces - keep
-          );
+          if (index === 2) {
+            (material as THREE.MeshBasicMaterial).color.set(0xff6666);
+          }
         });
       } else {
         // Normal colors - restore original face colors
         materials.forEach((material, index) => {
-          (material as THREE.MeshBasicMaterial).color.set(
-            index === 2
-              ? 0xffffff // Top face - light
-              : index === 0 || index === 4
-                ? 0xdce4e5 // Front/right faces - medium
-                : 0xa4b3b5, // Shadow faces - dark
-          );
+          if (index === 2) {
+            (material as THREE.MeshBasicMaterial).color.set(0xffffff);
+          }
         });
       }
     }
@@ -186,7 +432,8 @@ export function CubeScene() {
   onMount(() => {
     // Scene setup
     scene = new THREE.Scene();
-    scene.background = new THREE.Color(0xf0f0f0);
+    // Transparent background
+    scene.background = null;
 
     // Camera setup
     camera = new THREE.PerspectiveCamera(
@@ -199,7 +446,7 @@ export function CubeScene() {
     camera.lookAt(0, 0, 0);
 
     // Renderer setup
-    renderer = new THREE.WebGLRenderer({ antialias: true });
+    renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
     renderer.setSize(container.clientWidth, container.clientHeight);
     renderer.shadowMap.enabled = true;
     renderer.shadowMap.type = THREE.PCFSoftShadowMap;
@@ -378,14 +625,20 @@ export function CubeScene() {
     });
   });
 
-  // Effect to manage cube meshes
+  // Effect to manage cube meshes - this runs whenever cubes() changes
   createEffect(() => {
+    const currentCubes = cubes();
     const existing = new Set(meshMap.keys());
+    const deleting = deletingIds();
+    const creating = creatingIds();
 
     // Update existing cubes and create new ones
-    cubes().forEach((cube) => {
-      if (!meshMap.has(cube.id)) {
-        // Create cube mesh
+    currentCubes.forEach((cube) => {
+      const existingMesh = meshMap.get(cube.id);
+      const existingBase = baseMap.get(cube.id);
+
+      if (!existingMesh) {
+        // Create new cube mesh
         const cubeMaterials = createCubeMaterials();
         const mesh = new THREE.Mesh(sharedCubeGeometry, cubeMaterials);
         mesh.castShadow = true;
@@ -395,24 +648,123 @@ export function CubeScene() {
         scene.add(mesh);
         meshMap.set(cube.id, mesh);
 
-        // Create base mesh
+        // Create new base mesh
         const base = createCubeBase(cube.position);
         base.userData.id = cube.id;
         scene.add(base);
         baseMap.set(cube.id, base);
+
+        // Start create animation if this cube is being created
+        if (creating.has(cube.id)) {
+          animateCreate(mesh, base, () => {
+            // Animation complete callback - could add additional logic here
+          });
+        }
+      } else if (!deleting.has(cube.id)) {
+        // Only animate position if not being deleted
+        const targetPosition = cube.targetPosition || cube.position;
+        const currentPosition = existingMesh.position.toArray() as [
+          number,
+          number,
+          number,
+        ];
+        const target = targetPosition;
+
+        // Check if position actually changed
+        if (
+          Math.abs(currentPosition[0] - target[0]) > 0.01 ||
+          Math.abs(currentPosition[1] - target[1]) > 0.01 ||
+          Math.abs(currentPosition[2] - target[2]) > 0.01
+        ) {
+          animateToPosition(existingMesh, target);
+
+          if (existingBase) {
+            animateToPosition(existingBase, [
+              target[0],
+              target[1] - 0.5 - 0.025,
+              target[2],
+            ]);
+          }
+        }
       }
+
       existing.delete(cube.id);
     });
 
-    // Remove cubes that are no longer in the state
+    // Remove cubes that are no longer in the state and not being deleted
     existing.forEach((id) => {
-      deleteCube(id);
+      if (!deleting.has(id)) {
+        // Remove cube mesh
+        const mesh = meshMap.get(id);
+        if (mesh) {
+          scene.remove(mesh);
+          mesh.geometry.dispose();
+          // Dispose materials properly
+          if (Array.isArray(mesh.material)) {
+            mesh.material.forEach((material) => material.dispose());
+          } else {
+            mesh.material.dispose();
+          }
+          meshMap.delete(id);
+        }
+
+        // Remove base mesh
+        const base = baseMap.get(id);
+        if (base) {
+          scene.remove(base);
+          base.geometry.dispose();
+          // Dispose base materials properly
+          if (Array.isArray(base.material)) {
+            base.material.forEach((material) => material.dispose());
+          } else {
+            base.material.dispose();
+          }
+          baseMap.delete(id);
+        }
+      }
     });
 
     updateMeshColors();
   });
 
-  // Effect to update colors when selection changes
+  // Effect to clean up deleted cubes after animation
+  createEffect(() => {
+    const deleting = deletingIds();
+    const currentIds = ids();
+
+    // Clean up cubes that finished their delete animation
+    deleting.forEach((id) => {
+      if (!currentIds.includes(id)) {
+        // Check if this cube has finished its animation
+        const mesh = meshMap.get(id);
+        if (mesh && mesh.scale.x <= 0.01) {
+          // Remove cube mesh
+          scene.remove(mesh);
+          mesh.geometry.dispose();
+          if (Array.isArray(mesh.material)) {
+            mesh.material.forEach((material) => material.dispose());
+          } else {
+            mesh.material.dispose();
+          }
+          meshMap.delete(id);
+
+          // Remove base mesh
+          const base = baseMap.get(id);
+          if (base) {
+            scene.remove(base);
+            base.geometry.dispose();
+            if (Array.isArray(base.material)) {
+              base.material.forEach((material) => material.dispose());
+            } else {
+              base.material.dispose();
+            }
+            baseMap.delete(id);
+          }
+        }
+      }
+    });
+  });
+
   createEffect(() => {
     selectedIds(); // Track the signal
     updateMeshColors();
@@ -450,8 +802,11 @@ export function CubeScene() {
     <div>
       <div style={{ "margin-bottom": "10px" }}>
         <button onClick={addCube}>Add Cube</button>
+        <button onClick={() => deleteSelectedCubes(selectedIds())}>
+          Delete Selected
+        </button>
         <span style={{ "margin-left": "10px" }}>
-          Selected: {selectedIds().size} cubes
+          Selected: {selectedIds().size} cubes | Total: {ids().length} cubes
         </span>
       </div>
 
@@ -484,7 +839,7 @@ export function CubeScene() {
         ref={(el) => (container = el)}
         style={{
           width: "100%",
-          height: "500px",
+          height: "1000px",
           border: "1px solid #ccc",
           cursor: "grab",
         }}

pkgs/clan-cli/bin/clan

@@ -6,7 +6,7 @@ sys.path.insert(
     0, os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
 )
 
-from clan_cli import main  # NOQA
+from clan_cli.cli import main  # NOQA
 
 if __name__ == "__main__":
     main()

pkgs/clan-cli/bin/clan-config

@@ -6,7 +6,7 @@ sys.path.insert(
     0, os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
 )
 
-from clan_cli import config  # NOQA
+from clan_cli.cli import config  # NOQA
 
 if __name__ == "__main__":
     config.main()

pkgs/clan-cli/clan_cli/__init__.py

@@ -1,486 +0,0 @@
-import argparse
-import contextlib
-import logging
-import sys
-from pathlib import Path
-from types import ModuleType
-
-from clan_lib.custom_logger import setup_logging
-from clan_lib.dirs import get_clan_flake_toplevel_or_env
-from clan_lib.errors import ClanError
-from clan_lib.flake import Flake
-
-from . import (
-    backups,
-    clan,
-    secrets,
-    select,
-    state,
-    templates,
-    vms,
-)
-from .arg_actions import AppendOptionAction
-from .clan import show
-from .facts import cli as facts
-from .flash import cli as flash_cli
-from .hyperlink import help_hyperlink
-from .machines import cli as machines
-from .profiler import profile
-from .ssh import deploy_info as ssh_cli
-from .vars import cli as vars_cli
-
-log = logging.getLogger(__name__)
-
-argcomplete: ModuleType | None = None
-with contextlib.suppress(ImportError):
-    import argcomplete  # type: ignore[no-redef]
-
-
-def flake_path(arg: str) -> str:
-    flake_dir = Path(arg).resolve()
-    if flake_dir.exists() and flake_dir.is_dir():
-        return str(flake_dir)
-    return arg
-
-
-def default_flake() -> str | None:
-    val = get_clan_flake_toplevel_or_env()
-    if val:
-        return str(val)
-    return None
-
-
-def create_flake_from_args(args: argparse.Namespace) -> Flake:
-    """Create a Flake object from parsed arguments, including nix_options."""
-    flake_path_str = args.flake
-    nix_options = getattr(args, "option", [])
-    return Flake(flake_path_str, nix_options=nix_options)
-
-
-def add_common_flags(parser: argparse.ArgumentParser) -> None:
-    def argument_exists(parser: argparse.ArgumentParser, arg: str) -> bool:
-        """
-        Check if an argparse argument already exists.
-        This is needed because the aliases subcommand doesn't *really*
-        create an alias - it duplicates the actual parser in the tree
-        making duplication inevitable while naively traversing.
-
-        The error that would be thrown by argparse:
-        - argparse.ArgumentError
-        """
-        return any(
-            arg in action.option_strings
-            for action in parser._actions  # noqa: SLF001
-        )
-
-    if not argument_exists(parser, "--debug"):
-        parser.add_argument(
-            "--debug",
-            help="Enable debug logging",
-            action="store_true",
-            default=False,
-        )
-
-    if not argument_exists(parser, "--option"):
-        parser.add_argument(
-            "--option",
-            help="Nix option to set",
-            nargs=2,
-            metavar=("name", "value"),
-            action=AppendOptionAction,
-            default=[],
-        )
-
-    if not argument_exists(parser, "--flake"):
-        parser.add_argument(
-            "--flake",
-            help="path to the flake where the clan resides in, can be a remote flake or local, can be set through the [CLAN_DIR] environment variable",
-            default=default_flake(),
-            metavar="PATH",
-            type=flake_path,
-        )
-
-
-def register_common_flags(parser: argparse.ArgumentParser) -> None:
-    has_subparsers = False
-    for action in parser._actions:  # noqa: SLF001
-        if isinstance(action, argparse._SubParsersAction):  # noqa: SLF001
-            for _choice, child_parser in action.choices.items():
-                has_subparsers = True
-                register_common_flags(child_parser)
-
-    if not has_subparsers:
-        add_common_flags(parser)
-
-
-def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(
-        prog=prog,
-        usage="%(prog)s [-h] [SUBCOMMAND]",
-        description="The clan cli tool",
-        epilog=(
-            f"""
-Online reference for the clan cli tool: {help_hyperlink("cli reference", "https://docs.clan.lol/reference/cli")}
-For more detailed information, visit: {help_hyperlink("docs", "https://docs.clan.lol")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-
-    subparsers = parser.add_subparsers()
-
-    show_parser = subparsers.add_parser(
-        "show",
-        help="Show meta information about the clan",
-        description="Show meta information about the clan",
-        epilog=(
-            """
-This command prints the metadata of a clan.
-
-Examples:
-
-  $ clan show --flake [PATH]
-  Name: My Empty Clan
-  Description: some nice description
-  Icon: A path to the png
-"""
-        ),
-    )
-    show_parser.set_defaults(func=show.show_command)
-
-    parser_backups = subparsers.add_parser(
-        "backups",
-        aliases=["b"],
-        help="Manage backups of clan machines",
-        description="Manage backups of clan machines",
-        epilog=(
-            f"""
-This subcommand provides an interface to backups that clan machines expose.
-
-Examples:
-
-  $ clan backups list [MACHINE]
-  List backups for the machine [MACHINE]
-
-  $ clan backups create [MACHINE]
-  Create a backup for the machine [MACHINE].
-
-  $ clan backups restore [MACHINE] [PROVIDER] [NAME]
-  The backup to restore for the machine [MACHINE] with the configured [PROVIDER]
-  with the name [NAME].
-
-For more detailed information visit: {help_hyperlink("backups", "https://docs.clan.lol/guides/backups")}.
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    backups.register_parser(parser_backups)
-
-    parser_flake = subparsers.add_parser(
-        "flakes",
-        aliases=["f"],
-        help="Create a clan flake inside the current directory",
-        description="Create a clan flake inside the current directory",
-        epilog=(
-            f"""
-Examples:
-  $ clan flakes create [DIR]
-  Will create a new clan flake in the specified directory and create it if it
-  doesn't exist yet. The flake will be created from a default template.
-
-For more detailed information, visit: {help_hyperlink("getting-started", "https://docs.clan.lol/guides/getting-started/index.html")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-
-    clan.register_parser(parser_flake)
-
-    parser_templates = subparsers.add_parser(
-        "templates",
-        help="Subcommands to interact with templates",
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    templates.register_parser(parser_templates)
-
-    parser_flash = subparsers.add_parser(
-        "flash",
-        help="Flashes your machine to an USB drive",
-        description="Flashes your machine to an USB drive",
-        epilog=(
-            f"""
-Examples:
-  $ clan flash write mymachine --disk main /dev/sd<X> --ssh-pubkey ~/.ssh/id_rsa.pub
-  Will flash the machine 'mymachine' to the disk '/dev/sd<X>' with the ssh public key '~/.ssh/id_rsa.pub'.
-
-For more detailed information, visit: {help_hyperlink("getting-started", "https://docs.clan.lol/guides/getting-started/installer")}
-            """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    flash_cli.register_parser(parser_flash)
-
-    parser_ssh = subparsers.add_parser(
-        "ssh",
-        help="Ssh to a remote machine",
-        description="Ssh to a remote machine",
-        epilog=(
-            f"""
-This subcommand allows seamless ssh access to the nixos-image builders or a machine of your clan.
-
-Examples:
-
-  $ clan ssh [ssh_args ...] berlin`
-
-  Will ssh in to the machine called `berlin`, using the
-  `clan.core.networking.targetHost` specified in its configuration
-
-  $ clan ssh [ssh_args ...] --json [JSON]
-  Will ssh in to the machine based on the deployment information contained in
-  the json string. [JSON] can either be a json formatted string itself, or point
-  towards a file containing the deployment information
-
-For more detailed information, visit: {help_hyperlink("deploy", "https://docs.clan.lol/guides/getting-started/deploy")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    ssh_cli.register_parser(parser_ssh)
-
-    parser_secrets = subparsers.add_parser(
-        "secrets",
-        help="Manage secrets",
-        description="Manage secrets",
-        epilog=(
-            f"""
-This subcommand provides an interface to secrets.
-
-Examples:
-
-  $ clan secrets list [regex]
-  Will list secrets for all managed machines.
-  It accepts an optional regex, allowing easy filtering of returned secrets.
-
-  $ clan secrets get [SECRET]
-  Will display the content of the specified secret.
-
-For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    secrets.register_parser(parser_secrets)
-
-    parser_facts = subparsers.add_parser(
-        "facts",
-        help="Manage facts",
-        description="Manage facts",
-        epilog=(
-            f"""
-
-This subcommand provides an interface to facts of clan machines.
-Facts are artifacts that a service can generate.
-There are public and secret facts.
-Public facts can be referenced by other machines directly.
-Public facts can include: ip addresses, public keys.
-Secret facts can include: passwords, private keys.
-
-A service is an included clan-module that implements facts generation functionality.
-For example the zerotier module will generate private and public facts.
-In this case the public fact will be the resulting zerotier-ip of the machine.
-The secret fact will be the zerotier-identity-secret, which is used by zerotier
-to prove the machine has control of the zerotier-ip.
-
-Examples:
-
-  $ clan facts generate
-  Will generate facts for all machines.
-
-  $ clan facts generate --service [SERVICE] --regenerate
-  Will regenerate facts, if they are already generated for a specific service.
-  This is especially useful for resetting certain passwords while leaving the rest
-  of the facts for a machine in place.
-
-For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    facts.register_parser(parser_facts)
-
-    # like facts but with vars instead of facts
-    parser_vars = subparsers.add_parser(
-        "vars",
-        aliases=["va"],
-        help="Manage vars",
-        description="Manage vars",
-        epilog=(
-            f"""
-This subcommand provides an interface to `vars` of clan machines.
-Vars are variables that a service can generate.
-There are public and secret vars.
-Public vars can be referenced by other machines directly.
-Public vars can include: ip addresses, public keys.
-Secret vars can include: passwords, private keys.
-
-A service is an included clan-module that implements vars generation functionality.
-For example the zerotier module will generate private and public vars.
-In this case the public var will be the resulting zerotier-ip of the machine.
-The secret var will be the zerotier-identity-secret, which is used by zerotier
-to prove the machine has control of the zerotier-ip.
-
-Examples:
-
-    $ clan vars generate
-    Will generate vars for all machines.
-
-    $ clan vars generate --service [SERVICE] --regenerate
-    Will regenerate vars, if they are already generated for a specific service.
-    This is especially useful for resetting certain passwords while leaving the rest
-    of the vars for a machine in place.
-
-For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    vars_cli.register_parser(parser_vars)
-
-    parser_machine = subparsers.add_parser(
-        "machines",
-        aliases=["m"],
-        help="Manage machines and their configuration",
-        description="Manage machines and their configuration",
-        epilog=(
-            f"""
-This subcommand provides an interface to machines managed by Clan.
-
-Examples:
-
-  $ clan machines list
-  List all the machines managed by Clan.
-
-  $ clan machines update [MACHINES]
-  Will update the specified machines [MACHINES], if [MACHINES] is omitted, the command
-  will attempt to update every configured machine.
-
-  $ clan machines install [MACHINE] --target-host [TARGET_HOST]
-  Will install the specified machine [MACHINE] to the specified [TARGET_HOST].
-  If the `--target-host` flag is omitted will try to find host information by
-  checking the deployment configuration inside the specified machine.
-
-For more detailed information, visit: {help_hyperlink("deploy", "https://docs.clan.lol/guides/getting-started/deploy")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    machines.register_parser(parser_machine)
-
-    parser_vms = subparsers.add_parser(
-        "vms", help="Manage virtual machines", description="Manage virtual machines"
-    )
-    vms.register_parser(parser_vms)
-
-    parser_select = subparsers.add_parser(
-        "select",
-        aliases=["se"],
-        help="Select nixos values from the flake",
-        description="Select nixos values from the flake",
-        epilog=(
-            """
-This subcommand provides an interface nix values defined in the flake.
-
-Examples:
-
-  $ clan select nixosConfigurations.*.config.networking.hostName
-    List hostnames of all nixos configurations as JSON.
-
-  $ clan select nixosConfigurations.{jon,alice}.config.clan.core.vars.generators.*.name
-    List all vars generators for jon and alice.
-
-  $ clan select nixosConfigurations.jon.config.envirnonment.systemPackages.1
-    List the first system package for jon.
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    select.register_parser(parser_select)
-
-    parser_state = subparsers.add_parser(
-        "state",
-        aliases=["st"],
-        help="Query state information about machines",
-        description="Query state information about machines",
-        epilog=(
-            f"""
-This subcommand provides an interface to the state managed by Clan.
-
-State can be folders and databases that modules depend on managed by Clan.
-
-State directories can be added to on a per machine basis:
-```
-  config.clan.core.state.[SERVICE_NAME].folders = [
-    "/home"
-    "/root"
-  ];
-```
-Here [SERVICE_NAME] can be set freely, if the user sets them extra `userdata`
-can be a good choice.
-
-Examples:
-
-  $ clan state list [MACHINE]
-  List state of the machines managed by Clan.
-
-For more detailed information, visit: {help_hyperlink("getting-started", "https://docs.clan.lol/guides/backups")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    state.register_parser(parser_state)
-
-    if argcomplete:
-        argcomplete.autocomplete(parser, exclude=["morph"])
-
-    register_common_flags(parser)
-
-    return parser
-
-
-# this will be the entrypoint under /bin/clan (see pyproject.toml config)
-@profile
-def main() -> None:
-    parser = create_parser()
-    args = parser.parse_args()
-
-    if len(sys.argv) == 1:
-        parser.print_help()
-
-    if debug := getattr(args, "debug", False):
-        setup_logging(logging.DEBUG)
-        log.debug("Debug log activated")
-    else:
-        setup_logging(logging.INFO)
-
-    if not hasattr(args, "func"):
-        return
-
-    # Convert flake path to Flake object with nix_options if flake argument exists
-    if hasattr(args, "flake") and args.flake is not None:
-        args.flake = create_flake_from_args(args)
-
-    try:
-        args.func(args)
-    except ClanError as e:
-        if debug:
-            log.exception("Exited with error")
-        else:
-            log.error("%s", e)
-        sys.exit(1)
-    except KeyboardInterrupt as ex:
-        log.warning("Interrupted by user", exc_info=ex)
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

pkgs/clan-cli/clan_cli/__main__.py

@@ -1,4 +1,4 @@
-from . import main
+from .cli import main
 
 if __name__ == "__main__":
     main()

pkgs/clan-cli/clan_cli/clan/create.py

@@ -4,6 +4,7 @@ import logging
 from pathlib import Path
 
 from clan_lib.clan.create import CreateOptions, create_clan
+from clan_lib.errors import ClanError
 
 log = logging.getLogger(__name__)
 
@@ -26,10 +27,10 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
     )
 
     parser.add_argument(
-        "path",
-        type=Path,
-        help="Path where to write the clan template to",
-        default=Path(),
+        "name",
+        type=str,
+        nargs="?",
+        help="Name of the clan to create. If not provided, will prompt for a name.",
     )
 
     parser.add_argument(
@@ -40,9 +41,18 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
     )
 
     def create_flake_command(args: argparse.Namespace) -> None:
+        # Ask for a path interactively if none provided
+        if args.name is None:
+            user_input = input("Enter a name for the new clan: ").strip()
+            if not user_input:
+                msg = "Error: name is required."
+                raise ClanError(msg)
+
+            args.name = Path(user_input)
+
         create_clan(
             CreateOptions(
-                dest=args.path,
+                dest=Path(args.name),
                 template=args.template,
                 setup_git=not args.no_git,
                 src_flake=args.flake,

pkgs/clan-cli/clan_cli/cli.py

@@ -0,0 +1,506 @@
+import argparse
+import contextlib
+import logging
+import sys
+from pathlib import Path
+from types import ModuleType
+
+from clan_lib.custom_logger import setup_logging
+from clan_lib.dirs import get_clan_flake_toplevel_or_env
+from clan_lib.errors import ClanError
+from clan_lib.flake import Flake
+
+from . import (
+    backups,
+    clan,
+    secrets,
+    select,
+    state,
+    templates,
+    vms,
+)
+from .arg_actions import AppendOptionAction
+from .clan import show
+from .facts import cli as facts
+from .flash import cli as flash_cli
+from .hyperlink import help_hyperlink
+from .machines import cli as machines
+from .profiler import profile
+from .ssh import deploy_info as ssh_cli
+from .vars import cli as vars_cli
+
+log = logging.getLogger(__name__)
+
+argcomplete: ModuleType | None = None
+with contextlib.suppress(ImportError):
+    import argcomplete  # type: ignore[no-redef]
+
+
+def flake_path(arg: str) -> str:
+    flake_dir = Path(arg).resolve()
+    if flake_dir.exists() and flake_dir.is_dir():
+        return str(flake_dir)
+    return arg
+
+
+def default_flake() -> str | None:
+    val = get_clan_flake_toplevel_or_env()
+    if val:
+        return str(val)
+    return None
+
+
+def create_flake_from_args(args: argparse.Namespace) -> Flake:
+    """Create a Flake object from parsed arguments, including nix_options."""
+    flake_path_str = args.flake
+    nix_options = getattr(args, "option", [])
+    return Flake(flake_path_str, nix_options=nix_options)
+
+
+def add_common_flags(parser: argparse.ArgumentParser) -> None:
+    def argument_exists(parser: argparse.ArgumentParser, arg: str) -> bool:
+        """
+        Check if an argparse argument already exists.
+        This is needed because the aliases subcommand doesn't *really*
+        create an alias - it duplicates the actual parser in the tree
+        making duplication inevitable while naively traversing.
+
+        The error that would be thrown by argparse:
+        - argparse.ArgumentError
+        """
+        return any(
+            arg in action.option_strings
+            for action in parser._actions  # noqa: SLF001
+        )
+
+    if not argument_exists(parser, "--debug"):
+        parser.add_argument(
+            "--debug",
+            help="Enable debug logging",
+            action="store_true",
+            default=False,
+        )
+
+    if not argument_exists(parser, "--option"):
+        parser.add_argument(
+            "--option",
+            help="Nix option to set",
+            nargs=2,
+            metavar=("name", "value"),
+            action=AppendOptionAction,
+            default=[],
+        )
+
+    if not argument_exists(parser, "--flake"):
+        parser.add_argument(
+            "--flake",
+            help="path to the flake where the clan resides in, can be a remote flake or local, can be set through the [CLAN_DIR] environment variable",
+            default=default_flake(),
+            metavar="PATH",
+            type=flake_path,
+        )
+
+
+def register_common_flags(parser: argparse.ArgumentParser) -> None:
+    has_subparsers = False
+    for action in parser._actions:  # noqa: SLF001
+        if isinstance(action, argparse._SubParsersAction):  # noqa: SLF001
+            for _choice, child_parser in action.choices.items():
+                has_subparsers = True
+                register_common_flags(child_parser)
+
+    if not has_subparsers:
+        add_common_flags(parser)
+
+
+def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog=prog,
+        usage="%(prog)s [-h] [SUBCOMMAND]",
+        description="The clan cli tool",
+        epilog=(
+            f"""
+Online reference for the clan cli tool: {help_hyperlink("cli reference", "https://docs.clan.lol/reference/cli")}
+For more detailed information, visit: {help_hyperlink("docs", "https://docs.clan.lol")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+
+    subparsers = parser.add_subparsers()
+
+    show_parser = subparsers.add_parser(
+        "show",
+        help="Show meta information about the clan",
+        description="Show meta information about the clan",
+        epilog=(
+            """
+This command prints the metadata of a clan.
+
+Examples:
+
+  $ clan show --flake [PATH]
+  Name: My Empty Clan
+  Description: some nice description
+  Icon: A path to the png
+"""
+        ),
+    )
+    show_parser.set_defaults(func=show.show_command)
+
+    parser_backups = subparsers.add_parser(
+        "backups",
+        aliases=["b"],
+        help="Manage backups of clan machines",
+        description="Manage backups of clan machines",
+        epilog=(
+            f"""
+This subcommand provides an interface to backups that clan machines expose.
+
+Examples:
+
+  $ clan backups list [MACHINE]
+  List backups for the machine [MACHINE]
+
+  $ clan backups create [MACHINE]
+  Create a backup for the machine [MACHINE].
+
+  $ clan backups restore [MACHINE] [PROVIDER] [NAME]
+  The backup to restore for the machine [MACHINE] with the configured [PROVIDER]
+  with the name [NAME].
+
+For more detailed information visit: {help_hyperlink("backups", "https://docs.clan.lol/guides/backups")}.
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    backups.register_parser(parser_backups)
+
+    parser_flake = subparsers.add_parser(
+        "flakes",
+        aliases=["f"],
+        help="Create a clan flake inside the current directory",
+        description="Create a clan flake inside the current directory",
+        epilog=(
+            f"""
+Examples:
+  $ clan flakes create [DIR]
+  Will create a new clan flake in the specified directory and create it if it
+  doesn't exist yet. The flake will be created from a default template.
+
+For more detailed information, visit: {help_hyperlink("getting-started", "https://docs.clan.lol/guides/getting-started/index.html")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+
+    clan.register_parser(parser_flake)
+
+    parser_templates = subparsers.add_parser(
+        "templates",
+        help="Interact with templates",
+        description="Interact with templates",
+        epilog=(
+            """
+This subcommand provides an interface to templates provided by clan.
+
+Examples:
+
+  $ clan templates list
+  List all the machines managed by Clan.
+
+  $ clan templates apply disk [TEMPLATE] [MACHINE]
+  Will apply the specified [TEMPLATE] to the [MACHINE]
+
+  Many templates require to *set* variables via the `--set` flag.
+  $ clan templates apply disk [TEMPLATE] [MACHINE] --set key1 value1 --set key2 value2
+
+  Real world example
+  $ clan templates apply disk single-disk jon --set mainDisk "/dev/disk/by-id/nvme-WD_PC_SN740_SDDQNQD-512G-1201_232557804368"
+"""
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    templates.register_parser(parser_templates)
+
+    parser_flash = subparsers.add_parser(
+        "flash",
+        help="Flashes your machine to an USB drive",
+        description="Flashes your machine to an USB drive",
+        epilog=(
+            f"""
+Examples:
+  $ clan flash write mymachine --disk main /dev/sd<X> --ssh-pubkey ~/.ssh/id_rsa.pub
+  Will flash the machine 'mymachine' to the disk '/dev/sd<X>' with the ssh public key '~/.ssh/id_rsa.pub'.
+
+For more detailed information, visit: {help_hyperlink("getting-started", "https://docs.clan.lol/guides/getting-started/installer")}
+            """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    flash_cli.register_parser(parser_flash)
+
+    parser_ssh = subparsers.add_parser(
+        "ssh",
+        help="Ssh to a remote machine",
+        description="Ssh to a remote machine",
+        epilog=(
+            f"""
+This subcommand allows seamless ssh access to the nixos-image builders or a machine of your clan.
+
+Examples:
+
+  $ clan ssh [ssh_args ...] berlin`
+
+  Will ssh in to the machine called `berlin`, using the
+  `clan.core.networking.targetHost` specified in its configuration
+
+  $ clan ssh [ssh_args ...] --json [JSON]
+  Will ssh in to the machine based on the deployment information contained in
+  the json string. [JSON] can either be a json formatted string itself, or point
+  towards a file containing the deployment information
+
+For more detailed information, visit: {help_hyperlink("deploy", "https://docs.clan.lol/guides/getting-started/deploy")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    ssh_cli.register_parser(parser_ssh)
+
+    parser_secrets = subparsers.add_parser(
+        "secrets",
+        help="Manage secrets",
+        description="Manage secrets",
+        epilog=(
+            f"""
+This subcommand provides an interface to secrets.
+
+Examples:
+
+  $ clan secrets list [regex]
+  Will list secrets for all managed machines.
+  It accepts an optional regex, allowing easy filtering of returned secrets.
+
+  $ clan secrets get [SECRET]
+  Will display the content of the specified secret.
+
+For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    secrets.register_parser(parser_secrets)
+
+    parser_facts = subparsers.add_parser(
+        "facts",
+        help="Manage facts",
+        description="Manage facts",
+        epilog=(
+            f"""
+
+This subcommand provides an interface to facts of clan machines.
+Facts are artifacts that a service can generate.
+There are public and secret facts.
+Public facts can be referenced by other machines directly.
+Public facts can include: ip addresses, public keys.
+Secret facts can include: passwords, private keys.
+
+A service is an included clan-module that implements facts generation functionality.
+For example the zerotier module will generate private and public facts.
+In this case the public fact will be the resulting zerotier-ip of the machine.
+The secret fact will be the zerotier-identity-secret, which is used by zerotier
+to prove the machine has control of the zerotier-ip.
+
+Examples:
+
+  $ clan facts generate
+  Will generate facts for all machines.
+
+  $ clan facts generate --service [SERVICE] --regenerate
+  Will regenerate facts, if they are already generated for a specific service.
+  This is especially useful for resetting certain passwords while leaving the rest
+  of the facts for a machine in place.
+
+For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    facts.register_parser(parser_facts)
+
+    # like facts but with vars instead of facts
+    parser_vars = subparsers.add_parser(
+        "vars",
+        aliases=["va"],
+        help="Manage vars",
+        description="Manage vars",
+        epilog=(
+            f"""
+This subcommand provides an interface to `vars` of clan machines.
+Vars are variables that a service can generate.
+There are public and secret vars.
+Public vars can be referenced by other machines directly.
+Public vars can include: ip addresses, public keys.
+Secret vars can include: passwords, private keys.
+
+A service is an included clan-module that implements vars generation functionality.
+For example the zerotier module will generate private and public vars.
+In this case the public var will be the resulting zerotier-ip of the machine.
+The secret var will be the zerotier-identity-secret, which is used by zerotier
+to prove the machine has control of the zerotier-ip.
+
+Examples:
+
+    $ clan vars generate
+    Will generate vars for all machines.
+
+    $ clan vars generate --service [SERVICE] --regenerate
+    Will regenerate vars, if they are already generated for a specific service.
+    This is especially useful for resetting certain passwords while leaving the rest
+    of the vars for a machine in place.
+
+For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    vars_cli.register_parser(parser_vars)
+
+    parser_machine = subparsers.add_parser(
+        "machines",
+        aliases=["m"],
+        help="Manage machines and their configuration",
+        description="Manage machines and their configuration",
+        epilog=(
+            f"""
+This subcommand provides an interface to machines managed by Clan.
+
+Examples:
+
+  $ clan machines list
+  List all the machines managed by Clan.
+
+  $ clan machines update [MACHINES]
+  Will update the specified machines [MACHINES], if [MACHINES] is omitted, the command
+  will attempt to update every configured machine.
+
+  $ clan machines install [MACHINE] --target-host [TARGET_HOST]
+  Will install the specified machine [MACHINE] to the specified [TARGET_HOST].
+  If the `--target-host` flag is omitted will try to find host information by
+  checking the deployment configuration inside the specified machine.
+
+For more detailed information, visit: {help_hyperlink("deploy", "https://docs.clan.lol/guides/getting-started/deploy")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    machines.register_parser(parser_machine)
+
+    parser_vms = subparsers.add_parser(
+        "vms", help="Manage virtual machines", description="Manage virtual machines"
+    )
+    vms.register_parser(parser_vms)
+
+    parser_select = subparsers.add_parser(
+        "select",
+        aliases=["se"],
+        help="Select nixos values from the flake",
+        description="Select nixos values from the flake",
+        epilog=(
+            """
+This subcommand provides an interface nix values defined in the flake.
+
+Examples:
+
+  $ clan select nixosConfigurations.*.config.networking.hostName
+    List hostnames of all nixos configurations as JSON.
+
+  $ clan select nixosConfigurations.{jon,alice}.config.clan.core.vars.generators.*.name
+    List all vars generators for jon and alice.
+
+  $ clan select nixosConfigurations.jon.config.envirnonment.systemPackages.1
+    List the first system package for jon.
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    select.register_parser(parser_select)
+
+    parser_state = subparsers.add_parser(
+        "state",
+        aliases=["st"],
+        help="Query state information about machines",
+        description="Query state information about machines",
+        epilog=(
+            f"""
+This subcommand provides an interface to the state managed by Clan.
+
+State can be folders and databases that modules depend on managed by Clan.
+
+State directories can be added to on a per machine basis:
+```
+  config.clan.core.state.[SERVICE_NAME].folders = [
+    "/home"
+    "/root"
+  ];
+```
+Here [SERVICE_NAME] can be set freely, if the user sets them extra `userdata`
+can be a good choice.
+
+Examples:
+
+  $ clan state list [MACHINE]
+  List state of the machines managed by Clan.
+
+For more detailed information, visit: {help_hyperlink("getting-started", "https://docs.clan.lol/guides/backups")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    state.register_parser(parser_state)
+
+    if argcomplete:
+        argcomplete.autocomplete(parser, exclude=["morph"])
+
+    register_common_flags(parser)
+
+    return parser
+
+
+# this will be the entrypoint under /bin/clan (see pyproject.toml config)
+@profile
+def main() -> None:
+    parser = create_parser()
+    args = parser.parse_args()
+
+    if len(sys.argv) == 1:
+        parser.print_help()
+
+    if debug := getattr(args, "debug", False):
+        setup_logging(logging.DEBUG)
+        log.debug("Debug log activated")
+    else:
+        setup_logging(logging.INFO)
+
+    if not hasattr(args, "func"):
+        return
+
+    # Convert flake path to Flake object with nix_options if flake argument exists
+    if hasattr(args, "flake") and args.flake is not None:
+        args.flake = create_flake_from_args(args)
+
+    try:
+        args.func(args)
+    except ClanError as e:
+        if debug:
+            log.exception("Exited with error")
+        else:
+            log.error("%s", e)
+        sys.exit(1)
+    except KeyboardInterrupt as ex:
+        log.warning("Interrupted by user", exc_info=ex)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

pkgs/clan-cli/clan_cli/templates/__init__.py

@@ -1,6 +1,7 @@
 # !/usr/bin/env python3
 import argparse
 
+from .apply import register_apply_parser
 from .list import register_list_parser
 
 
@@ -12,5 +13,9 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
         help="the command to run",
         required=True,
     )
-    list_parser = subparser.add_parser("list", help="List avilable templates")
+    list_parser = subparser.add_parser("list", help="List available templates")
+    apply_parser = subparser.add_parser(
+        "apply", help="Apply a template of the specified type"
+    )
     register_list_parser(list_parser)
+    register_apply_parser(apply_parser)

pkgs/clan-cli/clan_cli/templates/apply.py

@@ -0,0 +1,15 @@
+import argparse
+
+from .apply_disk import register_apply_disk_template_parser
+
+
+def register_apply_parser(parser: argparse.ArgumentParser) -> None:
+    subparser = parser.add_subparsers(
+        title="template_type",
+        description="the template type to apply",
+        help="the template type to apply",
+        required=True,
+    )
+    disk_parser = subparser.add_parser("disk", help="Apply a disk template")
+
+    register_apply_disk_template_parser(disk_parser)

pkgs/clan-cli/clan_cli/templates/apply_disk.py

@@ -0,0 +1,82 @@
+import argparse
+import logging
+from collections.abc import Sequence
+from typing import Any
+
+from clan_lib.machines.machines import Machine
+from clan_lib.templates.disk import set_machine_disk_schema
+
+from clan_cli.completions import (
+    add_dynamic_completer,
+    complete_machines,
+)
+
+log = logging.getLogger(__name__)
+
+
+class AppendSetAction(argparse.Action):
+    def __init__(self, option_strings: str, dest: str, **kwargs: Any) -> None:
+        super().__init__(option_strings, dest, **kwargs)
+
+    def __call__(
+        self,
+        parser: argparse.ArgumentParser,
+        namespace: argparse.Namespace,
+        values: str | Sequence[str] | None,
+        option_string: str | None = None,
+    ) -> None:
+        lst = getattr(namespace, self.dest)
+        assert isinstance(values, list), "values must be a list"
+        lst.append((values[0], values[1]))
+
+
+def apply_command(args: argparse.Namespace) -> None:
+    """Apply a disk template to a machine."""
+    set_tuples: list[tuple[str, str]] = args.set
+
+    placeholders = dict(set_tuples)
+
+    set_machine_disk_schema(
+        Machine(args.machine, args.flake),
+        args.template,
+        placeholders,
+        force=args.force,
+        check_hw=not args.skip_hardware_check,
+    )
+    log.info(f"Applied disk template '{args.template}' to machine '{args.machine}' ")
+
+
+def register_apply_disk_template_parser(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "template",
+        type=str,
+        help="The name of the disk template to apply",
+    )
+    machine_action = parser.add_argument(
+        "machine",
+        type=str,
+        help="The machine to apply the template to",
+    )
+    add_dynamic_completer(machine_action, complete_machines)
+    parser.add_argument(
+        "--set",
+        help="Set a placeholder in the template to a value",
+        nargs=2,
+        metavar=("placeholder", "value"),
+        action=AppendSetAction,
+        default=[],
+    )
+    parser.add_argument(
+        "--force",
+        help="Force apply the template even if the machine already has a disk schema",
+        action="store_true",
+        default=False,
+    )
+    parser.add_argument(
+        "--skip-hardware-check",
+        help="Disables hardware checking. By default this command checks that the facter.json report exists and validates provided options",
+        action="store_true",
+        default=False,
+    )
+
+    parser.set_defaults(func=apply_command)

pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py

@@ -71,8 +71,8 @@ def substitute(
 
     with file.open() as f:
         for line in f:
+            line = line.replace("__NIXPKGS__", str(nixpkgs_source()))
             if clan_core_replacement:
-                line = line.replace("__NIXPKGS__", str(nixpkgs_source()))
                 line = line.replace("__CLAN_CORE__", clan_core_replacement)
                 line = line.replace(
                     "git+https://git.clan.lol/clan/clan-core", clan_core_replacement
@@ -385,6 +385,7 @@ def test_flake(
         flake_template="test_flake",
         monkeypatch=monkeypatch,
     )
+
     # check that git diff on ./sops is empty
     if (temporary_home / "test_flake" / "sops").exists():
         git_proc = sp.run(

pkgs/clan-cli/clan_cli/tests/helpers/cli.py

@@ -2,7 +2,7 @@ import argparse
 import logging
 import shlex
 
-from clan_cli import create_flake_from_args, create_parser
+from clan_cli.cli import create_flake_from_args, create_parser
 from clan_lib.custom_logger import print_trace
 
 log = logging.getLogger(__name__)

pkgs/clan-cli/clan_cli/tests/test_modules.py

@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING
 import pytest
 from clan_cli.machines.create import CreateOptions, create_machine
 from clan_cli.tests.fixtures_flakes import FlakeForTest
-from clan_lib.api.modules import list_modules
 from clan_lib.flake import Flake
 from clan_lib.nix import nix_eval, run
 from clan_lib.nix_models.clan import (
@@ -16,6 +15,7 @@ from clan_lib.nix_models.clan import (
 )
 from clan_lib.persist.inventory_store import InventoryStore
 from clan_lib.persist.util import set_value_by_path
+from clan_lib.services.modules import list_service_modules
 
 if TYPE_CHECKING:
     from .age_keys import KeyPair
@@ -27,10 +27,9 @@ from clan_lib.machines.machines import Machine as MachineMachine
 @pytest.mark.with_core
 def test_list_modules(test_flake_with_core: FlakeForTest) -> None:
     base_path = test_flake_with_core.path
-    modules_info = list_modules(str(base_path))
+    modules_info = list_service_modules(Flake(str(base_path)))
 
-    assert "localModules" in modules_info
-    assert "modulesPerSource" in modules_info
+    assert "modules" in modules_info
 
 
 @pytest.mark.impure