allow to persist zerotier identities/ips/meshnames for non-controller

nixosModules/clanCore/secrets/default.nix

@@ -18,7 +18,8 @@
   };
 
   options.clanCore.secretsUploadDirectory = lib.mkOption {
-    type = lib.types.path;
+    type = lib.types.nullOr lib.types.path;
+    default = null;
     description = ''
       The directory where secrets are uploaded into, This is backend specific.
     '';

nixosModules/clanCore/zerotier/default.nix

@@ -94,16 +94,38 @@ in
       # only the controller needs to have the key in the repo, the other clients can be dynamic
       # we generate the zerotier code manually for the controller, since it's part of the bootstrap command
       clanCore.secrets.zerotier = {
+        facts.zerotier-ip = { };
+        facts.zerotier-meshname = { };
         facts.zerotier-network-id = { };
         secrets.zerotier-identity-secret = { };
         generator = ''
           export PATH=${lib.makeBinPath [ config.services.zerotierone.package pkgs.fakeroot ]}
-          ${pkgs.python3.interpreter} ${./generate-network.py} "$facts/zerotier-network-id" "$secrets/zerotier-identity-secret"
+          ${pkgs.python3.interpreter} ${./generate.py} --mode network \
+            --ip "$facts/zerotier-ip" \
+            --meshname "$facts/zerotier-meshname" \
+            --identity-secret "$secrets/zerotier-identity-secret" \
+            --network-id "$facts/zerotier-network-id"
         '';
       };
       environment.systemPackages = [ config.clanCore.clanPkgs.zerotier-members ];
     })
-    (lib.mkIf ((config.clanCore.secrets ? zerotier) && (facts.zerotier-network-id.value != null)) {
+    (lib.mkIf (config.clanCore.secretsUploadDirectory != null && !cfg.controller.enable && cfg.networkId != null) {
+      clanCore.secrets.zerotier = {
+        facts.zerotier-ip = { };
+        facts.zerotier-meshname = { };
+        secrets.zerotier-identity-secret = { };
+
+        generator = ''
+          export PATH=${lib.makeBinPath [ config.services.zerotierone.package ]}
+          ${pkgs.python3.interpreter} ${./generate.py} --mode identity \
+            --ip "$facts/zerotier-ip" \
+            --meshname "$facts/zerotier-meshname" \
+            --identity-secret "$secrets/zerotier-identity-secret" \
+            --network-id ${cfg.networkId}
+        '';
+      };
+    })
+    (lib.mkIf (cfg.controller.enable && config.clanCore.secrets ? zerotier && facts.zerotier-network-id.value != null) {
       clan.networking.zerotier.networkId = facts.zerotier-network-id.value;
       environment.etc."zerotier/network-id".text = facts.zerotier-network-id.value;
 

nixosModules/clanCore/zerotier/generate.py

@@ -1,11 +1,14 @@
 import argparse
+import base64
 import contextlib
+import ipaddress
 import json
 import socket
 import subprocess
 import time
 import urllib.request
 from contextlib import contextmanager
+from dataclasses import dataclass
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any, Iterator, Optional
@@ -41,12 +44,25 @@ def find_free_port() -> Optional[int]:
         return sock.getsockname()[1]
 
 
+class Identity:
+    def __init__(self, path: Path) -> None:
+        self.public = (path / "identity.public").read_text()
+        self.private = (path / "identity.secret").read_text()
+
+    def node_id(self) -> str:
+        nid = self.public.split(":")[0]
+        assert (
+            len(nid) == 10
+        ), f"node_id must be 10 characters long, got {len(nid)}: {nid}"
+        return nid
+
+
 class ZerotierController:
     def __init__(self, port: int, home: Path) -> None:
         self.port = port
         self.home = home
         self.authtoken = (home / "authtoken.secret").read_text()
-        self.secret = (home / "identity.secret").read_text()
+        self.identity = Identity(home)
 
     def _http_request(
         self,
@@ -70,10 +86,10 @@ class ZerotierController:
         return self._http_request("/status")
 
     def create_network(self, data: dict[str, Any] = {}) -> dict[str, Any]:
-        identity = (self.home / "identity.public").read_text()
-        node_id = identity.split(":")[0]
         return self._http_request(
-            f"/controller/network/{node_id}______", method="POST", data=data
+            f"/controller/network/{self.identity.node_id()}______",
+            method="POST",
+            data=data,
         )
 
     def get_network(self, id: str) -> dict[str, Any]:
@@ -118,25 +134,91 @@ def zerotier_controller() -> Iterator[ZerotierController]:
                 p.wait()
 
 
+@dataclass
+class NetworkController:
+    networkid: str
+    identity: Identity
+
+
 # TODO: allow merging more network configuration here
-def create_network() -> dict:
+def create_network_controller() -> NetworkController:
     with zerotier_controller() as controller:
         network = controller.create_network()
-        return {
+        return NetworkController(network["nwid"], controller.identity)
-            "secret": controller.secret,
+
-            "networkid": network["nwid"],
+
-        }
+def create_identity() -> Identity:
+    with TemporaryDirectory() as d:
+        tmpdir = Path(d)
+        private = tmpdir / "identity.secret"
+        public = tmpdir / "identity.public"
+        subprocess.run(["zerotier-idtool", "generate", private, public])
+        return Identity(tmpdir)
+
+
+def compute_zerotier_ip(network_id: str, identity: Identity) -> ipaddress.IPv6Address:
+    assert (
+        len(network_id) == 16
+    ), "network_id must be 16 characters long, got {network_id}"
+    nwid = int(network_id, 16)
+    node_id = int(identity.node_id(), 16)
+    addr_parts = bytearray(
+        [
+            0xFD,
+            (nwid >> 56) & 0xFF,
+            (nwid >> 48) & 0xFF,
+            (nwid >> 40) & 0xFF,
+            (nwid >> 32) & 0xFF,
+            (nwid >> 24) & 0xFF,
+            (nwid >> 16) & 0xFF,
+            (nwid >> 8) & 0xFF,
+            (nwid) & 0xFF,
+            0x99,
+            0x93,
+            (node_id >> 32) & 0xFF,
+            (node_id >> 24) & 0xFF,
+            (node_id >> 16) & 0xFF,
+            (node_id >> 8) & 0xFF,
+            (node_id) & 0xFF,
+        ]
+    )
+    return ipaddress.IPv6Address(bytes(addr_parts))
+
+
+def compute_zerotier_meshname(ip: ipaddress.IPv6Address) -> str:
+    return base64.b32encode(ip.packed)[0:26].decode("ascii").lower()
 
 
 def main() -> None:
     parser = argparse.ArgumentParser()
-    parser.add_argument("network_id")
+    parser.add_argument(
-    parser.add_argument("identity_secret")
+        "--mode", choices=["network", "identity"], required=True, type=str
+    )
+    parser.add_argument("--ip", type=Path, required=True)
+    parser.add_argument("--meshname", type=Path, required=True)
+    parser.add_argument("--identity-secret", type=Path, required=True)
+    parser.add_argument("--network-id", type=str, required=False)
     args = parser.parse_args()
 
-    zerotier = create_network()
+    match args.mode:
-    Path(args.network_id).write_text(zerotier["networkid"])
+        case "network":
-    Path(args.identity_secret).write_text(zerotier["secret"])
+            if args.network_id is None:
+                raise ValueError("network_id parameter is required")
+            controller = create_network_controller()
+            identity = controller.identity
+            network_id = controller.networkid
+            Path(args.network_id).write_text(network_id)
+        case "identity":
+            identity = create_identity()
+            network_id = args.network_id
+        case _:
+            raise ValueError(f"unknown mode {args.mode}")
+    ip = compute_zerotier_ip(network_id, identity)
+    meshname = compute_zerotier_meshname(ip)
+
+    args.identity_secret.write_text(identity.private)
+    args.ip.write_text(ip.compressed)
+    args.meshname.write_text(meshname)
 
 
 if __name__ == "__main__":

pkgs/clan-cli/tests/test_flake_api.py

@@ -27,7 +27,7 @@ def test_inspect_ok(api: TestClient, test_flake_with_core: FlakeForTest) -> None
     assert response.status_code == 200, "Failed to inspect vm"
     data = response.json()
     print("Data: ", data)
-    assert data.get("flake_attrs") == ["vm1"]
+    assert data.get("flake_attrs") == ["vm1", "vm2"]
 
 
 @pytest.mark.impure

pkgs/clan-cli/tests/test_flake_with_core/flake.nix

@@ -31,6 +31,13 @@
               '';
             };
           };
+          vm2 = { lib, ... }: {
+            clan.networking.deploymentAddress = "__CLAN_DEPLOYMENT_ADDRESS__";
+            system.stateVersion = lib.version;
+            sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__";
+            clanCore.secretsUploadDirectory = "__CLAN_SOPS_KEY_DIR__";
+            clan.networking.zerotier.networkId = "82b44b162ec6c013";
+          };
         };
       };
     in

pkgs/clan-cli/tests/test_secrets_generate.py

@@ -1,3 +1,4 @@
+import ipaddress
 from typing import TYPE_CHECKING
 
 import pytest
@@ -39,16 +40,9 @@ def test_generate_secret(
         test_flake_with_core.name, "vm1", "zerotier-network-id"
     )
     assert len(network_id) == 16
-    age_key = (
+    secrets_folder = sops_secrets_folder(test_flake_with_core.path)
-        sops_secrets_folder(test_flake_with_core.path)
+    age_key = secrets_folder / "vm1-age.key" / "secret"
-        .joinpath("vm1-age.key")
+    identity_secret = secrets_folder / "vm1-zerotier-identity-secret" / "secret"
-        .joinpath("secret")
-    )
-    identity_secret = (
-        sops_secrets_folder(test_flake_with_core.path)
-        .joinpath("vm1-zerotier-identity-secret")
-        .joinpath("secret")
-    )
     age_key_mtime = age_key.lstat().st_mtime_ns
     secret1_mtime = identity_secret.lstat().st_mtime_ns
 
@@ -57,10 +51,14 @@ def test_generate_secret(
     assert age_key.lstat().st_mtime_ns == age_key_mtime
     assert identity_secret.lstat().st_mtime_ns == secret1_mtime
 
-    machine_path = (
+    assert (
-        sops_secrets_folder(test_flake_with_core.path)
+        secrets_folder / "vm1-zerotier-identity-secret" / "machines" / "vm1"
-        .joinpath("vm1-zerotier-identity-secret")
+    ).exists()
-        .joinpath("machines")
+
-        .joinpath("vm1")
+    cli.run(["secrets", "generate", "vm2"])
-    )
+    assert has_secret(test_flake_with_core.path, "vm2-age.key")
-    assert machine_path.exists()
+    assert has_secret(test_flake_with_core.path, "vm2-zerotier-identity-secret")
+    ip = machine_get_fact(test_flake_with_core.name, "vm1", "zerotier-ip")
+    assert ipaddress.IPv6Address(ip).is_private
+    meshname = machine_get_fact(test_flake_with_core.name, "vm1", "zerotier-meshname")
+    assert len(meshname) == 26