yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

WIP: zerotier refactor

Browse Source
This commit is contained in:
lassulus
2025-09-23 12:07:49 +02:00
parent 59aec481b2
commit 1b3f087079
2 changed files with 193 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

319
clanServices/zerotier/default.nix
View File

@@ -6,94 +6,74 @@
manifest.categories = [ "Utility" ];
manifest.readme = builtins.readFile ./README.md;
roles.peer = {
perInstance =
{
instanceName,
roles,
lib,
...
}:
{
exports.networking = {
priority = lib.mkDefault 900;
# TODO add user space network support to clan-cli
module = "clan_lib.network.zerotier";
peers = lib.mapAttrs (name: _machine: {
host.var = {
machine = name;
generator = "zerotier";
file = "zerotier-ip";
};
}) roles.peer.machines;
};
nixosModule =
{
config,
lib,
pkgs,
...
}:
{
imports = [
(import ./shared.nix {
inherit
instanceName
roles
config
lib
pkgs
;
})
];
};
};
};
roles.moon = {
roles.default = {
interface =
{ lib, ... }:
{
options.stableEndpoints = lib.mkOption {
type = lib.types.listOf lib.types.str;
options.networkId = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = ''
Make this machine a moon.
Other machines can join this moon by adding this moon in their config.
It will be reachable under the given stable endpoints.
'';
example = ''
[ "1.2.3.4" "10.0.0.3/9993" "2001:abcd:abcd::3/9993" ]
The zerotier network id to use. If not set, a network will be generated for you.
If you administrate your zerotier network via my.zerotier.com, you should set this.
'';
example = "8056c2e21c000001";
};
};
perInstance =
{
instanceName,
settings,
roles,
...
}:
{ instanceName, settings, ... }:
{
nixosModule =
{
config,
lib,
pkgs,
lib,
...
}:
{
config.clan.core.networking.zerotier.moon.stableEndpoints = settings.stableEndpoints;
config = lib.mkMerge [
# code to start/configure zerotier
({ config, ... }: {
services.zerotierone = {
enable = true;
joinNetworks = [ config.clan."zerotier_${instanceName}".networkId ];
};
systemd.network.networks."09-zerotier" = {
matchConfig.Name = "zt*";
networkConfig = {
LLDP = true;
MulticastDNS = true;
KeepConfiguration = "static";
};
};
imports = [
(import ./shared.nix {
inherit
instanceName
roles
config
lib
pkgs
;
systemd.services.zerotierone.serviceConfig.ExecStartPre = [
"+${pkgs.writeShellScript "init-zerotier" ''
# compare hashes of the current identity secret and the one in the config
hash1=$(sha256sum /var/lib/zerotier-one/identity.secret | cut -d ' ' -f 1)
hash2=$(sha256sum ${config.clan.core.vars.generators.zerotier.files.zerotier-identity-secret.path} | cut -d ' ' -f 1)
if [[ "$hash1" != "$hash2" ]]; then
echo "Identity secret has changed, backing up old identity to /var/lib/zerotier-one/identity.secret.bac"
cp /var/lib/zerotier-one/identity.secret /var/lib/zerotier-one/identity.secret.bac
cp /var/lib/zerotier-one/identity.public /var/lib/zerotier-one/identity.public.bac
cp ${config.clan.core.vars.generators.zerotier.files.zerotier-identity-secret.path} /var/lib/zerotier-one/identity.secret
zerotier-idtool getpublic /var/lib/zerotier-one/identity.secret > /var/lib/zerotier-one/identity.public
fi
# cleanup old networks
if [[ -d /var/lib/zerotier-one/networks.d ]]; then
find /var/lib/zerotier-one/networks.d \
-type f \
-name "*.conf" \
-not \( ${
lib.concatMapStringsSep " -o " (
netId: ''-name "${netId}.conf"''
) config.services.zerotierone.joinNetworks
} \) \
-delete
fi
''}"
];
})
];
};
@@ -104,16 +84,22 @@
interface =
{ lib, ... }:
{
options.allowedIps = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = ''
Extra machines by their zerotier ip that the zerotier controller
should accept. These could be external machines.
'';
example = ''
[ "fd5d:bbe3:cbc5:fe6b:f699:935d:bbe3:cbc5" ]
'';
options = {
allowedIps = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = ''
Extra machines by their zerotier ip that the zerotier controller
should accept. These could be external machines.
'';
example = ''
[ "fd5d:bbe3:cbc5:fe6b:f699:935d:bbe3:cbc5" ]
'';
};
settings = lib.mkOption {
description = "override the network config in /var/lib/zerotier/bla/$network.json";
type = lib.types.json;
};
};
};
@@ -136,54 +122,135 @@
uniqueStrings = list: builtins.attrNames (builtins.groupBy lib.id list);
in
{
imports = [
(import ./shared.nix {
inherit
instanceName
roles
config
lib
pkgs
;
})
];
config = {
systemd.services.zerotier-inventory-autoaccept =
let
machines = uniqueStrings (
(lib.optionals (roles ? moon) (lib.attrNames roles.moon.machines))
++ (lib.optionals (roles ? controller) (lib.attrNames roles.controller.machines))
++ (lib.optionals (roles ? peer) (lib.attrNames roles.peer.machines))
);
networkIps = builtins.foldl' (
ips: name:
if
builtins.pathExists "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value"
then
ips
++ [
(builtins.readFile "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value")
]
else
ips
) [ ] machines;
allHostIPs = settings.allowedIps ++ networkIps;
in
{
wantedBy = [ "multi-user.target" ];
after = [ "zerotierone.service" ];
path = [ config.clan.core.clanPkgs.zerotierone ];
serviceConfig.ExecStart = pkgs.writeShellScript "zerotier-inventory-autoaccept" ''
${lib.concatMapStringsSep "\n" (host: ''
${config.clan.core.clanPkgs.zerotier-members}/bin/zerotier-members allow --member-ip ${host}
'') allHostIPs}
'';
};
systemd.services.zerotierone.serviceConfig.ExecStartPre = [
"+${pkgs.writeShellScript "init-zerotier-${instanceName}" ''
mkdir -p /var/lib/zerotier-one/controller.d/${instanceName}
ln -sfT ${pkgs.writeText "net.json" (builtins.toJSON settings.settings)} /var/lib/zerotier-one/controller.d/network/${config.clan."zerotier_${instanceName}".networkId}.json
clan.core.networking.zerotier.controller.enable = lib.mkDefault true;
};
''}"
];
systemd.services.zerotierone.serviceConfig.ExecStartPost = [
"+${pkgs.writeShellScript "whitelist-controller" ''
${config.clan.core.clanPkgs.zerotier-members}/bin/zerotier-members allow ${
builtins.substring 0 10 config.clan."zerotier_${instanceName}".networkId
}
''}"
];
systemd.services.zerotier-inventory-autoaccept =
let
machines = uniqueStrings (
(lib.optionals (roles ? moon) (lib.attrNames roles.moon.machines))
++ (lib.optionals (roles ? controller) (lib.attrNames roles.controller.machines))
++ (lib.optionals (roles ? peer) (lib.attrNames roles.peer.machines))
);
networkIps = builtins.foldl' (
ips: name:
if
builtins.pathExists "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value"
then
ips
++ [
(builtins.readFile "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value")
]
else
ips
) [ ] machines;
allHostIPs = settings.allowedIps ++ networkIps;
in
{
wantedBy = [ "multi-user.target" ];
after = [ "zerotierone.service" ];
path = [ config.clan.core.clanPkgs.zerotierone ];
serviceConfig.ExecStart = pkgs.writeShellScript "zerotier-inventory-autoaccept" ''
${lib.concatMapStringsSep "\n" (host: ''
${config.clan.core.clanPkgs.zerotier-members}/bin/zerotier-members allow --member-ip ${host}
'') allHostIPs}
'';
};
};
};
};
# there a bunch of different scenarios that can happen which we need to take care of
## controller is not a peer (the controller doesn't have an ipv4 and is not part of the network)
## controller has multiple networks (currently the code only creates a single network per controller, so we need to figure out the API call to get another network created)
## controller has multiple networks but is peer in only some
## I guess we need to make an attrset of controllers to networks and then create the networks with the controller key
# every controller key will be shared, so all network specific ids can be shared
perMachine = { instances, machine, lib, ... }: let
# an attrset of { controller1 = { instance1 = {}; instance2 = {}; }; controller2 = { instance3 = {}; }; }
controllerNetworks = lib.foldlAttrs (acc: instanceName: instance: lib.recursiveUpdate acc { ${lib.head (lib.attrNames instance.roles.controller.machines)} = { ${instanceName} = {}; }; }) {} instances;
getInstanceController = instance: lib.head (lib.attrNames instance.roles.controller.machines);
in {
nixosModule = { pkgs, config, ... }: {
config = lib.mkMerge [
{ # every controller gets a shared network key, from which we can derive multiple network ids
# we have this var shared, so we can create it when evaluating any machine
clan.core.vars.generators = lib.mapAttrs' (controllerName: _: lib.nameValuePair "zerotier_controller_${controllerName}" {
shared = true;
files.zerotier-identity-secret.deploy = false;
runtimeInputs = [
config.services.zerotierone.package
pkgs.python3
];
script = ''
python3 ${./generate.py} --mode network \
--ip "$out/zerotier-ip" \
--identity-secret "$out/zerotier-identity-secret"
'';
}) controllerNetworks;
}
{ # every instance in a controller gets a network id, which is derived from the controller's shared network key
clan.core.vars.generators = lib.mapAttrs' (instanceName: instance: lib.nameValuePair "zerotier_network_${instanceName}" {
shared = true;
files.zerotier-network-id.secret = false;
dependencies = [
config.clan.core.vars.generators."zerotier_controller_${getInstanceController instance}"
];
runtimeInputs = [
config.services.zerotierone.package
pkgs.python3
];
script = ''
python3 ${./generate.py} --mode network-id \
--network-id "$out/zerotier-network-id"
# TODO we need to pass in the controller key
'';
}) instances;
}
{ # define nixos options, which we need to propagate certain information to other roles (like controller)
options.clan = lib.mapAttrs' (instanceName: _: lib.nameValuePair "zerotier_${instanceName}" {
networkId = lib.mkOption {
type = lib.types.str;
description = "The zerotier network id assigned to this machine";
default = if instances.${instanceName}.settings.networkId != null then instances.${instanceName}.settings.networkId else "generated-per-machine";
};
});
}
# if we have set null as networkId, we assume that we have a controller and generate a shared key for it
# (lib.mkIf (settings.networkId == null) {
# clan.core.vars.generators."zerotier_${instanceName}" = {
# files.zerotier-network-id.secret = false;
# files.zerotier-identity-secret.deploy = false;
# shared = true;
# runtimeInputs = [
# config.services.zerotierone.package
# pkgs.python3
# ];
# script = ''
# source ${(pkgs.callPackage ../../../pkgs/minifakeroot { })}/share/minifakeroot/rc
# python3 ${./generate.py} --mode network \
# --ip "$out/zerotier-ip" \
# --identity-secret "$out/zerotier-identity-secret" \
# --network-id "$out/zerotier-network-id"
# '';
# };
# clan."zerotier_${instanceName}".networkId =
# config.clan.core.vars.generators."zerotier_${instanceName}".files.zerotier-network-id.value;
# })
];
};
};
};
}

33
clanServices/zerotier/shared.nix
View File

@@ -28,38 +28,5 @@ in
config = {
clan.core.networking.zerotier.networkId = networkId;
clan.core.networking.zerotier.name = instanceName;
systemd.services.zerotierone.serviceConfig.ExecStartPost = lib.mkIf (moonIps != [ ]) (
lib.mkAfter [
"+${pkgs.writeScript "orbit-moons-by-ip" ''
#!${pkgs.python3.interpreter}
import json
import ipaddress
import subprocess
def compute_member_id(ipv6_addr: str) -> str:
addr = ipaddress.IPv6Address(ipv6_addr)
addr_bytes = bytearray(addr.packed)
# Extract the bytes corresponding to the member_id (node_id)
node_id_bytes = addr_bytes[10:16]
node_id = int.from_bytes(node_id_bytes, byteorder="big")
member_id = format(node_id, "x").zfill(10)[-10:]
return member_id
def main() -> None:
ips = json.loads(${builtins.toJSON (builtins.toJSON moonIps)})
for ip in ips:
member_id = compute_member_id(ip)
res = subprocess.run(["zerotier-cli", "orbit", member_id, member_id])
if res.returncode != 0:
print(f"Failed to add {member_id} to orbit")
if __name__ == "__main__":
main()
''}"
]
);
};
}