WIP: zerotier refactor

clanServices/zerotier/default.nix

@@ -6,94 +6,74 @@
   manifest.categories = [ "Utility" ];
   manifest.readme = builtins.readFile ./README.md;
 
-  roles.peer = {
+  roles.default = {
-    perInstance =
-      {
-        instanceName,
-        roles,
-        lib,
-        ...
-      }:
-      {
-        exports.networking = {
-          priority = lib.mkDefault 900;
-          # TODO add user space network support to clan-cli
-          module = "clan_lib.network.zerotier";
-          peers = lib.mapAttrs (name: _machine: {
-            host.var = {
-              machine = name;
-              generator = "zerotier";
-              file = "zerotier-ip";
-            };
-          }) roles.peer.machines;
-        };
-        nixosModule =
-          {
-            config,
-            lib,
-            pkgs,
-            ...
-          }:
-          {
-            imports = [
-              (import ./shared.nix {
-                inherit
-                  instanceName
-                  roles
-                  config
-                  lib
-                  pkgs
-                  ;
-              })
-            ];
-          };
-      };
-  };
-
-  roles.moon = {
     interface =
       { lib, ... }:
       {
-        options.stableEndpoints = lib.mkOption {
+        options.networkId = lib.mkOption {
-          type = lib.types.listOf lib.types.str;
+          type = lib.types.nullOr lib.types.str;
+          default = null;
           description = ''
-            Make this machine a moon.
+            The zerotier network id to use. If not set, a network will be generated for you.
-            Other machines can join this moon by adding this moon in their config.
+            If you administrate your zerotier network via my.zerotier.com, you should set this.
-            It will be reachable under the given stable endpoints.
-          '';
-          example = ''
-            [ "1.2.3.4" "10.0.0.3/9993" "2001:abcd:abcd::3/9993" ]
           '';
+          example = "8056c2e21c000001";
         };
-
       };
     perInstance =
-      {
+      { instanceName, settings, ... }:
-        instanceName,
-        settings,
-        roles,
-        ...
-      }:
       {
         nixosModule =
           {
             config,
-            lib,
             pkgs,
+            lib,
             ...
           }:
           {
-            config.clan.core.networking.zerotier.moon.stableEndpoints = settings.stableEndpoints;
+            config = lib.mkMerge [
+              # code to start/configure zerotier
+              ({ config, ... }: {
+                services.zerotierone = {
+                  enable = true;
+                  joinNetworks = [ config.clan."zerotier_${instanceName}".networkId ];
+                };
+                systemd.network.networks."09-zerotier" = {
+                  matchConfig.Name = "zt*";
+                  networkConfig = {
+                    LLDP = true;
+                    MulticastDNS = true;
+                    KeepConfiguration = "static";
+                  };
+                };
 
-            imports = [
+                systemd.services.zerotierone.serviceConfig.ExecStartPre = [
-              (import ./shared.nix {
+                  "+${pkgs.writeShellScript "init-zerotier" ''
-                inherit
+                    # compare hashes of the current identity secret and the one in the config
-                  instanceName
+                    hash1=$(sha256sum /var/lib/zerotier-one/identity.secret | cut -d ' ' -f 1)
-                  roles
+                    hash2=$(sha256sum ${config.clan.core.vars.generators.zerotier.files.zerotier-identity-secret.path} | cut -d ' ' -f 1)
-                  config
+                    if [[ "$hash1" != "$hash2" ]]; then
-                  lib
+                      echo "Identity secret has changed, backing up old identity to /var/lib/zerotier-one/identity.secret.bac"
-                  pkgs
+                      cp /var/lib/zerotier-one/identity.secret /var/lib/zerotier-one/identity.secret.bac
-                  ;
+                      cp /var/lib/zerotier-one/identity.public /var/lib/zerotier-one/identity.public.bac
+                      cp ${config.clan.core.vars.generators.zerotier.files.zerotier-identity-secret.path} /var/lib/zerotier-one/identity.secret
+                      zerotier-idtool getpublic /var/lib/zerotier-one/identity.secret > /var/lib/zerotier-one/identity.public
+                    fi
+
+                    # cleanup old networks
+                    if [[ -d /var/lib/zerotier-one/networks.d ]]; then
+                      find /var/lib/zerotier-one/networks.d \
+                        -type f \
+                        -name "*.conf" \
+                        -not \( ${
+                          lib.concatMapStringsSep " -o " (
+                            netId: ''-name "${netId}.conf"''
+                          ) config.services.zerotierone.joinNetworks
+                        } \) \
+                        -delete
+                    fi
+                  ''}"
+                ];
               })
             ];
           };
@@ -104,16 +84,22 @@
     interface =
       { lib, ... }:
       {
-        options.allowedIps = lib.mkOption {
+        options = {
-          type = lib.types.listOf lib.types.str;
+          allowedIps = lib.mkOption {
-          default = [ ];
+            type = lib.types.listOf lib.types.str;
-          description = ''
+            default = [ ];
-            Extra machines by their zerotier ip that the zerotier controller
+            description = ''
-            should accept. These could be external machines.
+              Extra machines by their zerotier ip that the zerotier controller
-          '';
+              should accept. These could be external machines.
-          example = ''
+            '';
-            [ "fd5d:bbe3:cbc5:fe6b:f699:935d:bbe3:cbc5" ]
+            example = ''
-          '';
+              [ "fd5d:bbe3:cbc5:fe6b:f699:935d:bbe3:cbc5" ]
+            '';
+          };
+          settings = lib.mkOption {
+            description = "override the network config in /var/lib/zerotier/bla/$network.json";
+            type = lib.types.json;
+          };
         };
       };
 
@@ -136,54 +122,135 @@
             uniqueStrings = list: builtins.attrNames (builtins.groupBy lib.id list);
           in
           {
-            imports = [
+            systemd.services.zerotierone.serviceConfig.ExecStartPre = [
-              (import ./shared.nix {
+              "+${pkgs.writeShellScript "init-zerotier-${instanceName}" ''
-                inherit
+                mkdir -p /var/lib/zerotier-one/controller.d/${instanceName}
-                  instanceName
+                ln -sfT ${pkgs.writeText "net.json" (builtins.toJSON settings.settings)} /var/lib/zerotier-one/controller.d/network/${config.clan."zerotier_${instanceName}".networkId}.json
-                  roles
-                  config
-                  lib
-                  pkgs
-                  ;
-              })
-            ];
-            config = {
-              systemd.services.zerotier-inventory-autoaccept =
-                let
-                  machines = uniqueStrings (
-                    (lib.optionals (roles ? moon) (lib.attrNames roles.moon.machines))
-                    ++ (lib.optionals (roles ? controller) (lib.attrNames roles.controller.machines))
-                    ++ (lib.optionals (roles ? peer) (lib.attrNames roles.peer.machines))
-                  );
-                  networkIps = builtins.foldl' (
-                    ips: name:
-                    if
-                      builtins.pathExists "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value"
-                    then
-                      ips
-                      ++ [
-                        (builtins.readFile "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value")
-                      ]
-                    else
-                      ips
-                  ) [ ] machines;
-                  allHostIPs = settings.allowedIps ++ networkIps;
-                in
-                {
-                  wantedBy = [ "multi-user.target" ];
-                  after = [ "zerotierone.service" ];
-                  path = [ config.clan.core.clanPkgs.zerotierone ];
-                  serviceConfig.ExecStart = pkgs.writeShellScript "zerotier-inventory-autoaccept" ''
-                    ${lib.concatMapStringsSep "\n" (host: ''
-                      ${config.clan.core.clanPkgs.zerotier-members}/bin/zerotier-members allow --member-ip ${host}
-                    '') allHostIPs}
-                  '';
-                };
 
-              clan.core.networking.zerotier.controller.enable = lib.mkDefault true;
+              ''}"
-            };
+            ];
+            systemd.services.zerotierone.serviceConfig.ExecStartPost = [
+              "+${pkgs.writeShellScript "whitelist-controller" ''
+                ${config.clan.core.clanPkgs.zerotier-members}/bin/zerotier-members allow ${
+                  builtins.substring 0 10 config.clan."zerotier_${instanceName}".networkId
+                }
+              ''}"
+            ];
+            systemd.services.zerotier-inventory-autoaccept =
+              let
+                machines = uniqueStrings (
+                  (lib.optionals (roles ? moon) (lib.attrNames roles.moon.machines))
+                  ++ (lib.optionals (roles ? controller) (lib.attrNames roles.controller.machines))
+                  ++ (lib.optionals (roles ? peer) (lib.attrNames roles.peer.machines))
+                );
+                networkIps = builtins.foldl' (
+                  ips: name:
+                  if
+                    builtins.pathExists "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value"
+                  then
+                    ips
+                    ++ [
+                      (builtins.readFile "${config.clan.core.settings.directory}/vars/per-machine/${name}/zerotier/zerotier-ip/value")
+                    ]
+                  else
+                    ips
+                ) [ ] machines;
+                allHostIPs = settings.allowedIps ++ networkIps;
+              in
+              {
+                wantedBy = [ "multi-user.target" ];
+                after = [ "zerotierone.service" ];
+                path = [ config.clan.core.clanPkgs.zerotierone ];
+                serviceConfig.ExecStart = pkgs.writeShellScript "zerotier-inventory-autoaccept" ''
+                  ${lib.concatMapStringsSep "\n" (host: ''
+                    ${config.clan.core.clanPkgs.zerotier-members}/bin/zerotier-members allow --member-ip ${host}
+                  '') allHostIPs}
+                '';
+              };
 
           };
       };
+    };
+
+    # there a bunch of different scenarios that can happen which we need to take care of
+    ## controller is not a peer (the controller doesn't have an ipv4 and is not part of the network)
+    ## controller has multiple networks (currently the code only creates a single network per controller, so we need to figure out the API call to get another network created)
+    ## controller has multiple networks but is peer in only some
+    ## I guess we need to make an attrset of controllers to networks and then create the networks with the controller key
+    # every controller key will be shared, so all network specific ids can be shared
+    perMachine = { instances, machine, lib, ... }: let
+      # an attrset of { controller1 = { instance1 = {}; instance2 = {}; }; controller2 = { instance3 = {}; }; }
+      controllerNetworks = lib.foldlAttrs (acc: instanceName: instance: lib.recursiveUpdate acc { ${lib.head (lib.attrNames instance.roles.controller.machines)} = { ${instanceName} = {}; }; }) {} instances;
+      getInstanceController = instance: lib.head (lib.attrNames instance.roles.controller.machines);
+    in {
+      nixosModule = { pkgs, config, ... }: {
+        config = lib.mkMerge [
+          { # every controller gets a shared network key, from which we can derive multiple network ids
+            # we have this var shared, so we can create it when evaluating any machine
+            clan.core.vars.generators = lib.mapAttrs' (controllerName: _: lib.nameValuePair "zerotier_controller_${controllerName}" {
+              shared = true;
+              files.zerotier-identity-secret.deploy = false;
+              runtimeInputs = [
+                config.services.zerotierone.package
+                pkgs.python3
+              ];
+              script = ''
+                python3 ${./generate.py} --mode network \
+                  --ip "$out/zerotier-ip" \
+                  --identity-secret "$out/zerotier-identity-secret"
+              '';
+            }) controllerNetworks;
+          }
+          { # every instance in a controller gets a network id, which is derived from the controller's shared network key
+            clan.core.vars.generators = lib.mapAttrs' (instanceName: instance: lib.nameValuePair "zerotier_network_${instanceName}" {
+              shared = true;
+              files.zerotier-network-id.secret = false;
+              dependencies = [
+                config.clan.core.vars.generators."zerotier_controller_${getInstanceController instance}"
+              ];
+              runtimeInputs = [
+                config.services.zerotierone.package
+                pkgs.python3
+              ];
+              script = ''
+                python3 ${./generate.py} --mode network-id \
+                  --network-id "$out/zerotier-network-id"
+                  # TODO we need to pass in the controller key
+              '';
+            }) instances;
+          }
+          { # define nixos options, which we need to propagate certain information to other roles (like controller)
+            options.clan = lib.mapAttrs' (instanceName: _: lib.nameValuePair "zerotier_${instanceName}" {
+              networkId = lib.mkOption {
+                type = lib.types.str;
+                description = "The zerotier network id assigned to this machine";
+                default = if instances.${instanceName}.settings.networkId != null then instances.${instanceName}.settings.networkId else "generated-per-machine";
+              };
+            });
+          }
+          # if we have set null as networkId, we assume that we have a controller and generate a shared key for it
+          # (lib.mkIf (settings.networkId == null) {
+          #   clan.core.vars.generators."zerotier_${instanceName}" = {
+          #     files.zerotier-network-id.secret = false;
+          #     files.zerotier-identity-secret.deploy = false;
+          #     shared = true;
+          #     runtimeInputs = [
+          #       config.services.zerotierone.package
+          #       pkgs.python3
+          #     ];
+          #     script = ''
+          #       source ${(pkgs.callPackage ../../../pkgs/minifakeroot { })}/share/minifakeroot/rc
+          #       python3 ${./generate.py} --mode network \
+          #         --ip "$out/zerotier-ip" \
+          #         --identity-secret "$out/zerotier-identity-secret" \
+          #         --network-id "$out/zerotier-network-id"
+          #     '';
+          #   };
+          #   clan."zerotier_${instanceName}".networkId =
+          #     config.clan.core.vars.generators."zerotier_${instanceName}".files.zerotier-network-id.value;
+          # })
+        ];
+      };
+    };
   };
 }

clanServices/zerotier/shared.nix

@@ -28,38 +28,5 @@ in
   config = {
     clan.core.networking.zerotier.networkId = networkId;
     clan.core.networking.zerotier.name = instanceName;
-
-    systemd.services.zerotierone.serviceConfig.ExecStartPost = lib.mkIf (moonIps != [ ]) (
-      lib.mkAfter [
-        "+${pkgs.writeScript "orbit-moons-by-ip" ''
-          #!${pkgs.python3.interpreter}
-          import json
-          import ipaddress
-          import subprocess
-
-          def compute_member_id(ipv6_addr: str) -> str:
-              addr = ipaddress.IPv6Address(ipv6_addr)
-              addr_bytes = bytearray(addr.packed)
-
-              # Extract the bytes corresponding to the member_id (node_id)
-              node_id_bytes = addr_bytes[10:16]
-              node_id = int.from_bytes(node_id_bytes, byteorder="big")
-
-              member_id = format(node_id, "x").zfill(10)[-10:]
-
-              return member_id
-          def main() -> None:
-              ips = json.loads(${builtins.toJSON (builtins.toJSON moonIps)})
-              for ip in ips:
-                  member_id = compute_member_id(ip)
-                  res = subprocess.run(["zerotier-cli", "orbit", member_id, member_id])
-                  if res.returncode != 0:
-                      print(f"Failed to add {member_id} to orbit")
-          if __name__ == "__main__":
-              main()
-        ''}"
-      ]
-    );
-
   };
 }