wip

Architecture.md

@@ -1,24 +1,27 @@
 premhome-gc1, 11GB Ram, 6 Cores, Public IP
 
-falcon-server-1, with X GB Ram, X Cores, No public IP, only private IP. 
+falcon-server-1, with X GB Ram, X Cores, No public IP, only private IP.
-eagle-server-1, with X GB Ram, X Cores, No public IP, only private IP. 
+eagle-server-1, with X GB Ram, X Cores, No public IP, only private IP.
 
 1. Why not use the flannel's wg interface?
 
-This requires the ndoes to have public IPs. But this wouldn't be the case for my system as the nodes at home only have private IPs. 
+This requires the ndoes to have public IPs. But this wouldn't be the case for my system as the nodes at home only have private IPs.
+
+Steps:
 
-Steps: 
 1. Install k3s on gc1
 2. Install flux on gc1
 3. Deploy zerotier controller on gc1
 4. Setup a zerotier interface on gc1
 5. Migrate flannel iface to zerotier interface
-6. Setup zerotier on the 
+6. Setup zerotier on the
 
 # Steps taken
+
 1. Setup a zerotier controller: https://docs.zerotier.com/controller
 
-On premhome-gc1, 
+On premhome-gc1,
+
 ```sh
 TOKEN=$(sudo cat /var/lib/zerotier-one/authtoken.secret)
 NODEID=$(sudo zerotier-cli info | cut -d " " -f 3)
@@ -37,4 +40,11 @@ curl -X POST "http://localhost:9993/controller/network/${NWID}/member/${NODEID}"
 
 Yay! you now have an interface, and an IP address to broadcast on :D
 
+# What I have
 
+1. premhome-gc1
+   IP: 167.253.159.47
+2. premhome-falcon-1
+   IP: 10.0.0.55
+3. premhome-eagle-1
+   IP: 10.0.0.248

nixos/common/k3s.nix

@@ -0,0 +1,17 @@
+{
+  config,
+  meta,
+  ...
+}: {
+  services.k3s = {
+    enable = true;
+    role = meta.role;
+    tokenFile = config.age.secrets.k3s.path;
+    clusterInit = config.networking.hostName == "premhome-gc1";
+    serverAddr =
+      if config.networking.hostName == "premhome-gc1"
+      then ""
+      else "https://${meta.server-addr}:6443";
+    extraFlags = ["--disable=servicelb" "--disable=traefik" "--node-ip ${meta.zt-ip}" "--flannel-iface ztxh6lvd6t" "--flannel-backend=host-gw" "--tls-san ${meta.zt-ip}"];
+  };
+}

nixos/common/zerotier.nix

@@ -1,24 +1,11 @@
 {config, ...}: {
-  age.secrets.zerotier.file = ../secrets/zerotier-network.age;
-
   services.zerotierone = {
     enable = true;
   };
 
   networking = {
     firewall = {
-      interfaces."zts23oi5io".allowedTCPPortRanges = [
+      trustedInterfaces = ["ztxh6lvd6t"];
-        {
-          from = 0;
-          to = 65535;
-        }
-      ];
-      interfaces."zts23oi5io".allowedUDPPortRanges = [
-        {
-          from = 0;
-          to = 65535;
-        }
-      ];
     };
   };
 }

nixos/flake.nix

@@ -72,7 +72,7 @@
       nodes
       // {
         premhome-gc1 = nixpkgs.lib.nixosSystem {
-          specialArgs.meta = (import ./server/nodes.nix).premhome-gc1;
+          specialArgs.meta = (import ./server/nodes.nix).premhome-gc1 // { server-addr = (import ./server/nodes.nix).premhome-gc1.zt-ip; };
           modules = [
             disko.nixosModules.disko
             agenix.nixosModules.default

nixos/proxmox/deno.json

@@ -0,0 +1,5 @@
+{
+  "imports": {
+    "@std/cli": "jsr:@std/cli@^1.0.6"
+  }
+}

nixos/proxmox/deno.lock

@@ -0,0 +1,16 @@
+{
+  "version": "4",
+  "specifiers": {
+    "jsr:@std/cli@^1.0.6": "1.0.6"
+  },
+  "jsr": {
+    "@std/cli@1.0.6": {
+      "integrity": "d22d8b38c66c666d7ad1f2a66c5b122da1704f985d3c47f01129f05abb6c5d3d"
+    }
+  },
+  "workspace": {
+    "dependencies": [
+      "jsr:@std/cli@^1.0.6"
+    ]
+  }
+}

nixos/proxmox/zerotier.ts

@@ -1,6 +1,9 @@
 const BASE_URL = "http://localhost:9993";
-const token = await Deno.readTextFile("/var/lib/zerotier-one/authtoken.secret");
+const token = Deno.env.get("ZT_TOKEN");
 async function main() {
+  if (!token) {
+    throw new Error("Token is empty");
+  }
   const nodeId = (await get("/status")).address;
   // check if networks exist
   const networks = await getNetworks();
@@ -11,18 +14,71 @@ async function main() {
     const network = await createNetwork(nodeId);
     networkId = network.nwid;
   }
+  switch (Deno.args[0]) {
+    case "join": {
+      console.log(
+        `Node should join the network ${networkId} Once joined, fill in the node address below`,
+      );
+      const nodeAddress = prompt("Node address")?.trim();
+      if (!nodeAddress) {
+        console.log("Node address is required");
+        return;
+      }
 
-  console.log(
+      await authorizeNode(networkId, nodeAddress);
-    `Node should join the network ${networkId} Once joined, fill in the node address below`,
+      console.log("Node authorized");
-  );
+      return;
-  const nodeAddress = prompt("Node address")?.trim();
+    }
-  if (!nodeAddress) {
+    case "configNetwork": {
-    console.log("Node address is required");
+      const data = await post(`/controller/network/${networkId}`, {
-    return;
+        name: "k3sNetwork",
+        "ipAssignmentPools": [{
+          "ipRangeStart": "10.222.0.0",
+          "ipRangeEnd": "10.222.0.254",
+        }],
+        "routes": [
+          { "target": "10.222.0.0/23", "via": null },
+          { "target": "10.42.0.0/16", "via": null },
+          // { "target": "10.42.0.0/24", "via": "10.222.0.52" },
+          // { "target": "10.42.1.0/24", "via": "10.222.0.63" },
+          // { "target": "10.42.2.0/24", "via": "10.222.0.62" },
+        ],
+        "rules": [
+          {
+            "etherType": 2048,
+            "not": true,
+            "or": false,
+            "type": "MATCH_ETHERTYPE",
+          },
+          {
+            "etherType": 2054,
+            "not": true,
+            "or": false,
+            "type": "MATCH_ETHERTYPE",
+          },
+          {
+            "etherType": 34525,
+            "not": true,
+            "or": false,
+            "type": "MATCH_ETHERTYPE",
+          },
+          { "type": "ACTION_DROP" },
+          { "type": "ACTION_ACCEPT" },
+        ],
+        "v4AssignMode": "zt",
+        "private": true,
+      });
+      break;
+    }
+
+    case "getNetwork": {
+      const data = await get(`/controller/network/${networkId}`) as string[];
+      console.log(JSON.stringify(data, null, 2));
+      break;
+    }
+    default:
+      throw new Error("unknown option");
   }
-
-  await authorizeNode(networkId, nodeAddress);
-  console.log("Node authorized");
 }
 
 async function getNetworks() {
@@ -78,9 +134,13 @@ async function _getNetwork(id: string) {
 
 async function authorizeNode(networkId: string, nodeId: string) {
   try {
-    const data = await post(`/controller/network/${networkId}/member/${nodeId}`, {
+    const data = await post(
-      authorized: true,
+      `/controller/network/${networkId}/member/${nodeId}`,
-    });
+      {
+        authorized: true,
+        activeBridge: true,
+      },
+    );
     return data;
   } catch (e) {
     console.error("ERROR", e);

nixos/secrets/keys.nix

@@ -1,7 +1,7 @@
 {
   yadunut = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJXOpmWsAnl2RtOuJJMRUx+iJTwf2RWJ1iS3FqXJFzFG";
   yadunut-mbp = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOlXV+TevruoYChk2XbqG5+yqEklRJvOx7YdTGFfXY/f yadunut@yadunut-mbp";
-premhome-gc1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBCEuV81mMpBCGkVniZ9MFUPv7Wls3tQs2eZsXmWYtfo yadunut@premhome-gc1";
+  premhome-gc1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBCEuV81mMpBCGkVniZ9MFUPv7Wls3tQs2eZsXmWYtfo yadunut@premhome-gc1";
-premhome-falcon-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA5JC6jX+aBcTTLVcUM+4mHzNgLkMs/fuP9YU/ngqld1 yadunut@premhome-falcon-1";
+  premhome-falcon-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA5JC6jX+aBcTTLVcUM+4mHzNgLkMs/fuP9YU/ngqld1 yadunut@premhome-falcon-1";
-premhome-eagle-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF/RiZ5Xymi0D7xWNItqxsdNmhlwlO4Sp1XwWP4BQHos yadunut@premhome-eagle-1";
+  premhome-eagle-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF/RiZ5Xymi0D7xWNItqxsdNmhlwlO4Sp1XwWP4BQHos yadunut@premhome-eagle-1";
 }

nixos/server/nodes.nix

@@ -1,8 +1,17 @@
 {
-  premhome-falcon-1 = { ip = "100.64.0.19"; };
+  premhome-falcon-1 = {
-  premhome-falcon-2 = { ip = "100.64.0.7"; };
+    role = "server";
-  premhome-falcon-3 = { ip = "100.64.0.8"; };
+    private-ip = "10.0.0.55";
-  premhome-eagle-1 = { ip = "100.64.0.9"; };
+    zt-ip = "10.222.0.63";
-  premhome-eagle-2 = { ip = "100.64.0.10"; };
+  };
+  premhome-eagle-1 = {
+    role = "server";
+    private-ip = "10.0.0.248";
+    zt-ip = "10.222.0.62";
+  };
+  premhome-gc1 = {
+    role = "server";
+    zt-ip = "10.222.0.52";
+  };
 }
 

nixos/server/premhome-gc1/configuration.nix

@@ -4,7 +4,7 @@
   pkgs,
   ...
 }: {
-  imports = [../../common/users.nix ../../common/zerotier.nix];
+  imports = [../../common/users.nix ../../common/zerotier.nix ../../common/k3s.nix];
   nix = {
     settings.experimental-features = ["nix-command" "flakes"];
   };
@@ -51,13 +51,5 @@
     "L+ /usr/local/bin - - - - /run/current-system/sw/bin/"
   ];
 
-  services.k3s = {
-    enable = true;
-    role = "server";
-    tokenFile = config.age.secrets.k3s.path;
-    clusterInit = true;
-    extraFlags = ["--disable=servicelb" "--disable=traefik" "--node-ip ${meta.zt-ip}" "--flannel-iface ztxh6lvd6t" "--tls-san ${meta.zt-ip}"];
-  };
-
   system.stateVersion = "24.11";
 }

nixos/server/proxmox/configuration.nix

@@ -4,7 +4,7 @@
   pkgs,
   ...
 }: {
-  imports = [../../common/users.nix ../../common/zerotier.nix];
+  imports = [../../common/users.nix ../../common/zerotier.nix ../../common/k3s.nix];
   nix = {
     settings.experimental-features = ["nix-command" "flakes"];
   };
@@ -44,14 +44,5 @@
     };
   };
 
-  services.k3s = {
-    enable = true;
-    role = meta.role;
-    tokenFile = config.age.secrets.k3s.path;
-    clusterInit = false;
-    serverAddr = "https://${meta.server-addr}:6443";
-    extraFlags = ["--disable=servicelb" "--disable=traefik" "--node-ip ${meta.zt-ip}" "--flannel-iface ztxh6lvd6t" "--tls-san ${meta.zt-ip}"];
-  };
-
   system.stateVersion = "24.11";
 }