From 31cb30804462e50c225299a883c3d4f77be11420 Mon Sep 17 00:00:00 2001 From: Yadunand Prem Date: Tue, 22 Oct 2024 16:56:21 -0400 Subject: [PATCH] feat: add zerotier --- Architecture.md | 40 +++++++++++++++++++++ nixos/common/zerotier.nix | 25 +++++++++++++ nixos/flake.nix | 3 +- nixos/secrets/keys.nix | 5 +-- nixos/secrets/secrets.nix | 1 + nixos/secrets/zerotier-network.age | 16 +++++++++ nixos/server/premhome-gc1/configuration.nix | 29 ++++++++++++++- nixos/server/proxmox/configuration.nix | 39 +++++++++----------- todos.txt | 5 +++ 9 files changed, 134 insertions(+), 29 deletions(-) create mode 100644 Architecture.md create mode 100644 nixos/common/zerotier.nix create mode 100644 nixos/secrets/zerotier-network.age create mode 100644 todos.txt diff --git a/Architecture.md b/Architecture.md new file mode 100644 index 0000000..189fc5e --- /dev/null +++ b/Architecture.md @@ -0,0 +1,40 @@ +premhome-gc1, 11GB Ram, 6 Cores, Public IP + +falcon-server-1, with X GB Ram, X Cores, No public IP, only private IP. +eagle-server-1, with X GB Ram, X Cores, No public IP, only private IP. + +1. Why not use the flannel's wg interface? + +This requires the ndoes to have public IPs. But this wouldn't be the case for my system as the nodes at home only have private IPs. + +Steps: +1. Install k3s on gc1 +2. Install flux on gc1 +3. Deploy zerotier controller on gc1 +4. Setup a zerotier interface on gc1 +5. Migrate flannel iface to zerotier interface +6. Setup zerotier on the + +# Steps taken +1. Setup a zerotier controller: https://docs.zerotier.com/controller + +On premhome-gc1, +```sh +TOKEN=$(sudo cat /var/lib/zerotier-one/authtoken.secret) +NODEID=$(sudo zerotier-cli info | cut -d " " -f 3) + +# Create a network +NWID=$(curl -X POST "http://localhost:9993/controller/network/${NODEID}______" -H "X-ZT1-AUTH: ${TOKEN}" -d {} | jq -r ".nwid") + +# Setup the IP address range and routes for this network +curl -X POST "http://localhost:9993/controller/network/${NWID}" -H "X-ZT1-AUTH: ${TOKEN}" \ + -d '{"ipAssignmentPools": [{"ipRangeStart": "10.222.0.0", "ipRangeEnd": "10.222.0.254"}], "routes": [{"target": "10.222.0.0/23", "via": null}], "rules": [ { "etherType": 2048, "not": true, "or": false, "type": "MATCH_ETHERTYPE" }, { "etherType": 2054, "not": true, "or": false, "type": "MATCH_ETHERTYPE" }, { "etherType": 34525, "not": true, "or": false, "type": "MATCH_ETHERTYPE" }, { "type": "ACTION_DROP" }, { "type": "ACTION_ACCEPT" } ], "v4AssignMode": "zt", "private": true }' + +# Authorize the current server +curl -X POST "http://localhost:9993/controller/network/${NWID}/member/${NODEID}" -H "X-ZT1-AUTH: ${TOKEN}" -d '{"authorized": true}' + +``` + +Yay! you now have an interface, and an IP address to broadcast on :D + + diff --git a/nixos/common/zerotier.nix b/nixos/common/zerotier.nix new file mode 100644 index 0000000..59c2b70 --- /dev/null +++ b/nixos/common/zerotier.nix @@ -0,0 +1,25 @@ +{config}: { + age.secrets.zerotier.file = ../../secrets/zerotier.age; + + services.zerotierone = { + enable = true; + joinNetworks = [builtins.readFile config.age.secrets.zerotier.path]; # Is an antipattern, but idc if this is readable on the servers + }; + + networking = { + firewall = { + interfaces."zts23oi5io".allowedTCPPortRanges = [ + { + from = 0; + to = 65535; + } + ]; + interfaces."zts23oi5io".allowedUDPPortRanges = [ + { + from = 0; + to = 65535; + } + ]; + }; + }; +} diff --git a/nixos/flake.nix b/nixos/flake.nix index 067a6d2..23e7dfd 100644 --- a/nixos/flake.nix +++ b/nixos/flake.nix @@ -35,8 +35,8 @@ specialArgs = { meta = { hostname = name; - ip = data.ip; private-ip = data.private-ip; + server-addr = (import ./server/nodes.nix).premhome-gc1.zt-ip; }; }; modules = [ @@ -50,6 +50,7 @@ nodes // { premhome-gc1 = nixpkgs.lib.nixosSystem { + specialArgs.meta = (import ./server/nodes.nix).premhome-gc1; modules = [ disko.nixosModules.disko agenix.nixosModules.default diff --git a/nixos/secrets/keys.nix b/nixos/secrets/keys.nix index 1702014..9972473 100644 --- a/nixos/secrets/keys.nix +++ b/nixos/secrets/keys.nix @@ -1,8 +1,5 @@ { yadunut = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJXOpmWsAnl2RtOuJJMRUx+iJTwf2RWJ1iS3FqXJFzFG"; yadunut-mbp = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOlXV+TevruoYChk2XbqG5+yqEklRJvOx7YdTGFfXY/f yadunut@yadunut-mbp"; - premhome-falcon-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHim2wAW8m6rbFqo6oLBOeprbljAQqghYkFahtFFMNqV yadunut@premhome-falcon-1"; - premhome-falcon-2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILOjiTk60H5LMHkEG7MOaysa0BRlul7qvhuWYpnMtme6 yadunut@premhome-falcon-2"; - premhome-falcon-3 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH4vBKSQyWYRmwuaYfIodOVc10veeH7V+EgobBI/5QHL yadunut@premhome-falcon-3"; -premhome-gc1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA3I/bCyi6rBzgJkfCFa8T9F+y1eOuZDB7l2Ly67slX3 yadunut@premhome-gc1"; + premhome-gc1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA3I/bCyi6rBzgJkfCFa8T9F+y1eOuZDB7l2Ly67slX3 yadunut@premhome-gc1"; } diff --git a/nixos/secrets/secrets.nix b/nixos/secrets/secrets.nix index dd43e0a..93b7868 100644 --- a/nixos/secrets/secrets.nix +++ b/nixos/secrets/secrets.nix @@ -4,4 +4,5 @@ in { "tailscale.age".publicKeys = builtins.attrValues keys; "k3s.age".publicKeys = builtins.attrValues keys; "flux.age".publicKeys = [keys.yadunut keys.yadunut-mbp]; + "zerotier-network.age".publicKeys = builtins.attrValues keys; } diff --git a/nixos/secrets/zerotier-network.age b/nixos/secrets/zerotier-network.age new file mode 100644 index 0000000..67d906f --- /dev/null +++ b/nixos/secrets/zerotier-network.age @@ -0,0 +1,16 @@ +age-encryption.org/v1 +-> ssh-ed25519 OOT7iQ jUogNJ9uREDJZEl4G5pb/2bNjiBHIB9IABgXQfo0g1M +InYXkJls2Sdd+jnQ9Z8ifoUGznwktmstsM8avHFfTuU +-> ssh-ed25519 7Lat4Q O6JMNKXRwRWjFZxJM/agtJ922KR+74u8a0WmWJdSaQM +5tPV0awfn5djn2c50xloyDRkeu2Aon3/z+6kfoA/eHk +-> ssh-ed25519 dPFwiQ HeOAeTzItJMkTPW2ODs/Z/E9nZycqtJnjGaKPigZ/CM +MLTkDKg0hLOfDplWb33hvGQahvEgjqy+S9w+UCKZNXU +-> ssh-ed25519 OV7A4A 4mxMknf9sJCZ8NiENMJqGd2lEBB7dmvzGqVLirHvWGo +fJ+4MY9oSFEdvjGYm2NoWviC9J4ocSUhUWuGEa71pFc +-> ssh-ed25519 Gc/MTQ pjuOkv7iMuSkrFccMGd5Usz/a0bcOJYikvHeuYg5ATA +ydWemX28ZNygAYR/MsOezz81haHj2XhvHlFcZMwsgjo +-> ssh-ed25519 0ckKSg hV+hpxVdfr2xOfNYZkbrGNMu5GOASlHDch4AYhqlWnQ +MeZdLC12XrF9sSy1q28dpdqjYtIEKcTzJq7/vfhILf0 +--- 0rgW5rFnvhi1LMOcC3vl70s9Vq9S+PQ5Pu8Apgxu0v4 +Îß¡ÓÚPÜ7‚-9dqцíb>?2ôÝà}ÈN’Û}Þ5ñÌuu\çw”ƒ • +O \ No newline at end of file diff --git a/nixos/server/premhome-gc1/configuration.nix b/nixos/server/premhome-gc1/configuration.nix index 5252d76..7293678 100644 --- a/nixos/server/premhome-gc1/configuration.nix +++ b/nixos/server/premhome-gc1/configuration.nix @@ -1,13 +1,16 @@ { config, + meta, pkgs, ... }: { - imports = [../../common/users.nix]; + imports = [../../common/users.nix ../../common/zerotier.nix]; nix = { settings.experimental-features = ["nix-command" "flakes"]; }; + nixpkgs.config.allowUnfree = true; + networking.hostName = "premhome-gc1"; age.secrets.k3s.file = ../../secrets/k3s.age; @@ -16,6 +19,10 @@ # grub.device is set by disko boot.tmp.cleanOnBoot = true; + boot.kernel.sysctl = { + "net.ipv4.ip_forward" = 1; + }; + services.openssh.enable = true; services.qemuGuest.enable = true; @@ -23,7 +30,27 @@ git neovim wget + jq + ripgrep + fd ]; + networking = { + nftables.enable = true; + firewall = { + enable = true; + allowedTCPPorts = [22 80 443]; + trustedInterfaces = ["zts23oi5io"]; + }; + }; + + services.k3s = { + enable = true; + role = "server"; + tokenFile = config.age.secrets.k3s.path; + clusterInit = true; + extraFlags = ["--disable=servicelb" "--disable=traefik" "--node-ip ${meta.zt-ip}" "--flannel-iface zts23oi5io"]; + }; + system.stateVersion = "24.11"; } diff --git a/nixos/server/proxmox/configuration.nix b/nixos/server/proxmox/configuration.nix index 5ca6b4c..303c5ae 100644 --- a/nixos/server/proxmox/configuration.nix +++ b/nixos/server/proxmox/configuration.nix @@ -1,27 +1,17 @@ { - config, meta, pkgs, ... }: { - imports = [../../common/users.nix]; + imports = [../../common/users.nix ../../common/zerotier.nix]; nix = { settings.experimental-features = ["nix-command" "flakes"]; }; networking.hostName = meta.hostname; - age.secrets.tailscale.file = ../../secrets/tailscale.age; age.secrets.k3s.file = ../../secrets/k3s.age; - services.tailscale = { - enable = true; - authKeyFile = config.age.secrets.tailscale.path; - useRoutingFeatures = "both"; - extraUpFlags = ["--advertise-routes=10.0.1.0/24" "--login-server=http://ts.yadunut.com:444" "--accept-routes"]; - interfaceName = "tailscale0"; - }; - boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; boot.tmp.cleanOnBoot = true; @@ -44,20 +34,23 @@ wget ]; - services.k3s = { - enable = true; - role = "server"; - tokenFile = config.age.secrets.k3s.path; - clusterInit = meta.hostname == "premhome-falcon-1"; - serverAddr = - if meta.hostname == "premhome-falcon-1" - then "" - else "https://premhome-falcon-1:6443"; - extraFlags = ["--disable=servicelb" "--disable=traefik"]; + networking = { + nftables.enable = true; + firewall = { + enable = true; + allowedTCPPorts = [22]; + trustedInterfaces = ["zts23oi5io"]; + }; }; - networking.firewall.trustedInterfaces = ["tailscale0"]; - networking.firewall.enable = false; + # services.k3s = { + # enable = true; + # role = "server"; + # tokenFile = config.age.secrets.k3s.path; + # clusterInit = false; + # serverAddr = "https://${meta.server-addr}:6443"; + # extraFlags = ["--disable=servicelb" "--disable=traefik" "--node-ip ${meta.zt-ip}" "--flannel-iface zts23oi5io"]; + # }; system.stateVersion = "24.11"; } diff --git a/todos.txt b/todos.txt new file mode 100644 index 0000000..a0ace0f --- /dev/null +++ b/todos.txt @@ -0,0 +1,5 @@ +X Delete ingress digitalocean container +X Install nixos on greencloud server +Setup ztnet on greencloud server +setup zerotier on greencloud server +Delete nodes from