From 8ed186ce55e2b3098b4a5db82031a6e0329bc3d4 Mon Sep 17 00:00:00 2001 From: Pablo Ovelleiro Corral Date: Mon, 10 Mar 2025 13:23:04 +0100 Subject: [PATCH] data-mesher: init module Co-authored-by: Brian McGee --- checks/data-mesher/default.nix | 138 ++++++++++++++++ .../data-mesher-host-key/public_key/value | 3 + .../data-mesher-host-key/public_key/value | 3 + .../data-mesher-host-key/public_key/value | 3 + .../admin/data-mesher-host-key/private_key | 3 + .../admin/data-mesher-network-key/private_key | 3 + .../peer/data-mesher-host-key/private_key | 3 + .../signer/data-mesher-host-key/private_key | 3 + .../data-mesher-network-key/public_key/value | 3 + checks/flake-module.nix | 1 + clanModules/data-mesher/README.md | 10 ++ clanModules/data-mesher/lib.nix | 19 +++ clanModules/data-mesher/roles/admin.nix | 51 ++++++ clanModules/data-mesher/roles/peer.nix | 5 + clanModules/data-mesher/roles/signer.nix | 5 + clanModules/data-mesher/shared.nix | 154 ++++++++++++++++++ clanModules/flake-module.nix | 1 + devShell.nix | 36 ++-- docs/mkdocs.yml | 1 + flake.lock | 30 ++++ flake.nix | 10 ++ formatter.nix | 2 + nixosModules/clanCore/vars/secret/vm.nix | 5 +- nixosModules/flake-module.nix | 1 + 24 files changed, 478 insertions(+), 15 deletions(-) create mode 100644 checks/data-mesher/default.nix create mode 100644 checks/data-mesher/vars/per-machine/admin/data-mesher-host-key/public_key/value create mode 100644 checks/data-mesher/vars/per-machine/peer/data-mesher-host-key/public_key/value create mode 100644 checks/data-mesher/vars/per-machine/signer/data-mesher-host-key/public_key/value create mode 100644 checks/data-mesher/vars/secret/admin/data-mesher-host-key/private_key create mode 100644 checks/data-mesher/vars/secret/admin/data-mesher-network-key/private_key create mode 100644 checks/data-mesher/vars/secret/peer/data-mesher-host-key/private_key create mode 100644 checks/data-mesher/vars/secret/signer/data-mesher-host-key/private_key create mode 100644 checks/data-mesher/vars/shared/data-mesher-network-key/public_key/value create mode 100644 clanModules/data-mesher/README.md create mode 100644 clanModules/data-mesher/lib.nix create mode 100644 clanModules/data-mesher/roles/admin.nix create mode 100644 clanModules/data-mesher/roles/peer.nix create mode 100644 clanModules/data-mesher/roles/signer.nix create mode 100644 clanModules/data-mesher/shared.nix diff --git a/checks/data-mesher/default.nix b/checks/data-mesher/default.nix new file mode 100644 index 000000000..636b1ef7f --- /dev/null +++ b/checks/data-mesher/default.nix @@ -0,0 +1,138 @@ +(import ../lib/test-base.nix) ( + { self, lib, ... }: + let + + inherit (self.lib.inventory) buildInventory; + + machines = [ + "signer" + "admin" + "peer" + ]; + + serviceConfigs = buildInventory { + inventory = { + machines = lib.genAttrs machines (_: { }); + services = { + data-mesher.default = { + roles.peer.machines = [ "peer" ]; + roles.admin.machines = [ "admin" ]; + roles.signer.machines = [ "signer" ]; + }; + }; + modules = { + data-mesher = self.clanModules.data-mesher; + }; + }; + directory = ./.; + }; + + commonConfig = + { config, ... }: + { + + imports = [ self.nixosModules.clanCore ]; + + clan.core.settings.directory = builtins.toString ./.; + + environment.systemPackages = [ + config.services.data-mesher.package + ]; + + clan.core.vars.settings.publicStore = "in_repo"; + clan.core.vars.settings.secretStore = "vm"; + + clan.data-mesher.network.interface = "eth1"; + clan.data-mesher.bootstrapNodes = [ + "[2001:db8:1::1]:7946" # peer1 + "[2001:db8:1::2]:7946" # peer2 + ]; + + # speed up for testing + services.data-mesher.settings = { + cluster.join_interval = lib.mkForce "2s"; + cluster.push_pull_interval = lib.mkForce "5s"; + }; + + systemd.tmpfiles.settings."vmsecrets" = { + "/etc/secrets" = { + C.argument = "${./vars/secret/${config.clan.core.settings.machine.name}}"; + z = { + mode = "0700"; + user = "data-mesher"; + }; + }; + }; + }; + + adminConfig = { + imports = serviceConfigs.machines.admin.machineImports; + + config.clan.data-mesher.network.tld = "foo"; + config.clan.core.settings.machine.name = "admin"; + }; + + peerConfig = { + imports = serviceConfigs.machines.peer.machineImports; + config.clan.core.settings.machine.name = "peer"; + }; + + signerConfig = { + imports = serviceConfigs.machines.signer.machineImports; + clan.core.settings.machine.name = "signer"; + }; + + in + { + name = "data-mesher"; + + nodes = { + peer = { + imports = [ + peerConfig + commonConfig + ]; + }; + + admin = { + imports = [ + adminConfig + commonConfig + ]; + }; + + signer = { + imports = [ + signerConfig + commonConfig + ]; + }; + }; + + # TODO Add better test script. + testScript = '' + + def resolve(node, success = {}, fail = [], timeout = 60): + for hostname, ips in success.items(): + for ip in ips: + node.wait_until_succeeds(f"getent ahosts {hostname} | grep {ip}", timeout) + + for hostname in fail: + node.wait_until_fails(f"getent ahosts {hostname}") + + start_all() + + admin.wait_for_unit("data-mesher") + signer.wait_for_unit("data-mesher") + peer.wait_for_unit("data-mesher") + + # check dns resolution + for node in [admin, signer, peer]: + resolve(node, { + "admin.foo": ["2001:db8:1::1", "192.168.1.1"], + "peer.foo": ["2001:db8:1::2", "192.168.1.2"], + "signer.foo": ["2001:db8:1::3", "192.168.1.3"] + }) + ''; + } +) diff --git a/checks/data-mesher/vars/per-machine/admin/data-mesher-host-key/public_key/value b/checks/data-mesher/vars/per-machine/admin/data-mesher-host-key/public_key/value new file mode 100644 index 000000000..8fac9de61 --- /dev/null +++ b/checks/data-mesher/vars/per-machine/admin/data-mesher-host-key/public_key/value @@ -0,0 +1,3 @@ +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEAV/XZHv1UQEEzfD2YbJP1Q2jd1ZDG+CP5wvGf/1hcR+Q= +-----END PUBLIC KEY----- diff --git a/checks/data-mesher/vars/per-machine/peer/data-mesher-host-key/public_key/value b/checks/data-mesher/vars/per-machine/peer/data-mesher-host-key/public_key/value new file mode 100644 index 000000000..d4fc1d1d2 --- /dev/null +++ b/checks/data-mesher/vars/per-machine/peer/data-mesher-host-key/public_key/value @@ -0,0 +1,3 @@ +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEAKSSUXJCftt5Vif6ek57CNKBcDRNfrWrxZUHjAIFW9HY= +-----END PUBLIC KEY----- diff --git a/checks/data-mesher/vars/per-machine/signer/data-mesher-host-key/public_key/value b/checks/data-mesher/vars/per-machine/signer/data-mesher-host-key/public_key/value new file mode 100644 index 000000000..728e5242c --- /dev/null +++ b/checks/data-mesher/vars/per-machine/signer/data-mesher-host-key/public_key/value @@ -0,0 +1,3 @@ +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEAvLD0mHQA+hf9ItlUHD0ml3i5XEArmmjwCC5rYEOmzWs= +-----END PUBLIC KEY----- diff --git a/checks/data-mesher/vars/secret/admin/data-mesher-host-key/private_key b/checks/data-mesher/vars/secret/admin/data-mesher-host-key/private_key new file mode 100644 index 000000000..b0d3adbf5 --- /dev/null +++ b/checks/data-mesher/vars/secret/admin/data-mesher-host-key/private_key @@ -0,0 +1,3 @@ +-----BEGIN PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIFX+AzHy821hHqWLPeK3nzRuHod3FNrnPfaDoFvpz6LX +-----END PRIVATE KEY----- diff --git a/checks/data-mesher/vars/secret/admin/data-mesher-network-key/private_key b/checks/data-mesher/vars/secret/admin/data-mesher-network-key/private_key new file mode 100644 index 000000000..58f4d7474 --- /dev/null +++ b/checks/data-mesher/vars/secret/admin/data-mesher-network-key/private_key @@ -0,0 +1,3 @@ +-----BEGIN PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIMwuDntiLoC7cFFyttGDf7cQWlOXOR0q90Jz3lEiuLg+ +-----END PRIVATE KEY----- diff --git a/checks/data-mesher/vars/secret/peer/data-mesher-host-key/private_key b/checks/data-mesher/vars/secret/peer/data-mesher-host-key/private_key new file mode 100644 index 000000000..91315309c --- /dev/null +++ b/checks/data-mesher/vars/secret/peer/data-mesher-host-key/private_key @@ -0,0 +1,3 @@ +-----BEGIN PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIPmH2+vjYG6UOp+/g0Iqu7yZZKId5jffrfsySE36yO+D +-----END PRIVATE KEY----- diff --git a/checks/data-mesher/vars/secret/signer/data-mesher-host-key/private_key b/checks/data-mesher/vars/secret/signer/data-mesher-host-key/private_key new file mode 100644 index 000000000..3bfcdb5af --- /dev/null +++ b/checks/data-mesher/vars/secret/signer/data-mesher-host-key/private_key @@ -0,0 +1,3 @@ +-----BEGIN PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEINS0tSnjHPG8IfpzQAS3wzoJA+4mYM70DIpltN8O4YD7 +-----END PRIVATE KEY----- diff --git a/checks/data-mesher/vars/shared/data-mesher-network-key/public_key/value b/checks/data-mesher/vars/shared/data-mesher-network-key/public_key/value new file mode 100644 index 000000000..b0cc850a8 --- /dev/null +++ b/checks/data-mesher/vars/shared/data-mesher-network-key/public_key/value @@ -0,0 +1,3 @@ +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEA3P18+R5Gt+Jn7wYXpWNTXM5pyWn2WiOWekYCzXqWPwg= +-----END PUBLIC KEY----- diff --git a/checks/flake-module.nix b/checks/flake-module.nix index 1c7dd42ab..d083a8905 100644 --- a/checks/flake-module.nix +++ b/checks/flake-module.nix @@ -41,6 +41,7 @@ in borgbackup = import ./borgbackup nixosTestArgs; matrix-synapse = import ./matrix-synapse nixosTestArgs; mumble = import ./mumble nixosTestArgs; + data-mesher = import ./data-mesher nixosTestArgs; syncthing = import ./syncthing nixosTestArgs; zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs; postgresql = import ./postgresql nixosTestArgs; diff --git a/clanModules/data-mesher/README.md b/clanModules/data-mesher/README.md new file mode 100644 index 000000000..172430861 --- /dev/null +++ b/clanModules/data-mesher/README.md @@ -0,0 +1,10 @@ +--- +description = "Set up data-mesher" +categories = ["System"] +features = [ "inventory" ] + +[constraints] +roles.admin.min = 1 +roles.admin.max = 1 +--- + diff --git a/clanModules/data-mesher/lib.nix b/clanModules/data-mesher/lib.nix new file mode 100644 index 000000000..80a284ab4 --- /dev/null +++ b/clanModules/data-mesher/lib.nix @@ -0,0 +1,19 @@ +lib: { + + machines = + config: + let + instanceNames = builtins.attrNames config.clan.inventory.services.data-mesher; + instanceName = builtins.head instanceNames; + dataMesherInstances = config.clan.inventory.services.data-mesher.${instanceName}; + + uniqueStrings = list: builtins.attrNames (builtins.groupBy lib.id list); + in + rec { + admins = dataMesherInstances.roles.admin.machines or [ ]; + signers = dataMesherInstances.roles.signer.machines or [ ]; + peers = dataMesherInstances.roles.peer.machines or [ ]; + bootstrap = uniqueStrings (admins ++ signers); + }; + +} diff --git a/clanModules/data-mesher/roles/admin.nix b/clanModules/data-mesher/roles/admin.nix new file mode 100644 index 000000000..93a4c2ff0 --- /dev/null +++ b/clanModules/data-mesher/roles/admin.nix @@ -0,0 +1,51 @@ +{ lib, config, ... }: +let + cfg = config.clan.data-mesher; + + dmLib = import ../lib.nix lib; +in +{ + imports = [ + ../shared.nix + ]; + + options.clan.data-mesher = { + network = { + tld = lib.mkOption { + type = lib.types.str; + default = (config.networking.domain or "clan"); + description = "Top level domain to use for the network"; + }; + + hostTTL = lib.mkOption { + type = lib.types.str; + default = "672h"; # 28 days + example = "24h"; + description = "The TTL for hosts in the network, in the form of a Go time.Duration"; + }; + }; + }; + + config = { + services.data-mesher.initNetwork = + let + # for a given machine, read it's public key and remove any new lines + readHostKey = + machine: + let + path = "${config.clan.core.settings.directory}/vars/per-machine/${machine}/data-mesher-host-key/public_key/value"; + in + builtins.elemAt (lib.splitString "\n" (builtins.readFile path)) 1; + in + { + enable = true; + keyPath = config.clan.core.vars.generators.data-mesher-network-key.files.private_key.path; + + tld = cfg.network.tld; + hostTTL = cfg.network.hostTTL; + + # admin and signer host public keys + signingKeys = builtins.map readHostKey (dmLib.machines config).bootstrap; + }; + }; +} diff --git a/clanModules/data-mesher/roles/peer.nix b/clanModules/data-mesher/roles/peer.nix new file mode 100644 index 000000000..a56780406 --- /dev/null +++ b/clanModules/data-mesher/roles/peer.nix @@ -0,0 +1,5 @@ +{ + imports = [ + ../shared.nix + ]; +} diff --git a/clanModules/data-mesher/roles/signer.nix b/clanModules/data-mesher/roles/signer.nix new file mode 100644 index 000000000..a56780406 --- /dev/null +++ b/clanModules/data-mesher/roles/signer.nix @@ -0,0 +1,5 @@ +{ + imports = [ + ../shared.nix + ]; +} diff --git a/clanModules/data-mesher/shared.nix b/clanModules/data-mesher/shared.nix new file mode 100644 index 000000000..49f43f6fe --- /dev/null +++ b/clanModules/data-mesher/shared.nix @@ -0,0 +1,154 @@ +{ + config, + lib, + ... +}: +let + cfg = config.clan.data-mesher; + dmLib = import ./lib.nix lib; + + # the default bootstrap nodes are any machines with the admin or signers role + # we iterate through those machines, determining an IP address for them based on their VPN + # currently only supports zerotier + defaultBootstrapNodes = builtins.foldl' ( + urls: name: + if + builtins.pathExists "${config.clan.core.settings.directory}/machines/${name}/facts/zerotier-ip" + then + let + ip = builtins.readFile "${config.clan.core.settings.directory}/machines/${name}/facts/zerotier-ip"; + in + urls ++ "${ip}:${cfg.network.port}" + else + urls + ) [ ] (dmLib.machines config).bootstrap; +in +{ + options.clan.data-mesher = { + + bootstrapNodes = lib.mkOption { + type = lib.types.nullOr (lib.types.listOf lib.types.str); + default = null; + description = '' + A list of bootstrap nodes that act as an initial gateway when joining + the cluster. + ''; + example = [ + "192.168.1.1:7946" + "192.168.1.2:7946" + ]; + }; + + network = { + + interface = lib.mkOption { + type = lib.types.str; + description = '' + The interface over which cluster communication should be performed. + All the ip addresses associate with this interface will be part of + our host claim, including both ipv4 and ipv6. + + This should be set to an internal/VPN interface. + ''; + example = "tailscale0"; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 7946; + description = '' + Port to listen on for cluster communication. + ''; + }; + }; + }; + + config = { + + services.data-mesher = { + enable = true; + openFirewall = true; + + settings = { + log_level = "warn"; + state_dir = "/var/lib/data-mesher"; + + # read network id from vars + network.id = config.clan.core.vars.generators.data-mesher-network-key.files.public_key.value; + + host = { + names = [ config.networking.hostName ]; + key_path = config.clan.core.vars.generators.data-mesher-host-key.files.private_key.path; + }; + + cluster = { + port = cfg.network.port; + join_interval = "30s"; + push_pull_interval = "30s"; + + interface = cfg.network.interface; + bootstrap_nodes = cfg.bootstrapNodes or defaultBootstrapNodes; + }; + + http.port = 7331; + http.interface = "lo"; + }; + }; + + # Generate host key. + clan.core.vars.generators.data-mesher-host-key = { + files = + let + owner = config.users.users.data-mesher.name; + in + { + private_key = { + inherit owner; + }; + public_key = { + inherit owner; + secret = false; + }; + }; + + runtimeInputs = [ + config.services.data-mesher.package + ]; + + script = '' + data-mesher generate keypair \ + --public-key-path $out/public_key \ + --private-key-path $out/private_key + ''; + }; + + clan.core.vars.generators.data-mesher-network-key = { + # generated once per clan + share = true; + + files = + let + owner = config.users.users.data-mesher.name; + in + { + private_key = { + inherit owner; + }; + public_key = { + inherit owner; + secret = false; + }; + }; + + runtimeInputs = [ + config.services.data-mesher.package + ]; + + script = '' + data-mesher generate keypair \ + --public-key-path $out/public_key \ + --private-key-path $out/private_key + ''; + }; + }; +} diff --git a/clanModules/flake-module.nix b/clanModules/flake-module.nix index 8fbc13244..5ebb7a2ac 100644 --- a/clanModules/flake-module.nix +++ b/clanModules/flake-module.nix @@ -13,6 +13,7 @@ in borgbackup = ./borgbackup; borgbackup-static = ./borgbackup-static; deltachat = ./deltachat; + data-mesher = ./data-mesher; disk-id = ./disk-id; dyndns = ./dyndns; ergochat = ./ergochat; diff --git a/devShell.nix b/devShell.nix index c23c49627..4abaf22c9 100644 --- a/devShell.nix +++ b/devShell.nix @@ -1,10 +1,12 @@ -{ ... }: +{ inputs, ... }: { perSystem = { + lib, pkgs, self', config, + system, ... }: let @@ -24,18 +26,26 @@ in { devShells.default = pkgs.mkShell { - packages = [ - select-shell - pkgs.nix-unit - pkgs.tea - # Better error messages than nix 2.18 - pkgs.nixVersions.latest - self'.packages.tea-create-pr - self'.packages.merge-after-ci - self'.packages.pending-reviews - # treefmt with config defined in ./flake-parts/formatting.nix - config.treefmt.build.wrapper - ]; + packages = + [ + select-shell + pkgs.nix-unit + pkgs.tea + # Better error messages than nix 2.18 + pkgs.nixVersions.latest + self'.packages.tea-create-pr + self'.packages.merge-after-ci + self'.packages.pending-reviews + # treefmt with config defined in ./flake-parts/formatting.nix + config.treefmt.build.wrapper + ] + # bring in data-mesher for the cli which can help with things like key generation + ++ ( + let + data-mesher = inputs.data-mesher.packages.${system}.data-mesher or null; + in + lib.optional (data-mesher != null) data-mesher + ); shellHook = '' echo -e "${ansiEscapes.green}switch to another dev-shell using: select-shell${ansiEscapes.reset}" export PRJ_ROOT=$(git rev-parse --show-toplevel) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index bb9aa3a3c..02282bb1a 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -79,6 +79,7 @@ nav: # This is the module overview and should stay at the top - reference/clanModules/admin.md - reference/clanModules/borgbackup-static.md + - reference/clanModules/data-mesher.md - reference/clanModules/borgbackup.md - reference/clanModules/deltachat.md - reference/clanModules/disk-id.md diff --git a/flake.lock b/flake.lock index 37a3ad5d2..cd4309ec1 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,34 @@ { "nodes": { + "data-mesher": { + "inputs": { + "flake-parts": [ + "flake-parts" + ], + "nixpkgs": [ + "nixpkgs" + ], + "systems": [ + "systems" + ], + "treefmt-nix": [ + "treefmt-nix" + ] + }, + "locked": { + "lastModified": 1743174807, + "narHash": "sha256-seuF0N/I1Drz41RckL8e2DH3ympiQInAD9SsCnBEgEg=", + "ref": "refs/heads/main", + "rev": "562b8ad4f85bf3f2b9c2fde708a5894143a96311", + "revCount": 370, + "type": "git", + "url": "https://git.clan.lol/clan/data-mesher" + }, + "original": { + "type": "git", + "url": "https://git.clan.lol/clan/data-mesher" + } + }, "disko": { "inputs": { "nixpkgs": [ @@ -70,6 +99,7 @@ }, "root": { "inputs": { + "data-mesher": "data-mesher", "disko": "disko", "flake-parts": "flake-parts", "nixos-facter-modules": "nixos-facter-modules", diff --git a/flake.nix b/flake.nix index 7af7300a4..406b1a677 100644 --- a/flake.nix +++ b/flake.nix @@ -19,6 +19,16 @@ treefmt-nix.url = "github:numtide/treefmt-nix"; treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; + + data-mesher = { + url = "git+https://git.clan.lol/clan/data-mesher"; + inputs = { + flake-parts.follows = "flake-parts"; + nixpkgs.follows = "nixpkgs"; + systems.follows = "systems"; + treefmt-nix.follows = "treefmt-nix"; + }; + }; }; outputs = diff --git a/formatter.nix b/formatter.nix index a1cd9a1d9..3ffe6e2fd 100644 --- a/formatter.nix +++ b/formatter.nix @@ -23,6 +23,7 @@ "*.clan-flake" "*.code-workspace" "*.pub" + "*.priv" "*.typed" "*.age" "*.list" @@ -37,6 +38,7 @@ # prettier messes up our mkdocs flavoured markdown "*.md" + "checks/data-mesher/vars/*" "checks/lib/ssh/privkey" "checks/lib/ssh/pubkey" "checks/matrix-synapse/synapse-registration_shared_secret" diff --git a/nixosModules/clanCore/vars/secret/vm.nix b/nixosModules/clanCore/vars/secret/vm.nix index 006ee0d45..0e40d4c8e 100644 --- a/nixosModules/clanCore/vars/secret/vm.nix +++ b/nixosModules/clanCore/vars/secret/vm.nix @@ -6,11 +6,12 @@ { config.clan.core.vars.settings = lib.mkIf (config.clan.core.vars.settings.secretStore == "vm") { fileModule = file: { - path = + path = lib.mkIf (file.config.secret == true) ( if file.config.neededFor == "partitioning" then "/run/partitioning-secrets/${file.config.generatorName}/${file.config.name}" else - "/etc/secrets/${file.config.generatorName}/${file.config.name}"; + "/etc/secrets/${file.config.generatorName}/${file.config.name}" + ); }; secretModule = "clan_cli.vars.secret_modules.vm"; }; diff --git a/nixosModules/flake-module.nix b/nixosModules/flake-module.nix index cbf930274..13f4278d5 100644 --- a/nixosModules/flake-module.nix +++ b/nixosModules/flake-module.nix @@ -12,6 +12,7 @@ inputs.sops-nix.nixosModules.sops inputs.nixos-facter-modules.nixosModules.facter inputs.disko.nixosModules.default + inputs.data-mesher.nixosModules.data-mesher ./clanCore ( { pkgs, lib, ... }: