Merge pull request 'data-mesher-module' (#3086) from data-mesher-module into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3086

checks/data-mesher/default.nix

@@ -0,0 +1,138 @@
+(import ../lib/test-base.nix) (
+  { self, lib, ... }:
+  let
+
+    inherit (self.lib.inventory) buildInventory;
+
+    machines = [
+      "signer"
+      "admin"
+      "peer"
+    ];
+
+    serviceConfigs = buildInventory {
+      inventory = {
+        machines = lib.genAttrs machines (_: { });
+        services = {
+          data-mesher.default = {
+            roles.peer.machines = [ "peer" ];
+            roles.admin.machines = [ "admin" ];
+            roles.signer.machines = [ "signer" ];
+          };
+        };
+        modules = {
+          data-mesher = self.clanModules.data-mesher;
+        };
+      };
+      directory = ./.;
+    };
+
+    commonConfig =
+      { config, ... }:
+      {
+
+        imports = [ self.nixosModules.clanCore ];
+
+        clan.core.settings.directory = builtins.toString ./.;
+
+        environment.systemPackages = [
+          config.services.data-mesher.package
+        ];
+
+        clan.core.vars.settings.publicStore = "in_repo";
+        clan.core.vars.settings.secretStore = "vm";
+
+        clan.data-mesher.network.interface = "eth1";
+        clan.data-mesher.bootstrapNodes = [
+          "[2001:db8:1::1]:7946" # peer1
+          "[2001:db8:1::2]:7946" # peer2
+        ];
+
+        # speed up for testing
+        services.data-mesher.settings = {
+          cluster.join_interval = lib.mkForce "2s";
+          cluster.push_pull_interval = lib.mkForce "5s";
+        };
+
+        systemd.tmpfiles.settings."vmsecrets" = {
+          "/etc/secrets" = {
+            C.argument = "${./vars/secret/${config.clan.core.settings.machine.name}}";
+            z = {
+              mode = "0700";
+              user = "data-mesher";
+            };
+          };
+        };
+      };
+
+    adminConfig = {
+      imports = serviceConfigs.machines.admin.machineImports;
+
+      config.clan.data-mesher.network.tld = "foo";
+      config.clan.core.settings.machine.name = "admin";
+    };
+
+    peerConfig = {
+      imports = serviceConfigs.machines.peer.machineImports;
+      config.clan.core.settings.machine.name = "peer";
+    };
+
+    signerConfig = {
+      imports = serviceConfigs.machines.signer.machineImports;
+      clan.core.settings.machine.name = "signer";
+    };
+
+  in
+  {
+    name = "data-mesher";
+
+    nodes = {
+      peer = {
+        imports = [
+          peerConfig
+          commonConfig
+        ];
+      };
+
+      admin = {
+        imports = [
+          adminConfig
+          commonConfig
+        ];
+      };
+
+      signer = {
+        imports = [
+          signerConfig
+          commonConfig
+        ];
+      };
+    };
+
+    # TODO Add better test script.
+    testScript = ''
+
+      def resolve(node, success = {}, fail = [], timeout = 60):
+        for hostname, ips in success.items():
+            for ip in ips:
+                node.wait_until_succeeds(f"getent ahosts {hostname} | grep {ip}", timeout)
+
+        for hostname in fail:
+            node.wait_until_fails(f"getent ahosts {hostname}")
+
+      start_all()
+
+      admin.wait_for_unit("data-mesher")
+      signer.wait_for_unit("data-mesher")
+      peer.wait_for_unit("data-mesher")
+
+      # check dns resolution
+      for node in [admin, signer, peer]:
+        resolve(node, {
+            "admin.foo": ["2001:db8:1::1", "192.168.1.1"],
+            "peer.foo": ["2001:db8:1::2", "192.168.1.2"],
+            "signer.foo": ["2001:db8:1::3", "192.168.1.3"]
+        })
+    '';
+  }
+)

checks/data-mesher/vars/per-machine/admin/data-mesher-host-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEAV/XZHv1UQEEzfD2YbJP1Q2jd1ZDG+CP5wvGf/1hcR+Q=
+-----END PUBLIC KEY-----

checks/data-mesher/vars/per-machine/peer/data-mesher-host-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEAKSSUXJCftt5Vif6ek57CNKBcDRNfrWrxZUHjAIFW9HY=
+-----END PUBLIC KEY-----

checks/data-mesher/vars/per-machine/signer/data-mesher-host-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEAvLD0mHQA+hf9ItlUHD0ml3i5XEArmmjwCC5rYEOmzWs=
+-----END PUBLIC KEY-----

checks/data-mesher/vars/secret/admin/data-mesher-host-key/private_key

@@ -0,0 +1,3 @@
+-----BEGIN PRIVATE KEY-----
+MC4CAQAwBQYDK2VwBCIEIFX+AzHy821hHqWLPeK3nzRuHod3FNrnPfaDoFvpz6LX
+-----END PRIVATE KEY-----

checks/data-mesher/vars/secret/admin/data-mesher-network-key/private_key

@@ -0,0 +1,3 @@
+-----BEGIN PRIVATE KEY-----
+MC4CAQAwBQYDK2VwBCIEIMwuDntiLoC7cFFyttGDf7cQWlOXOR0q90Jz3lEiuLg+
+-----END PRIVATE KEY-----

checks/data-mesher/vars/secret/peer/data-mesher-host-key/private_key

@@ -0,0 +1,3 @@
+-----BEGIN PRIVATE KEY-----
+MC4CAQAwBQYDK2VwBCIEIPmH2+vjYG6UOp+/g0Iqu7yZZKId5jffrfsySE36yO+D
+-----END PRIVATE KEY-----

checks/data-mesher/vars/secret/signer/data-mesher-host-key/private_key

@@ -0,0 +1,3 @@
+-----BEGIN PRIVATE KEY-----
+MC4CAQAwBQYDK2VwBCIEINS0tSnjHPG8IfpzQAS3wzoJA+4mYM70DIpltN8O4YD7
+-----END PRIVATE KEY-----

checks/data-mesher/vars/shared/data-mesher-network-key/public_key/value

@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MCowBQYDK2VwAyEA3P18+R5Gt+Jn7wYXpWNTXM5pyWn2WiOWekYCzXqWPwg=
+-----END PUBLIC KEY-----

checks/flake-module.nix

@@ -41,6 +41,7 @@ in
             borgbackup = import ./borgbackup nixosTestArgs;
             matrix-synapse = import ./matrix-synapse nixosTestArgs;
             mumble = import ./mumble nixosTestArgs;
+            data-mesher = import ./data-mesher nixosTestArgs;
             syncthing = import ./syncthing nixosTestArgs;
             zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs;
             postgresql = import ./postgresql nixosTestArgs;

clanModules/data-mesher/README.md

@@ -0,0 +1,10 @@
+---
+description = "Set up data-mesher"
+categories = ["System"]
+features = [ "inventory" ]
+
+[constraints]
+roles.admin.min = 1
+roles.admin.max = 1
+---
+

clanModules/data-mesher/lib.nix

@@ -0,0 +1,19 @@
+lib: {
+
+  machines =
+    config:
+    let
+      instanceNames = builtins.attrNames config.clan.inventory.services.data-mesher;
+      instanceName = builtins.head instanceNames;
+      dataMesherInstances = config.clan.inventory.services.data-mesher.${instanceName};
+
+      uniqueStrings = list: builtins.attrNames (builtins.groupBy lib.id list);
+    in
+    rec {
+      admins = dataMesherInstances.roles.admin.machines or [ ];
+      signers = dataMesherInstances.roles.signer.machines or [ ];
+      peers = dataMesherInstances.roles.peer.machines or [ ];
+      bootstrap = uniqueStrings (admins ++ signers);
+    };
+
+}

clanModules/data-mesher/roles/admin.nix

@@ -0,0 +1,51 @@
+{ lib, config, ... }:
+let
+  cfg = config.clan.data-mesher;
+
+  dmLib = import ../lib.nix lib;
+in
+{
+  imports = [
+    ../shared.nix
+  ];
+
+  options.clan.data-mesher = {
+    network = {
+      tld = lib.mkOption {
+        type = lib.types.str;
+        default = (config.networking.domain or "clan");
+        description = "Top level domain to use for the network";
+      };
+
+      hostTTL = lib.mkOption {
+        type = lib.types.str;
+        default = "672h"; # 28 days
+        example = "24h";
+        description = "The TTL for hosts in the network, in the form of a Go time.Duration";
+      };
+    };
+  };
+
+  config = {
+    services.data-mesher.initNetwork =
+      let
+        # for a given machine, read it's public key and remove any new lines
+        readHostKey =
+          machine:
+          let
+            path = "${config.clan.core.settings.directory}/vars/per-machine/${machine}/data-mesher-host-key/public_key/value";
+          in
+          builtins.elemAt (lib.splitString "\n" (builtins.readFile path)) 1;
+      in
+      {
+        enable = true;
+        keyPath = config.clan.core.vars.generators.data-mesher-network-key.files.private_key.path;
+
+        tld = cfg.network.tld;
+        hostTTL = cfg.network.hostTTL;
+
+        # admin and signer host public keys
+        signingKeys = builtins.map readHostKey (dmLib.machines config).bootstrap;
+      };
+  };
+}

clanModules/data-mesher/roles/peer.nix

@@ -0,0 +1,5 @@
+{
+  imports = [
+    ../shared.nix
+  ];
+}

clanModules/data-mesher/roles/signer.nix

@@ -0,0 +1,5 @@
+{
+  imports = [
+    ../shared.nix
+  ];
+}

clanModules/data-mesher/shared.nix

@@ -0,0 +1,154 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.clan.data-mesher;
+  dmLib = import ./lib.nix lib;
+
+  # the default bootstrap nodes are any machines with the admin or signers role
+  # we iterate through those machines, determining an IP address for them based on their VPN
+  # currently only supports zerotier
+  defaultBootstrapNodes = builtins.foldl' (
+    urls: name:
+    if
+      builtins.pathExists "${config.clan.core.settings.directory}/machines/${name}/facts/zerotier-ip"
+    then
+      let
+        ip = builtins.readFile "${config.clan.core.settings.directory}/machines/${name}/facts/zerotier-ip";
+      in
+      urls ++ "${ip}:${cfg.network.port}"
+    else
+      urls
+  ) [ ] (dmLib.machines config).bootstrap;
+in
+{
+  options.clan.data-mesher = {
+
+    bootstrapNodes = lib.mkOption {
+      type = lib.types.nullOr (lib.types.listOf lib.types.str);
+      default = null;
+      description = ''
+        A list of bootstrap nodes that act as an initial gateway when joining
+        the cluster.
+      '';
+      example = [
+        "192.168.1.1:7946"
+        "192.168.1.2:7946"
+      ];
+    };
+
+    network = {
+
+      interface = lib.mkOption {
+        type = lib.types.str;
+        description = ''
+          The interface over which cluster communication should be performed.
+          All the ip addresses associate with this interface will be part of
+          our host claim, including both ipv4 and ipv6.
+
+          This should be set to an internal/VPN interface.
+        '';
+        example = "tailscale0";
+      };
+
+      port = lib.mkOption {
+        type = lib.types.port;
+        default = 7946;
+        description = ''
+          Port to listen on for cluster communication.
+        '';
+      };
+    };
+  };
+
+  config = {
+
+    services.data-mesher = {
+      enable = true;
+      openFirewall = true;
+
+      settings = {
+        log_level = "warn";
+        state_dir = "/var/lib/data-mesher";
+
+        # read network id from vars
+        network.id = config.clan.core.vars.generators.data-mesher-network-key.files.public_key.value;
+
+        host = {
+          names = [ config.networking.hostName ];
+          key_path = config.clan.core.vars.generators.data-mesher-host-key.files.private_key.path;
+        };
+
+        cluster = {
+          port = cfg.network.port;
+          join_interval = "30s";
+          push_pull_interval = "30s";
+
+          interface = cfg.network.interface;
+          bootstrap_nodes = cfg.bootstrapNodes or defaultBootstrapNodes;
+        };
+
+        http.port = 7331;
+        http.interface = "lo";
+      };
+    };
+
+    # Generate host key.
+    clan.core.vars.generators.data-mesher-host-key = {
+      files =
+        let
+          owner = config.users.users.data-mesher.name;
+        in
+        {
+          private_key = {
+            inherit owner;
+          };
+          public_key = {
+            inherit owner;
+            secret = false;
+          };
+        };
+
+      runtimeInputs = [
+        config.services.data-mesher.package
+      ];
+
+      script = ''
+        data-mesher generate keypair \
+            --public-key-path $out/public_key \
+            --private-key-path $out/private_key
+      '';
+    };
+
+    clan.core.vars.generators.data-mesher-network-key = {
+      # generated once per clan
+      share = true;
+
+      files =
+        let
+          owner = config.users.users.data-mesher.name;
+        in
+        {
+          private_key = {
+            inherit owner;
+          };
+          public_key = {
+            inherit owner;
+            secret = false;
+          };
+        };
+
+      runtimeInputs = [
+        config.services.data-mesher.package
+      ];
+
+      script = ''
+        data-mesher generate keypair \
+            --public-key-path $out/public_key \
+            --private-key-path $out/private_key
+      '';
+    };
+  };
+}

clanModules/flake-module.nix

@@ -13,6 +13,7 @@ in
     borgbackup = ./borgbackup;
     borgbackup-static = ./borgbackup-static;
     deltachat = ./deltachat;
+    data-mesher = ./data-mesher;
     disk-id = ./disk-id;
     dyndns = ./dyndns;
     ergochat = ./ergochat;

devShell.nix

@@ -1,10 +1,12 @@
-{ ... }:
+{ inputs, ... }:
 {
   perSystem =
     {
+      lib,
       pkgs,
       self',
       config,
+      system,
       ...
     }:
     let
@@ -24,18 +26,26 @@
     in
     {
       devShells.default = pkgs.mkShell {
-        packages = [
+        packages =
-          select-shell
+          [
-          pkgs.nix-unit
+            select-shell
-          pkgs.tea
+            pkgs.nix-unit
-          # Better error messages than nix 2.18
+            pkgs.tea
-          pkgs.nixVersions.latest
+            # Better error messages than nix 2.18
-          self'.packages.tea-create-pr
+            pkgs.nixVersions.latest
-          self'.packages.merge-after-ci
+            self'.packages.tea-create-pr
-          self'.packages.pending-reviews
+            self'.packages.merge-after-ci
-          # treefmt with config defined in ./flake-parts/formatting.nix
+            self'.packages.pending-reviews
-          config.treefmt.build.wrapper
+            # treefmt with config defined in ./flake-parts/formatting.nix
-        ];
+            config.treefmt.build.wrapper
+          ]
+          # bring in data-mesher for the cli which can help with things like key generation
+          ++ (
+            let
+              data-mesher = inputs.data-mesher.packages.${system}.data-mesher or null;
+            in
+            lib.optional (data-mesher != null) data-mesher
+          );
         shellHook = ''
           echo -e "${ansiEscapes.green}switch to another dev-shell using: select-shell${ansiEscapes.reset}"
           export PRJ_ROOT=$(git rev-parse --show-toplevel)

docs/mkdocs.yml

@@ -79,6 +79,7 @@ nav:
           # This is the module overview and should stay at the top
           - reference/clanModules/admin.md
           - reference/clanModules/borgbackup-static.md
+          - reference/clanModules/data-mesher.md
           - reference/clanModules/borgbackup.md
           - reference/clanModules/deltachat.md
           - reference/clanModules/disk-id.md

flake.lock

@@ -1,5 +1,34 @@
 {
   "nodes": {
+    "data-mesher": {
+      "inputs": {
+        "flake-parts": [
+          "flake-parts"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "systems": [
+          "systems"
+        ],
+        "treefmt-nix": [
+          "treefmt-nix"
+        ]
+      },
+      "locked": {
+        "lastModified": 1743174807,
+        "narHash": "sha256-seuF0N/I1Drz41RckL8e2DH3ympiQInAD9SsCnBEgEg=",
+        "ref": "refs/heads/main",
+        "rev": "562b8ad4f85bf3f2b9c2fde708a5894143a96311",
+        "revCount": 370,
+        "type": "git",
+        "url": "https://git.clan.lol/clan/data-mesher"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.clan.lol/clan/data-mesher"
+      }
+    },
     "disko": {
       "inputs": {
         "nixpkgs": [
@@ -70,6 +99,7 @@
     },
     "root": {
       "inputs": {
+        "data-mesher": "data-mesher",
         "disko": "disko",
         "flake-parts": "flake-parts",
         "nixos-facter-modules": "nixos-facter-modules",

flake.nix

@@ -19,6 +19,16 @@
 
     treefmt-nix.url = "github:numtide/treefmt-nix";
     treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
+
+    data-mesher = {
+      url = "git+https://git.clan.lol/clan/data-mesher";
+      inputs = {
+        flake-parts.follows = "flake-parts";
+        nixpkgs.follows = "nixpkgs";
+        systems.follows = "systems";
+        treefmt-nix.follows = "treefmt-nix";
+      };
+    };
   };
 
   outputs =

formatter.nix

@@ -23,6 +23,7 @@
         "*.clan-flake"
         "*.code-workspace"
         "*.pub"
+        "*.priv"
         "*.typed"
         "*.age"
         "*.list"
@@ -37,6 +38,7 @@
         # prettier messes up our mkdocs flavoured markdown
         "*.md"
 
+        "checks/data-mesher/vars/*"
         "checks/lib/ssh/privkey"
         "checks/lib/ssh/pubkey"
         "checks/matrix-synapse/synapse-registration_shared_secret"

nixosModules/clanCore/vars/secret/vm.nix

@@ -6,11 +6,12 @@
 {
   config.clan.core.vars.settings = lib.mkIf (config.clan.core.vars.settings.secretStore == "vm") {
     fileModule = file: {
-      path =
+      path = lib.mkIf (file.config.secret == true) (
         if file.config.neededFor == "partitioning" then
           "/run/partitioning-secrets/${file.config.generatorName}/${file.config.name}"
         else
-          "/etc/secrets/${file.config.generatorName}/${file.config.name}";
+          "/etc/secrets/${file.config.generatorName}/${file.config.name}"
+      );
     };
     secretModule = "clan_cli.vars.secret_modules.vm";
   };

nixosModules/flake-module.nix

@@ -12,6 +12,7 @@
       inputs.sops-nix.nixosModules.sops
       inputs.nixos-facter-modules.nixosModules.facter
       inputs.disko.nixosModules.default
+      inputs.data-mesher.nixosModules.data-mesher
       ./clanCore
       (
         { pkgs, lib, ... }: