Merge pull request 'network module + CLI' (#4344) from networking into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4344

clanServices/internet/default.nix

@@ -0,0 +1,47 @@
+{ ... }:
+{
+  _class = "clan.service";
+  manifest.name = "clan-core/internet";
+  manifest.description = "direct access (or via ssh jumphost) to machines";
+  manifest.categories = [
+    "System"
+    "Network"
+  ];
+  roles.default = {
+    interface =
+      { lib, ... }:
+      {
+        options = {
+          host = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              ip address or hostname (domain) of the machine
+            '';
+          };
+          jumphosts = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [ ];
+            description = ''
+              optional list of jumphosts to use to connect to the machine
+            '';
+          };
+        };
+      };
+    perInstance =
+      {
+        roles,
+        lib,
+        settings,
+        ...
+      }:
+      {
+        exports.networking = {
+          # TODO add user space network support to clan-cli
+          peers = lib.mapAttrs (_name: machine: {
+            host.plain = machine.settings.host;
+            SSHOptions = map (_x: "-J x") machine.settings.jumphosts;
+          }) roles.default.machines;
+        };
+      };
+  };
+}

clanServices/internet/flake-module.nix

@@ -0,0 +1,9 @@
+{ lib, ... }:
+let
+  module = lib.modules.importApply ./default.nix { };
+in
+{
+  clan.modules = {
+    internet = module;
+  };
+}

clanServices/tor/default.nix

@@ -0,0 +1,88 @@
+{ ... }:
+{
+  _class = "clan.service";
+  manifest.name = "clan-core/tor";
+  manifest.description = "Onion routing, use Hidden services to connect your machines";
+  manifest.categories = [
+    "System"
+    "Network"
+  ];
+
+  roles.default = {
+    # interface =
+    #   { lib, ... }:
+    #   {
+    #     options = {
+    #       OciSettings = lib.mkOption {
+    #         type = lib.types.raw;
+    #         default = null;
+    #         description = "NixOS settings for virtualisation.oci-container.<name>.settings";
+    #       };
+    #       buildContainer = lib.mkOption {
+    #         type = lib.types.nullOr lib.types.str;
+    #         default = null;
+    #       };
+    #     };
+    #   };
+    perInstance =
+      {
+        instanceName,
+        roles,
+        lib,
+        ...
+      }:
+      {
+        exports.networking = {
+          priority = lib.mkDefault 10;
+          # TODO add user space network support to clan-cli
+          module = "clan_lib.network.tor";
+          peers = lib.mapAttrs (name: machine: {
+            host.var = {
+              machine = name;
+              generator = "tor_${instanceName}";
+              file = "hostname";
+            };
+          }) roles.default.machines;
+        };
+        nixosModule =
+          {
+            pkgs,
+            config,
+            ...
+          }:
+          {
+            config = {
+              services.tor = {
+                enable = true;
+                relay.onionServices."clan_${instanceName}" = {
+                  version = 3;
+                  # TODO get ports from instance machine config
+                  map = [
+                    {
+                      port = 22;
+                      target.port = 22;
+                    }
+                  ];
+                  secretKey = config.clan.core.vars.generators."tor_${instanceName}".files.hs_ed25519_secret_key.path;
+                };
+              };
+              clan.core.vars.generators."tor_${instanceName}" = {
+                files.hs_ed25519_secret_key = { };
+                files.hostname = { };
+                runtimeInputs = with pkgs; [
+                  coreutils
+                  tor
+                ];
+                script = ''
+                  mkdir -p data
+                  echo -e "DataDirectory ./data\nSocksPort 0\nHiddenServiceDir ./hs\nHiddenServicePort 80 127.0.0.1:80" > torrc
+                  timeout 2 tor -f torrc || :
+                  mv hs/hs_ed25519_secret_key $out/hs_ed25519_secret_key
+                  mv hs/hostname $out/hostname
+                '';
+              };
+            };
+          };
+      };
+  };
+}

clanServices/tor/flake-module.nix

@@ -0,0 +1,9 @@
+{ lib, ... }:
+let
+  module = lib.modules.importApply ./default.nix { };
+in
+{
+  clan.modules = {
+    tor = module;
+  };
+}

docs/nix/render_options/__init__.py

@@ -465,6 +465,10 @@ Learn how to use `clanServices` in practice in the [Using clanServices guide](..
         service_links: dict[str, dict[str, dict[str, Any]]] = json.load(f3)
 
     for module_name, module_info in service_links.items():
+        # Skip specific modules that are not ready for documentation
+        if module_name in ["internet", "tor"]:
+            continue
+
         output = f"# {module_name}\n\n"
         # output += f"`clan.modules.{module_name}`\n"
         output += f"*{module_info['manifest']['description']}*\n"

lib/modules/clan/interface.nix

@@ -78,7 +78,87 @@ in
       internal = true;
       visible = false;
       type = types.deferredModule;
-      default = { };
+      default = {
+        options.networking = lib.mkOption {
+          default = null;
+          type = lib.types.nullOr (
+            lib.types.submodule {
+              options = {
+                priority = lib.mkOption {
+                  type = lib.types.int;
+                  default = 1000;
+                  description = ''
+                    priority with which this network should be tried.
+                    higher priority means it gets used earlier in the chain
+                  '';
+                };
+                module = lib.mkOption {
+                  # type = lib.types.enum [
+                  #   "clan_lib.network.direct"
+                  #   "clan_lib.network.tor"
+                  # ];
+                  type = lib.types.str;
+                  default = "clan_lib.network.direct";
+                  description = ''
+                    the technology this network uses to connect to the target
+                    This is used for userspace networking with socks proxies.
+                  '';
+                };
+                # should we call this machines? hosts?
+                peers = lib.mkOption {
+                  # <name>
+                  type = lib.types.attrsOf (
+                    lib.types.submodule (
+                      { name, ... }:
+                      {
+                        options = {
+                          name = lib.mkOption {
+                            type = lib.types.str;
+                            default = name;
+                          };
+                          SSHOptions = lib.mkOption {
+                            type = lib.types.listOf lib.types.str;
+                            default = [ ];
+                          };
+                          host = lib.mkOption {
+                            description = '''';
+                            type = lib.types.attrTag {
+                              plain = lib.mkOption {
+                                type = lib.types.str;
+                                description = ''
+                                  a plain value, which can be read directly from the config
+                                '';
+                              };
+                              var = lib.mkOption {
+                                type = lib.types.submodule {
+                                  options = {
+                                    machine = lib.mkOption {
+                                      type = lib.types.str;
+                                      example = "jon";
+                                    };
+                                    generator = lib.mkOption {
+                                      type = lib.types.str;
+                                      example = "tor-ssh";
+                                    };
+                                    file = lib.mkOption {
+                                      type = lib.types.str;
+                                      example = "hostname";
+                                    };
+                                  };
+                                };
+                              };
+                            };
+                          };
+                        };
+                      }
+                    )
+                  );
+                };
+              };
+            }
+          );
+        };
+      };
       description = ''
         A module that is used to define the module of flake level exports -
 

pkgs/clan-cli/clan_cli/cli.py

@@ -25,6 +25,7 @@ from .facts import cli as facts
 from .flash import cli as flash_cli
 from .hyperlink import help_hyperlink
 from .machines import cli as machines
+from .network import cli as network_cli
 from .profiler import profile
 from .ssh import deploy_info as ssh_cli
 from .vars import cli as vars_cli
@@ -428,6 +429,26 @@ Examples:
     )
     select.register_parser(parser_select)
 
+    parser_network = subparsers.add_parser(
+        "network",
+        aliases=["net"],
+        # TODO: Add help="Manage networks" when network code is ready
+        # help="Manage networks",
+        description="Manage networks",
+        epilog=(
+            """
+show information about configured networks
+
+Examples:
+
+    $ clan network list
+    Will list networks
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    network_cli.register_parser(parser_network)
+
     parser_state = subparsers.add_parser(
         "state",
         aliases=["st"],
@@ -462,7 +483,7 @@ For more detailed information, visit: {help_hyperlink("getting-started", "https:
     state.register_parser(parser_state)
 
     if argcomplete:
-        argcomplete.autocomplete(parser, exclude=["morph"])
+        argcomplete.autocomplete(parser, exclude=["morph", "network", "net"])
 
     register_common_flags(parser)
 

pkgs/clan-cli/clan_cli/network/cli.py

@@ -0,0 +1,72 @@
+# !/usr/bin/env python3
+import argparse
+
+from .list import register_list_parser
+from .overview import register_overview_parser
+from .ping import register_ping_parser
+
+
+# takes a (sub)parser and configures it
+def register_parser(parser: argparse.ArgumentParser) -> None:
+    subparser = parser.add_subparsers(
+        title="command",
+        description="the command to run",
+        help="the command to run",
+        required=True,
+    )
+
+    list_parser = subparser.add_parser(
+        "list",
+        help="list all networks",
+        epilog=(
+            """
+This subcommand allows listing all networks
+```
+[NETWORK1] [PRIORITY] [MODULE] [PEER1, PEER2]
+[NETOWKR2] [PRIORITY] [MODULE] [PEER1, PEER2]
+```
+
+Examples:
+
+  $ clan network list
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    register_list_parser(list_parser)
+
+    ping_parser = subparser.add_parser(
+        "ping",
+        help="ping a machine to check if it's online",
+        epilog=(
+            """
+This subcommand allows pinging a machine to check if it's online
+
+Examples:
+
+  $ clan network ping machine1
+  Check machine1 on all networks (in priority order)
+
+  $ clan network ping machine1 --network tor
+  Check machine1 only on the tor network
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    register_ping_parser(ping_parser)
+
+    overview_parser = subparser.add_parser(
+        "overview",
+        help="show the overview of all network and hosts",
+        epilog=(
+            """
+This command shows the complete state of all networks
+
+Examples:
+
+  $ clan network overview
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    register_overview_parser(overview_parser)

pkgs/clan-cli/clan_cli/network/list.py

@@ -0,0 +1,64 @@
+import argparse
+import logging
+
+from clan_lib.flake import Flake
+from clan_lib.network.network import networks_from_flake
+
+log = logging.getLogger(__name__)
+
+
+def list_command(args: argparse.Namespace) -> None:
+    flake: Flake = args.flake
+    networks = networks_from_flake(flake)
+
+    if not networks:
+        print("No networks found")
+        return
+
+    # Calculate column widths
+    col_network = max(12, max(len(name) for name in networks))
+    col_priority = 8
+    col_module = max(
+        10, max(len(net.module_name.split(".")[-1]) for net in networks.values())
+    )
+    col_running = 8
+
+    # Print header
+    header = f"{'Network':<{col_network}}  {'Priority':<{col_priority}}  {'Module':<{col_module}}  {'Running':<{col_running}}  {'Peers'}"
+    print(header)
+    print("-" * len(header))
+
+    # Print network entries
+    for network_name, network in sorted(
+        networks.items(), key=lambda network: -network[1].priority
+    ):
+        # Extract simple module name from full module path
+        module_name = network.module_name.split(".")[-1]
+
+        # Create peer list with truncation
+        peer_names = list(network.peers.keys())
+        max_peers_shown = 3
+
+        if not peer_names:
+            peers_str = "No peers"
+        elif len(peer_names) <= max_peers_shown:
+            peers_str = ", ".join(peer_names)
+        else:
+            shown_peers = peer_names[:max_peers_shown]
+            remaining = len(peer_names) - max_peers_shown
+            peers_str = f"{', '.join(shown_peers)} ...({remaining} more)"
+
+        # Check if network is running
+        try:
+            is_running = network.is_running()
+            running_status = "Yes" if is_running else "No"
+        except Exception:
+            running_status = "Error"
+
+        print(
+            f"{network_name:<{col_network}}  {network.priority:<{col_priority}}  {module_name:<{col_module}}  {running_status:<{col_running}}  {peers_str}"
+        )
+
+
+def register_list_parser(parser: argparse.ArgumentParser) -> None:
+    parser.set_defaults(func=list_command)

pkgs/clan-cli/clan_cli/network/overview.py

@@ -0,0 +1,21 @@
+import argparse
+import logging
+
+from clan_lib.flake import Flake
+from clan_lib.network.network import get_network_overview, networks_from_flake
+
+log = logging.getLogger(__name__)
+
+
+def overview_command(args: argparse.Namespace) -> None:
+    flake: Flake = args.flake
+    networks = networks_from_flake(flake)
+    overview = get_network_overview(networks)
+    for network_name, network in overview.items():
+        print(f"{network_name} {'[ONLINE]' if network['status'] else '[OFFLINE]'}")
+        for peer_name, peer in network["peers"].items():
+            print(f"\t{peer_name}: {'[OFFLINE]' if not peer else f'[{peer}]'}")
+
+
+def register_overview_parser(parser: argparse.ArgumentParser) -> None:
+    parser.set_defaults(func=overview_command)

pkgs/clan-cli/clan_cli/network/ping.py

@@ -0,0 +1,67 @@
+import argparse
+import logging
+
+from clan_lib.errors import ClanError
+from clan_lib.flake import Flake
+from clan_lib.network.network import networks_from_flake
+
+log = logging.getLogger(__name__)
+
+
+def ping_command(args: argparse.Namespace) -> None:
+    flake: Flake = args.flake
+    machine = args.machine
+    network_name = args.network
+
+    networks = networks_from_flake(flake)
+
+    if not networks:
+        print("No networks found in the flake")
+
+    # If network is specified, only check that network
+    if network_name:
+        networks_to_check = [(network_name, networks[network_name])]
+
+    else:
+        # Sort networks by priority (highest first)
+        networks_to_check = sorted(networks.items(), key=lambda x: -x[1].priority)
+
+    found = False
+    results = []
+    for net_name, network in networks_to_check:
+        if machine in network.peers:
+            found = True
+
+            # Check if network technology is running
+            if not network.is_running():
+                results.append(f"{machine} ({net_name}): network not running")
+                continue
+
+            # Check if peer is online
+            ping = network.ping(machine)
+            results.append(f"{machine} ({net_name}): {ping}")
+
+    if not found:
+        msg = f"Machine '{machine}' not found in any network"
+        raise ClanError(msg)
+
+    # Print all results
+    for result in results:
+        print(result)
+
+
+def register_ping_parser(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "machine",
+        type=str,
+        help="Machine name to ping",
+    )
+
+    parser.add_argument(
+        "--network",
+        "-n",
+        type=str,
+        help="Specific network to use for ping (if not specified, checks all networks)",
+    )
+
+    parser.set_defaults(func=ping_command)

pkgs/clan-cli/clan_lib/network/direct.py

@@ -0,0 +1,9 @@
+from clan_lib.network.network import NetworkTechnologyBase
+
+
+class NetworkTechnology(NetworkTechnologyBase):
+    """Direct network connection technology - checks SSH connectivity"""
+
+    def is_running(self) -> bool:
+        """Direct connections are always 'running' as they don't require a daemon"""
+        return True

pkgs/clan-cli/clan_lib/network/network.py

@@ -0,0 +1,133 @@
+import importlib
+import logging
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from functools import cached_property
+from typing import Any
+
+from clan_cli.vars.get import get_machine_var
+from clan_lib.errors import ClanError
+from clan_lib.flake import Flake
+from clan_lib.ssh.parse import parse_ssh_uri
+from clan_lib.ssh.remote import Remote, check_machine_ssh_reachable
+
+log = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class Peer:
+    _host: dict[str, str | dict[str, str]]
+    flake: Flake
+
+    @cached_property
+    def host(self) -> str:
+        if "plain" in self._host and isinstance(self._host["plain"], str):
+            return self._host["plain"]
+        if "var" in self._host and isinstance(self._host["var"], dict):
+            _var: dict[str, str] = self._host["var"]
+            var = get_machine_var(
+                str(self.flake),
+                _var["machine"],
+                f"{_var['generator']}/{_var['file']}",
+            )
+            return var.value.decode()
+        msg = f"Unknown Var Type {self._host}"
+        raise ClanError(msg)
+
+
+class NetworkTechnologyBase(ABC):
+    @abstractmethod
+    def is_running(self) -> bool:
+        pass
+
+    # TODO this will depend on the network implementation if we do user networking at some point, so it should be abstractmethod
+    def ping(self, peer: Peer) -> None | float:
+        if self.is_running():
+            try:
+                # Parse the peer's host address to create a Remote object, use peer here since we don't have the machine_name here
+                remote = parse_ssh_uri(machine_name="peer", address=peer.host)
+
+                # Use the existing SSH reachability check
+                now = time.time()
+                result = check_machine_ssh_reachable(remote)
+
+                if result.ok:
+                    return (time.time() - now) * 1000
+                return None
+
+            except Exception as e:
+                log.debug(f"Error checking peer {peer.host}: {e}")
+                return None
+        return None
+
+
+@dataclass(frozen=True)
+class Network:
+    peers: dict[str, Peer]
+    module_name: str
+    priority: int = 1000
+
+    @cached_property
+    def module(self) -> NetworkTechnologyBase:
+        module = importlib.import_module(self.module_name)
+        return module.NetworkTechnology()
+
+    def is_running(self) -> bool:
+        return self.module.is_running()
+
+    def ping(self, peer: str) -> float | None:
+        return self.module.ping(self.peers[peer])
+
+
+def networks_from_flake(flake: Flake) -> dict[str, Network]:
+    networks: dict[str, Network] = {}
+    networks_ = flake.select("clan.exports.instances.*.networking")
+    for network_name, network in networks_.items():
+        if network:
+            peers: dict[str, Peer] = {}
+            for _peer in network["peers"].values():
+                peers[_peer["name"]] = Peer(_host=_peer["host"], flake=flake)
+            networks[network_name] = Network(
+                peers=peers,
+                module_name=network["module"],
+                priority=network["priority"],
+            )
+    return networks
+
+
+def get_best_remote(machine_name: str, networks: dict[str, Network]) -> Remote | None:
+    for network_name, network in sorted(
+        networks.items(), key=lambda network: -network[1].priority
+    ):
+        if machine_name in network.peers:
+            if network.is_running() and network.ping(machine_name):
+                print(f"connecting via {network_name}")
+                return Remote.from_ssh_uri(
+                    machine_name=machine_name,
+                    address=network.peers[machine_name].host,
+                )
+    return None
+
+
+def get_network_overview(networks: dict[str, Network]) -> dict:
+    result: dict[str, dict[str, Any]] = {}
+    for network_name, network in networks.items():
+        result[network_name] = {}
+        result[network_name]["status"] = None
+        result[network_name]["peers"] = {}
+        network_online = False
+        if network.module.is_running():
+            result[network_name]["status"] = True
+            network_online = True
+        for peer_name in network.peers:
+            if network_online:
+                try:
+                    result[network_name]["peers"][peer_name] = network.ping(peer_name)
+                except ClanError:
+                    log.warning(
+                        f"getting host for machine: {peer_name} in network: {network_name} failed"
+                    )
+            else:
+                result[network_name]["peers"][peer_name] = None
+    return result

pkgs/clan-cli/clan_lib/network/network_test.py

@@ -0,0 +1,106 @@
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+from clan_lib.flake import Flake
+from clan_lib.network.network import Network, Peer, networks_from_flake
+
+
+@patch("clan_lib.network.network.get_machine_var")
+def test_networks_from_flake(mock_get_machine_var: MagicMock) -> None:
+    # Create a mock flake
+    flake = MagicMock(spec=Flake)
+
+    # Mock the var decryption
+    def mock_var_side_effect(flake_path: str, machine: str, var_path: str) -> Any:
+        if machine == "machine1" and var_path == "wireguard/address":
+            mock_var = MagicMock()
+            mock_var.value.decode.return_value = "192.168.1.10"
+            return mock_var
+        if machine == "machine2" and var_path == "wireguard/address":
+            mock_var = MagicMock()
+            mock_var.value.decode.return_value = "192.168.1.11"
+            return mock_var
+        return None
+
+    mock_get_machine_var.side_effect = mock_var_side_effect
+
+    # Define the expected return value from flake.select
+    mock_networking_data = {
+        "vpn-network": {
+            "peers": {
+                "machine1": {
+                    "name": "machine1",
+                    "host": {
+                        "var": {
+                            "machine": "machine1",
+                            "generator": "wireguard",
+                            "file": "address",
+                        }
+                    },
+                },
+                "machine2": {
+                    "name": "machine2",
+                    "host": {
+                        "var": {
+                            "machine": "machine2",
+                            "generator": "wireguard",
+                            "file": "address",
+                        }
+                    },
+                },
+            },
+            "module": "clan_lib.network.tor",
+            "priority": 1000,
+        },
+        "local-network": {
+            "peers": {
+                "machine1": {
+                    "name": "machine1",
+                    "host": {"plain": "10.0.0.10"},
+                },
+                "machine3": {
+                    "name": "machine3",
+                    "host": {"plain": "10.0.0.12"},
+                },
+            },
+            "module": "clan_lib.network.direct",
+            "priority": 500,
+        },
+    }
+
+    # Mock the select method
+    flake.select.return_value = mock_networking_data
+
+    # Call the function
+    networks = networks_from_flake(flake)
+
+    # Verify the flake.select was called with the correct pattern
+    flake.select.assert_called_once_with("clan.exports.instances.*.networking")
+
+    # Verify the returned networks
+    assert len(networks) == 2
+    assert "vpn-network" in networks
+    assert "local-network" in networks
+
+    # Check vpn-network
+    vpn_network = networks["vpn-network"]
+    assert isinstance(vpn_network, Network)
+    assert vpn_network.module_name == "clan_lib.network.tor"
+    assert vpn_network.priority == 1000
+    assert len(vpn_network.peers) == 2
+    assert "machine1" in vpn_network.peers
+    assert "machine2" in vpn_network.peers
+
+    # Check peer details - this will call get_machine_var to decrypt the var
+    machine1_peer = vpn_network.peers["machine1"]
+    assert isinstance(machine1_peer, Peer)
+    assert machine1_peer.host == "192.168.1.10"
+    assert machine1_peer.flake == flake
+
+    # Check local-network
+    local_network = networks["local-network"]
+    assert local_network.module_name == "clan_lib.network.direct"
+    assert local_network.priority == 500
+    assert len(local_network.peers) == 2
+    assert "machine1" in local_network.peers
+    assert "machine3" in local_network.peers

pkgs/clan-cli/clan_lib/network/tor.py

@@ -0,0 +1,20 @@
+from urllib.error import URLError
+from urllib.request import urlopen
+
+from .network import NetworkTechnologyBase
+
+
+class NetworkTechnology(NetworkTechnologyBase):
+    socks_port: int
+    command_port: int
+
+    def is_running(self) -> bool:
+        """Check if Tor is running by sending HTTP request to SOCKS port."""
+        try:
+            response = urlopen("http://127.0.0.1:9050", timeout=5)
+            content = response.read().decode("utf-8", errors="ignore")
+            return "tor" in content.lower()
+        except URLError as e:
+            return "tor" in str(e).lower()
+        except Exception:
+            return False

pkgs/clan-cli/docs.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 from clan_cli.cli import create_parser
 
-hidden_subcommands = ["machine", "b", "f", "m", "se", "st", "va"]
+hidden_subcommands = ["machine", "b", "f", "m", "se", "st", "va", "net", "network"]
 
 
 @dataclass