Merge pull request 'clan-cli: use automatic networking for vars upload and machines update' (#4792) from networking_4 into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4792

docs/mkdocs.yml

@@ -64,7 +64,7 @@ nav:
       - Disk Encryption: guides/disk-encryption.md
       - Age Plugins: guides/age-plugins.md
       - Secrets management: guides/secrets.md
-      - Target Host: guides/target-host.md
+      - Networking: guides/networking.md
       - Zerotier VPN: guides/mesh-vpn.md
       - Secure Boot: guides/secure-boot.md
       - Flake-parts: guides/flake-parts.md

docs/site/guides/networking.md

@@ -0,0 +1,184 @@
+# Connecting to Your Machines
+
+Clan provides automatic networking with fallback mechanisms to reliably connect to your machines.
+
+## Option 1: Automatic Networking with Fallback (Recommended)
+
+Clan's networking module automatically manages connections through various network technologies with intelligent fallback. When you run `clan ssh` or `clan machines update`, Clan tries each configured network by priority until one succeeds.
+
+### Basic Setup with Internet Service
+
+For machines with public IPs or DNS names, use the `internet` service to configure direct SSH while keeping fallback options:
+
+```{.nix title="flake.nix" hl_lines="7-10 14-16"}
+{
+  outputs = { self, clan-core, ... }:
+    let
+      clan = clan-core.lib.clan {
+        inventory.instances = {
+          # Direct SSH with fallback support
+          internet = {
+            roles.default.machines.server1 = {
+              settings.address = "server1.example.com";
+            };
+            roles.default.machines.server2 = {
+              settings.address = "192.168.1.100";
+            };
+          };
+
+          # Fallback: Secure connections via Tor
+          tor = {
+            roles.server.tags.nixos = { };
+          };
+        };
+      };
+    in
+    {
+      inherit (clan.config) nixosConfigurations;
+    };
+}
+```
+
+### Advanced Setup with Multiple Networks
+
+```{.nix title="flake.nix" hl_lines="7-10 13-16 19-21"}
+{
+  outputs = { self, clan-core, ... }:
+    let
+      clan = clan-core.lib.clan {
+        inventory.instances = {
+          # Priority 1: Try direct connection first
+          internet = {
+            roles.default.machines.publicserver = {
+              settings.address = "public.example.com";
+            };
+          };
+
+          # Priority 2: VPN for internal machines
+          zerotier = {
+            roles.controller.machines."controller" = { };
+            roles.peer.tags.nixos = { };
+          };
+
+          # Priority 3: Tor as universal fallback
+          tor = {
+            roles.server.tags.nixos = { };
+          };
+        };
+      };
+    in
+    {
+      inherit (clan.config) nixosConfigurations;
+    };
+}
+```
+
+### How It Works
+
+Clan automatically tries networks in order of priority:
+1. Direct internet connections (if configured)
+2. VPN networks (ZeroTier, Tailscale, etc.)
+3. Tor hidden services
+4. Any other configured networks
+
+If one network fails, Clan automatically tries the next.
+
+### Useful Commands
+
+```bash
+# View all configured networks and their status
+clan network list
+
+# Test connectivity through all networks
+clan network ping machine1
+
+# Show complete network topology
+clan network overview
+```
+
+## Option 2: Manual targetHost (Bypasses Fallback!)
+
+!!! warning
+    Setting `targetHost` directly **disables all automatic networking and fallback**. Only use this if you need complete control and don't want Clan's intelligent connection management.
+
+### Using Inventory (For Static Addresses)
+
+Use inventory-level `targetHost` when the address is **static** and doesn't depend on NixOS configuration:
+
+```{.nix title="flake.nix" hl_lines="8"}
+{
+  outputs = { self, clan-core, ... }:
+    let
+      clan = clan-core.lib.clan {
+        inventory.machines.server = {
+          # WARNING: This bypasses all networking modules!
+          # Use for: Static IPs, DNS names, known hostnames
+          deploy.targetHost = "root@192.168.1.100";
+        };
+      };
+    in
+    {
+      inherit (clan.config) nixosConfigurations;
+    };
+}
+```
+
+**When to use inventory-level:**
+- Static IP addresses: `"root@192.168.1.100"`
+- DNS names: `"user@server.example.com"`
+- Any address that doesn't change based on machine configuration
+
+### Using NixOS Configuration (For Dynamic Addresses)
+
+Use machine-level `targetHost` when you need to **interpolate values from the NixOS configuration**:
+
+```{.nix title="flake.nix" hl_lines="7"}
+{
+  outputs = { self, clan-core, ... }:
+    let
+      clan = clan-core.lib.clan {
+        machines.server = { config, ... }: {
+          # WARNING: This also bypasses all networking modules!
+          # REQUIRED for: Addresses that depend on NixOS config
+          clan.core.networking.targetHost = "root@${config.networking.hostName}.local";
+        };
+      };
+    in
+    {
+      inherit (clan.config) nixosConfigurations;
+    };
+}
+```
+
+**When to use machine-level (NixOS config):**
+- Using hostName from config: `"root@${config.networking.hostName}.local"`
+- Building from multiple config values: `"${config.users.users.deploy.name}@${config.networking.hostName}"`
+- Any address that depends on evaluated NixOS configuration
+
+!!! info "Key Difference"
+    **Inventory-level** (`deploy.targetHost`) is evaluated immediately and works with static strings.
+    **Machine-level** (`clan.core.networking.targetHost`) is evaluated after NixOS configuration and can access `config.*` values.
+
+## Quick Decision Guide
+
+| Scenario | Recommended Approach | Why |
+|----------|---------------------|-----|
+| Public servers | `internet` service | Keeps fallback options |
+| Mixed infrastructure | Multiple networks | Automatic failover |
+| Machines behind NAT | ZeroTier/Tor | NAT traversal with fallback |
+| Testing/debugging | Manual targetHost | Full control, no magic |
+| Single static machine | Manual targetHost | Simple, no overhead |
+
+## Command-Line Override
+
+The `--target-host` flag bypasses ALL networking configuration:
+
+```bash
+# Emergency access - ignores all networking config
+clan machines update server --target-host root@backup-ip.com
+
+# Direct SSH - no fallback attempted
+clan ssh laptop --target-host user@10.0.0.5
+```
+
+Use this for debugging or emergency access when automatic networking isn't working.

docs/site/guides/target-host.md

@@ -1,84 +0,0 @@
-# How to Set `targetHost` for a Machine
-
-The `targetHost` defines where the machine can be reached for operations like SSH or deployment. You can set it in two ways, depending on your use case.
-
----
-
-## ✅ Option 1: Use the Inventory (Recommended for Static Hosts)
-
-If the hostname is **static**, like `server.example.com`, set it in the **inventory**:
-
-```{.nix title="flake.nix" hl_lines="8"}
-{
-  # edlided
-  outputs =
-    { self, clan-core, ... }:
-    let
-      # Sometimes this attribute set is defined in clan.nix
-      clan = clan-core.lib.clan {
-        inventory.machines.jon = {
-            deploy.targetHost = "root@server.example.com";
-        };
-      };
-    in
-    {
-      inherit (clan.config) nixosConfigurations nixosModules clanInternals;
-      # elided
-    };
-}
-```
-
-This is fast, simple and explicit, and doesn’t require evaluating the NixOS config. We can also displayed it in the clan-cli or clan-app.
-
----
-
-## ✅ Option 2: Use NixOS (Only for Dynamic Hosts)
-
-If your target host depends on a **dynamic expression** (like using the machine’s evaluated FQDN), set it inside the NixOS module:
-
-```{.nix title="flake.nix" hl_lines="8"}
-{
-  # edlided
-  outputs =
-    { self, clan-core, ... }:
-    let
-      # Sometimes this attribute set is defined in clan.nix
-      clan = clan-core.lib.clan {
-        machines.jon = {config, ...}: {
-            clan.core.networking.targetHost = "jon@${config.networking.fqdn}";
-        };
-      };
-    in
-    {
-      inherit (clan.config) nixosConfigurations nixosModules clanInternals;
-      # elided
-    };
-}
-```
-
-Use this **only if the value cannot be made static**, because it’s slower and won't be displayed in the clan-cli or clan-app yet.
-
----
-
-## 📝 TL;DR
-
-| Use Case                  | Use Inventory? | Example                          |
-| ------------------------- | -------------- | -------------------------------- |
-| Static hostname           | ✅ Yes          | `root@server.example.com`        |
-| Dynamic config expression | ❌ No           | `jon@${config.networking.fqdn}` |
-
----
-
-## 🚀 Coming Soon: Unified Networking Module
-
-We’re working on a new networking module that will automatically do all of this for you.
-
-- Easier to use
-- Sane defaults: You’ll always be able to reach the machine — no need to worry about hostnames.
-- ✨ Migration from **either method** will be supported and simple.
-
-## Summary
-
-- Ask: *Does this hostname dynamically change based on NixOS config?*
-- If **no**, use the inventory.
-- If **yes**, then use NixOS config.

pkgs/clan-cli/clan_cli/cli.py

@@ -442,17 +442,31 @@ Examples:
     parser_network = subparsers.add_parser(
         "network",
         aliases=["net"],
-        # TODO: Add help="Manage networks" when network code is ready
+        help="Manage networks",
-        # help="Manage networks",
         description="Manage networks",
         epilog=(
             """
-show information about configured networks
+Manage and monitor network connections for machines.
+
+Clan supports multiple network technologies (direct SSH, Tor, etc.) that can be
+configured with different priorities. When connecting to a machine, Clan will:
+1. Check for targetHost in inventory
+2. Try configured networks by priority
+3. Fall back to targetHost from machine config
+
+Commands like 'ssh' and 'machines update' automatically use the best
+available network connection unless overridden with --target-host.
 
 Examples:
 
     $ clan network list
-    Will list networks
+    List all configured networks and their peers
+
+    $ clan network ping machine1
+    Check connectivity to machine1 across all networks
+
+    $ clan network overview
+    Show complete network status and connectivity
         """
         ),
         formatter_class=argparse.RawTextHelpFormatter,
@@ -495,7 +509,7 @@ For more detailed information, visit: {help_hyperlink("getting-started", "https:
     register_common_flags(parser)
 
     if argcomplete:
-        argcomplete.autocomplete(parser, exclude=["morph", "network", "net"])
+        argcomplete.autocomplete(parser, exclude=["morph"])
 
     return parser
 

pkgs/clan-cli/clan_cli/machines/update.py

@@ -12,6 +12,7 @@ from clan_lib.machines.list import instantiate_inventory_to_machines
 from clan_lib.machines.machines import Machine
 from clan_lib.machines.suggestions import validate_machine_names
 from clan_lib.machines.update import run_machine_update
+from clan_lib.network.network import get_best_remote
 from clan_lib.nix import nix_config
 from clan_lib.ssh.host import Host
 from clan_lib.ssh.host_key import HostKeyCheck
@@ -27,6 +28,42 @@ from clan_cli.completions import (
 log = logging.getLogger(__name__)
 
 
+def run_update_with_network(
+    machine: Machine,
+    build_host: Remote | LocalHost | None,
+    upload_inputs: bool,
+    host_key_check: HostKeyCheck,
+    target_host_override: str | None = None,
+) -> None:
+    """Run machine update with proper network context handling.
+
+    If target_host_override is provided, use it directly.
+    Otherwise, use get_best_remote to establish network connection.
+    """
+    if target_host_override:
+        # Direct connection without network context
+        target_host = Remote.from_ssh_uri(
+            machine_name=machine.name,
+            address=target_host_override,
+        ).override(host_key_check=host_key_check)
+        run_machine_update(
+            machine=machine,
+            target_host=target_host,
+            build_host=build_host,
+            upload_inputs=upload_inputs,
+        )
+    else:
+        # Use network context
+        with get_best_remote(machine) as remote:
+            target_host = remote.override(host_key_check=host_key_check)
+            run_machine_update(
+                machine=machine,
+                target_host=target_host,
+                build_host=build_host,
+                upload_inputs=upload_inputs,
+            )
+
+
 def requires_explicit_update(m: Machine) -> bool:
     try:
         if m.select("config.clan.deployment.requireExplicitUpdate"):
@@ -144,27 +181,19 @@ def update_command(args: argparse.Namespace) -> None:
                         ).override(host_key_check=host_key_check)
                 else:
                     build_host = machine.build_host()
-                # Figure out the target host
+
-                if args.target_host:
+                # Schedule the update with network handling
-                    target_host = Remote.from_ssh_uri(
-                        machine_name=machine.name,
-                        address=args.target_host,
-                    ).override(host_key_check=host_key_check)
-                else:
-                    target_host = machine.target_host().override(
-                        host_key_check=host_key_check
-                    )
-                # run the update
                 runtime.async_run(
                     AsyncOpts(
                         tid=machine.name,
                         async_ctx=AsyncContext(prefix=machine.name),
                     ),
-                    run_machine_update,
+                    run_update_with_network,
                     machine=machine,
-                    target_host=target_host,
                     build_host=build_host,
                     upload_inputs=args.upload_inputs,
+                    host_key_check=host_key_check,
+                    target_host_override=args.target_host,
                 )
             runtime.join_all()
             runtime.check_all()

pkgs/clan-cli/clan_cli/vars/upload.py

@@ -5,6 +5,7 @@ from pathlib import Path
 from clan_cli.completions import add_dynamic_completer, complete_machines
 from clan_lib.flake import require_flake
 from clan_lib.machines.machines import Machine
+from clan_lib.network.network import get_best_remote
 from clan_lib.ssh.host import Host
 
 log = logging.getLogger(__name__)
@@ -32,7 +33,12 @@ def upload_command(args: argparse.Namespace) -> None:
         populate_secret_vars(machine, directory)
         return
 
-    with machine.target_host().host_connection() as host, host.become_root() as host:
+    # Use get_best_remote to handle networking
+    with (
+        get_best_remote(machine) as remote,
+        remote.host_connection() as host,
+        host.become_root() as host,
+    ):
         upload_secret_vars(machine, host)
 
 

templates/clan/default/clan.nix

@@ -37,6 +37,13 @@
       # tags.all means 'all machines' will joined
       roles.peer.tags.all = { };
     };
+
+    # Docs: https://docs.clan.lol/reference/clanServices/tor/
+    # Tor network provides secure, anonymous connections to your machines
+    # All machines will be accessible via Tor as a fallback connection method
+    tor = {
+      roles.server.tags.nixos = { };
+    };
   };
 
   # Additional NixOS configuration can be added here.

templates/clan/flake-parts/clan.nix

@@ -37,6 +37,13 @@
       # tags.all means 'all machines' will joined
       roles.peer.tags.all = { };
     };
+
+    # Docs: https://docs.clan.lol/reference/clanServices/tor/
+    # Tor network provides secure, anonymous connections to your machines
+    # All machines will be accessible via Tor as a fallback connection method
+    tor = {
+      roles.server.tags.nixos = { };
+    };
   };
 
   # Additional NixOS configuration can be added here.