Merge branch 'main' into docs/clanservices-borgbackup

README.md

@@ -24,7 +24,7 @@ If you're new to Clan and eager to dive in, start with our quickstart guide and
 
 In the Clan ecosystem, security is paramount. Learn how to handle secrets effectively:
 
-- **Secrets Management**: Securely manage secrets by consulting [Vars](https://docs.clan.lol/guides/vars-backend/)<!-- [secrets.md](docs/site/guides/vars-backend.md) -->.
+- **Secrets Management**: Securely manage secrets by consulting [Vars](https://docs.clan.lol/concepts/generators/)<!-- [secrets.md](docs/site/concepts/generators.md) -->.
 
 ### Contributing to Clan
 

clanModules/borgbackup-static/README.md

@@ -4,7 +4,7 @@ description = "Statically configure borgbackup with sane defaults."
 !!! Danger "Deprecated"
     Use [borgbackup](borgbackup.md) instead.
 
-    Don't use borgbackup-static through [inventory](../../guides/inventory.md).
+    Don't use borgbackup-static through [inventory](../../concepts/inventory.md).
 
 This module implements the `borgbackup` backend and implements sane defaults
 for backup management through `borgbackup` for members of the clan.

clanModules/root-password/README.md

@@ -12,7 +12,7 @@ After the system was installed/deployed the following command can be used to dis
 clan vars get [machine_name] root-password/root-password
 ```
 
-See also: [Vars](../../guides/vars-backend.md)
+See also: [Vars](../../concepts/generators.md)
 
 To regenerate the password run:
 ```

clanModules/user-password/README.md

@@ -16,7 +16,7 @@ After the system was installed/deployed the following command can be used to dis
 clan vars get [machine_name] root-password/root-password
 ```
 
-See also: [Vars](../../guides/vars-backend.md)
+See also: [Vars](../../concepts/generators.md)
 
 To regenerate the password run:
 ```

docs/mkdocs.yml

@@ -55,29 +55,39 @@ nav:
           - Add Services: guides/getting-started/add-services.md
           - Deploy Machine: guides/getting-started/deploy.md
           - Continuous Integration: guides/getting-started/check.md
-      - Inventory: guides/inventory.md
       - Using Services: guides/clanServices.md
       - Backup & Restore: guides/backups.md
       - Disk Encryption: guides/disk-encryption.md
-      - Vars: guides/vars-backend.md
       - Age Plugins: guides/age-plugins.md
-      - Advanced Secrets: guides/secrets.md
+      - Secrets management: guides/secrets.md
-      - Machine Autoincludes: guides/more-machines.md
       - Target Host: guides/target-host.md
       - Zerotier VPN: guides/mesh-vpn.md
       - Secure Boot: guides/secure-boot.md
       - Flake-parts: guides/flake-parts.md
+      - macOS: guides/macos.md
+      - Contributing:
+          - Contributing: guides/contributing/CONTRIBUTING.md
+          - Debugging: guides/contributing/debugging.md
+          - Testing: guides/contributing/testing.md
+
+      - Writing a Service Module: guides/services/community.md
+      - Writing a Disko Template: guides/disko-templates/community.md
       - Migrations:
           - Migrate existing Flakes: guides/migrations/migration-guide.md
           - Migrate inventory Services: guides/migrations/migrate-inventory-services.md
           - Facts Vars Migration: guides/migrations/migration-facts-vars.md
           - Disk id: guides/migrations/disk-id.md
-      - macOS: guides/macos.md
+  - Concepts:
+      - Inventory: concepts/inventory.md
+      - Generators: concepts/generators.md
+      - Autoincludes: concepts/autoincludes.md
   - Reference:
       - Overview: reference/index.md
+      - Clan Options: options.md
       - Services:
-          - List:
+          - Overview:
-              - Overview: reference/clanServices/index.md
+              - reference/clanServices/index.md
+
               - reference/clanServices/admin.md
               - reference/clanServices/borgbackup.md
               - reference/clanServices/data-mesher.md
@@ -94,66 +104,7 @@ nav:
               - reference/clanServices/wifi.md
               - reference/clanServices/zerotier.md
           - API: reference/clanServices/clan-service-author-interface.md
-          - Writing a Service Module: developer/extensions/clanServices/index.md
-      - Modules:
-          - List:
-              - Overview: reference/clanModules/index.md
-              - reference/clanModules/frontmatter/index.md
-              # TODO: display the docs of the clan.service modules
-              - reference/clanModules/admin.md
-              # This is the module overview and should stay at the top
-              - reference/clanModules/borgbackup-static.md
-              - reference/clanModules/data-mesher.md
-              - reference/clanModules/borgbackup.md
-              - reference/clanModules/deltachat.md
-              - reference/clanModules/disk-id.md
-              - reference/clanModules/dyndns.md
-              - reference/clanModules/ergochat.md
-              - reference/clanModules/garage.md
-              - reference/clanModules/heisenbridge.md
-              - reference/clanModules/importer.md
-              - reference/clanModules/iwd.md
-              - reference/clanModules/localbackup.md
-              - reference/clanModules/localsend.md
-              - reference/clanModules/matrix-synapse.md
-              - reference/clanModules/moonlight.md
-              - reference/clanModules/mumble.md
-              - reference/clanModules/mycelium.md
-              - reference/clanModules/nginx.md
-              - reference/clanModules/packages.md
-              - reference/clanModules/postgresql.md
-              - reference/clanModules/root-password.md
-              - reference/clanModules/single-disk.md
-              - reference/clanModules/sshd.md
-              - reference/clanModules/state-version.md
-              - reference/clanModules/static-hosts.md
-              - reference/clanModules/sunshine.md
-              - reference/clanModules/syncthing-static-peers.md
-              - reference/clanModules/syncthing.md
-              - reference/clanModules/thelounge.md
-              - reference/clanModules/trusted-nix-caches.md
-              - reference/clanModules/user-password.md
-              - reference/clanModules/auto-upgrade.md
-              - reference/clanModules/vaultwarden.md
-              - reference/clanModules/xfce.md
-              - reference/clanModules/zerotier-static-peers.md
-              - reference/clanModules/zerotier.md
-              - reference/clanModules/zt-tcp-relay.md
-          - Writing a Clan Module: developer/extensions/clanModules/index.md
 
-      - Nix API:
-          - inputs.clan-core.lib.clan: reference/nix-api/clan.md
-          - config.clan.core:
-              - Overview: reference/clan.core/index.md
-              - reference/clan.core/backups.md
-              - reference/clan.core/deployment.md
-              - reference/clan.core/facts.md
-              - reference/clan.core/networking.md
-              - reference/clan.core/settings.md
-              - reference/clan.core/sops.md
-              - reference/clan.core/state.md
-              - reference/clan.core/vars.md
-          - Inventory: reference/nix-api/inventory.md
       - CLI:
           - Overview: reference/cli/index.md
 
@@ -170,8 +121,63 @@ nav:
           - reference/cli/templates.md
           - reference/cli/vars.md
           - reference/cli/vms.md
+      - Modules (deprecated):
+          - Overview: reference/clanModules/index.md
+          - reference/clanModules/frontmatter/index.md
+          # TODO: display the docs of the clan.service modules
+          - reference/clanModules/admin.md
+          # This is the module overview and should stay at the top
+          - reference/clanModules/borgbackup-static.md
+          - reference/clanModules/data-mesher.md
+          - reference/clanModules/borgbackup.md
+          - reference/clanModules/deltachat.md
+          - reference/clanModules/disk-id.md
+          - reference/clanModules/dyndns.md
+          - reference/clanModules/ergochat.md
+          - reference/clanModules/garage.md
+          - reference/clanModules/heisenbridge.md
+          - reference/clanModules/importer.md
+          - reference/clanModules/iwd.md
+          - reference/clanModules/localbackup.md
+          - reference/clanModules/localsend.md
+          - reference/clanModules/matrix-synapse.md
+          - reference/clanModules/moonlight.md
+          - reference/clanModules/mumble.md
+          - reference/clanModules/mycelium.md
+          - reference/clanModules/nginx.md
+          - reference/clanModules/packages.md
+          - reference/clanModules/postgresql.md
+          - reference/clanModules/root-password.md
+          - reference/clanModules/single-disk.md
+          - reference/clanModules/sshd.md
+          - reference/clanModules/state-version.md
+          - reference/clanModules/static-hosts.md
+          - reference/clanModules/sunshine.md
+          - reference/clanModules/syncthing-static-peers.md
+          - reference/clanModules/syncthing.md
+          - reference/clanModules/thelounge.md
+          - reference/clanModules/trusted-nix-caches.md
+          - reference/clanModules/user-password.md
+          - reference/clanModules/auto-upgrade.md
+          - reference/clanModules/vaultwarden.md
+          - reference/clanModules/xfce.md
+          - reference/clanModules/zerotier-static-peers.md
+          - reference/clanModules/zerotier.md
+          - reference/clanModules/zt-tcp-relay.md
+
+      - clan.core (NixOS Options):
+          - Overview: reference/clan.core/index.md
+          - reference/clan.core/backups.md
+          - reference/clan.core/deployment.md
+          - reference/clan.core/facts.md
+          - reference/clan.core/networking.md
+          - reference/clan.core/settings.md
+          - reference/clan.core/sops.md
+          - reference/clan.core/state.md
+          - reference/clan.core/vars.md
+
+      - Developer-api: api.md
 
-      - Glossary: reference/glossary.md
       - Decisions:
           - Architecture Decisions: decisions/README.md
           - 01-clanModules: decisions/01-ClanModules.md
@@ -180,16 +186,7 @@ nav:
           - 04-fetching-nix-from-python: decisions/04-fetching-nix-from-python.md
           - 05-deployment-parameters: decisions/05-deployment-parameters.md
           - Template: decisions/_template.md
-  - Options: options.md
+      - Glossary: reference/glossary.md
-  - Developer:
-      - Introduction: developer/index.md
-      - Dev Setup: developer/contributing/CONTRIBUTING.md
-      - Writing a Service Module: developer/extensions/clanServices/index.md
-      - Writing a Clan Module: developer/extensions/clanModules/index.md
-      - Writing a Disko Template: developer/extensions/templates/disk/disko-templates.md
-      - Debugging: developer/contributing/debugging.md
-      - Testing: developer/contributing/testing.md
-      - Python API: developer/api.md
 
 docs_dir: site
 site_dir: out
@@ -249,4 +246,4 @@ plugins:
   - redoc-tag
   - redirects:
       redirect_maps:
-        guides/getting-started/secrets.md: guides/vars-backend.md
+        guides/getting-started/secrets.md: concepts/generators.md

docs/nix/options/flake-module.nix

@@ -114,9 +114,6 @@
         in
         {
           options = {
-            _ = mkOption {
-              type = types.raw;
-            };
             instances.${name} = lib.mkOption {
               inherit description;
               type = types.submodule {
@@ -149,20 +146,29 @@
           };
         };
 
-      mkScope = name: modules: {
+      docModules = [
-        inherit name;
+        {
-        modules = [
+          inherit self;
-          {
+        }
-            _module.args = { inherit clanLib; };
+        self.modules.clan.default
-            _file = "docs mkScope";
+        {
-          }
+          options.inventory = lib.mkOption {
-          { noInstanceOptions = true; }
+            type = types.submoduleWith {
-          ../../../lib/modules/inventoryClass/interface.nix
+              modules = [
-        ] ++ mapAttrsToList fakeInstanceOptions modules;
+                { noInstanceOptions = true; }
-        urlPrefix = "https://github.com/nix-community/dream2nix/blob/main/";
+              ] ++ mapAttrsToList fakeInstanceOptions serviceModules;
-      };
+            };
+          };
+        }
+      ];
+
     in
     {
+      # Uncomment for debugging
+      # legacyPackages.docModules = lib.evalModules {
+      #   modules = docModules;
+      # };
+
       packages = lib.optionalAttrs ((privateInputs ? nuschtos) || (inputs ? nuschtos)) {
         docs-options =
           (privateInputs.nuschtos or inputs.nuschtos)
@@ -171,7 +177,13 @@
               inherit baseHref;
               title = "Clan Options";
               # scopes = mapAttrsToList mkScope serviceModules;
-              scopes = [ (mkScope "Clan Inventory" serviceModules) ];
+              scopes = [
+                {
+                  name = "Clan";
+                  modules = docModules;
+                  urlPrefix = "https://git.clan.lol/clan/clan-core/src/branch/main/";
+                }
+              ];
             };
       };
     };

docs/nix/render_options/__init__.py

@@ -193,7 +193,7 @@ def module_header(module_name: str, has_inventory_feature: bool = False) -> str:
 def module_nix_usage(module_name: str) -> str:
     return f"""## Usage via Nix
 
-**This module can be also imported directly in your nixos configuration. Although it is recommended to use the [inventory](../../reference/nix-api/inventory.md) interface if available.**
+**This module can be also imported directly in your nixos configuration. Although it is recommended to use the [inventory](../../concepts/inventory.md) interface if available.**
 
 Some modules are considered 'low-level' or 'expert modules' and are not available via the inventory interface.
 
@@ -373,7 +373,7 @@ This module can be used via predefined roles
             """
 Every role has its own configuration options, which are each listed below.
 
-For more information, see the [inventory guide](../../guides/inventory.md).
+For more information, see the [inventory guide](../../concepts/inventory.md).
 
 ??? Example
     For example the `admin` module adds the following options globally to all machines where it is used.
@@ -402,7 +402,7 @@ certain option types restricted to enable configuration through a graphical
 interface.
 
 !!! note "🔹"
-    Modules with this indicator support the [inventory](../../guides/inventory.md) feature.
+    Modules with this indicator support the [inventory](../../concepts/inventory.md) feature.
 
 """
 
@@ -679,86 +679,6 @@ def build_option_card(module_name: str, frontmatter: Frontmatter) -> str:
     return f"{to_md_li(module_name, frontmatter)}\n\n"
 
 
-def produce_build_clan_docs() -> None:
-    if not BUILD_CLAN_PATH:
-        msg = f"Environment variables are not set correctly: BUILD_CLAN_PATH={BUILD_CLAN_PATH}. Expected a path to the optionsJSON"
-        raise ClanError(msg)
-
-    if not OUT:
-        msg = f"Environment variables are not set correctly: $out={OUT}"
-        raise ClanError(msg)
-
-    output = """# Clan
-
-This provides an overview of the available arguments of the `clan` interface.
-
-Each attribute is documented below
-
-- **clan-core.lib.clan**: A function that takes an attribute set.
-
-    ??? example "clan Example"
-
-        ```nix
-        clan {
-            self = self;
-            machines = {
-                jon = { };
-                sara = { };
-            };
-        };
-        ```
-
-- **clan with flake-parts**: Import the FlakeModule
-
-    After importing the FlakeModule you can define your `clan` as a flake attribute
-
-    All attribute can be defined via `clan.*`
-
-    Further information see: [flake-parts](../../guides/flake-parts.md) guide.
-
-    ??? example "flake-parts Example"
-
-        ```nix
-        flake-parts.lib.mkFlake { inherit inputs; } ({
-            systems = [];
-            imports = [
-                clan-core.flakeModules.default
-            ];
-            clan = {
-                machines = {
-                    jon = { };
-                    sara = { };
-                };
-            };
-        });
-        ```
-
-"""
-    with Path(BUILD_CLAN_PATH).open() as f:
-        options: dict[str, dict[str, Any]] = json.load(f)
-
-        split = split_options_by_root(options)
-        for option_name, options in split.items():
-            # Skip underscore options
-            if option_name.startswith("_"):
-                continue
-            # Skip inventory sub options
-            # Inventory model has its own chapter
-            if option_name.startswith("inventory."):
-                continue
-
-            print(f"[build_clan_docs] Rendering option of {option_name}...")
-            root = options_to_tree(options)
-
-            for option in root.suboptions:
-                output += options_docs_from_tree(option, init_level=2)
-
-    outfile = Path(OUT) / "nix-api/clan.md"
-    outfile.parent.mkdir(parents=True, exist_ok=True)
-    with Path.open(outfile, "w") as of:
-        of.write(output)
-
-
 def split_options_by_root(options: dict[str, Any]) -> dict[str, dict[str, Any]]:
     """
     Split the flat dictionary of options into a dict of which each entry will construct complete option trees.
@@ -805,7 +725,7 @@ Typically needed by module authors to define roles, behavior and metadata for di
 !!! Note
     This is not a user-facing documentation, but rather meant as a reference for *module authors*
 
-    See: [clanService Authoring Guide](../../developer/extensions/clanServices/index.md)
+    See: [clanService Authoring Guide](../../guides/services/community.md)
 """
     # Inventory options are already included under the clan attribute
     # We just omitted them in the clan docs, because we want a separate output for the inventory model
@@ -834,48 +754,6 @@ class Option:
     suboptions: list["Option"] = field(default_factory=list)
 
 
-def produce_inventory_docs() -> None:
-    if not BUILD_CLAN_PATH:
-        msg = f"Environment variables are not set correctly: BUILD_CLAN_PATH={BUILD_CLAN_PATH}. Expected a path to the optionsJSON"
-        raise ClanError(msg)
-
-    if not OUT:
-        msg = f"Environment variables are not set correctly: $out={OUT}"
-        raise ClanError(msg)
-
-    output = """# Inventory
-This provides an overview of the available attributes of the `inventory` model.
-
-It can be set via the `inventory` attribute of the [`clan`](./clan.md#inventory) function, or via the [`clan.inventory`](./clan.md#inventory) attribute of flake-parts.
-
-"""
-    # Inventory options are already included under the clan attribute
-    # We just omitted them in the clan docs, because we want a separate output for the inventory model
-    with Path(BUILD_CLAN_PATH).open() as f:
-        options: dict[str, dict[str, Any]] = json.load(f)
-
-        clan_root_option = options_to_tree(options)
-        # Find the inventory options
-        inventory_opt: None | Option = None
-        for opt in clan_root_option.suboptions:
-            if opt.name == "inventory":
-                inventory_opt = opt
-                break
-
-        if not inventory_opt:
-            print("No inventory options found.")
-            exit(1)
-        # Render the inventory options
-        # This for loop excludes the root node
-        for option in inventory_opt.suboptions:
-            output += options_docs_from_tree(option, init_level=2)
-
-    outfile = Path(OUT) / "nix-api/inventory.md"
-    outfile.parent.mkdir(parents=True, exist_ok=True)
-    with Path.open(outfile, "w") as of:
-        of.write(output)
-
-
 def option_short_name(option_name: str) -> str:
     parts = option_name.split(".")
     short_name = ""
@@ -984,9 +862,6 @@ def options_docs_from_tree(
 if __name__ == "__main__":  #
     produce_clan_core_docs()
 
-    produce_build_clan_docs()
-    produce_inventory_docs()
-
     produce_clan_service_author_docs()
 
     produce_clan_modules_docs()

docs/site/concepts/autoincludes.md

@@ -0,0 +1,15 @@
+
+Clan automatically imports the following files from a directory and registers them.
+
+## Machine registration
+
+Every folder `machines/{machineName}` will be registered automatically as a Clan machine.
+
+!!! info "Automatically loaded files"
+
+    The following files are loaded automatically for each Clan machine:
+
+    - [x] `machines/{machineName}/configuration.nix`
+    - [x] `machines/{machineName}/hardware-configuration.nix`
+    - [x] `machines/{machineName}/facter.json` Automatically configured, for further information see [nixos-facter](https://clan.lol/blog/nixos-facter/)
+    - [x] `machines/{machineName}/disko.nix` Automatically loaded, for further information see the [disko docs](https://github.com/nix-community/disko/blob/master/docs/quickstart.md).

docs/site/concepts/generators.md

@@ -1,7 +1,4 @@
-
+# Generators
-!!! Note
-    Vars is the new secret backend that will soon replace the Facts backend
-
 
 Defining a linux user's password via the nixos configuration previously required running `mkpasswd ...` and then copying the hash back into the nix configuration.
 
@@ -11,7 +8,7 @@ For a more general explanation of what clan vars are and how it works, see the i
 
 This guide assumes
 - Clan is set up already (see [Getting Started](../guides/getting-started/index.md))
-- a machine has been added to the clan (see [Adding Machines](./more-machines.md))
+- a machine has been added to the clan (see [Adding Machines](../guides/getting-started/add-machines.md))
 
 This section will walk you through the following steps:
 
@@ -23,7 +20,7 @@ This section will walk you through the following steps:
 6. share the root password between machines
 7. change the password
 
-## Declare the generator
+## Declare a generator
 
 In this example, a `vars` `generator` is used to:
 

docs/site/concepts/inventory.md

@@ -9,8 +9,6 @@ The inventory logic will automatically derive the modules and configurations to
 
 The following tutorial will walk through setting up a Backup service where the terms `Service` and `Role` will become more clear.
 
-See also: [Inventory API Documentation](../reference/nix-api/inventory.md)
-
 !!! example "Experimental status"
     The inventory implementation is not considered stable yet.
     We are actively soliciting feedback from users.
@@ -19,7 +17,7 @@ See also: [Inventory API Documentation](../reference/nix-api/inventory.md)
 
 ## Prerequisites
 
-- [x] [Add multiple machines](./more-machines.md) to your Clan.
+- [x] [Add some machines](../guides/getting-started/add-machines.md) to your Clan.
 
 ## Services
 

docs/site/developer/extensions/clanModules/index.md

@@ -1,229 +0,0 @@
-# Authoring a clanModule
-
-!!! Danger "Will get deprecated soon"
-    Please consider twice creating new modules in this format
-
-    [`clan.service` module](../clanServices/index.md) will be the new standard soon.
-
-This site will guide you through authoring your first module. Explaining which conventions must be followed, such that others will have an enjoyable experience and the module can be used with minimal effort.
-
-
-!!! Tip
-    External ClanModules can be ad-hoc loaded via [`clan.inventory.modules`](../../../reference/nix-api/inventory.md#inventory.modules)
-
-## Bootstrapping the `clanModule`
-
-A ClanModule is a specific subset of a [NixOS Module](https://nix.dev/tutorials/module-system/index.html), but it has some constraints and might be used via the [Inventory](../../../guides/inventory.md) interface.
-In fact a `ClanModule` can be thought of as a layer of abstraction on-top of NixOS and/or other ClanModules. It may configure sane defaults and provide an ergonomic interface that is easy to use and can also be used via a UI that is under development currently.
-
-Because ClanModules should be configurable via `json`/`API` all of its interface (`options`) must be serializable.
-
-!!! Tip
-    ClanModules interface can be checked by running the json schema converter as follows.
-
-    `nix build .#legacyPackages.x86_64-linux.schemas.inventory`
-
-    If the build succeeds the module is compatible.
-
-## Directory structure
-
-Each module SHOULD be a directory of the following format:
-
-```sh
-# Example: borgbackup
-clanModules/borgbackup
-├── README.md
-└── roles
-    ├── client.nix
-    └── server.nix
-```
-
-!!! Tip
-    `README.md` is always required. See section [Readme](#readme) for further details.
-
-    The `roles` folder is strictly required for `features = [ "inventory" ]`.
-
-## Registering the module
-
-=== "User module"
-
-    If the module should be ad-hoc loaded.
-    It can be made available in any project via the [`clan.inventory.modules`](../../../reference/nix-api/inventory.md#inventory.modules) attribute.
-
-    ```nix title="flake.nix"
-    # ...
-    # Sometimes this attribute set is defined in clan.nix
-    clan-core.lib.clan {
-        # 1. Add the module to the available clanModules with inventory support
-        inventory.modules = {
-            custom-module = ./modules/my_module;
-        };
-        # 2. Use the module in the inventory
-        inventory.services = {
-            custom-module.instance_1 = {
-                roles.default.machines = [ "machineA" ];
-            };
-        };
-    };
-    ```
-
-=== "Upstream module"
-
-    If the module will be contributed to [`clan-core`](https://git.clan.lol/clan-core)
-    The clanModule must be registered within the `clanModules` attribute in `clan-core`
-
-    ```nix title="clanModules/flake-module.nix"
-    --8<-- "clanModules/flake-module.nix:0:5"
-        # Register our new module here
-        # ...
-    ```
-
-## Readme
-
-The `README.md` is a required file for all modules. It MUST contain frontmatter in [`toml`](https://toml.io) format.
-
-```markdown
----
-description = "Module A"
----
-
-This is the example module that does xyz.
-```
-
-See the [Full Frontmatter reference](../../../reference/clanModules/frontmatter/index.md) further details and all supported attributes.
-
-## Roles
-
-If the module declares to implement `features = [ "inventory" ]` then it MUST contain a roles directory.
-
-Each `.nix` file in the `roles` directory is added as a role to the inventory service.
-
-Other files can also be placed alongside the `.nix` files
-
-```sh
-└── roles
-    ├── client.nix
-    └── server.nix
-```
-
-Adds the roles: `client` and `server`
-
-??? Tip "Good to know"
-    Sometimes a `ClanModule` should be usable via both clan's `inventory` concept but also natively as a NixOS module.
-
-    > In the long term, we want most modules to implement support for the inventory,
-    > but we are also aware that there are certain low-level modules that always serve as a backend for other higher-level `clanModules` with inventory support.
-    > These modules may not want to implement inventory interfaces as they are always used directly by other modules.
-
-    This can be achieved by placing an additional `default.nix` into the root of the ClanModules directory as shown:
-
-    ```sh
-    # ModuleA
-    ├── README.md
-    ├── default.nix
-    └── roles
-        └── default.nix
-    ```
-
-    ```nix title="default.nix"
-    {...}:{
-        imports = [ ./roles/default.nix ];
-    }
-    ```
-
-    By utilizing this pattern the module (`moduleA`) can then be imported into any regular NixOS module via:
-
-    ```nix
-    {...}:{
-        imports  = [ clanModules.moduleA ];
-    }
-    ```
-
-## Adding configuration options
-
-While we recommend to keep the interface as minimal as possible and deriving all required information from the `roles` model it might sometimes be required or convenient to expose customization options beyond `roles`.
-
-The following shows how to add options to your module.
-
-**It is important to understand that every module has its own namespace where it should declare options**
-
-**`clan.{moduleName}`**
-
-???+ Example
-    The following example shows how to register options in the module interface
-
-    and how it can be set via the inventory
-
-
-    ```nix title="/default.nix"
-    custom-module = ./modules/custom-module;
-    ```
-
-    Since the module is called `custom-module` all of its exposed options should be added to `options.clan.custom-module.*...*`
-
-    ```nix title="custom-module/roles/default.nix"
-    {
-        options = {
-            clan.custom-module.foo = mkOption {
-                type = types.str;
-                default = "bar";
-            };
-        };
-    }
-    ```
-
-    If the module is [registered](#registering-the-module).
-    Configuration can be set as follows.
-
-    ```nix title="flake.nix"
-    # Sometimes this attribute set is defined in clan.nix
-    clan-core.lib.clan {
-        inventory.services = {
-            custom-module.instance_1 = {
-                roles.default.machines = [ "machineA" ];
-                roles.default.config = {
-                    # All configuration here is scoped to `clan.custom-module`
-                    foo = "foobar";
-                };
-            };
-        };
-    }
-    ```
-
-## Organizing the ClanModule
-
-Each `{role}.nix` is included into the machine if the machine is declared to have the role.
-
-For example
-
-```nix
-roles.client.machines = ["MachineA"];
-```
-
-Then `roles/client.nix` will be added to the machine `MachineA`.
-
-This behavior makes it possible to split the interface and common code paths when using multiple roles.
-In the concrete example of `borgbackup` this allows a `server` to declare a different interface than the corresponding `client`.
-
-The client offers configuration option, to exclude certain local directories from being backed up:
-
-```nix title="roles/client.nix"
-# Example client interface
-  options.clan.borgbackup.exclude = ...
-```
-
-The server doesn't offer any configuration option. Because everything is set-up automatically.
-
-```nix title="roles/server.nix"
-# Example server interface
-  options.clan.borgbackup = {};
-```
-
-Assuming that there is a common code path or a common interface between `server` and `client` this can be structured as:
-
-```nix title="roles/server.nix, roles/client.nix"
-{...}: {
-    # ...
-    imports = [ ../common.nix ];
-}
-```

docs/site/developer/index.md

@@ -1,25 +0,0 @@
-# Developer Documentation
-
-!!! Danger
-    This documentation is **not** intended for external users. It may contain low-level details and internal-only interfaces.*
-
-Welcome to the internal developer documentation.
-
-This section is intended for contributors, engineers, and internal stakeholders working directly with our system, tooling, and APIs. It provides a technical overview of core components, internal APIs, conventions, and patterns that support the platform.
-
-Our goal is to make the internal workings of the system **transparent, discoverable, and consistent** — helping you contribute confidently, troubleshoot effectively, and build faster.
-
-## What's Here?
-
-!!! note "docs migration ongoing"
-
-- [ ] **API Reference**: 🚧🚧🚧 Detailed documentation of internal API functions, inputs, and expected outputs. 🚧🚧🚧
-- [ ] **System Concepts**: Architectural overviews and domain-specific guides.
-- [ ] **Development Guides**: How to test, extend, or integrate with key components.
-- [ ] **Design Notes**: Rationales behind major design decisions or patterns.
-
-## Who is This For?
-
-* Developers contributing to the platform
-* Engineers debugging or extending internal systems
-* Anyone needing to understand **how** and **why** things work under the hood

docs/site/guides/clanServices.md

@@ -138,7 +138,7 @@ You can use services exposed by Clan’s core module library, `clan-core`.
 
 You can also author your own `clanService` modules.
 
-🔗 Learn how to write your own service: [Authoring a clanService](../developer/extensions/clanServices/index.md)
+🔗 Learn how to write your own service: [Authoring a service](../guides/services/community.md)
 
 You might expose your service module from your flake — this makes it easy for other people to also use your module in their clan.
 
@@ -154,6 +154,6 @@ You might expose your service module from your flake — this makes it easy for
 
 ## What’s Next?
 
-* [Author your own clanService →](../developer/extensions/clanServices/index.md)
+* [Author your own clanService →](../guides/services/community.md)
 * [Migrate from clanModules →](../guides/migrations/migrate-inventory-services.md)
 <!-- TODO: * [Understand the architecture →](../explanation/clan-architecture.md) -->

docs/site/guides/flake-parts.md

@@ -27,7 +27,7 @@ inputs = {
 
 ## Import the Clan flake-parts Module
 
-After updating your flake inputs, the next step is to import the Clan flake-parts module. This will make the [Clan options](../reference/nix-api/clan.md) available within `mkFlake`.
+After updating your flake inputs, the next step is to import the Clan flake-parts module. This will make the [Clan options](../options.md) available within `mkFlake`.
 
 ```nix
 {

docs/site/guides/getting-started/add-machines.md

@@ -6,7 +6,7 @@ Machines can be added using the following methods
 - Editing machines/`machine_name`/configuration.nix (automatically included if it exists)
 - `clan machines create` (imperative)
 
-See the complete [list](../../guides/more-machines.md#automatic-registration) of auto-loaded files.
+See the complete [list](../../concepts/autoincludes.md) of auto-loaded files.
 
 ## Create a machine
 

docs/site/guides/getting-started/add-services.md

@@ -41,7 +41,7 @@ To learn more: [Guide about clanService](../clanServices.md)
 ```
 
 1. See [reference/clanServices](../../reference/clanServices/index.md) for all available services and how to configure them.
-   Or read [authoring/clanServices](../../developer/extensions/clanServices/index.md) if you want to bring your own
+   Or read [authoring/clanServices](../../guides/services/community.md) if you want to bring your own
 
 2. Replace `__YOUR_CONTROLLER_` with the *name* of your machine.
 

docs/site/guides/macos.md

@@ -7,7 +7,7 @@ This guide explains how to manage macOS machines using Clan.
 Currently, Clan supports the following features for macOS:
 
 - `clan machines update` for existing [nix-darwin](https://github.com/nix-darwin/nix-darwin) installations
-- Support for [vars](../guides/vars-backend.md)
+- Support for [vars](../concepts/generators.md)
 
 ## Add Your Machine to Your Clan Flake
 

docs/site/guides/migrations/migrate-inventory-services.md

@@ -1,7 +1,7 @@
 # Migrating from using `clanModules` to `clanServices`
 
 **Audience**: This is a guide for **people using `clanModules`**.
-If you are a **module author** and need to migrate your modules please consult our **new** [clanServices authoring guide](../../developer/extensions/clanServices/index.md)
+If you are a **module author** and need to migrate your modules please consult our **new** [clanServices authoring guide](../../guides/services/community.md)
 
 ## What's Changing?
 
@@ -329,6 +329,6 @@ instances = {
 
 ## Further reference
 
-* [Authoring a 'clan.service' module](../../developer/extensions/clanServices/index.md)
+* [Inventory Concept](../../concepts/inventory.md)
+* [Authoring a 'clan.service' module](../../guides/services/community.md)
 * [ClanServices](../clanServices.md)
-* [Inventory Reference](../../reference/nix-api/inventory.md)

docs/site/guides/migrations/migration-facts-vars.md

@@ -3,7 +3,7 @@
 For a high level overview about `vars` see our [blog post](https://clan.lol/blog/vars/).
 
 This guide will help you migrate your modules that still use our [`facts`](../../guides/secrets.md) backend
-to the [`vars`](../../guides/vars-backend.md) backend.
+to the [`vars`](../../concepts/generators.md) backend.
 
 The `vars` [module](../../reference/clan.core/vars.md) and the clan [command](../../reference/cli/vars.md) work in tandem, they should ideally be kept in sync.
 

docs/site/guides/more-machines.md

@@ -1,50 +0,0 @@
-
-Clan has two general methods of adding machines:
-
-- **Automatic**: Detects every folder in the `machines` folder.
-- **Declarative**: Explicit declarations in Nix.
-
-## Automatic registration
-
-Every folder `machines/{machineName}` will be registered automatically as a Clan machine.
-
-!!! info "Automatically loaded files"
-
-    The following files are loaded automatically for each Clan machine:
-
-    - [x] `machines/{machineName}/configuration.nix`
-    - [x] `machines/{machineName}/hardware-configuration.nix`
-    - [x] `machines/{machineName}/facter.json` Automatically configured, for further information see [nixos-facter](https://clan.lol/blog/nixos-facter/)
-    - [x] `machines/{machineName}/disko.nix` Automatically loaded, for further information see the [disko docs](https://github.com/nix-community/disko/blob/master/docs/quickstart.md).
-
-## Manual declaration
-
-Machines can be added via [`clan.inventory.machines`](../guides/inventory.md) or in `clan.machines`, which allows for defining NixOS options.
-
-=== "**Individual Machine Configuration**"
-
-    ```{.nix}
-    clan-core.lib.clan {
-        machines = {
-            "jon" = {
-                # Any valid nixos config
-            };
-        };
-    }
-    ```
-
-=== "**Inventory Configuration**"
-
-    ```{.nix}
-    clan-core.lib.clan {
-        inventory = {
-            machines = {
-                "jon" = {
-                    # Inventory can set tags and other metadata
-                    tags = [ "zone1" ];
-                    deploy.targetHost = "root@jon";
-                };
-            };
-        };
-    }
-    ```

docs/site/guides/secrets.md

@@ -1,5 +1,5 @@
-This article provides an overview over the underlying secrets system which is used by [Vars](../guides/vars-backend.md).
+This article provides an overview over the underlying secrets system which is used by [Vars](../concepts/generators.md).
-Under most circumstances you should use [Vars](../guides/vars-backend.md) directly instead.
+Under most circumstances you should use [Vars](../concepts/generators.md) directly instead.
 
 Consider using `clan secrets` only for managing admin users and groups, as well as a debugging tool.
 

docs/site/guides/services/community.md

@@ -1,16 +1,16 @@
 # Authoring a 'clan.service' module
 
 !!! Tip
-    This is the successor format to the older [clanModules](../clanModules/index.md)
+    This is the successor format to the older [clanModules](../../reference/clanModules/index.md)
 
     While some features might still be missing we recommend to adapt this format early and give feedback.
 
 ## Service Module Specification
 
 This section explains how to author a clan service module.
-We discussed the initial architecture in [01-clan-service-modules](../../../decisions/01-ClanModules.md) and decided to rework the format.
+We discussed the initial architecture in [01-clan-service-modules](../../decisions/01-ClanModules.md) and decided to rework the format.
 
-For the full specification and current state see: **[Service Author Reference](../../../reference/clanServices/clan-service-author-interface.md)**
+For the full specification and current state see: **[Service Author Reference](../../reference/clanServices/clan-service-author-interface.md)**
 
 ### A Minimal module
 
@@ -52,7 +52,7 @@ The imported module file must fulfill at least the following requirements:
 }
 ```
 
-For more attributes see: **[Service Author Reference](../../../reference/clanServices/clan-service-author-interface.md)**
+For more attributes see: **[Service Author Reference](../../reference/clanServices/clan-service-author-interface.md)**
 
 ### Adding functionality to the module
 
@@ -266,6 +266,6 @@ The benefit of this approach is that downstream users can override the value of
 
 ## Further
 
-- [Reference Documentation for Service Authors](../../../reference/clanServices/clan-service-author-interface.md)
+- [Reference Documentation for Service Authors](../../reference/clanServices/clan-service-author-interface.md)
-- [Migration Guide from ClanModules to ClanServices](../../../guides/migrations/migrate-inventory-services.md)
+- [Migration Guide from ClanModules to ClanServices](../../guides/migrations/migrate-inventory-services.md)
-- [Decision that lead to ClanServices](../../../decisions/01-ClanModules.md)
+- [Decision that lead to ClanServices](../../decisions/01-ClanModules.md)

docs/site/index.md

@@ -28,47 +28,49 @@ services](./guides/clanServices.md) tailored to your specific needs.
 
 <div class="grid cards" markdown>
 
--   [Adding more machines](./guides/more-machines.md)
+-   [Create a Machine](./guides/getting-started/add-machines.md)
 
     ---
 
-    Learn how Clan automatically includes machines and Nix files.
+    How to create your first machine
 
--   [Vars Backend](./guides/vars-backend.md)
+-   [macOS](./guides/macos.md)
 
     ---
 
-    Learn how to manage secrets with vars.
+    How to manage macOS machines with nix-darwin
 
--   [Inventory](./guides/inventory.md)
+-   [Contribute](./guides/contributing/CONTRIBUTING.md)
 
     ---
 
-    Clan's declaration format for running **services** on one or multiple **machines**.
+    How to set up a development environment
 
--   [Flake-parts](./guides/flake-parts.md)
+</div>
+
+## Concepts
+
+Explore the foundational ideas.
+
+<div class="grid cards" markdown>
+
+-   [Generators](./concepts/generators.md)
 
     ---
 
-    Use Clan with [https://flake.parts/]()
+    Learn about Generators
 
--   [Contribute](./developer/contributing/CONTRIBUTING.md)
+-   [Inventory](./concepts/inventory.md)
 
     ---
 
-    Discover how to set up a development environment to contribute to Clan!
+    Learn about Inventory
-
--   [macOS machines](./guides/macos.md)
-
-    ---
-
-    Manage macOS machines with nix-darwin
 
 </div>
 
 ## API Reference
 
-**Reference API Documentation**
+Technical reference for Clan's CLI and Nix modules
 
 <div class="grid cards" markdown>
 
@@ -76,29 +78,22 @@ services](./guides/clanServices.md) tailored to your specific needs.
 
     ---
 
-    The `clan` CLI command
+    Command-line interface.
+
+    Full reference for the `clan` CLI tool.
 
 -   [Service Modules](./reference/clanServices/index.md)
 
     ---
 
-    An overview of available service modules
+    Overview of built-in service modules that provide composable functionality across machines.
 
--   [Core](./reference/clan.core/index.md)
+-   [Core NixOS-module](./reference/clan.core/index.md)
 
     ---
 
-    The clan core nix module.
+    The foundation of Clan's functionality
-    This is imported when using clan and is the basis of the extra functionality
-    that can be provided.
-
--   [(Legacy) Modules](./reference/clanModules/index.md)
-
-    ---
-
-    An overview of available clanModules
-
-    !!! Example "These will be deprecated soon"
 
+    Reference for the `clan-core` NixOS module — automatically part of any machine to enable Clan's core features.
 
 </div>

docs/site/reference/index.md

@@ -4,10 +4,10 @@ This section of the site provides an overview of available options and commands
 
 ---
 
+- [Clan Configuration Option](../options.md) - for defining a Clan
 - Learn how to use the [Clan CLI](./cli/index.md)
 - Explore available services and application [modules](./clanModules/index.md)
-- Discover [configuration options](./clan.core/index.md) that manage essential features
+- [NixOS Configuration Options](./clan.core/index.md) - Additional options avilable on a NixOS machine.
-- Find descriptions of the [Nix interfaces](./nix-api/clan.md) for defining a Clan
 
 ---
 

lib/modules/clan/interface.nix

@@ -229,8 +229,8 @@ in
     };
 
     inventory = lib.mkOption {
-      type = types.submodule {
+      type = types.submoduleWith {
-        imports = [
+        modules = [
           {
             _module.args = { inherit clanLib; };
             _file = "clan interface";

lib/modules/inventoryClass/interface.nix

@@ -142,7 +142,7 @@ in
             - The module MUST have at least `features = [ "inventory" ]` in the frontmatter section.
             - The module MUST have a subfolder `roles` with at least one `{roleName}.nix` file.
 
-            For further information see: [Module Authoring Guide](../../developer/extensions/clanServices/index.md).
+            For further information see: [Module Authoring Guide](../../guides/services/community.md).
 
         ???+ example
             ```nix
@@ -179,8 +179,7 @@ in
               map (m: "'${m}'") (lib.attrNames (lib.filterAttrs (n: _v: !builtins.elem n allowedNames) moduleSet))
             )}
 
-            See: https://docs.clan.lol/developer/extensions/clanServices/
+            See: https://docs.clan.lol/guides/services/community/
-            And: https://docs.clan.lol/developer/extensions/clanServices/
           '' moduleSet;
     };
 

nixosModules/clanCore/outputs.nix

@@ -31,6 +31,7 @@
             The deployment data is now accessed directly from the configuration
             instead of being written to a separate JSON file.
           '';
+          defaultText = "error: deployment.json file generation has been removed in favor of direct selectors.";
         };
         deployment.buildHost = lib.mkOption {
           type = lib.types.nullOr lib.types.str;
@@ -54,10 +55,10 @@
         deployment.nixosMobileWorkaround = lib.mkOption {
           type = lib.types.bool;
           description = ''
-            if true, the deployment will first do a nixos-rebuild switch 
+            if true, the deployment will first do a nixos-rebuild switch
             to register the boot profile the command will fail applying it to the running system
-            which is why afterwards we execute a nixos-rebuild test to apply 
+            which is why afterwards we execute a nixos-rebuild test to apply
-            the new config without having to reboot. 
+            the new config without having to reboot.
             This is a nixos-mobile deployment bug and will be removed in the future
           '';
           default = false;

pkgs/clan-app/clan_app/api/api_bridge.py

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any
 
 from clan_lib.api import ApiResponse
 from clan_lib.api.tasks import WebThread
-from clan_lib.async_run import set_should_cancel
+from clan_lib.async_run import set_current_thread_opkey, set_should_cancel
 
 if TYPE_CHECKING:
     from .middleware import Middleware
@@ -98,7 +98,7 @@ class ApiBridge(ABC):
         *,
         thread_name: str = "ApiBridgeThread",
         wait_for_completion: bool = False,
-        timeout: float = 60.0,
+        timeout: float = 60.0 * 60,  # 1 hour default timeout
     ) -> None:
         """Process an API request in a separate thread with cancellation support.
 
@@ -112,6 +112,7 @@ class ApiBridge(ABC):
 
         def thread_task(stop_event: threading.Event) -> None:
             set_should_cancel(lambda: stop_event.is_set())
+            set_current_thread_opkey(op_key)
             try:
                 log.debug(
                     f"Processing {request.method_name} with args {request.args} "

pkgs/clan-app/clan_app/api/file_gtk.py

@@ -9,6 +9,7 @@ gi.require_version("Gtk", "4.0")
 
 from clan_lib.api import ApiError, ErrorDataClass, SuccessDataClass
 from clan_lib.api.directory import FileRequest
+from clan_lib.async_run import get_current_thread_opkey
 from clan_lib.clan.check import check_clan_valid
 from clan_lib.flake import Flake
 from gi.repository import Gio, GLib, Gtk
@@ -24,7 +25,7 @@ def remove_none(_list: list) -> list:
 RESULT: dict[str, SuccessDataClass[list[str] | None] | ErrorDataClass] = {}
 
 
-def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass:
+def get_clan_folder() -> SuccessDataClass[Flake] | ErrorDataClass:
     """
     Opens the clan folder using the GTK file dialog.
     Returns the path to the clan folder or an error if it fails.
@@ -34,7 +35,10 @@ def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass:
         title="Select Clan Folder",
         initial_folder=str(Path.home()),
     )
-    response = get_system_file(file_request, op_key=op_key)
+
+    response = get_system_file(file_request)
+
+    op_key = response.op_key
 
     if isinstance(response, ErrorDataClass):
         return response
@@ -70,8 +74,13 @@ def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass:
 
 
 def get_system_file(
-    file_request: FileRequest, *, op_key: str
+    file_request: FileRequest,
 ) -> SuccessDataClass[list[str] | None] | ErrorDataClass:
+    op_key = get_current_thread_opkey()
+
+    if not op_key:
+        msg = "No operation key found in the current thread context."
+        raise RuntimeError(msg)
     GLib.idle_add(gtk_open_file, file_request, op_key)
 
     while RESULT.get(op_key) is None:

pkgs/clan-app/clan_app/api/middleware/argument_parsing.py

@@ -21,18 +21,12 @@ class ArgumentParsingMiddleware(Middleware):
             # Convert dictionary arguments to dataclass instances
             reconciled_arguments = {}
             for k, v in context.request.args.items():
-                if k == "op_key":
-                    continue
-
                 # Get the expected argument type from the API
                 arg_class = self.api.get_method_argtype(context.request.method_name, k)
 
                 # Convert dictionary to dataclass instance
                 reconciled_arguments[k] = from_dict(arg_class, v)
 
-            # Add op_key to arguments
-            reconciled_arguments["op_key"] = context.request.op_key
-
             # Create a new request with reconciled arguments
 
             updated_request = BackendRequest(

pkgs/clan-app/clan_app/deps/http/http_bridge.py

@@ -1,13 +1,22 @@
 import json
 import logging
+import threading
 import uuid
 from http.server import BaseHTTPRequestHandler
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 from urllib.parse import urlparse
 
-from clan_lib.api import MethodRegistry, SuccessDataClass, dataclass_to_dict
+from clan_lib.api import (
+    MethodRegistry,
+    SuccessDataClass,
+    dataclass_to_dict,
+)
 from clan_lib.api.tasks import WebThread
+from clan_lib.async_run import (
+    set_current_thread_opkey,
+    set_should_cancel,
+)
 
 from clan_app.api.api_bridge import ApiBridge, BackendRequest, BackendResponse
 
@@ -324,17 +333,34 @@ class HttpBridge(ApiBridge, BaseHTTPRequestHandler):
             msg = f"Operation key '{op_key}' is already in use. Please try again."
             raise ValueError(msg)
 
+    def process_request_in_thread(
+        self,
+        request: BackendRequest,
+        *,
+        thread_name: str = "ApiBridgeThread",
+        wait_for_completion: bool = False,
+        timeout: float = 60.0 * 60,  # 1 hour default timeout
+    ) -> None:
+        pass
+
     def _process_api_request_in_thread(
         self, api_request: BackendRequest, method_name: str
     ) -> None:
         """Process the API request in a separate thread."""
-        # Use the inherited thread processing method
+        stop_event = threading.Event()
-        self.process_request_in_thread(
+        request = api_request
-            api_request,
+        op_key = request.op_key or "unknown"
-            thread_name="HttpThread",
+        set_should_cancel(lambda: stop_event.is_set())
-            wait_for_completion=True,
+        set_current_thread_opkey(op_key)
-            timeout=60.0,
+
+        curr_thread = threading.current_thread()
+        self.threads[op_key] = WebThread(thread=curr_thread, stop_event=stop_event)
+
+        log.debug(
+            f"Processing {request.method_name} with args {request.args} "
+            f"and header {request.header}"
         )
+        self.process_request(request)
 
     def log_message(self, format: str, *args: Any) -> None:  # noqa: A002
         """Override default logging to use our logger."""

pkgs/clan-app/ui/src/components/Sidebar/Sidebar.css

@@ -1,5 +1,5 @@
 div.sidebar {
-  @apply w-60 border-none;
+  @apply w-60 border-none z-10;
 
   & > div.header {
   }

pkgs/clan-app/ui/src/components/Sidebar/SidebarBody.tsx

@@ -3,7 +3,7 @@ import { A } from "@solidjs/router";
 import { Accordion } from "@kobalte/core/accordion";
 import Icon from "../Icon/Icon";
 import { Typography } from "@/src/components/Typography/Typography";
-import { For, Suspense } from "solid-js";
+import { For } from "solid-js";
 import { MachineStatus } from "@/src/components/MachineStatus/MachineStatus";
 import { buildMachinePath, useClanURI } from "@/src/hooks/clan";
 import { useMachinesQuery } from "@/src/queries/queries";
@@ -89,21 +89,19 @@ export const SidebarBody = (props: SidebarProps) => {
             </Accordion.Trigger>
           </Accordion.Header>
           <Accordion.Content class="content">
-            <Suspense fallback={"Loading..."}>
+            <nav>
-              <nav>
+              <For each={Object.entries(machineList.data || {})}>
-                <For each={Object.entries(machineList.data || {})}>
+                {([id, machine]) => (
-                  {([id, machine]) => (
+                  <MachineRoute
-                    <MachineRoute
+                    clanURI={clanURI}
-                      clanURI={clanURI}
+                    machineID={id}
-                      machineID={id}
+                    name={machine.name || id}
-                      name={machine.name || id}
+                    status="Not Installed"
-                      status="Not Installed"
+                    serviceCount={0}
-                      serviceCount={0}
+                  />
-                    />
+                )}
-                  )}
+              </For>
-                </For>
+            </nav>
-              </nav>
-            </Suspense>
           </Accordion.Content>
         </Accordion.Item>
 

pkgs/clan-app/ui/src/components/Sidebar/SidebarHeader.css

@@ -39,7 +39,7 @@ div.sidebar-header {
 }
 
 .sidebar-dropdown-content {
-  @apply flex flex-col w-full px-2 py-1.5;
+  @apply flex flex-col w-full px-2 py-1.5 z-10;
   @apply bg-def-1 rounded-bl-md rounded-br-md;
   @apply border border-def-2;
 

pkgs/clan-app/ui/src/components/Sidebar/SidebarHeader.tsx

@@ -4,7 +4,7 @@ import { DropdownMenu } from "@kobalte/core/dropdown-menu";
 import { useNavigate } from "@solidjs/router";
 import { Typography } from "../Typography/Typography";
 import { createSignal, For, Suspense } from "solid-js";
-import { useAllClanDetailsQuery } from "@/src/queries/queries";
+import { useClanListQuery } from "@/src/queries/queries";
 import { navigateToClan, useClanURI } from "@/src/hooks/clan";
 import { clanURIs } from "@/src/stores/clan";
 
@@ -15,7 +15,7 @@ export const SidebarHeader = () => {
 
   // get information about the current active clan
   const clanURI = useClanURI();
-  const allClans = useAllClanDetailsQuery(clanURIs());
+  const allClans = useClanListQuery(clanURIs());
 
   const activeClan = () => allClans.find(({ data }) => data?.uri === clanURI);
 

pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.css

@@ -1,5 +1,16 @@
 div.sidebar-pane {
-  @apply w-full max-w-60 border-none;
+  @apply border-none z-10;
+
+  animation: sidebarPaneShow 250ms ease-in forwards;
+
+  &.closing {
+    animation: sidebarPaneHide 250ms ease-out 300ms forwards;
+
+    & > div.header > *,
+    & > div.body > * {
+      animation: sidebarFadeOut 250ms ease-out forwards;
+    }
+  }
 
   & > div.header {
     @apply flex items-center justify-between px-3 py-2 rounded-t-[0.5rem];
@@ -7,11 +18,17 @@ div.sidebar-pane {
         border-r-[1px] border-r-bg-inv-3
         border-b-2 border-b-bg-inv-4
         border-l-[1px] border-l-bg-inv-3;
+
     background: linear-gradient(
       90deg,
       theme(colors.bg.inv.3) 0%,
       theme(colors.bg.inv.4) 100%
     );
+
+    & > * {
+      @apply opacity-0;
+      animation: sidebarFadeIn 250ms ease-in 250ms forwards;
+    }
   }
 
   & > div.body {
@@ -29,5 +46,54 @@ div.sidebar-pane {
         theme(colors.bg.inv.2) 0%,
         theme(colors.bg.inv.3) 100%
       );
+
+    & > * {
+      @apply opacity-0;
+      animation: sidebarFadeIn 250ms ease-in 350ms forwards;
+    }
+  }
+}
+
+@keyframes sidebarPaneShow {
+  0% {
+    @apply w-0;
+    @apply opacity-0;
+  }
+  10% {
+    @apply w-8;
+  }
+  30% {
+    @apply opacity-100;
+  }
+  100% {
+    @apply w-60;
+  }
+}
+
+@keyframes sidebarPaneHide {
+  90% {
+    @apply w-8;
+  }
+  100% {
+    @apply w-0;
+    @apply opacity-0;
+  }
+}
+
+@keyframes sidebarFadeIn {
+  from {
+    opacity: 0;
+  }
+  to {
+    opacity: 1;
+  }
+}
+
+@keyframes sidebarFadeOut {
+  from {
+    opacity: 1;
+  }
+  to {
+    opacity: 0;
   }
 }

pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.tsx

@@ -1,8 +1,9 @@
-import { JSX } from "solid-js";
+import { createSignal, JSX } from "solid-js";
 import "./SidebarPane.css";
 import { Typography } from "@/src/components/Typography/Typography";
 import Icon from "../Icon/Icon";
 import { Button as KButton } from "@kobalte/core/button";
+import cx from "classnames";
 
 export interface SidebarPaneProps {
   title: string;
@@ -11,13 +12,20 @@ export interface SidebarPaneProps {
 }
 
 export const SidebarPane = (props: SidebarPaneProps) => {
+  const [closing, setClosing] = createSignal(false);
+
+  const onClose = () => {
+    setClosing(true);
+    setTimeout(() => props.onClose(), 550);
+  };
+
   return (
-    <div class="sidebar-pane">
+    <div class={cx("sidebar-pane", { closing: closing() })}>
       <div class="header">
         <Typography hierarchy="body" size="s" weight="bold" inverted={true}>
           {props.title}
         </Typography>
-        <KButton onClick={props.onClose}>
+        <KButton onClick={onClose}>
           <Icon icon="Close" color="primary" size="0.75rem" inverted={true} />
         </KButton>
       </div>

pkgs/clan-app/ui/src/hooks/api.ts

@@ -20,6 +20,7 @@ export type SuccessData<T extends OperationNames> = SuccessQuery<T>["data"];
 
 interface SendHeaderType {
   logging?: { group_path: string[] };
+  op_key?: string;
 }
 interface BackendSendType<K extends OperationNames> {
   body: OperationArgs<K>;
@@ -64,9 +65,14 @@ export const callApi = <K extends OperationNames>(
     };
   }
 
+  const op_key = backendOpts?.op_key ?? crypto.randomUUID();
+
   const req: BackendSendType<OperationNames> = {
     body: args,
-    header: backendOpts,
+    header: {
+      ...backendOpts,
+      op_key,
+    },
   };
 
   const result = (
@@ -78,9 +84,6 @@ export const callApi = <K extends OperationNames>(
     >
   )[method](req) as Promise<BackendReturnType<K>>;
 
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  const op_key = (result as any)._webviewMessageId as string;
-
   return {
     uuid: op_key,
     result: result.then(({ body }) => body),

pkgs/clan-app/ui/src/queries/queries.ts

@@ -3,8 +3,12 @@ import { callApi, SuccessData } from "../hooks/api";
 import { encodeBase64 } from "@/src/hooks/clan";
 
 export type ClanDetails = SuccessData<"get_clan_details">;
+export type ClanDetailsWithURI = ClanDetails & { uri: string };
+
 export type ListMachines = SuccessData<"list_machines">;
+
 export type MachinesQueryResult = UseQueryResult<ListMachines>;
+export type ClanListQueryResult = UseQueryResult<ClanDetailsWithURI>[];
 
 export const useMachinesQuery = (clanURI: string) =>
   useQuery<ListMachines>(() => ({
@@ -48,7 +52,7 @@ export const useClanDetailsQuery = (clanURI: string) =>
     },
   }));
 
-export const useAllClanDetailsQuery = (clanURIs: string[]) =>
+export const useClanListQuery = (clanURIs: string[]): ClanListQueryResult =>
   useQueries(() => ({
     queries: clanURIs.map((clanURI) => ({
       queryKey: ["clans", encodeBase64(clanURI), "details"],

pkgs/clan-app/ui/src/routes/Clan/Clan.tsx

@@ -15,9 +15,14 @@ import {
   useClanURI,
 } from "@/src/hooks/clan";
 import { CubeScene } from "@/src/scene/cubes";
-import { MachinesQueryResult, useMachinesQuery } from "@/src/queries/queries";
+import {
+  ClanListQueryResult,
+  MachinesQueryResult,
+  useClanListQuery,
+  useMachinesQuery,
+} from "@/src/queries/queries";
 import { callApi } from "@/src/hooks/api";
-import { store, setStore } from "@/src/stores/clan";
+import { store, setStore, clanURIs } from "@/src/stores/clan";
 import { produce } from "solid-js/store";
 import { Button } from "@/src/components/Button/Button";
 import { Splash } from "@/src/scene/splash";
@@ -42,10 +47,12 @@ export const Clan: Component<RouteSectionProps> = (props) => {
 interface CreateFormValues extends FieldValues {
   name: string;
 }
+
 interface MockProps {
   onClose: () => void;
   onSubmit: (formValues: CreateFormValues) => void;
 }
+
 const MockCreateMachine = (props: MockProps) => {
   let container: Node;
 
@@ -173,7 +180,26 @@ const ClanSceneController = (props: RouteSectionProps) => {
 
   return (
     <SceneDataProvider clanURI={clanURI}>
-      {({ query }) => {
+      {({ clansQuery, machinesQuery }) => {
+        // a combination of the individual clan details query status and the machines query status
+        // the cube scene needs the machines query, the sidebar needs the clans query and machines query results
+        // so we wait on both before removing the loader to avoid any loading artefacts
+        const isLoading = (): boolean => {
+          // check the machines query first
+          if (machinesQuery.isLoading) {
+            return true;
+          }
+
+          // otherwise iterate the clans query and return early if we find a queries that is still loading
+          for (const query of clansQuery) {
+            if (query.isLoading) {
+              return true;
+            }
+          }
+
+          return false;
+        };
+
         return (
           <>
             <Show when={showModal()}>
@@ -217,7 +243,7 @@ const ClanSceneController = (props: RouteSectionProps) => {
                 ghost
                 onClick={() => {
                   console.log("Refetching API");
-                  query.refetch();
+                  machinesQuery.refetch();
                 }}
               >
                 Refetch API
@@ -225,7 +251,9 @@ const ClanSceneController = (props: RouteSectionProps) => {
             </div>
             {/* TODO: Add minimal display time */}
             <div
-              class={cx({ "fade-out": !query.isLoading && loadingCooldown() })}
+              class={cx({
+                "fade-out": !machinesQuery.isLoading && loadingCooldown(),
+              })}
             >
               <Splash />
             </div>
@@ -233,8 +261,8 @@ const ClanSceneController = (props: RouteSectionProps) => {
             <CubeScene
               selectedIds={selectedIds}
               onSelect={onMachineSelect}
-              isLoading={query.isLoading}
+              isLoading={isLoading()}
-              cubesQuery={query}
+              cubesQuery={machinesQuery}
               onCreate={onCreate}
               sceneStore={() => {
                 const clanURI = useClanURI();
@@ -268,10 +296,14 @@ const ClanSceneController = (props: RouteSectionProps) => {
 
 const SceneDataProvider = (props: {
   clanURI: string;
-  children: (sceneData: { query: MachinesQueryResult }) => JSX.Element;
+  children: (sceneData: {
+    clansQuery: ClanListQueryResult;
+    machinesQuery: MachinesQueryResult;
+  }) => JSX.Element;
 }) => {
+  const clansQuery = useClanListQuery(clanURIs());
   const machinesQuery = useMachinesQuery(props.clanURI);
 
   // This component can be used to provide scene data or context if needed
-  return props.children({ query: machinesQuery });
+  return props.children({ clansQuery, machinesQuery });
 };

pkgs/clan-app/ui/src/scene/cubes.css

@@ -8,3 +8,19 @@
   @apply absolute bottom-8 z-10 w-full;
   @apply flex justify-center items-center;
 }
+
+.machine-label {
+  @apply text-white bg-inv-4 py-1 px-2 rounded-sm;
+  font-size: 0.75rem;
+}
+
+.machine-label::after {
+  content: "";
+  position: absolute;
+  top: 100%;
+  left: 50%;
+  margin-left: -5px;
+  border-width: 5px;
+  border-style: solid;
+  border-color: #203637 transparent transparent transparent;
+}

pkgs/clan-app/ui/src/scene/cubes.tsx

@@ -10,6 +10,10 @@ import "./cubes.css";
 
 import * as THREE from "three";
 import { MapControls } from "three/examples/jsm/controls/MapControls.js";
+import {
+  CSS2DRenderer,
+  CSS2DObject,
+} from "three/examples/jsm/renderers/CSS2DRenderer.js";
 
 import { Toolbar } from "../components/Toolbar/Toolbar";
 import { ToolbarButton } from "../components/Toolbar/ToolbarButton";
@@ -77,6 +81,7 @@ export function CubeScene(props: {
   let scene: THREE.Scene;
   let camera: THREE.OrthographicCamera;
   let renderer: THREE.WebGLRenderer;
+  let labelRenderer: CSS2DRenderer;
   let floor: THREE.Mesh;
   let controls: MapControls;
   // Raycaster for clicking
@@ -195,8 +200,11 @@ export function CubeScene(props: {
 
     renderer.autoClear = false;
     renderer.render(bgScene, bgCamera);
+
     controls.update(); // optional; see note below
+
     renderer.render(scene, camera);
+    labelRenderer.render(scene, camera);
 
     if (frameCount % 30 === 0) logMemoryUsage();
   }
@@ -523,6 +531,15 @@ export function CubeScene(props: {
     renderer.shadowMap.type = THREE.PCFSoftShadowMap;
     container.appendChild(renderer.domElement);
 
+    // Label renderer
+    labelRenderer = new CSS2DRenderer();
+    labelRenderer.setSize(container.clientWidth, container.clientHeight);
+    labelRenderer.domElement.style.position = "absolute";
+    labelRenderer.domElement.style.top = "0px";
+    labelRenderer.domElement.style.pointerEvents = "none";
+    labelRenderer.domElement.style.zIndex = "0";
+    container.appendChild(labelRenderer.domElement);
+
     controls = new MapControls(camera, renderer.domElement);
     controls.enableDamping = true; // an animation loop is required when either damping or auto-rotation are enabled
     // Enable the context menu,
@@ -546,7 +563,7 @@ export function CubeScene(props: {
     const ambientLight = new THREE.AmbientLight(0xffffff, 1);
     scene.add(ambientLight);
 
-    const directionalLight = new THREE.DirectionalLight(0xffffff, 1);
+    const directionalLight = new THREE.DirectionalLight(0xffffff, 2);
 
     // scene.add(new THREE.DirectionalLightHelper(directionalLight));
     // scene.add(new THREE.CameraHelper(directionalLight.shadow.camera));
@@ -570,7 +587,7 @@ export function CubeScene(props: {
     directionalLight.shadow.camera.far = 2000;
     directionalLight.shadow.mapSize.width = 4096; // Higher resolution for sharper shadows
     directionalLight.shadow.mapSize.height = 4096;
-    directionalLight.shadow.radius = 0; // Hard shadows (low radius)
+    directionalLight.shadow.radius = 1; // Hard shadows (low radius)
     directionalLight.shadow.blurSamples = 4; // Fewer samples for harder edges
     scene.add(directionalLight);
     scene.add(directionalLight.target);
@@ -716,6 +733,7 @@ export function CubeScene(props: {
       camera.updateProjectionMatrix();
 
       renderer.setSize(container.clientWidth, container.clientHeight);
+      labelRenderer.setSize(container.clientWidth, container.clientHeight);
 
       // Update background shader resolution
       uniforms.resolution.value.set(
@@ -791,6 +809,14 @@ export function CubeScene(props: {
     const baseMesh = createCubeBase([0, BASE_HEIGHT / 2, 0]);
     baseMesh.name = "base"; // Name for easy identification
 
+    const nameDiv = document.createElement("div");
+    nameDiv.className = "machine-label";
+    nameDiv.textContent = `${userData.id}`;
+
+    const nameLabel = new CSS2DObject(nameDiv);
+    nameLabel.position.set(0, CUBE_Y + CUBE_SIZE / 2 - 0.2, 0);
+    cubeMesh.add(nameLabel);
+
     // TODO: Destroy Group in onCleanup
     const group = new THREE.Group();
     group.add(cubeMesh);

pkgs/clan-cli/clan_cli/machines/install.py

@@ -35,7 +35,7 @@ def install_command(args: argparse.Namespace) -> None:
         use_tor = False
         if deploy_info:
             host = find_reachable_host(deploy_info)
-            if host is None or host.tor_socks:
+            if host is None or host.socks_port:
                 use_tor = True
                 target_host_str = deploy_info.tor.target
             else:
@@ -74,7 +74,9 @@ def install_command(args: argparse.Namespace) -> None:
             target_host = target_host.override(password=password)
 
         if use_tor:
-            target_host = target_host.override(tor_socks=True)
+            target_host = target_host.override(
+                socks_port=9050, socks_wrapper=["torify"]
+            )
 
         return run_machine_install(
             InstallOptions(

pkgs/clan-cli/clan_cli/network/ping.py

@@ -27,28 +27,27 @@ def ping_command(args: argparse.Namespace) -> None:
         networks_to_check = sorted(networks.items(), key=lambda x: -x[1].priority)
 
     found = False
-    results = []
     for net_name, network in networks_to_check:
         if machine in network.peers:
             found = True
 
-            # Check if network technology is running
+            with network.module.connection(network) as network:
-            if not network.is_running():
+                log.info(f"Pinging '{machine}' in network '{net_name}' ...")
-                results.append(f"{machine} ({net_name}): network not running")
+                res = ""
-                continue
+                # Check if peer is online
-
+                ping = network.ping(machine)
-            # Check if peer is online
+                if ping is None:
-            ping = network.ping(machine)
+                    res = "not reachable"
-            results.append(f"{machine} ({net_name}): {ping}")
+                    log.info(f"{machine} ({net_name}): {res}")
+                else:
+                    res = f"reachable, ping: {ping:.2f} ms"
+                    log.info(f"{machine} ({net_name}): {res}")
+                    break
 
     if not found:
         msg = f"Machine '{machine}' not found in any network"
         raise ClanError(msg)
 
-    # Print all results
-    for result in results:
-        print(result)
-
 
 def register_ping_parser(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(

pkgs/clan-cli/clan_cli/ssh/deploy_info.py

@@ -1,4 +1,5 @@
 import argparse
+import contextlib
 import json
 import logging
 import textwrap
@@ -9,6 +10,7 @@ from typing import Any, get_args
 from clan_lib.cmd import run
 from clan_lib.errors import ClanError
 from clan_lib.machines.machines import Machine
+from clan_lib.network.tor.lib import spawn_tor
 from clan_lib.nix import nix_shell
 from clan_lib.ssh.remote import HostKeyCheck, Remote
 
@@ -16,7 +18,6 @@ from clan_cli.completions import (
     add_dynamic_completer,
     complete_machines,
 )
-from clan_cli.ssh.tor import TorTarget, spawn_tor, ssh_tor_reachable
 
 log = logging.getLogger(__name__)
 
@@ -27,15 +28,15 @@ class DeployInfo:
 
     @property
     def tor(self) -> Remote:
-        """Return a list of Remote objects that are configured for Tor."""
+        """Return a list of Remote objects that are configured for SOCKS5 proxy."""
-        addrs = [addr for addr in self.addrs if addr.tor_socks]
+        addrs = [addr for addr in self.addrs if addr.socks_port]
 
         if not addrs:
-            msg = "No tor address provided, please provide a tor address."
+            msg = "No socks5 proxy address provided, please provide a socks5 proxy address."
             raise ClanError(msg)
 
         if len(addrs) > 1:
-            msg = "Multiple tor addresses provided, expected only one."
+            msg = "Multiple socks5 proxy addresses provided, expected only one."
             raise ClanError(msg)
         return addrs[0]
 
@@ -76,7 +77,12 @@ class DeployInfo:
             remote = Remote.from_ssh_uri(
                 machine_name="clan-installer",
                 address=tor_addr,
-            ).override(host_key_check=host_key_check, tor_socks=True, password=password)
+            ).override(
+                host_key_check=host_key_check,
+                socks_port=9050,
+                socks_wrapper=["torify"],
+                password=password,
+            )
             addrs.append(remote)
 
         return DeployInfo(addrs=addrs)
@@ -103,7 +109,8 @@ def find_reachable_host(deploy_info: DeployInfo) -> Remote | None:
         return deploy_info.addrs[0]
 
     for addr in deploy_info.addrs:
-        if addr.check_machine_ssh_reachable():
+        with contextlib.suppress(ClanError):
+            addr.check_machine_ssh_reachable()
             return addr
     return None
 
@@ -129,7 +136,7 @@ def ssh_shell_from_deploy(
     log.info("Could not reach host via clearnet 'addrs'")
     log.info(f"Trying to reach host via tor '{deploy_info}'")
 
-    tor_addrs = [addr for addr in deploy_info.addrs if addr.tor_socks]
+    tor_addrs = [addr for addr in deploy_info.addrs if addr.socks_port]
     if not tor_addrs:
         msg = "No tor address provided, please provide a tor address."
         raise ClanError(msg)
@@ -137,11 +144,10 @@ def ssh_shell_from_deploy(
     with spawn_tor():
         for tor_addr in tor_addrs:
             log.info(f"Trying to reach host via tor address: {tor_addr}")
-            if ssh_tor_reachable(
+
-                TorTarget(
+            with contextlib.suppress(ClanError):
-                    onion=tor_addr.address, port=tor_addr.port if tor_addr.port else 22
+                tor_addr.check_machine_ssh_reachable()
-                )
+
-            ):
                 log.info(
                     "Host reachable via tor address, starting interactive ssh session."
                 )

pkgs/clan-cli/clan_cli/ssh/test_deploy_info.py

@@ -37,7 +37,7 @@ def test_qrcode_scan(temp_dir: Path) -> None:
         tor_host.address
         == "qjeerm4r6t55hcfum4pinnvscn5njlw2g3k7ilqfuu7cdt3ahaxhsbid.onion"
     )
-    assert tor_host.tor_socks is True
+    assert tor_host.socks_port == 9050
     assert tor_host.password == "scabbed-defender-headlock"
     assert tor_host.user == "root"
     assert (
@@ -59,7 +59,7 @@ def test_from_json() -> None:
         tor_host.address
         == "qjeerm4r6t55hcfum4pinnvscn5njlw2g3k7ilqfuu7cdt3ahaxhsbid.onion"
     )
-    assert tor_host.tor_socks is True
+    assert tor_host.socks_port == 9050
     assert tor_host.password == "scabbed-defender-headlock"
     assert tor_host.user == "root"
     assert (

pkgs/clan-cli/clan_cli/ssh/tor.py

@@ -1,233 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import logging
-import socket
-import struct
-import time
-from collections.abc import Iterator
-from contextlib import contextmanager
-from dataclasses import dataclass
-from subprocess import Popen
-
-from clan_lib.errors import TorConnectionError, TorSocksError
-from clan_lib.nix import nix_shell
-
-log = logging.getLogger(__name__)
-
-
-@dataclass
-class TorTarget:
-    onion: str
-    port: int
-    proxy_host: str = "127.0.0.1"
-    proxy_port: int = 9050
-
-
-def connect_to_tor_socks(sock: socket.socket, target: TorTarget) -> socket.socket:
-    """
-    Connects to a .onion host through Tor's SOCKS5 proxy using the standard library.
-
-    Args:
-        target (TorTarget): An object containing the .onion address, port, proxy host, and proxy port.
-
-    Returns:
-        socket.socket: A socket connected to the .onion address via Tor.
-    """
-    try:
-        # 1. Create a socket to the Tor SOCKS proxy
-        sock.connect((target.proxy_host, target.proxy_port))
-    except ConnectionRefusedError as ex:
-        msg = f"Failed to connect to Tor SOCKS proxy at {target.proxy_host}:{target.proxy_port}: {ex}"
-        raise TorSocksError(msg) from ex
-
-    # 2. SOCKS5 handshake
-    sock.sendall(
-        b"\x05\x01\x00"
-    )  # SOCKS version (0x05), number of authentication methods (0x01), no-authentication (0x00)
-    response = sock.recv(2)
-
-    # Validate the SOCKS5 handshake response
-    if response != b"\x05\x00":  # SOCKS version = 0x05, no-authentication = 0x00
-        msg = f"SOCKS5 handshake failed, unexpected response: {response.hex()}"
-        raise TorSocksError(msg)
-
-    # 3. Connection request
-    request = (
-        b"\x05\x01\x00\x03"  # SOCKS version, connect command, reserved, address type = domainname
-        + bytes([len(target.onion)])
-        + target.onion.encode("utf-8")  # Add domain name length and domain name
-        + struct.pack(">H", target.port)  # Add destination port in network byte order
-    )
-    sock.sendall(request)
-
-    # Read the connection request response
-    response = sock.recv(10)
-    if response[1] != 0x00:  # 0x00 indicates success
-        msg = f".onion address not reachable: {response[1]}"
-        raise TorConnectionError(msg)
-
-    return sock
-
-
-def fetch_onion_content(target: TorTarget) -> str:
-    """
-    Fetches the HTTP response from a .onion service through a Tor SOCKS5 proxy.
-
-    Args:
-        target (TorTarget): An object containing the .onion address, port, proxy host, and proxy port.
-
-    Returns:
-        str: The HTTP response text, or an error message if something goes wrong.
-    """
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        # Connect to the .onion service via the SOCKS proxy
-        sock = connect_to_tor_socks(sock, target)
-
-        # 1. Send an HTTP GET request
-        request = f"GET / HTTP/1.1\r\nHost: {target.onion}\r\nConnection: close\r\n\r\n"
-        sock.sendall(request.encode("utf-8"))
-
-        # 2. Read the HTTP response
-        response = b""
-        while True:
-            chunk = sock.recv(4096)
-            if not chunk:
-                break
-            response += chunk
-
-        return response.decode("utf-8", errors="replace")
-
-
-def is_tor_running() -> bool:
-    """Checks if Tor is online."""
-    try:
-        tor_online_test()
-    except TorSocksError:
-        return False
-    else:
-        return True
-
-
-@contextmanager
-def spawn_tor() -> Iterator[None]:
-    """
-    Spawns a Tor process using `nix-shell` if Tor is not already running.
-    """
-
-    # Check if Tor is already running
-    if is_tor_running():
-        log.info("Tor is running")
-        return
-    cmd_args = ["tor", "--HardwareAccel", "1"]
-    packages = ["tor"]
-    cmd = nix_shell(packages, cmd_args)
-    process = Popen(cmd)
-    try:
-        while not is_tor_running():
-            log.debug("Waiting for Tor to start...")
-            time.sleep(0.2)
-        log.info("Tor is now running")
-        yield
-    finally:
-        log.info("Terminating Tor process...")
-        process.terminate()
-        process.wait()
-        log.info("Tor process terminated")
-
-
-def tor_online_test() -> bool:
-    """
-    Tests if Tor is online by attempting to fetch content from a known .onion service.
-    Returns True if successful, False otherwise.
-    """
-    target = TorTarget(
-        onion="duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion", port=80
-    )
-    try:
-        response = fetch_onion_content(target)
-    except TorConnectionError:
-        return False
-    else:
-        return "duckduckgo" in response
-
-
-def ssh_tor_reachable(target: TorTarget) -> bool:
-    """
-    Tests if SSH is reachable via Tor by attempting to connect to a known .onion service.
-    Returns True if successful, False otherwise.
-    """
-    try:
-        response = fetch_onion_content(target)
-    except TorConnectionError:
-        return False
-    else:
-        return "SSH-" in response
-
-
-def main() -> None:
-    """
-    Main function to handle command-line arguments and execute the script.
-    """
-    parser = argparse.ArgumentParser(
-        description="Interact with a .onion service through Tor SOCKS5 proxy."
-    )
-    parser.add_argument(
-        "onion_url", type=str, help=".onion URL to connect to (e.g., 'example.onion')"
-    )
-    parser.add_argument(
-        "--port", type=int, help="Port to connect to on the .onion URL (default: 80)"
-    )
-    parser.add_argument(
-        "--proxy-host",
-        type=str,
-        default="127.0.0.1",
-        help="Address of the Tor SOCKS5 proxy (default: 127.0.0.1)",
-    )
-    parser.add_argument(
-        "--proxy-port",
-        type=int,
-        default=9050,
-        help="Port of the Tor SOCKS5 proxy (default: 9050)",
-    )
-    parser.add_argument(
-        "--ssh-tor-reachable",
-        action="store_true",
-        help="Test if SSH is reachable via Tor",
-    )
-
-    args = parser.parse_args()
-    default_port = 22 if args.ssh_tor_reachable else 80
-
-    # Create a TorTarget instance
-    target = TorTarget(
-        onion=args.onion_url,
-        port=args.port or default_port,
-        proxy_host=args.proxy_host,
-        proxy_port=args.proxy_port,
-    )
-
-    if args.ssh_tor_reachable:
-        print(f"Testing if SSH is reachable via Tor for {target.onion}...")
-        reachable = ssh_tor_reachable(target)
-        print(f"SSH is {'reachable' if reachable else 'not reachable'} via Tor.")
-        return
-
-    print(
-        f"Connecting to {target.onion} on port {target.port} via proxy {target.proxy_host}:{target.proxy_port}..."
-    )
-    try:
-        response = fetch_onion_content(target)
-        print("Response:")
-        print(response)
-    except TorSocksError:
-        log.error("Failed to connect to the Tor SOCKS proxy.")
-        log.error(
-            "Is Tor running? If not, you can start it by running 'tor' in a nix-shell."
-        )
-    except TorConnectionError:
-        log.error("The onion address is not reachable via Tor.")
-
-
-if __name__ == "__main__":
-    main()

pkgs/clan-cli/clan_cli/vars/get.py

@@ -17,6 +17,7 @@ log = logging.getLogger(__name__)
 
 
 def get_machine_var(base_dir: str, machine_name: str, var_id: str) -> Var:
+    log.debug(f"getting var: {var_id} from machine: {machine_name}")
     vars_ = get_machine_vars(base_dir=base_dir, machine_name=machine_name)
     results = []
     for var in vars_:

pkgs/clan-cli/clan_lib/api/__init__.py

@@ -15,6 +15,7 @@ from typing import (
 )
 
 from clan_lib.api.util import JSchemaTypeError
+from clan_lib.async_run import get_current_thread_opkey
 from clan_lib.errors import ClanError
 
 from .serde import dataclass_to_dict, from_dict, sanitize_string
@@ -54,26 +55,6 @@ class ErrorDataClass:
 ApiResponse = SuccessDataClass[ResponseDataType] | ErrorDataClass
 
 
-def update_wrapper_signature(wrapper: Callable, wrapped: Callable) -> None:
-    sig = signature(wrapped)
-    params = list(sig.parameters.values())
-
-    # Add 'op_key' parameter
-    op_key_param = Parameter(
-        "op_key",
-        Parameter.KEYWORD_ONLY,
-        # we add a None default value so that typescript code gen drops the parameter
-        # FIXME: this is a hack, we should filter out op_key in the typescript code gen
-        default=None,
-        annotation=str,
-    )
-    params.append(op_key_param)
-
-    # Create a new signature
-    new_sig = sig.replace(parameters=params)
-    wrapper.__signature__ = new_sig  # type: ignore
-
-
 class MethodRegistry:
     def __init__(self) -> None:
         self._orig_signature: dict[str, Signature] = {}
@@ -130,18 +111,8 @@ API.register(get_system_file)
         fn_signature = signature(fn)
         abstract_signature = signature(self._registry[fn_name])
 
-        # Remove the default argument of op_key from abstract_signature
-        # FIXME: This is a hack to make the signature comparison work
-        # because the other hack above where default value of op_key is None in the wrapper
-        abstract_params = list(abstract_signature.parameters.values())
-        for i, param in enumerate(abstract_params):
-            if param.name == "op_key":
-                abstract_params[i] = param.replace(default=Parameter.empty)
-                break
-        abstract_signature = abstract_signature.replace(parameters=abstract_params)
-
         if fn_signature != abstract_signature:
-            msg = f"Expected signature: {abstract_signature}\nActual signature: {fn_signature}"
+            msg = f"For function: {fn_name}. Expected signature: {abstract_signature}\nActual signature: {fn_signature}"
             raise ClanError(msg)
 
         self._registry[fn_name] = fn
@@ -159,7 +130,11 @@ API.register(get_system_file)
         self._orig_signature[fn.__name__] = signature(fn)
 
         @wraps(fn)
-        def wrapper(*args: Any, op_key: str, **kwargs: Any) -> ApiResponse[T]:
+        def wrapper(*args: Any, **kwargs: Any) -> ApiResponse[T]:
+            op_key = get_current_thread_opkey()
+            if op_key is None:
+                msg = f"While executing {fn.__name__}. Middleware forgot to set_current_thread_opkey()"
+                raise RuntimeError(msg)
             try:
                 data: T = fn(*args, **kwargs)
                 return SuccessDataClass(status="success", data=data, op_key=op_key)
@@ -196,11 +171,6 @@ API.register(get_system_file)
         orig_return_type = get_type_hints(fn).get("return")
         wrapper.__annotations__["return"] = ApiResponse[orig_return_type]  # type: ignore
 
-        # Add additional argument for the operation key
-        wrapper.__annotations__["op_key"] = str  # type: ignore
-
-        update_wrapper_signature(wrapper, fn)
-
         self._registry[fn.__name__] = wrapper
 
         return fn

pkgs/clan-cli/clan_lib/async_run/__init__.py

@@ -74,6 +74,7 @@ class AsyncContext:
     should_cancel: Callable[[], bool] = (
         lambda: False
     )  # Used to signal cancellation of task
+    op_key: str | None = None
 
 
 @dataclass
@@ -90,6 +91,22 @@ class AsyncOpts:
 ASYNC_CTX_THREAD_LOCAL = threading.local()
 
 
+def set_current_thread_opkey(op_key: str) -> None:
+    """
+    Set the current thread's operation key.
+    """
+    ctx = get_async_ctx()
+    ctx.op_key = op_key
+
+
+def get_current_thread_opkey() -> str | None:
+    """
+    Get the current thread's operation key.
+    """
+    ctx = get_async_ctx()
+    return ctx.op_key
+
+
 def is_async_cancelled() -> bool:
     """
     Check if the current task has been cancelled.

pkgs/clan-cli/clan_lib/errors/__init__.py

@@ -189,13 +189,3 @@ class ClanCmdError(ClanError):
 
     def __repr__(self) -> str:
         return f"ClanCmdError({self.cmd})"
-
-
-class TorSocksError(ClanError):
-    def __init__(self, msg: str) -> None:
-        super().__init__(msg)
-
-
-class TorConnectionError(ClanError):
-    def __init__(self, msg: str) -> None:
-        super().__init__(msg)

pkgs/clan-cli/clan_lib/flake/flake.py

@@ -2,7 +2,7 @@ import json
 import logging
 import os
 import re
-import textwrap
+import shlex
 from dataclasses import asdict, dataclass, field
 from enum import Enum
 from functools import cache
@@ -307,6 +307,7 @@ class FlakeCacheEntry:
     is_list: bool = False
     exists: bool = True
     fetched_all: bool = False
+    _num_accessed: int = field(default=0, init=False)
 
     def insert(
         self,
@@ -476,6 +477,80 @@ class FlakeCacheEntry:
 
         return False
 
+    def is_path_unaccessed(self, selectors: list[Selector]) -> bool:
+        """Check if the leaf entry has _num_accessed == 0"""
+        if selectors == []:
+            # This is the leaf, check if it's unaccessed
+            return self._num_accessed == 0
+
+        selector = selectors[0]
+
+        # Navigate to leaf for string/maybe selectors
+        if (
+            selector.type == SelectorType.STR or selector.type == SelectorType.MAYBE
+        ) and isinstance(self.value, dict):
+            assert isinstance(selector.value, str)
+            if selector.value in self.value:
+                return self.value[selector.value].is_path_unaccessed(selectors[1:])
+            return True  # Non-existent path is considered unaccessed
+
+        # For set selectors, check if any leaf is unaccessed
+        if (
+            selector.type == SelectorType.SET
+            and isinstance(selector.value, list)
+            and isinstance(self.value, dict)
+        ):
+            for subselector in selector.value:
+                if subselector.value in self.value:
+                    if not self.value[subselector.value].is_path_unaccessed(
+                        selectors[1:]
+                    ):
+                        return False
+            return True
+
+        # For all selectors, check if any existing key is unaccessed
+        if selector.type == SelectorType.ALL and isinstance(self.value, dict):
+            for key in self.value:
+                if self.value[key].exists:
+                    if not self.value[key].is_path_unaccessed(selectors[1:]):
+                        return False
+            return True
+
+        return True
+
+    def mark_path_accessed(self, selectors: list[Selector]) -> None:
+        """Mark only the leaf entry as accessed"""
+        if selectors == []:
+            # This is the leaf, increment access count
+            self._num_accessed += 1
+            return
+
+        selector = selectors[0]
+
+        # Navigate to leaf for string/maybe selectors
+        if (
+            selector.type == SelectorType.STR or selector.type == SelectorType.MAYBE
+        ) and isinstance(self.value, dict):
+            assert isinstance(selector.value, str)
+            if selector.value in self.value:
+                self.value[selector.value].mark_path_accessed(selectors[1:])
+
+        # For set selectors, mark all leaf paths
+        elif (
+            selector.type == SelectorType.SET
+            and isinstance(selector.value, list)
+            and isinstance(self.value, dict)
+        ):
+            for subselector in selector.value:
+                if subselector.value in self.value:
+                    self.value[subselector.value].mark_path_accessed(selectors[1:])
+
+        # For all selectors, mark all existing keys
+        elif selector.type == SelectorType.ALL and isinstance(self.value, dict):
+            for key in self.value:
+                if self.value[key].exists:
+                    self.value[key].mark_path_accessed(selectors[1:])
+
     def select(self, selectors: list[Selector]) -> Any:
         selector: Selector
         if selectors == []:
@@ -593,6 +668,7 @@ class FlakeCacheEntry:
         entry = FlakeCacheEntry(
             value=value, is_list=is_list, exists=exists, fetched_all=fetched_all
         )
+        entry._num_accessed = 0
         return entry
 
     def __repr__(self) -> str:
@@ -620,12 +696,21 @@ class FlakeCache:
 
     def select(self, selector_str: str) -> Any:
         selectors = parse_selector(selector_str)
+        self.mark_path_accessed(selectors)
         return self.cache.select(selectors)
 
     def is_cached(self, selector_str: str) -> bool:
         selectors = parse_selector(selector_str)
+        if self.is_path_unaccessed(selectors):
+            log.debug(f"$ clan select {shlex.quote(selector_str)}")
         return self.cache.is_cached(selectors)
 
+    def is_path_unaccessed(self, selectors: list[Selector]) -> bool:
+        return self.cache.is_path_unaccessed(selectors)
+
+    def mark_path_accessed(self, selectors: list[Selector]) -> None:
+        self.cache.mark_path_accessed(selectors)
+
     def save_to_file(self, path: Path) -> None:
         path.parent.mkdir(parents=True, exist_ok=True)
         with NamedTemporaryFile(mode="w", dir=path.parent, delete=False) as temp_file:
@@ -636,7 +721,7 @@ class FlakeCache:
 
     def load_from_file(self, path: Path) -> None:
         with path.open("r") as f:
-            log.debug("Loading flake cache from file")
+            log.debug(f"Loading flake cache from file {path}")
             data = json.load(f)
             self.cache = FlakeCacheEntry.from_json(data["cache"])
 
@@ -729,7 +814,7 @@ class Flake:
             self.identifier,
         ]
 
-        trace_prefetch = os.environ.get("CLAN_DEBUG_NIX_PREFETCH", "0") == "1"
+        trace_prefetch = os.environ.get("CLAN_DEBUG_NIX_PREFETCH", False) == "1"
         if not trace_prefetch:
             log.debug(f"Prefetching flake {self.identifier}")
         flake_prefetch = run(nix_command(cmd), RunOpts(trace=trace_prefetch))
@@ -862,47 +947,11 @@ class Flake:
                 ];
               }}
         """
-        if len(selectors) > 1 :
+        trace = os.environ.get("CLAN_DEBUG_NIX_SELECTORS", False) == "1"
-            msg = textwrap.dedent(f"""
-                clan select "{selectors}"
-            """).lstrip("\n").rstrip("\n")
-            if os.environ.get("CLAN_DEBUG_NIX_SELECTORS"):
-                msg += textwrap.dedent(f"""
-                    to debug run:
-                    nix repl --expr 'rec {{
-                    flake = builtins.getFlake "{self.identifier}";
-                    selectLib = (builtins.getFlake "path:{select_source()}?narHash={select_hash}").lib;
-                    query = [
-                        {" ".join(
-                            [
-                                f"(selectLib.select ''{selector}'' flake)"
-                                for selector in selectors
-                            ]
-                        )}
-                    ];
-                    }}'
-                """).lstrip("\n")
-            log.debug(msg)
-        # fmt: on
-        elif len(selectors) == 1:
-            msg = textwrap.dedent(f"""
-               $ clan select "{selectors[0]}"
-            """).lstrip("\n").rstrip("\n")
-            if os.environ.get("CLAN_DEBUG_NIX_SELECTORS"):
-                msg += textwrap.dedent(f"""
-                to debug run:
-                    nix repl --expr 'rec {{
-                    flake = builtins.getFlake "{self.identifier}";
-                    selectLib = (builtins.getFlake "path:{select_source()}?narHash={select_hash}").lib;
-                    query = selectLib.select '"''{selectors[0]}''"' flake;
-                    }}'
-                    """).lstrip("\n")
-            log.debug(msg)
-
         build_output = Path(
             run(
                 nix_build(["--expr", nix_code, *nix_options]),
-                RunOpts(log=Log.NONE, trace=False),
+                RunOpts(log=Log.NONE, trace=trace),
             ).stdout.strip()
         )
 
@@ -929,14 +978,21 @@ class Flake:
         Args:
             selectors (list[str]): A list of attribute selectors to check and cache.
         """
+
         if self._cache is None:
             self.invalidate_cache()
         assert self._cache is not None
         assert self.flake_cache_path is not None
         not_fetched_selectors = []
         for selector in selectors:
+            parsed_selectors = parse_selector(selector)
+
             if not self._cache.is_cached(selector):
                 not_fetched_selectors.append(selector)
+
+            # Mark path as accessed after checking
+            self._cache.mark_path_accessed(parsed_selectors)
+
         if not_fetched_selectors:
             self.get_from_nix(not_fetched_selectors)
 
@@ -959,6 +1015,7 @@ class Flake:
         if not self._cache.is_cached(selector):
             log.debug(f"Cache miss for {selector}")
             self.get_from_nix([selector])
+
         value = self._cache.select(selector)
         return value
 

pkgs/clan-cli/clan_lib/flake/test_flake_access.py

@@ -0,0 +1,488 @@
+from clan_lib.flake.flake import FlakeCache, parse_selector
+
+
+class TestFlakeAccessTracking:
+    """Test the leaf-only access tracking feature for FlakeCache"""
+
+    def test_leaf_access_tracking_simple(self) -> None:
+        """Test that only leaf nodes have their access count incremented"""
+        cache = FlakeCache()
+
+        # Insert some data
+        data = {"level1": {"level2": {"leaf": "value"}}}
+        cache.insert(data, "")
+
+        # Initially, the leaf should be unaccessed
+        assert cache.is_path_unaccessed(parse_selector("level1.level2.leaf"))
+
+        # Mark the path as accessed
+        cache.mark_path_accessed(parse_selector("level1.level2.leaf"))
+
+        # Now the leaf should be accessed
+        assert not cache.is_path_unaccessed(parse_selector("level1.level2.leaf"))
+
+        # But intermediate nodes should still have _num_accessed == 0
+        # We'll need to check this directly on the cache entries
+        entry = cache.cache
+        assert isinstance(entry.value, dict)
+        assert entry._num_accessed == 0  # Root should not be incremented
+
+        level1_entry = entry.value["level1"]
+        assert level1_entry._num_accessed == 0  # level1 should not be incremented
+
+        level2_entry = level1_entry.value["level2"]
+        assert level2_entry._num_accessed == 0  # level2 should not be incremented
+
+        leaf_entry = level2_entry.value["leaf"]
+        assert leaf_entry._num_accessed == 1  # Only leaf should be incremented
+
+    def test_leaf_access_tracking_multiple_paths(self) -> None:
+        """Test access tracking with multiple paths"""
+        cache = FlakeCache()
+
+        # Insert data with multiple branches
+        data = {"branch1": {"leaf1": "value1"}, "branch2": {"leaf2": "value2"}}
+        cache.insert(data, "")
+
+        # Initially both leaves should be unaccessed
+        assert cache.is_path_unaccessed(parse_selector("branch1.leaf1"))
+        assert cache.is_path_unaccessed(parse_selector("branch2.leaf2"))
+
+        # Access first leaf
+        cache.mark_path_accessed(parse_selector("branch1.leaf1"))
+
+        # First leaf should be accessed, second should not
+        assert not cache.is_path_unaccessed(parse_selector("branch1.leaf1"))
+        assert cache.is_path_unaccessed(parse_selector("branch2.leaf2"))
+
+        # Access second leaf
+        cache.mark_path_accessed(parse_selector("branch2.leaf2"))
+
+        # Both should now be accessed
+        assert not cache.is_path_unaccessed(parse_selector("branch1.leaf1"))
+        assert not cache.is_path_unaccessed(parse_selector("branch2.leaf2"))
+
+    def test_leaf_access_tracking_with_lists(self) -> None:
+        """Test access tracking with list entries"""
+        cache = FlakeCache()
+
+        # Insert data with a list
+        data = {"items": ["item0", "item1", "item2"]}
+        cache.insert(data, "")
+
+        # Check initial state
+        assert cache.is_path_unaccessed(parse_selector("items.0"))
+        assert cache.is_path_unaccessed(parse_selector("items.1"))
+        assert cache.is_path_unaccessed(parse_selector("items.2"))
+
+        # Access one item
+        cache.mark_path_accessed(parse_selector("items.1"))
+
+        # Only that item should be marked as accessed
+        assert cache.is_path_unaccessed(parse_selector("items.0"))
+        assert not cache.is_path_unaccessed(parse_selector("items.1"))
+        assert cache.is_path_unaccessed(parse_selector("items.2"))
+
+    def test_leaf_access_tracking_nonexistent_path(self) -> None:
+        """Test that non-existent paths are considered unaccessed"""
+        cache = FlakeCache()
+
+        # Insert minimal data
+        data = {"exists": "value"}
+        cache.insert(data, "")
+
+        # Non-existent path should be considered unaccessed
+        assert cache.is_path_unaccessed(parse_selector("does.not.exist"))
+
+        # Marking non-existent path should not crash
+        cache.mark_path_accessed(parse_selector("does.not.exist"))
+
+        # It should still be considered unaccessed since it doesn't exist
+        assert cache.is_path_unaccessed(parse_selector("does.not.exist"))
+
+    def test_leaf_access_tracking_integration_with_select(self) -> None:
+        """Test that the access tracking works correctly with the select operation"""
+        cache = FlakeCache()
+
+        # Insert nested data
+        data = {
+            "config": {
+                "database": {"host": "localhost", "port": 5432},
+                "cache": {"enabled": True},
+            }
+        }
+        cache.insert(data, "")
+
+        # Verify all paths are initially unaccessed
+        assert cache.is_path_unaccessed(parse_selector("config.database.host"))
+        assert cache.is_path_unaccessed(parse_selector("config.database.port"))
+        assert cache.is_path_unaccessed(parse_selector("config.cache.enabled"))
+
+        # Select a value (this would happen in Flake.select)
+        value = cache.select("config.database.host")
+        assert value == "localhost"
+
+        # Mark the path as accessed (simulating what Flake.select does)
+        cache.mark_path_accessed(parse_selector("config.database.host"))
+
+        # Only the accessed path should be marked
+        assert not cache.is_path_unaccessed(parse_selector("config.database.host"))
+        assert cache.is_path_unaccessed(parse_selector("config.database.port"))
+        assert cache.is_path_unaccessed(parse_selector("config.cache.enabled"))
+
+    def test_leaf_access_multiple_accesses(self) -> None:
+        """Test that multiple accesses increment the counter"""
+        cache = FlakeCache()
+
+        data = {"key": "value"}
+        cache.insert(data, "")
+
+        # Initially unaccessed
+        assert cache.is_path_unaccessed(parse_selector("key"))
+
+        # Access once
+        cache.mark_path_accessed(parse_selector("key"))
+        assert not cache.is_path_unaccessed(parse_selector("key"))
+
+        # Access again
+        cache.mark_path_accessed(parse_selector("key"))
+        assert not cache.is_path_unaccessed(parse_selector("key"))
+
+        # Check the actual count
+        assert isinstance(cache.cache.value, dict)
+        entry = cache.cache.value["key"]
+        assert entry._num_accessed == 2
+
+    def test_leaf_access_tracking_with_set_selectors(self) -> None:
+        """Test access tracking with SET selectors (subselectors)"""
+        cache = FlakeCache()
+
+        # Insert data with multiple keys at the same level
+        data = {"apps": {"web": "web-app", "api": "api-service", "db": "database"}}
+        cache.insert(data, "")
+
+        # Initially all should be unaccessed
+        assert cache.is_path_unaccessed(parse_selector("apps.web"))
+        assert cache.is_path_unaccessed(parse_selector("apps.api"))
+        assert cache.is_path_unaccessed(parse_selector("apps.db"))
+
+        # Use SET selector to access multiple keys at once
+        cache.mark_path_accessed(parse_selector("apps.{web,api}"))
+
+        # Only the keys in the set should be marked as accessed
+        assert not cache.is_path_unaccessed(parse_selector("apps.web"))
+        assert not cache.is_path_unaccessed(parse_selector("apps.api"))
+        assert cache.is_path_unaccessed(parse_selector("apps.db"))  # Not in set
+
+        # Verify the intermediate node is not marked as accessed
+        assert isinstance(cache.cache.value, dict)
+        apps_entry = cache.cache.value["apps"]
+        assert (
+            apps_entry._num_accessed == 0
+        )  # Intermediate node should not be incremented
+
+    def test_leaf_access_tracking_set_selectors_nested(self) -> None:
+        """Test SET selectors with nested paths"""
+        cache = FlakeCache()
+
+        # Insert nested data
+        data = {
+            "services": {
+                "frontend": {"port": 3000, "host": "localhost"},
+                "backend": {"port": 8080, "host": "0.0.0.0"},
+            }
+        }
+        cache.insert(data, "")
+
+        # Use SET selector at different nesting levels
+        cache.mark_path_accessed(parse_selector("services.{frontend,backend}.port"))
+
+        # Both port leaves should be accessed
+        assert not cache.is_path_unaccessed(parse_selector("services.frontend.port"))
+        assert not cache.is_path_unaccessed(parse_selector("services.backend.port"))
+
+        # Host leaves should not be accessed
+        assert cache.is_path_unaccessed(parse_selector("services.frontend.host"))
+        assert cache.is_path_unaccessed(parse_selector("services.backend.host"))
+
+    def test_leaf_access_tracking_set_selectors_partial_match(self) -> None:
+        """Test SET selectors when some keys don't exist"""
+        cache = FlakeCache()
+
+        # Insert data with only some of the keys that will be requested
+        data = {
+            "config": {
+                "database": "postgres",
+                "cache": "redis",
+                # Note: 'queue' key is missing
+            }
+        }
+        cache.insert(data, "")
+
+        # Use SET selector that includes non-existent key
+        cache.mark_path_accessed(parse_selector("config.{database,cache,queue}"))
+
+        # Existing keys should be marked as accessed
+        assert not cache.is_path_unaccessed(parse_selector("config.database"))
+        assert not cache.is_path_unaccessed(parse_selector("config.cache"))
+
+        # Non-existent key should still be considered unaccessed
+        assert cache.is_path_unaccessed(parse_selector("config.queue"))
+
+    def test_leaf_access_tracking_set_selectors_is_path_unaccessed(self) -> None:
+        """Test is_path_unaccessed with SET selectors"""
+        cache = FlakeCache()
+
+        # Insert data
+        data = {"metrics": {"cpu": 50, "memory": 80, "disk": 30}}
+        cache.insert(data, "")
+
+        # Initially, SET selector path should be unaccessed
+        assert cache.is_path_unaccessed(parse_selector("metrics.{cpu,memory}"))
+
+        # Access one of the keys in the set
+        cache.mark_path_accessed(parse_selector("metrics.cpu"))
+
+        # Now the SET selector should be considered accessed (since at least one key is accessed)
+        assert not cache.is_path_unaccessed(parse_selector("metrics.{cpu,memory}"))
+
+        # But a different SET selector should still be unaccessed
+        assert cache.is_path_unaccessed(parse_selector("metrics.{memory,disk}"))
+
+    def test_leaf_access_tracking_set_selectors_mixed_access(self) -> None:
+        """Test SET selectors with mixed individual and set access patterns"""
+        cache = FlakeCache()
+
+        # Insert data
+        data = {
+            "endpoints": {
+                "users": "/api/users",
+                "posts": "/api/posts",
+                "comments": "/api/comments",
+                "auth": "/api/auth",
+            }
+        }
+        cache.insert(data, "")
+
+        # Access individual key first
+        cache.mark_path_accessed(parse_selector("endpoints.users"))
+
+        # Then access using SET selector
+        cache.mark_path_accessed(parse_selector("endpoints.{posts,comments}"))
+
+        # Check individual access states
+        assert not cache.is_path_unaccessed(
+            parse_selector("endpoints.users")
+        )  # Individual access
+        assert not cache.is_path_unaccessed(
+            parse_selector("endpoints.posts")
+        )  # SET access
+        assert not cache.is_path_unaccessed(
+            parse_selector("endpoints.comments")
+        )  # SET access
+        assert cache.is_path_unaccessed(
+            parse_selector("endpoints.auth")
+        )  # Not accessed
+
+        # Verify access counts
+        assert isinstance(cache.cache.value, dict)
+        endpoints_entry = cache.cache.value["endpoints"]
+        assert isinstance(endpoints_entry.value, dict)
+
+        # Each accessed leaf should have _num_accessed == 1
+        assert endpoints_entry.value["users"]._num_accessed == 1
+        assert endpoints_entry.value["posts"]._num_accessed == 1
+        assert endpoints_entry.value["comments"]._num_accessed == 1
+        assert endpoints_entry.value["auth"]._num_accessed == 0
+
+    def test_leaf_access_tracking_with_all_selectors(self) -> None:
+        """Test access tracking with ALL selectors (*)"""
+        cache = FlakeCache()
+
+        # Insert data with multiple keys at the same level
+        data = {
+            "packages": {
+                "python": "3.11",
+                "nodejs": "18.0",
+                "java": "17.0",
+                "rust": "1.70",
+            }
+        }
+        cache.insert(data, "")
+
+        # Initially all should be unaccessed
+        assert cache.is_path_unaccessed(parse_selector("packages.python"))
+        assert cache.is_path_unaccessed(parse_selector("packages.nodejs"))
+        assert cache.is_path_unaccessed(parse_selector("packages.java"))
+        assert cache.is_path_unaccessed(parse_selector("packages.rust"))
+
+        # Use ALL selector to access all keys at once
+        cache.mark_path_accessed(parse_selector("packages.*"))
+
+        # All keys should now be marked as accessed
+        assert not cache.is_path_unaccessed(parse_selector("packages.python"))
+        assert not cache.is_path_unaccessed(parse_selector("packages.nodejs"))
+        assert not cache.is_path_unaccessed(parse_selector("packages.java"))
+        assert not cache.is_path_unaccessed(parse_selector("packages.rust"))
+
+        # Verify the intermediate node is not marked as accessed
+        assert isinstance(cache.cache.value, dict)
+        packages_entry = cache.cache.value["packages"]
+        assert (
+            packages_entry._num_accessed == 0
+        )  # Intermediate node should not be incremented
+
+        # Verify each leaf has been accessed exactly once
+        assert isinstance(packages_entry.value, dict)
+        assert packages_entry.value["python"]._num_accessed == 1
+        assert packages_entry.value["nodejs"]._num_accessed == 1
+        assert packages_entry.value["java"]._num_accessed == 1
+        assert packages_entry.value["rust"]._num_accessed == 1
+
+    def test_leaf_access_tracking_all_selectors_nested(self) -> None:
+        """Test ALL selectors with nested paths"""
+        cache = FlakeCache()
+
+        # Insert nested data
+        data = {
+            "environments": {
+                "dev": {"database": "dev-db", "cache": "dev-cache"},
+                "prod": {"database": "prod-db", "cache": "prod-cache"},
+                "test": {"database": "test-db", "cache": "test-cache"},
+            }
+        }
+        cache.insert(data, "")
+
+        # Use ALL selector at different nesting levels
+        cache.mark_path_accessed(parse_selector("environments.*.database"))
+
+        # All database leaves should be accessed
+        assert not cache.is_path_unaccessed(parse_selector("environments.dev.database"))
+        assert not cache.is_path_unaccessed(
+            parse_selector("environments.prod.database")
+        )
+        assert not cache.is_path_unaccessed(
+            parse_selector("environments.test.database")
+        )
+
+        # Cache leaves should not be accessed
+        assert cache.is_path_unaccessed(parse_selector("environments.dev.cache"))
+        assert cache.is_path_unaccessed(parse_selector("environments.prod.cache"))
+        assert cache.is_path_unaccessed(parse_selector("environments.test.cache"))
+
+    def test_leaf_access_tracking_all_selectors_is_path_unaccessed(self) -> None:
+        """Test is_path_unaccessed with ALL selectors"""
+        cache = FlakeCache()
+
+        # Insert data
+        data = {"services": {"web": "nginx", "api": "fastapi", "worker": "celery"}}
+        cache.insert(data, "")
+
+        # Initially, ALL selector path should be unaccessed
+        assert cache.is_path_unaccessed(parse_selector("services.*"))
+
+        # Access one of the keys individually
+        cache.mark_path_accessed(parse_selector("services.web"))
+
+        # Now the ALL selector should be considered accessed (since at least one key is accessed)
+        assert not cache.is_path_unaccessed(parse_selector("services.*"))
+
+    def test_leaf_access_tracking_comprehensive_mixed_selectors(self) -> None:
+        """Comprehensive test combining ALL, SET, individual, and list selectors"""
+        cache = FlakeCache()
+
+        # Insert complex nested data structure
+        data = {
+            "cluster": {
+                "nodes": {
+                    "master": {"cpu": 4, "memory": "8GB", "disk": "100GB"},
+                    "worker1": {"cpu": 2, "memory": "4GB", "disk": "50GB"},
+                    "worker2": {"cpu": 2, "memory": "4GB", "disk": "50GB"},
+                },
+                "config": {
+                    "network": "10.0.0.0/8",
+                    "storage": "nfs",
+                    "monitoring": "prometheus",
+                },
+                "services": ["api", "web", "database", "cache"],
+                "metadata": {"version": "1.0", "created": "2023-01-01"},
+            }
+        }
+        cache.insert(data, "")
+
+        # Phase 1: Use individual selector
+        cache.mark_path_accessed(parse_selector("cluster.config.network"))
+        assert not cache.is_path_unaccessed(parse_selector("cluster.config.network"))
+        assert cache.is_path_unaccessed(parse_selector("cluster.config.storage"))
+
+        # Phase 2: Use SET selector
+        cache.mark_path_accessed(parse_selector("cluster.nodes.{master,worker1}.cpu"))
+        assert not cache.is_path_unaccessed(parse_selector("cluster.nodes.master.cpu"))
+        assert not cache.is_path_unaccessed(parse_selector("cluster.nodes.worker1.cpu"))
+        assert cache.is_path_unaccessed(parse_selector("cluster.nodes.worker2.cpu"))
+
+        # Phase 3: Use ALL selector
+        cache.mark_path_accessed(parse_selector("cluster.metadata.*"))
+        assert not cache.is_path_unaccessed(parse_selector("cluster.metadata.version"))
+        assert not cache.is_path_unaccessed(parse_selector("cluster.metadata.created"))
+
+        # Phase 4: Use list selector
+        cache.mark_path_accessed(parse_selector("cluster.services.1"))  # "web"
+        assert not cache.is_path_unaccessed(parse_selector("cluster.services.1"))
+        assert cache.is_path_unaccessed(parse_selector("cluster.services.0"))  # "api"
+
+        # Phase 5: Complex combination - ALL selector followed by SET selector
+        cache.mark_path_accessed(parse_selector("cluster.nodes.*.{memory,disk}"))
+
+        # All memory and disk leaves should be accessed
+        assert not cache.is_path_unaccessed(
+            parse_selector("cluster.nodes.master.memory")
+        )
+        assert not cache.is_path_unaccessed(parse_selector("cluster.nodes.master.disk"))
+        assert not cache.is_path_unaccessed(
+            parse_selector("cluster.nodes.worker1.memory")
+        )
+        assert not cache.is_path_unaccessed(
+            parse_selector("cluster.nodes.worker1.disk")
+        )
+        assert not cache.is_path_unaccessed(
+            parse_selector("cluster.nodes.worker2.memory")
+        )
+        assert not cache.is_path_unaccessed(
+            parse_selector("cluster.nodes.worker2.disk")
+        )
+
+        # Verify intermediate nodes are not accessed
+        assert isinstance(cache.cache.value, dict)
+        cluster_entry = cache.cache.value["cluster"]
+        assert cluster_entry._num_accessed == 0
+
+        assert isinstance(cluster_entry.value, dict)
+        nodes_entry = cluster_entry.value["nodes"]
+        assert nodes_entry._num_accessed == 0
+
+        config_entry = cluster_entry.value["config"]
+        assert config_entry._num_accessed == 0
+
+        # Verify specific leaf access counts
+        assert isinstance(nodes_entry.value, dict)
+        master_entry = nodes_entry.value["master"]
+        assert master_entry._num_accessed == 0  # Intermediate node
+
+        assert isinstance(master_entry.value, dict)
+        # CPU was accessed once via SET selector
+        assert master_entry.value["cpu"]._num_accessed == 1
+        # Memory and disk were accessed once via ALL+SET combination
+        assert master_entry.value["memory"]._num_accessed == 1
+        assert master_entry.value["disk"]._num_accessed == 1
+
+        # Verify that mixed selector patterns work correctly
+        # Test a complex query that should be unaccessed
+        assert cache.is_path_unaccessed(
+            parse_selector("cluster.config.{storage,monitoring}")
+        )
+
+        # Test a complex query that should be accessed
+        assert not cache.is_path_unaccessed(
+            parse_selector("cluster.nodes.{master,worker2}.memory")
+        )

pkgs/clan-cli/clan_lib/import_utils/import_utils_test.py

@@ -22,7 +22,13 @@ def test_import_with_source(tmp_path: Path) -> None:
     test_module_path = module_dir / "test_tech.py"
     test_module_path.write_text(
         dedent("""
-        from clan_lib.network.network import NetworkTechnologyBase
+        from clan_lib.network.network import NetworkTechnologyBase, Peer, Network
+        from contextlib import contextmanager
+        from collections.abc import Iterator
+        from typing import TYPE_CHECKING
+
+        if TYPE_CHECKING:
+            from clan_lib.ssh.remote import Remote
 
         class NetworkTechnology(NetworkTechnologyBase):
             def __init__(self, source):
@@ -31,6 +37,17 @@ def test_import_with_source(tmp_path: Path) -> None:
 
             def is_running(self) -> bool:
                 return True
+
+            def remote(self, peer: Peer) -> "Remote":
+                from clan_lib.ssh.remote import Remote
+                return Remote(host=peer.host)
+
+            def ping(self, peer: Peer) -> None | float:
+                return 0.1
+
+            @contextmanager
+            def connection(self, network: Network) -> Iterator[Network]:
+                yield network
     """)
     )
 
@@ -57,7 +74,7 @@ def test_import_with_source(tmp_path: Path) -> None:
         assert instance.source.module_name == "test_module.test_tech"
         assert instance.source.file_path.name == "test_tech.py"
         assert instance.source.object_name == "NetworkTechnology"
-        assert instance.source.line_number == 4  # Line where class is defined
+        assert instance.source.line_number == 10  # Line where class is defined
 
         # Test string representations
         str_repr = str(instance)
@@ -81,7 +98,13 @@ def test_import_with_source_with_args() -> None:
     with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
         f.write(
             dedent("""
-            from clan_lib.network.network import NetworkTechnologyBase
+            from clan_lib.network.network import NetworkTechnologyBase, Peer, Network
+            from contextlib import contextmanager
+            from collections.abc import Iterator
+            from typing import TYPE_CHECKING
+
+            if TYPE_CHECKING:
+                from clan_lib.ssh.remote import Remote
 
             class NetworkTechnology(NetworkTechnologyBase):
                 def __init__(self, source, extra_arg, keyword_arg=None):
@@ -91,6 +114,17 @@ def test_import_with_source_with_args() -> None:
 
                 def is_running(self) -> bool:
                     return False
+
+                def remote(self, peer: Peer) -> "Remote":
+                    from clan_lib.ssh.remote import Remote
+                    return Remote(host=peer.host)
+
+                def ping(self, peer: Peer) -> None | float:
+                    return None
+
+                @contextmanager
+                def connection(self, network: Network) -> Iterator[Network]:
+                    yield network
         """)
         )
         temp_file = Path(f.name)

pkgs/clan-cli/clan_lib/machines/install.py

@@ -133,16 +133,15 @@ def run_machine_install(opts: InstallOptions, target_host: Remote) -> None:
         cmd.extend(opts.machine.flake.nix_options or [])
 
         cmd.append(target_host.target)
-        if target_host.tor_socks:
+        if target_host.socks_port:
-            # nix copy does not support tor socks proxy
+            # nix copy does not support socks5 proxy, use wrapper command
-            # cmd.append("--ssh-option")
+            wrapper_cmd = target_host.socks_wrapper or ["torify"]
-            # cmd.append("ProxyCommand=nc -x 127.0.0.1:9050 -X 5 %h %p")
             cmd = nix_shell(
                 [
                     "nixos-anywhere",
-                    "tor",
+                    *wrapper_cmd,
                 ],
-                ["torify", *cmd],
+                [*wrapper_cmd, *cmd],
             )
         else:
             cmd = nix_shell(

pkgs/clan-cli/clan_lib/network/__init__.py

@@ -0,0 +1,3 @@
+from .network import Network, NetworkTechnologyBase, Peer
+
+__all__ = ["Network", "NetworkTechnologyBase", "Peer"]

pkgs/clan-cli/clan_lib/network/check.py

@@ -0,0 +1,131 @@
+import logging
+from dataclasses import dataclass
+
+from clan_lib.api import API
+from clan_lib.cmd import ClanCmdError, ClanCmdTimeoutError, RunOpts, run
+from clan_lib.errors import ClanError  # Assuming these are available
+from clan_lib.ssh.remote import Remote
+
+cmdlog = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class ConnectionOptions:
+    timeout: int = 20
+
+
+@API.register
+def check_machine_ssh_login(
+    remote: Remote, opts: ConnectionOptions | None = None
+) -> None:
+    """Checks if a remote machine is reachable via SSH by attempting to run a simple command.
+    Args:
+        remote (Remote): The remote host to check for SSH login.
+        opts (ConnectionOptions, optional): Connection options such as timeout.
+            If not provided, default values are used.
+    Usage:
+        result = check_machine_ssh_login(remote)
+        if result.ok:
+            print("SSH login successful")
+        else:
+            print(f"SSH login failed: {result.reason}")
+    Raises:
+        ClanError: If the SSH login fails.
+    """
+    if opts is None:
+        opts = ConnectionOptions()
+
+    with remote.ssh_control_master() as ssh:
+        try:
+            ssh.run(
+                ["true"],
+                RunOpts(timeout=opts.timeout, needs_user_terminal=True),
+            )
+            return
+        except ClanCmdTimeoutError as e:
+            msg = f"SSH login timeout after {opts.timeout}s"
+            raise ClanError(msg) from e
+        except ClanCmdError as e:
+            if "Host key verification failed." in e.cmd.stderr:
+                raise ClanError(e.cmd.stderr.strip()) from e
+            msg = f"SSH login failed: {e}"
+            raise ClanError(msg) from e
+
+
+@API.register
+def check_machine_ssh_reachable(
+    remote: Remote, opts: ConnectionOptions | None = None
+) -> None:
+    """
+    Checks if a remote machine is reachable via SSH by attempting to open a TCP connection
+    to the specified address and port.
+    Args:
+        remote (Remote): The remote host to check for SSH reachability.
+        opts (ConnectionOptions, optional): Connection options such as timeout.
+            If not provided, default values are used.
+    Returns:
+        CheckResult: An object indicating whether the SSH port is reachable (`ok=True`) or not (`ok=False`),
+        and a reason if the check failed.
+    Usage:
+        result = check_machine_ssh_reachable(remote)
+        if result.ok:
+            print("SSH port is reachable")
+            print(f"SSH port is not reachable: {result.reason}")
+    """
+    if opts is None:
+        opts = ConnectionOptions()
+
+    cmdlog.debug(
+        f"Checking SSH reachability for {remote.target} on port {remote.port or 22}",
+    )
+
+    # Use ssh with ProxyCommand to check through SOCKS5
+    cmd = [
+        "ssh",
+    ]
+
+    # If using SOCKS5 proxy, add ProxyCommand
+    if remote.socks_port:
+        cmd.extend(
+            [
+                "-o",
+                f"ProxyCommand=nc -X 5 -x localhost:{remote.socks_port} %h %p",
+            ]
+        )
+
+    cmd.extend(
+        [
+            "-o",
+            "BatchMode=yes",
+            "-o",
+            "StrictHostKeyChecking=no",
+            "-o",
+            "UserKnownHostsFile=/dev/null",
+            "-o",
+            f"ConnectTimeout={opts.timeout}",
+            "-o",
+            "PreferredAuthentications=none",
+            "-p",
+            str(remote.port or 22),
+            f"dummy@{remote.address.strip()}",
+            "true",
+        ]
+    )
+
+    try:
+        res = run(cmd, options=RunOpts(timeout=opts.timeout, check=False))
+
+        # SSH will fail with authentication error if server is reachable
+        # Check for SSH-related errors in stderr
+        if (
+            "Permission denied" in res.stderr
+            or "No supported authentication" in res.stderr
+        ):
+            return  # Server is reachable, auth failed as expected
+
+        msg = "Connection failed: SSH server not reachable"
+        raise ClanError(msg)
+
+    except ClanCmdTimeoutError as e:
+        msg = f"Connection timeout after {opts.timeout}s"
+        raise ClanError(msg) from e

pkgs/clan-cli/clan_lib/network/direct.py

@@ -1,9 +1,47 @@
-from clan_lib.network.network import NetworkTechnologyBase
+import logging
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+
+from clan_lib.errors import ClanError
+from clan_lib.network.network import Network, NetworkTechnologyBase, Peer
+from clan_lib.ssh.remote import Remote
+
+log = logging.getLogger(__name__)
 
 
+@dataclass(frozen=True)
 class NetworkTechnology(NetworkTechnologyBase):
     """Direct network connection technology - checks SSH connectivity"""
 
     def is_running(self) -> bool:
         """Direct connections are always 'running' as they don't require a daemon"""
         return True
+
+    def ping(self, peer: Peer) -> None | float:
+        if self.is_running():
+            try:
+                # Parse the peer's host address to create a Remote object, use peer here since we don't have the machine_name here
+                remote = Remote.from_ssh_uri(machine_name="peer", address=peer.host)
+
+                # Use the existing SSH reachability check
+                now = time.time()
+
+                remote.check_machine_ssh_reachable()
+
+                return (time.time() - now) * 1000
+
+            except ClanError as e:
+                log.debug(f"Error checking peer {peer.host}: {e}")
+                return None
+        return None
+
+    @contextmanager
+    def connection(self, network: Network) -> Iterator[Network]:
+        # direct connections are always online and don't use SOCKS, so we just return the original network
+        # TODO maybe we want to setup jumphosts for network access? but sounds complicated
+        yield network
+
+    def remote(self, peer: Peer) -> Remote:
+        return Remote.from_ssh_uri(machine_name=peer.name, address=peer.host)

pkgs/clan-cli/clan_lib/network/network.py

@@ -1,23 +1,27 @@
 import logging
 import textwrap
-import time
 from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from contextlib import contextmanager
 from dataclasses import dataclass
 from functools import cached_property
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from clan_cli.vars.get import get_machine_var
+
 from clan_lib.errors import ClanError
 from clan_lib.flake import Flake
 from clan_lib.import_utils import ClassSource, import_with_source
-from clan_lib.ssh.parse import parse_ssh_uri
+
-from clan_lib.ssh.remote import Remote, check_machine_ssh_reachable
+if TYPE_CHECKING:
+    from clan_lib.ssh.remote import Remote
 
 log = logging.getLogger(__name__)
 
 
 @dataclass(frozen=True)
 class Peer:
+    name: str
     _host: dict[str, str | dict[str, str]]
     flake: Flake
 
@@ -30,7 +34,9 @@ class Peer:
             machine_name = _var["machine"]
             generator = _var["generator"]
             var = get_machine_var(
-                str(self.flake),
+                str(
+                    self.flake
+                ),  # TODO we should really pass the flake instance here instead of a str representation
                 machine_name,
                 f"{generator}/{_var['file']}",
             )
@@ -50,35 +56,6 @@ class Peer:
         raise ClanError(msg)
 
 
-@dataclass
-class NetworkTechnologyBase(ABC):
-    source: ClassSource
-
-    @abstractmethod
-    def is_running(self) -> bool:
-        pass
-
-    # TODO this will depend on the network implementation if we do user networking at some point, so it should be abstractmethod
-    def ping(self, peer: Peer) -> None | float:
-        if self.is_running():
-            try:
-                # Parse the peer's host address to create a Remote object, use peer here since we don't have the machine_name here
-                remote = parse_ssh_uri(machine_name="peer", address=peer.host)
-
-                # Use the existing SSH reachability check
-                now = time.time()
-                result = check_machine_ssh_reachable(remote)
-
-                if result.ok:
-                    return (time.time() - now) * 1000
-                return None
-
-            except Exception as e:
-                log.debug(f"Error checking peer {peer.host}: {e}")
-                return None
-        return None
-
-
 @dataclass(frozen=True)
 class Network:
     peers: dict[str, Peer]
@@ -86,7 +63,7 @@ class Network:
     priority: int = 1000
 
     @cached_property
-    def module(self) -> NetworkTechnologyBase:
+    def module(self) -> "NetworkTechnologyBase":
         res = import_with_source(
             self.module_name,
             "NetworkTechnology",
@@ -100,15 +77,49 @@ class Network:
     def ping(self, peer: str) -> float | None:
         return self.module.ping(self.peers[peer])
 
+    def remote(self, peer: str) -> "Remote":
+        # TODO raise exception if peer is not in peers
+        return self.module.remote(self.peers[peer])
+
+
+@dataclass(frozen=True)
+class NetworkTechnologyBase(ABC):
+    source: ClassSource
+
+    @abstractmethod
+    def is_running(self) -> bool:
+        pass
+
+    @abstractmethod
+    def remote(self, peer: Peer) -> "Remote":
+        pass
+
+    @abstractmethod
+    def ping(self, peer: Peer) -> None | float:
+        pass
+
+    @contextmanager
+    @abstractmethod
+    def connection(self, network: Network) -> Iterator[Network]:
+        pass
+
 
 def networks_from_flake(flake: Flake) -> dict[str, Network]:
+    # TODO more precaching, for example for vars
+    flake.precache(
+        [
+            "clan.exports.instances.*.networking",
+        ]
+    )
     networks: dict[str, Network] = {}
     networks_ = flake.select("clan.exports.instances.*.networking")
     for network_name, network in networks_.items():
         if network:
             peers: dict[str, Peer] = {}
             for _peer in network["peers"].values():
-                peers[_peer["name"]] = Peer(_host=_peer["host"], flake=flake)
+                peers[_peer["name"]] = Peer(
+                    name=_peer["name"], _host=_peer["host"], flake=flake
+                )
             networks[network_name] = Network(
                 peers=peers,
                 module_name=network["module"],
@@ -117,17 +128,14 @@ def networks_from_flake(flake: Flake) -> dict[str, Network]:
     return networks
 
 
-def get_best_remote(machine_name: str, networks: dict[str, Network]) -> Remote | None:
+def get_best_network(machine_name: str, networks: dict[str, Network]) -> Network | None:
     for network_name, network in sorted(
         networks.items(), key=lambda network: -network[1].priority
     ):
         if machine_name in network.peers:
             if network.is_running() and network.ping(machine_name):
                 print(f"connecting via {network_name}")
-                return Remote.from_ssh_uri(
+                return network
-                    machine_name=machine_name,
-                    address=network.peers[machine_name].host,
-                )
     return None
 
 
@@ -137,20 +145,19 @@ def get_network_overview(networks: dict[str, Network]) -> dict:
         result[network_name] = {}
         result[network_name]["status"] = None
         result[network_name]["peers"] = {}
-        network_online = False
         module = network.module
         log.debug(f"Using network module: {module}")
         if module.is_running():
             result[network_name]["status"] = True
-            network_online = True
+        else:
-        for peer_name in network.peers:
+            with module.connection(network) as network:
-            if network_online:
+                for peer_name in network.peers:
-                try:
+                    try:
-                    result[network_name]["peers"][peer_name] = network.ping(peer_name)
+                        result[network_name]["peers"][peer_name] = network.ping(
-                except ClanError:
+                            peer_name
-                    log.warning(
+                        )
-                        f"getting host for machine: {peer_name} in network: {network_name} failed"
+                    except ClanError:
-                    )
+                        log.warning(
-            else:
+                            f"getting host for machine: {peer_name} in network: {network_name} failed"
-                result[network_name]["peers"][peer_name] = None
+                        )
     return result

pkgs/clan-cli/clan_lib/network/tor.py

@@ -1,20 +0,0 @@
-from urllib.error import HTTPError
-from urllib.request import urlopen
-
-from .network import NetworkTechnologyBase
-
-
-class NetworkTechnology(NetworkTechnologyBase):
-    socks_port: int
-    command_port: int
-
-    def is_running(self) -> bool:
-        """Check if Tor is running by sending HTTP request to SOCKS port."""
-        try:
-            response = urlopen("http://127.0.0.1:9050", timeout=5)
-            content = response.read().decode("utf-8", errors="ignore")
-            return "tor" in content.lower()
-        except HTTPError as e:
-            return "tor" in str(e).lower()
-        except Exception:
-            return False

pkgs/clan-cli/clan_lib/network/tor/__init__.py

@@ -0,0 +1,62 @@
+import logging
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from clan_lib.errors import ClanError
+from clan_lib.network import Network, NetworkTechnologyBase, Peer
+from clan_lib.network.tor.lib import is_tor_running, spawn_tor
+
+if TYPE_CHECKING:
+    from clan_lib.ssh.remote import Remote
+
+
+log = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class NetworkTechnology(NetworkTechnologyBase):
+    @property
+    def proxy(self) -> int:
+        """Return the SOCKS5 proxy port for this network technology."""
+        return 9050
+
+    def is_running(self) -> bool:
+        """Check if Tor is running by sending HTTP request to SOCKS port."""
+        return is_tor_running(self.proxy)
+
+    def ping(self, peer: Peer) -> None | float:
+        if self.is_running():
+            try:
+                remote = self.remote(peer)
+
+                # Use the existing SSH reachability check
+                now = time.time()
+                remote.check_machine_ssh_reachable()
+
+                return (time.time() - now) * 1000
+
+            except ClanError as e:
+                log.debug(f"Error checking peer {peer.host}: {e}")
+                return None
+        return None
+
+    @contextmanager
+    def connection(self, network: Network) -> Iterator[Network]:
+        if self.is_running():
+            yield network
+        else:
+            with spawn_tor() as _:
+                yield network
+
+    def remote(self, peer: Peer) -> "Remote":
+        from clan_lib.ssh.remote import Remote
+
+        return Remote(
+            address=peer.host,
+            command_prefix=peer.name,
+            socks_port=self.proxy,
+            socks_wrapper=["torify"],
+        )

pkgs/clan-cli/clan_lib/network/tor/lib.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+import logging
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from subprocess import Popen
+
+from clan_lib.errors import ClanError
+from clan_lib.nix import nix_shell
+
+log = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class TorTarget:
+    onion: str
+    port: int
+
+
+def is_tor_running(proxy_port: int | None = None) -> bool:
+    """Checks if Tor is online."""
+    if proxy_port is None:
+        proxy_port = 9050
+    try:
+        tor_online_test(proxy_port)
+    except ClanError as err:
+        log.debug(f"Tor is not running: {err}")
+        return False
+
+    return True
+
+
+# TODO: Move this to network technology tor module
+@contextmanager
+def spawn_tor() -> Iterator[None]:
+    """
+    Spawns a Tor process using `nix-shell` if Tor is not already running.
+    """
+
+    # Check if Tor is already running
+    if is_tor_running():
+        log.info("Tor is running")
+        return
+    cmd_args = ["tor", "--HardwareAccel", "1"]
+    packages = ["tor"]
+    cmd = nix_shell(packages, cmd_args)
+    process = Popen(cmd)
+    try:
+        while not is_tor_running():
+            log.debug("Waiting for Tor to start...")
+            time.sleep(0.2)
+        log.info("Tor is now running")
+        yield
+    finally:
+        log.info("Terminating Tor process...")
+        process.terminate()
+        process.wait()
+        log.info("Tor process terminated")
+
+
+@dataclass(frozen=True)
+class TorCheck:
+    onion: str
+    expected_content: str
+    port: int = 80
+
+
+def tor_online_test(proxy_port: int) -> None:
+    """
+    Tests if Tor is online by checking if we can establish a SOCKS5 connection.
+    """
+    import socket
+
+    # Try to establish a SOCKS5 handshake with the Tor proxy
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.settimeout(2.0)  # Short timeout for local connection
+
+    try:
+        # Connect to the SOCKS5 proxy
+        sock.connect(("localhost", proxy_port))
+
+        # Send SOCKS5 handshake
+        sock.sendall(b"\x05\x01\x00")  # SOCKS5, 1 auth method, no auth
+        response = sock.recv(2)
+
+        # Check if we got a valid SOCKS5 response
+        if response == b"\x05\x00":  # SOCKS5, no auth accepted
+            return
+        msg = f"Invalid SOCKS5 response from Tor: {response.hex()}"
+        raise ClanError(msg)
+
+    except (TimeoutError, ConnectionRefusedError, OSError) as e:
+        msg = f"Cannot connect to Tor SOCKS5 proxy at localhost:{proxy_port}: {e}"
+        raise ClanError(msg) from e
+    finally:
+        sock.close()

pkgs/clan-cli/clan_lib/ssh/remote.py

@@ -2,19 +2,17 @@ import ipaddress
 import logging
 import os
 import shlex
-import socket
 import subprocess
 import sys
-import time
 from collections.abc import Iterator
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 from pathlib import Path
 from shlex import quote
 from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING
 
-from clan_lib.api import API
+from clan_lib.cmd import CmdOut, RunOpts, run
-from clan_lib.cmd import ClanCmdError, ClanCmdTimeoutError, CmdOut, RunOpts, run
 from clan_lib.colors import AnsiColor
 from clan_lib.errors import ClanError, indent_command  # Assuming these are available
 from clan_lib.nix import nix_shell
@@ -22,6 +20,9 @@ from clan_lib.ssh.host_key import HostKeyCheck, hostkey_to_ssh_opts
 from clan_lib.ssh.parse import parse_ssh_uri
 from clan_lib.ssh.sudo_askpass_proxy import SudoAskpassProxy
 
+if TYPE_CHECKING:
+    from clan_lib.network.check import ConnectionOptions
+
 cmdlog = logging.getLogger(__name__)
 
 # Seconds until a message is printed when _run produces no output.
@@ -40,7 +41,8 @@ class Remote:
     host_key_check: HostKeyCheck = "ask"
     verbose_ssh: bool = False
     ssh_options: dict[str, str] = field(default_factory=dict)
-    tor_socks: bool = False
+    socks_port: int | None = None
+    socks_wrapper: list[str] | None = None
 
     _control_path_dir: Path | None = None
     _askpass_path: str | None = None
@@ -60,7 +62,8 @@ class Remote:
         host_key_check: HostKeyCheck | None = None,
         private_key: Path | None = None,
         password: str | None = None,
-        tor_socks: bool | None = None,
+        socks_port: int | None = None,
+        socks_wrapper: list[str] | None = None,
         command_prefix: str | None = None,
         port: int | None = None,
         ssh_options: dict[str, str] | None = None,
@@ -81,7 +84,10 @@ class Remote:
             ),
             verbose_ssh=self.verbose_ssh,
             ssh_options=ssh_options or self.ssh_options,
-            tor_socks=tor_socks if tor_socks is not None else self.tor_socks,
+            socks_port=socks_port if socks_port is not None else self.socks_port,
+            socks_wrapper=socks_wrapper
+            if socks_wrapper is not None
+            else self.socks_wrapper,
             _control_path_dir=self._control_path_dir,
             _askpass_path=self._askpass_path,
         )
@@ -152,7 +158,7 @@ class Remote:
                 host_key_check=self.host_key_check,
                 verbose_ssh=self.verbose_ssh,
                 ssh_options=self.ssh_options,
-                tor_socks=self.tor_socks,
+                socks_port=self.socks_port,
                 _control_path_dir=Path(temp_dir),
                 _askpass_path=self._askpass_path,
             )
@@ -220,7 +226,7 @@ class Remote:
                 host_key_check=self.host_key_check,
                 verbose_ssh=self.verbose_ssh,
                 ssh_options=self.ssh_options,
-                tor_socks=self.tor_socks,
+                socks_port=self.socks_port,
                 _control_path_dir=self._control_path_dir,
                 _askpass_path=askpass_path,
             )
@@ -373,10 +379,13 @@ class Remote:
         if tty:
             current_ssh_opts.extend(["-t"])
 
-        if self.tor_socks:
+        if self.socks_port:
             packages.append("netcat")
             current_ssh_opts.extend(
-                ["-o", "ProxyCommand=nc -x 127.0.0.1:9050 -X 5 %h %p"]
+                [
+                    "-o",
+                    f"ProxyCommand=nc -x localhost:{self.socks_port} -X 5 %h %p",
+                ]
             )
 
         cmd = [
@@ -447,100 +456,14 @@ class Remote:
         if self.password:
             self.check_sshpass_errorcode(res)
 
-    def check_machine_ssh_reachable(self) -> bool:
+    def check_machine_ssh_reachable(
-        return check_machine_ssh_reachable(self).ok
+        self, opts: "ConnectionOptions | None" = None
+    ) -> None:
+        from clan_lib.network.check import check_machine_ssh_reachable
 
+        return check_machine_ssh_reachable(self, opts)
 
-@dataclass(frozen=True)
+    def check_machine_ssh_login(self) -> None:
-class ConnectionOptions:
+        from clan_lib.network.check import check_machine_ssh_login
-    timeout: int = 2
-    retries: int = 5
 
-
+        return check_machine_ssh_login(self)
-@dataclass
-class CheckResult:
-    ok: bool
-    reason: str | None = None
-
-
-@API.register
-def check_machine_ssh_login(
-    remote: Remote, opts: ConnectionOptions | None = None
-) -> CheckResult:
-    """Checks if a remote machine is reachable via SSH by attempting to run a simple command.
-    Args:
-        remote (Remote): The remote host to check for SSH login.
-        opts (ConnectionOptions, optional): Connection options such as timeout and number of retries.
-            If not provided, default values are used.
-    Returns:
-        CheckResult: An object indicating whether the SSH login is successful (`ok=True`) or not (`ok=False`),
-        and a reason if the check failed.
-    Usage:
-        result = check_machine_ssh_login(remote)
-        if result.ok:
-            print("SSH login successful")
-        else:
-            print(f"SSH login failed: {result.reason}")
-    """
-    if opts is None:
-        opts = ConnectionOptions()
-
-    for _ in range(opts.retries):
-        with remote.ssh_control_master() as ssh:
-            try:
-                res = ssh.run(
-                    ["true"],
-                    RunOpts(timeout=opts.timeout, needs_user_terminal=True),
-                )
-                return CheckResult(True)
-            except ClanCmdTimeoutError:
-                pass
-            except ClanCmdError as e:
-                if "Host key verification failed." in e.cmd.stderr:
-                    raise ClanError(res.stderr.strip()) from e
-            else:
-                time.sleep(opts.timeout)
-
-    return CheckResult(False, f"failed after {opts.retries} attempts")
-
-
-@API.register
-def check_machine_ssh_reachable(
-    remote: Remote, opts: ConnectionOptions | None = None
-) -> CheckResult:
-    """
-    Checks if a remote machine is reachable via SSH by attempting to open a TCP connection
-    to the specified address and port.
-    Args:
-        remote (Remote): The remote host to check for SSH reachability.
-        opts (ConnectionOptions, optional): Connection options such as timeout and number of retries.
-            If not provided, default values are used.
-    Returns:
-        CheckResult: An object indicating whether the SSH port is reachable (`ok=True`) or not (`ok=False`),
-        and a reason if the check failed.
-    Usage:
-        result = check_machine_ssh_reachable(remote)
-        if result.ok:
-            print("SSH port is reachable")
-            print(f"SSH port is not reachable: {result.reason}")
-    """
-    if opts is None:
-        opts = ConnectionOptions()
-
-    cmdlog.debug(
-        f"Checking SSH reachability for {remote.target} on port {remote.port or 22}",
-    )
-
-    address_family = socket.AF_INET6 if remote.is_ipv6() else socket.AF_INET
-    for _ in range(opts.retries):
-        with socket.socket(address_family, socket.SOCK_STREAM) as sock:
-            sock.settimeout(opts.timeout)
-            try:
-                sock.connect((remote.address, remote.port or 22))
-                return CheckResult(True)
-            except (TimeoutError, OSError):
-                pass
-            else:
-                time.sleep(opts.timeout)
-
-    return CheckResult(False, f"failed after {opts.retries} attempts")

pkgs/clan-cli/clan_lib/tests/test_create.py

@@ -32,7 +32,7 @@ from clan_lib.nix_models.clan import InventoryMachineDeploy as MachineDeploy
 from clan_lib.persist.inventory_store import InventoryStore
 from clan_lib.persist.util import set_value_by_path
 from clan_lib.services.modules import list_service_modules
-from clan_lib.ssh.remote import Remote, check_machine_ssh_login
+from clan_lib.ssh.remote import Remote
 from clan_lib.templates.disk import hw_main_disk_options, set_machine_disk_schema
 
 log = logging.getLogger(__name__)
@@ -189,9 +189,9 @@ def test_clan_create_api(
     target_host = machine.target_host().override(
         private_key=private_key, host_key_check="none"
     )
-    assert check_machine_ssh_login(target_host).ok, (
+
-        f"Machine {machine.name} is not online"
+    target_host.check_machine_ssh_reachable()
-    )
+    target_host.check_machine_ssh_login()
 
     ssh_keys = [
         SSHKeyPair(

pyproject.toml

@@ -52,6 +52,7 @@ lint.ignore = [
   "TRY301",
   "TRY300",
   "ANN401",
+  "SLF001",
   "RUF100",
   "TRY400",
   "E402",