Add server and client module

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4512

README.md

@@ -24,7 +24,7 @@ If you're new to Clan and eager to dive in, start with our quickstart guide and
 
 In the Clan ecosystem, security is paramount. Learn how to handle secrets effectively:
 
-- **Secrets Management**: Securely manage secrets by consulting [secrets](https://docs.clan.lol/guides/getting-started/secrets/)<!-- [secrets.md](docs/site/guides/getting-started/secrets.md) -->.
+- **Secrets Management**: Securely manage secrets by consulting [Vars](https://docs.clan.lol/concepts/generators/)<!-- [secrets.md](docs/site/concepts/generators.md) -->.
 
 ### Contributing to Clan
 

clanModules/borgbackup-static/README.md

@@ -4,7 +4,7 @@ description = "Statically configure borgbackup with sane defaults."
 !!! Danger "Deprecated"
     Use [borgbackup](borgbackup.md) instead.
 
-    Don't use borgbackup-static through [inventory](../../guides/inventory.md).
+    Don't use borgbackup-static through [inventory](../../concepts/inventory.md).
 
 This module implements the `borgbackup` backend and implements sane defaults
 for backup management through `borgbackup` for members of the clan.

clanModules/root-password/README.md

@@ -12,7 +12,7 @@ After the system was installed/deployed the following command can be used to dis
 clan vars get [machine_name] root-password/root-password
 ```
 
-See also: [Vars](../../guides/vars-backend.md)
+See also: [Vars](../../concepts/generators.md)
 
 To regenerate the password run:
 ```

clanModules/user-password/README.md

@@ -16,7 +16,7 @@ After the system was installed/deployed the following command can be used to dis
 clan vars get [machine_name] root-password/root-password
 ```
 
-See also: [Vars](../../guides/vars-backend.md)
+See also: [Vars](../../concepts/generators.md)
 
 To regenerate the password run:
 ```

clanServices/borgbackup/README.md

@@ -1,9 +1,59 @@
-BorgBackup (short: Borg) gives you:
+## Usage
 
-- Space efficient storage of backups.
-- Secure, authenticated encryption.
-- Compression: lz4, zstd, zlib, lzma or none.
-- Mountable backups with FUSE.
+```nix
+inventory.instances = {
+  borgbackup = {
+    module = {
+      name = "borgbackup";
+      input = "clan";
+    };
+    roles.client.machines."jon".settings = {
+      destinations."storagebox" = {
+        repo = "username@$hostname:/./borgbackup";
+        rsh = ''ssh -oPort=23 -i /run/secrets/vars/borgbackup/borgbackup.ssh'';
+      };
+    };
+    roles.server.machines = { };
+  };
+};
+```
+
+The input should be named according to your flake input. Jon is configured as a
+client machine with a destination pointing to a Hetzner Storage Box.
+
+## Overview
+
+This guide explains how to set up and manage
+[BorgBackup](https://borgbackup.readthedocs.io/) for secure, efficient backups
+in a clan network. BorgBackup provides:
+
+- Space efficient storage of backups with deduplication
+- Secure, authenticated encryption
+- Compression: lz4, zstd, zlib, lzma or none
+- Mountable backups with FUSE
 - Easy installation on multiple platforms: Linux, macOS, BSD, …
 - Free software (BSD license).
 - Backed by a large and active open-source community.
+
+## Roles
+
+### 1. Client
+
+Clients are machines that create and send backups to various destinations. Each
+client can have multiple backup destinations configured.
+
+### 2. Server
+
+Servers act as backup repositories, receiving and storing backups from client
+machines. They can be dedicated backup servers within your clan network.
+
+## Backup destinations
+
+This service allows you to perform backups to multiple `destinations`.
+Destinations can be:
+
+- **Local**: Local disk storage
+- **Server**: Your own borgbackup server (using the `server` role)
+- **Third-party services**: Such as Hetzner's Storage Box
+
+For a more comprehensive guide on backups look into the guide section.

clanServices/restic/default.nix

@@ -0,0 +1,215 @@
+{ nixpkgs, ... }:
+
+let
+
+  mkResticGenerator = machineName: rec {
+
+    files."restic-rest-pass" = { };
+    files."restic-rest-env" = { };
+    files."restic-repokey" = { };
+
+    share = true;
+
+    validation.name = script;
+
+    runtimeInputs = [ nixpkgs.xkcdpass ];
+
+    script = ''
+      xkcdpass -n 4 -d - > "$out/restic-rest-pass"
+      xkcdpass -n 4 -d - > "$out/restic-repokey"
+      echo "RESTIC_REST_USERNAME=${machineName}" > $out/restic-rest-env
+      echo "RESTIC_REST_PASSWORD=$(cat $out/restic-rest-pass)" >> $out/restic-rest-env
+    '';
+  };
+
+in
+
+{
+  _class = "clan.service";
+  manifest.name = "restic";
+
+  # Define what roles exist
+
+  roles.server = {
+    interface =
+      { lib, ... }:
+      # system ,
+      # {
+      #   ...
+      # }:
+      # # let pkgs = import nixpkgs { inherit system; }; in
+      {
+        # These options can be set via 'roles.server.settings'
+
+        # options.restic.package = lib.mkPackageOption pkgs "restic" { };
+
+        options.directory = lib.mkOption {
+          type = lib.types.str;
+          default = "/var/lib/restic";
+          description = ''
+            The directory where the restic repositories are stored.
+          '';
+        };
+
+      };
+    perInstance =
+      { settings, roles, ... }:
+      {
+        nixosModule =
+          {
+            config,
+            lib,
+            pkgs,
+            ...
+          }:
+          {
+
+            clan.core.vars.generators = {
+
+                restic-cert = rec {
+
+                  files."restic-key" = { };
+                  files."restic-cert".secret = false;
+
+                  share = true;
+
+                  runtimeInputs = with pkgs; [
+                    coreutils
+                    openssl
+                  ];
+
+                  validation.script = script;
+
+                  # TODO openssl will ask for Country, City, etc.
+                  # Can we pass those as prompts?
+
+                  # We use a wildcard cert for *.restic
+                  script = ''
+                    openssl req -newkey rsa:2048 -nodes -x509 \
+                      -keyout $out/restic-key \
+                      -out $out/restic-cert \
+                      -addext "subjectAltName = DNS:*.restic"
+                  '';
+                };
+
+                restic-server = {
+                  files."htpasswd" = { };
+
+                  runtimeInputs = [ pkgs.apacheHttpd ];
+
+                  # depend on all client's generators
+                  dependencies = lib.unique (
+                    map (machine: "restic-${machine}") (lib.attrNames roles.client.machines)
+                  );
+
+                  script = lib.concatMapStringsSep "\n" (machine: ''
+                    cat $in/restic-${machine}/restic-rest-pass | \
+                    htpasswd -B -i -n ${machine} >> $out/htpasswd
+                  '') (lib.attrNames roles.client.machines);
+                };
+
+              }
+              // builtins.listToAttrs (
+                map (machine: {
+                  name = "restic-${machine}";
+                  value = (mkResticGenerator machine);
+                }) (lib.attrNames roles.client.machines)
+              );
+
+            systemd.paths.restic-htpasswd = {
+              description = "Watch restic-rest .htpasswd for changes";
+              pathConfig = {
+                PathChanged = config.clan.core.vars.generators."restic-server".files."htpasswd".path;
+                Unit = "restic-rest-server.service";
+              };
+            };
+
+            systemd.services.restic-rest-server.serviceConfig = {
+
+              # TODO not sure if this is the correct fix
+              RestrictAddressFamilies = lib.mkForce "AF_INET AF_INET6";
+
+              LoadCredential = [
+                "restic-cert:${config.clan.core.vars.generators."restic-cert".files."restic-cert".path}"
+                "restic-key:${config.clan.core.vars.generators."restic-cert".files."restic-key".path}"
+                "htpasswd:${config.clan.core.vars.generators."restic-server".files."htpasswd".path}"
+              ];
+            };
+
+            networking.firewall.allowedTCPPorts = [ 8124 ];
+
+            services.restic.server = rec {
+              enable = true;
+              # prometheus = true;
+              privateRepos = true;
+              listenAddress = "0.0.0.0:8124";
+              dataDir = settings.directory;
+              extraFlags = [
+                "--htpasswd-file=%d/htpasswd"
+                "--listen=${listenAddress}"
+                "--tls"
+                "--tls-cert=%d/restic-cert"
+                "--tls-key=%d/restic-key"
+              ];
+            };
+          };
+
+      };
+  };
+
+  roles.client = { 
+    # TODO
+  };
+  # roles.client = {
+  #     interface = {
+  #         # These options can be set via 'roles.client.settings'
+  #         options.ipRanges = mkOption { type = listOf str; };
+  #     };
+  #
+  #     # Maps over all instances and produces one result per instance.
+  #     perInstance = { instanceName, settings, machine, roles, ... }: {
+  #     # Analog to 'perSystem' of flake-parts.
+  #         # For every instance of this service we will add a nixosModule to a client-machine
+  #         nixosModule = { config, ... }: {
+  #             # Interaction examples what you could do here:
+  #             # - Get some settings of this machine
+  #             # settings.ipRanges
+  #             #
+  #             # - Get all controller names:
+  #             # allControllerNames = lib.attrNames roles.controller.machines
+  #             #
+  #             # - Get all roles of the machine:
+  #             # machine.roles
+  #             #
+  #             # - Get the settings that where applied to a specific controller machine:
+  #             # roles.controller.machines.jon.settings
+  #             #
+  #             # Add one systemd service for every instance
+  #             systemd.services.zerotier-client-${instanceName} = {
+  #                 # ... depend on the '.config' and 'perInstance arguments'
+  #             };
+  #         };
+  #     }
+  # };
+
+  # Maps over all machines and produces one result per machine.
+  # perMachine = { instances, machine, ... }: {
+  # # Analog to 'perSystem' of flake-parts.
+  #     # For every machine of this service we will add exactly one nixosModule to a machine
+  #     nixosModule = { config, ... }: {
+  #         # Interaction examples what you could do here:
+  #         # - Get the name of this machine
+  #         # machine.name
+  #         #
+  #         # - Get all roles of this machine across all instances:
+  #         # machine.roles
+  #         #
+  #         # - Get the settings of a specific instance of a specific machine
+  #         # instances.foo.roles.peer.machines.jon.settings
+  #         #
+  #         # Globally enable something
+  #         networking.enable = true;
+  #     };
+  # };
+
+}

clanServices/restic/flake-module.nix

@@ -0,0 +1,25 @@
+{ self, inputs, ... }:
+let
+  restic-module = import ../restic { inherit (inputs) nixpkgs; };
+in
+{
+  perSystem =
+    { pkgs, ... }:
+    let
+      nixosTestArgs = {
+        inherit pkgs;
+        inherit self;
+      };
+    in
+    {
+      checks = {
+        restic = import ./tests {
+          # inherit (self) clanLib;
+          inherit restic-module
+          pkgs self;
+        } nixosTestArgs;
+      };
+    };
+
+  clan.inventory.modules.restic = restic-module;
+}

clanServices/restic/tests/default.nix

@@ -0,0 +1,81 @@
+{
+  pkgs,
+  self,
+  clanLib,
+  restic-module,
+  ...
+}:
+clanLib.test.makeTestClan {
+  inherit pkgs self;
+  nixosTest = (
+    { ... }:
+    {
+
+      name = "restic";
+
+      clan = {
+
+        /*
+          # Desired inventory for a complete test:
+          inventory = {
+            machines = {
+              # - Two backup servers (destinations)
+              # - Three backup clients:
+              #    - client_1 and client_2 should backup to both servers
+              #    - client_3 should only backup to server_2
+              server_1 = { };
+              server_2 = { };
+              client_1.tags = [ "backup_to1" "backup_to2" ];
+              client_2.tags = [ "backup_to1" "backup_to2" ];
+              client_3.tags = [ "backup_to1" ];
+            };
+
+            services = {
+              # Apply roles based on tags for clients
+              restic.backup_dest1.roles.client.tags = [ "backup_to1" ];
+              restic.backup_dest2.roles.client.tags = [ "backup_to2" ];
+
+              # Apply roles based on machine names for servers
+              restic.backup_dest1.roles.server.machines = ["server_1"];
+              restic.backup_dest2.roles.server.machines = ["server_2"];
+            };
+          };
+        */
+
+        inventory = {
+
+          # Define machines
+          machines.client_machine = { };
+          machines.server_machine = { };
+          modules.restic = restic-module;
+
+          instances.test-restic = {
+
+            module.name = "restic";
+
+            roles.server.machines.server_machine = {
+              # Server settings
+              settings.directory = "/var/lib/restic";
+            };
+
+            roles.client.machines.client_machine = {
+              # Client settings
+            };
+
+          };
+
+        };
+        directory = ./.;
+      };
+
+      # defaults = { };
+      #
+      # nodes = { };
+
+      testScript = ''
+        start_all()
+        server_machine.wait_for_unit("restic-rest-server")
+      '';
+    }
+  );
+}

clanServices/restic/tests/sops/users/admin/key.json

@@ -0,0 +1 @@
+{"publickey": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg", "type": "age"}

clanServices/zerotier/default.nix

@@ -134,9 +134,9 @@
               systemd.services.zerotier-inventory-autoaccept =
                 let
                   machines = uniqueStrings (
-                    (lib.attrNames roles.moon.machines)
-                    ++ (lib.attrNames roles.controller.machines)
-                    ++ (lib.attrNames roles.peer.machines)
+                    (lib.optionals (roles ? moon) (lib.attrNames roles.moon.machines))
+                    ++ (lib.optionals (roles ? controller) (lib.attrNames roles.controller.machines))
+                    ++ (lib.optionals (roles ? peer) (lib.attrNames roles.peer.machines))
                   );
                   networkIps = builtins.foldl' (
                     ips: name:

clanServices/zerotier/tests/eval-tests.nix

@@ -32,6 +32,33 @@ let
         };
       };
     }).config;
+  testFlakeNoMoon =
+    (clanLib.clan {
+      self = { };
+      directory = ./vm;
+
+      machines.jon = {
+        nixpkgs.hostPlatform = "x86_64-linux";
+      };
+      machines.sara = {
+        nixpkgs.hostPlatform = "x86_64-linux";
+      };
+      machines.bam = {
+        nixpkgs.hostPlatform = "x86_64-linux";
+      };
+
+      modules.zerotier = module;
+
+      inventory.instances = {
+        zerotier = {
+          module.name = "zerotier";
+          module.input = "self";
+
+          roles.peer.tags.all = { };
+          roles.controller.machines.bam = { };
+        };
+      };
+    }).config;
 in
 {
   test_peers = {
@@ -73,4 +100,30 @@ in
       networkName = "zerotier";
     };
   };
+  test_peers_no_moon = {
+    expr = {
+      hasNetworkIds = testFlakeNoMoon.nixosConfigurations.jon.config.services.zerotierone.joinNetworks;
+      isController =
+        testFlakeNoMoon.nixosConfigurations.jon.config.clan.core.networking.zerotier.controller.enable;
+      networkName = testFlakeNoMoon.nixosConfigurations.jon.config.clan.core.networking.zerotier.name;
+    };
+    expected = {
+      hasNetworkIds = [ "0e28cb903344475e" ];
+      isController = false;
+      networkName = "zerotier";
+    };
+  };
+  test_controller_no_moon = {
+    expr = {
+      hasNetworkIds = testFlakeNoMoon.nixosConfigurations.bam.config.services.zerotierone.joinNetworks;
+      isController =
+        testFlakeNoMoon.nixosConfigurations.bam.config.clan.core.networking.zerotier.controller.enable;
+      networkName = testFlakeNoMoon.nixosConfigurations.bam.config.clan.core.networking.zerotier.name;
+    };
+    expected = {
+      hasNetworkIds = [ "0e28cb903344475e" ];
+      isController = true;
+      networkName = "zerotier";
+    };
+  };
 }

docs/mkdocs.yml

@@ -48,61 +48,80 @@ nav:
   - Home: index.md
   - Guides:
       - Getting Started:
-          - 🚀 Creating Your First Clan: guides/getting-started/index.md
-          - 📀 Create USB Installer (optional): guides/getting-started/installer.md
-          - ⚙️ Add Machines: guides/getting-started/add-machines.md
-          - ⚙️ Add User: guides/getting-started/add-user.md
-          - ⚙️ Add Services: guides/getting-started/add-services.md
-          - 🔐 Secrets & Facts: guides/getting-started/secrets.md
-          - 🚢 Deploy Machine: guides/getting-started/deploy.md
-          - 🧪 Continuous Integration: guides/getting-started/check.md
-      - clanServices: guides/clanServices.md
-      - Disk Encryption: guides/disk-encryption.md
-      - Mesh VPN: guides/mesh-vpn.md
+          - Creating Your First Clan: guides/getting-started/index.md
+          - Create USB Installer: guides/getting-started/installer.md
+          - Add Machines: guides/getting-started/add-machines.md
+          - Add User: guides/getting-started/add-user.md
+          - Add Services: guides/getting-started/add-services.md
+          - Deploy Machine: guides/getting-started/deploy.md
+          - Continuous Integration: guides/getting-started/check.md
+      - Using Services: guides/clanServices.md
       - Backup & Restore: guides/backups.md
-      - Vars Backend: guides/vars-backend.md
-      - Facts Backend: guides/secrets.md
-      - Adding more machines: guides/more-machines.md
+      - Disk Encryption: guides/disk-encryption.md
+      - Age Plugins: guides/age-plugins.md
+      - Secrets management: guides/secrets.md
       - Target Host: guides/target-host.md
-      - Inventory:
-          - Inventory: guides/inventory.md
+      - Zerotier VPN: guides/mesh-vpn.md
       - Secure Boot: guides/secure-boot.md
       - Flake-parts: guides/flake-parts.md
-      - Authoring:
-          - clanService: guides/authoring/clanServices/index.md
-          - Disk Template: guides/authoring/templates/disk/disko-templates.md
-          - clanModule: guides/authoring/clanModules/index.md
+      - macOS: guides/macos.md
       - Contributing:
-          - Contribute: guides/contributing/CONTRIBUTING.md
+          - Contributing: guides/contributing/CONTRIBUTING.md
           - Debugging: guides/contributing/debugging.md
           - Testing: guides/contributing/testing.md
+
+      - Writing a Service Module: guides/services/community.md
+      - Writing a Disko Template: guides/disko-templates/community.md
       - Migrations:
           - Migrate existing Flakes: guides/migrations/migration-guide.md
           - Migrate inventory Services: guides/migrations/migrate-inventory-services.md
           - Facts Vars Migration: guides/migrations/migration-facts-vars.md
           - Disk id: guides/migrations/disk-id.md
-      - macOS: guides/macos.md
+  - Concepts:
+      - Inventory: concepts/inventory.md
+      - Generators: concepts/generators.md
+      - Autoincludes: concepts/autoincludes.md
   - Reference:
       - Overview: reference/index.md
+      - Clan Options: options.md
       - Services:
-          - Overview: reference/clanServices/index.md
-          - reference/clanServices/admin.md
-          - reference/clanServices/borgbackup.md
-          - reference/clanServices/data-mesher.md
-          - reference/clanServices/emergency-access.md
-          - reference/clanServices/garage.md
-          - reference/clanServices/hello-world.md
-          - reference/clanServices/importer.md
-          - reference/clanServices/mycelium.md
-          - reference/clanServices/packages.md
-          - reference/clanServices/sshd.md
-          - reference/clanServices/state-version.md
-          - reference/clanServices/trusted-nix-caches.md
-          - reference/clanServices/users.md
-          - reference/clanServices/wifi.md
-          - reference/clanServices/zerotier.md
-      - Interface for making Services: reference/clanServices/clan-service-author-interface.md
-      - Modules:
+          - Overview:
+              - reference/clanServices/index.md
+
+              - reference/clanServices/admin.md
+              - reference/clanServices/borgbackup.md
+              - reference/clanServices/data-mesher.md
+              - reference/clanServices/emergency-access.md
+              - reference/clanServices/garage.md
+              - reference/clanServices/hello-world.md
+              - reference/clanServices/importer.md
+              - reference/clanServices/mycelium.md
+              - reference/clanServices/packages.md
+              - reference/clanServices/sshd.md
+              - reference/clanServices/state-version.md
+              - reference/clanServices/trusted-nix-caches.md
+              - reference/clanServices/users.md
+              - reference/clanServices/wifi.md
+              - reference/clanServices/zerotier.md
+          - API: reference/clanServices/clan-service-author-interface.md
+
+      - CLI:
+          - Overview: reference/cli/index.md
+
+          - reference/cli/backups.md
+          - reference/cli/facts.md
+          - reference/cli/flakes.md
+          - reference/cli/flash.md
+          - reference/cli/machines.md
+          - reference/cli/select.md
+          - reference/cli/secrets.md
+          - reference/cli/show.md
+          - reference/cli/ssh.md
+          - reference/cli/state.md
+          - reference/cli/templates.md
+          - reference/cli/vars.md
+          - reference/cli/vms.md
+      - Modules (deprecated):
           - Overview: reference/clanModules/index.md
           - reference/clanModules/frontmatter/index.md
           # TODO: display the docs of the clan.service modules
@@ -145,38 +164,20 @@ nav:
           - reference/clanModules/zerotier-static-peers.md
           - reference/clanModules/zerotier.md
           - reference/clanModules/zt-tcp-relay.md
-      - CLI:
-          - Overview: reference/cli/index.md
 
-          - reference/cli/backups.md
-          - reference/cli/facts.md
-          - reference/cli/flakes.md
-          - reference/cli/flash.md
-          - reference/cli/machines.md
-          - reference/cli/select.md
-          - reference/cli/secrets.md
-          - reference/cli/show.md
-          - reference/cli/ssh.md
-          - reference/cli/state.md
-          - reference/cli/templates.md
-          - reference/cli/vars.md
-          - reference/cli/vms.md
-      - NixOS Modules:
-          - clan.core:
-              - Overview: reference/clan.core/index.md
+      - clan.core (NixOS Options):
+          - Overview: reference/clan.core/index.md
+          - reference/clan.core/backups.md
+          - reference/clan.core/deployment.md
+          - reference/clan.core/facts.md
+          - reference/clan.core/networking.md
+          - reference/clan.core/settings.md
+          - reference/clan.core/sops.md
+          - reference/clan.core/state.md
+          - reference/clan.core/vars.md
+
+      - Developer-api: api.md
 
-              - reference/clan.core/backups.md
-              - reference/clan.core/deployment.md
-              - reference/clan.core/facts.md
-              - reference/clan.core/networking.md
-              - reference/clan.core/settings.md
-              - reference/clan.core/sops.md
-              - reference/clan.core/state.md
-              - reference/clan.core/vars.md
-      - Nix API:
-          - clan: reference/nix-api/clan.md
-          - Inventory: reference/nix-api/inventory.md
-      - Glossary: reference/glossary.md
       - Decisions:
           - Architecture Decisions: decisions/README.md
           - 01-clanModules: decisions/01-ClanModules.md
@@ -185,10 +186,7 @@ nav:
           - 04-fetching-nix-from-python: decisions/04-fetching-nix-from-python.md
           - 05-deployment-parameters: decisions/05-deployment-parameters.md
           - Template: decisions/_template.md
-  - Options: options.md
-  - Developer:
-      - Introduction: intern/index.md
-      - API: intern/api.md
+      - Glossary: reference/glossary.md
 
 docs_dir: site
 site_dir: out
@@ -199,6 +197,7 @@ theme:
   favicon: https://clan.lol/favicon.svg
   name: material
   features:
+    - navigation.footer
     - navigation.instant
     - navigation.tabs
     - navigation.tabs.sticky
@@ -246,3 +245,6 @@ plugins:
   - search
   - macros
   - redoc-tag
+  - redirects:
+      redirect_maps:
+        guides/getting-started/secrets.md: concepts/generators.md

docs/nix/default.nix

@@ -40,6 +40,7 @@ pkgs.stdenv.mkDerivation {
       mkdocs-material
       mkdocs-macros
       mkdocs-redoc-tag
+      mkdocs-redirects
     ]);
   configurePhase = ''
     pushd docs
@@ -54,6 +55,7 @@ pkgs.stdenv.mkDerivation {
     chmod -R +w ./site/reference
     echo "Generated API documentation in './site/reference/' "
 
+    rm -r ./site/options-page || true
     cp -r ${docs-options} ./site/options-page
     chmod -R +w ./site/options-page
 

docs/nix/options/flake-module.nix

@@ -114,9 +114,6 @@
         in
         {
           options = {
-            _ = mkOption {
-              type = types.raw;
-            };
             instances.${name} = lib.mkOption {
               inherit description;
               type = types.submodule {
@@ -149,20 +146,29 @@
           };
         };
 
-      mkScope = name: modules: {
-        inherit name;
-        modules = [
-          {
-            _module.args = { inherit clanLib; };
-            _file = "docs mkScope";
-          }
-          { noInstanceOptions = true; }
-          ../../../lib/modules/inventoryClass/interface.nix
-        ] ++ mapAttrsToList fakeInstanceOptions modules;
-        urlPrefix = "https://github.com/nix-community/dream2nix/blob/main/";
-      };
+      docModules = [
+        {
+          inherit self;
+        }
+        self.modules.clan.default
+        {
+          options.inventory = lib.mkOption {
+            type = types.submoduleWith {
+              modules = [
+                { noInstanceOptions = true; }
+              ] ++ mapAttrsToList fakeInstanceOptions serviceModules;
+            };
+          };
+        }
+      ];
+
     in
     {
+      # Uncomment for debugging
+      # legacyPackages.docModules = lib.evalModules {
+      #   modules = docModules;
+      # };
+
       packages = lib.optionalAttrs ((privateInputs ? nuschtos) || (inputs ? nuschtos)) {
         docs-options =
           (privateInputs.nuschtos or inputs.nuschtos)
@@ -171,7 +177,13 @@
               inherit baseHref;
               title = "Clan Options";
               # scopes = mapAttrsToList mkScope serviceModules;
-              scopes = [ (mkScope "Clan Inventory" serviceModules) ];
+              scopes = [
+                {
+                  name = "Clan";
+                  modules = docModules;
+                  urlPrefix = "https://git.clan.lol/clan/clan-core/src/branch/main/";
+                }
+              ];
             };
       };
     };

docs/nix/render_options/__init__.py

@@ -193,7 +193,7 @@ def module_header(module_name: str, has_inventory_feature: bool = False) -> str:
 def module_nix_usage(module_name: str) -> str:
     return f"""## Usage via Nix
 
-**This module can be also imported directly in your nixos configuration. Although it is recommended to use the [inventory](../../reference/nix-api/inventory.md) interface if available.**
+**This module can be also imported directly in your nixos configuration. Although it is recommended to use the [inventory](../../concepts/inventory.md) interface if available.**
 
 Some modules are considered 'low-level' or 'expert modules' and are not available via the inventory interface.
 
@@ -373,7 +373,7 @@ This module can be used via predefined roles
             """
 Every role has its own configuration options, which are each listed below.
 
-For more information, see the [inventory guide](../../guides/inventory.md).
+For more information, see the [inventory guide](../../concepts/inventory.md).
 
 ??? Example
     For example the `admin` module adds the following options globally to all machines where it is used.
@@ -402,7 +402,7 @@ certain option types restricted to enable configuration through a graphical
 interface.
 
 !!! note "🔹"
-    Modules with this indicator support the [inventory](../../guides/inventory.md) feature.
+    Modules with this indicator support the [inventory](../../concepts/inventory.md) feature.
 
 """
 
@@ -679,86 +679,6 @@ def build_option_card(module_name: str, frontmatter: Frontmatter) -> str:
     return f"{to_md_li(module_name, frontmatter)}\n\n"
 
 
-def produce_build_clan_docs() -> None:
-    if not BUILD_CLAN_PATH:
-        msg = f"Environment variables are not set correctly: BUILD_CLAN_PATH={BUILD_CLAN_PATH}. Expected a path to the optionsJSON"
-        raise ClanError(msg)
-
-    if not OUT:
-        msg = f"Environment variables are not set correctly: $out={OUT}"
-        raise ClanError(msg)
-
-    output = """# Clan
-
-This provides an overview of the available arguments of the `clan` interface.
-
-Each attribute is documented below
-
-- **clan-core.lib.clan**: A function that takes an attribute set.
-
-    ??? example "clan Example"
-
-        ```nix
-        clan {
-            self = self;
-            machines = {
-                jon = { };
-                sara = { };
-            };
-        };
-        ```
-
-- **clan with flake-parts**: Import the FlakeModule
-
-    After importing the FlakeModule you can define your `clan` as a flake attribute
-
-    All attribute can be defined via `clan.*`
-
-    Further information see: [flake-parts](../../guides/flake-parts.md) guide.
-
-    ??? example "flake-parts Example"
-
-        ```nix
-        flake-parts.lib.mkFlake { inherit inputs; } ({
-            systems = [];
-            imports = [
-                clan-core.flakeModules.default
-            ];
-            clan = {
-                machines = {
-                    jon = { };
-                    sara = { };
-                };
-            };
-        });
-        ```
-
-"""
-    with Path(BUILD_CLAN_PATH).open() as f:
-        options: dict[str, dict[str, Any]] = json.load(f)
-
-        split = split_options_by_root(options)
-        for option_name, options in split.items():
-            # Skip underscore options
-            if option_name.startswith("_"):
-                continue
-            # Skip inventory sub options
-            # Inventory model has its own chapter
-            if option_name.startswith("inventory."):
-                continue
-
-            print(f"[build_clan_docs] Rendering option of {option_name}...")
-            root = options_to_tree(options)
-
-            for option in root.suboptions:
-                output += options_docs_from_tree(option, init_level=2)
-
-    outfile = Path(OUT) / "nix-api/clan.md"
-    outfile.parent.mkdir(parents=True, exist_ok=True)
-    with Path.open(outfile, "w") as of:
-        of.write(output)
-
-
 def split_options_by_root(options: dict[str, Any]) -> dict[str, dict[str, Any]]:
     """
     Split the flat dictionary of options into a dict of which each entry will construct complete option trees.
@@ -805,7 +725,7 @@ Typically needed by module authors to define roles, behavior and metadata for di
 !!! Note
     This is not a user-facing documentation, but rather meant as a reference for *module authors*
 
-    See: [clanService Authoring Guide](../../guides/authoring/clanServices/index.md)
+    See: [clanService Authoring Guide](../../guides/services/community.md)
 """
     # Inventory options are already included under the clan attribute
     # We just omitted them in the clan docs, because we want a separate output for the inventory model
@@ -834,48 +754,6 @@ class Option:
     suboptions: list["Option"] = field(default_factory=list)
 
 
-def produce_inventory_docs() -> None:
-    if not BUILD_CLAN_PATH:
-        msg = f"Environment variables are not set correctly: BUILD_CLAN_PATH={BUILD_CLAN_PATH}. Expected a path to the optionsJSON"
-        raise ClanError(msg)
-
-    if not OUT:
-        msg = f"Environment variables are not set correctly: $out={OUT}"
-        raise ClanError(msg)
-
-    output = """# Inventory
-This provides an overview of the available attributes of the `inventory` model.
-
-It can be set via the `inventory` attribute of the [`clan`](./clan.md#inventory) function, or via the [`clan.inventory`](./clan.md#inventory) attribute of flake-parts.
-
-"""
-    # Inventory options are already included under the clan attribute
-    # We just omitted them in the clan docs, because we want a separate output for the inventory model
-    with Path(BUILD_CLAN_PATH).open() as f:
-        options: dict[str, dict[str, Any]] = json.load(f)
-
-        clan_root_option = options_to_tree(options)
-        # Find the inventory options
-        inventory_opt: None | Option = None
-        for opt in clan_root_option.suboptions:
-            if opt.name == "inventory":
-                inventory_opt = opt
-                break
-
-        if not inventory_opt:
-            print("No inventory options found.")
-            exit(1)
-        # Render the inventory options
-        # This for loop excludes the root node
-        for option in inventory_opt.suboptions:
-            output += options_docs_from_tree(option, init_level=2)
-
-    outfile = Path(OUT) / "nix-api/inventory.md"
-    outfile.parent.mkdir(parents=True, exist_ok=True)
-    with Path.open(outfile, "w") as of:
-        of.write(output)
-
-
 def option_short_name(option_name: str) -> str:
     parts = option_name.split(".")
     short_name = ""
@@ -984,9 +862,6 @@ def options_docs_from_tree(
 if __name__ == "__main__":  #
     produce_clan_core_docs()
 
-    produce_build_clan_docs()
-    produce_inventory_docs()
-
     produce_clan_service_author_docs()
 
     produce_clan_modules_docs()

docs/site/concepts/autoincludes.md

@@ -0,0 +1,15 @@
+
+Clan automatically imports the following files from a directory and registers them.
+
+## Machine registration
+
+Every folder `machines/{machineName}` will be registered automatically as a Clan machine.
+
+!!! info "Automatically loaded files"
+
+    The following files are loaded automatically for each Clan machine:
+
+    - [x] `machines/{machineName}/configuration.nix`
+    - [x] `machines/{machineName}/hardware-configuration.nix`
+    - [x] `machines/{machineName}/facter.json` Automatically configured, for further information see [nixos-facter](https://clan.lol/blog/nixos-facter/)
+    - [x] `machines/{machineName}/disko.nix` Automatically loaded, for further information see the [disko docs](https://github.com/nix-community/disko/blob/master/docs/quickstart.md).

docs/site/concepts/generators.md

@@ -1,7 +1,4 @@
-
-!!! Note
-    Vars is the new secret backend that will soon replace the Facts backend
-
+# Generators
 
 Defining a linux user's password via the nixos configuration previously required running `mkpasswd ...` and then copying the hash back into the nix configuration.
 
@@ -11,7 +8,7 @@ For a more general explanation of what clan vars are and how it works, see the i
 
 This guide assumes
 - Clan is set up already (see [Getting Started](../guides/getting-started/index.md))
-- a machine has been added to the clan (see [Adding Machines](./more-machines.md))
+- a machine has been added to the clan (see [Adding Machines](../guides/getting-started/add-machines.md))
 
 This section will walk you through the following steps:
 
@@ -23,7 +20,7 @@ This section will walk you through the following steps:
 6. share the root password between machines
 7. change the password
 
-## Declare the generator
+## Declare a generator
 
 In this example, a `vars` `generator` is used to:
 

docs/site/concepts/inventory.md

@@ -9,8 +9,6 @@ The inventory logic will automatically derive the modules and configurations to
 
 The following tutorial will walk through setting up a Backup service where the terms `Service` and `Role` will become more clear.
 
-See also: [Inventory API Documentation](../reference/nix-api/inventory.md)
-
 !!! example "Experimental status"
     The inventory implementation is not considered stable yet.
     We are actively soliciting feedback from users.
@@ -19,7 +17,7 @@ See also: [Inventory API Documentation](../reference/nix-api/inventory.md)
 
 ## Prerequisites
 
-- [x] [Add multiple machines](./more-machines.md) to your Clan.
+- [x] [Add some machines](../guides/getting-started/add-machines.md) to your Clan.
 
 ## Services
 

docs/site/decisions/01-ClanModules.md

@@ -6,6 +6,8 @@ Accepted
 
 ## Context
 
+Current state as of writing:
+
 To define a service in Clan, you need to define two things:
 
 - `clanModule` - defined by module authors

docs/site/guides/age-plugins.md

@@ -0,0 +1,59 @@
+## Using Age Plugins
+
+If you wish to use a key generated using an [age plugin] as your admin key, extra care is needed.
+
+You must **precede your secret key with a comment that contains its corresponding recipient**.
+
+This is usually output as part of the generation process
+and is only required because there is no unified mechanism for recovering a recipient from a plugin secret key.
+
+Here is an example:
+
+```title="~/.config/sops/age/keys.txt"
+# public key: age1zdy49ek6z60q9r34vf5mmzkx6u43pr9haqdh5lqdg7fh5tpwlfwqea356l
+AGE-PLUGIN-FIDO2-HMAC-1QQPQZRFR7ZZ2WCV...
+```
+
+!!! note
+    The comment that precedes the plugin secret key need only contain the recipient.
+    Any other text is ignored.
+
+    In the example above, you can specify `# recipient: age1zdy...`, `# public: age1zdy....` or even
+    just `# age1zdy....`
+
+You will need to add an entry into your `flake.nix` to ensure that the necessary `age` plugins
+are loaded when using Clan:
+
+```nix title="flake.nix"
+{
+  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
+  inputs.nixpkgs.follows = "clan-core/nixpkgs";
+
+  outputs =
+    { self, clan-core, ... }:
+    let
+      # Sometimes this attribute set is defined in clan.nix
+      clan = clan-core.lib.clan {
+        inherit self;
+
+        meta.name = "myclan";
+
+        # Add Yubikey and FIDO2 HMAC plugins
+        # Note: the plugins listed here must be available in nixpkgs.
+        secrets.age.plugins = [
+          "age-plugin-yubikey"
+          "age-plugin-fido2-hmac"
+        ];
+
+        machines = {
+          # elided for brevity
+        };
+      };
+    in
+    {
+      inherit (clan) nixosConfigurations nixosModules clanInternals;
+
+      # elided for brevity
+    };
+}
+```

docs/site/guides/authoring/clanModules/index.md

@@ -1,229 +0,0 @@
-# Authoring a clanModule
-
-!!! Danger "Will get deprecated soon"
-    Please consider twice creating new modules in this format
-
-    [`clan.service` module](../clanServices/index.md) will be the new standard soon.
-
-This site will guide you through authoring your first module. Explaining which conventions must be followed, such that others will have an enjoyable experience and the module can be used with minimal effort.
-
-
-!!! Tip
-    External ClanModules can be ad-hoc loaded via [`clan.inventory.modules`](../../../reference/nix-api/inventory.md#inventory.modules)
-
-## Bootstrapping the `clanModule`
-
-A ClanModule is a specific subset of a [NixOS Module](https://nix.dev/tutorials/module-system/index.html), but it has some constraints and might be used via the [Inventory](../../../guides/inventory.md) interface.
-In fact a `ClanModule` can be thought of as a layer of abstraction on-top of NixOS and/or other ClanModules. It may configure sane defaults and provide an ergonomic interface that is easy to use and can also be used via a UI that is under development currently.
-
-Because ClanModules should be configurable via `json`/`API` all of its interface (`options`) must be serializable.
-
-!!! Tip
-    ClanModules interface can be checked by running the json schema converter as follows.
-
-    `nix build .#legacyPackages.x86_64-linux.schemas.inventory`
-
-    If the build succeeds the module is compatible.
-
-## Directory structure
-
-Each module SHOULD be a directory of the following format:
-
-```sh
-# Example: borgbackup
-clanModules/borgbackup
-├── README.md
-└── roles
-    ├── client.nix
-    └── server.nix
-```
-
-!!! Tip
-    `README.md` is always required. See section [Readme](#readme) for further details.
-
-    The `roles` folder is strictly required for `features = [ "inventory" ]`.
-
-## Registering the module
-
-=== "User module"
-
-    If the module should be ad-hoc loaded.
-    It can be made available in any project via the [`clan.inventory.modules`](../../../reference/nix-api/inventory.md#inventory.modules) attribute.
-
-    ```nix title="flake.nix"
-    # ...
-    # Sometimes this attribute set is defined in clan.nix
-    clan-core.lib.clan {
-        # 1. Add the module to the available clanModules with inventory support
-        inventory.modules = {
-            custom-module = ./modules/my_module;
-        };
-        # 2. Use the module in the inventory
-        inventory.services = {
-            custom-module.instance_1 = {
-                roles.default.machines = [ "machineA" ];
-            };
-        };
-    };
-    ```
-
-=== "Upstream module"
-
-    If the module will be contributed to [`clan-core`](https://git.clan.lol/clan-core)
-    The clanModule must be registered within the `clanModules` attribute in `clan-core`
-
-    ```nix title="clanModules/flake-module.nix"
-    --8<-- "clanModules/flake-module.nix:0:5"
-        # Register our new module here
-        # ...
-    ```
-
-## Readme
-
-The `README.md` is a required file for all modules. It MUST contain frontmatter in [`toml`](https://toml.io) format.
-
-```markdown
----
-description = "Module A"
----
-
-This is the example module that does xyz.
-```
-
-See the [Full Frontmatter reference](../../../reference/clanModules/frontmatter/index.md) further details and all supported attributes.
-
-## Roles
-
-If the module declares to implement `features = [ "inventory" ]` then it MUST contain a roles directory.
-
-Each `.nix` file in the `roles` directory is added as a role to the inventory service.
-
-Other files can also be placed alongside the `.nix` files
-
-```sh
-└── roles
-    ├── client.nix
-    └── server.nix
-```
-
-Adds the roles: `client` and `server`
-
-??? Tip "Good to know"
-    Sometimes a `ClanModule` should be usable via both clan's `inventory` concept but also natively as a NixOS module.
-
-    > In the long term, we want most modules to implement support for the inventory,
-    > but we are also aware that there are certain low-level modules that always serve as a backend for other higher-level `clanModules` with inventory support.
-    > These modules may not want to implement inventory interfaces as they are always used directly by other modules.
-
-    This can be achieved by placing an additional `default.nix` into the root of the ClanModules directory as shown:
-
-    ```sh
-    # ModuleA
-    ├── README.md
-    ├── default.nix
-    └── roles
-        └── default.nix
-    ```
-
-    ```nix title="default.nix"
-    {...}:{
-        imports = [ ./roles/default.nix ];
-    }
-    ```
-
-    By utilizing this pattern the module (`moduleA`) can then be imported into any regular NixOS module via:
-
-    ```nix
-    {...}:{
-        imports  = [ clanModules.moduleA ];
-    }
-    ```
-
-## Adding configuration options
-
-While we recommend to keep the interface as minimal as possible and deriving all required information from the `roles` model it might sometimes be required or convenient to expose customization options beyond `roles`.
-
-The following shows how to add options to your module.
-
-**It is important to understand that every module has its own namespace where it should declare options**
-
-**`clan.{moduleName}`**
-
-???+ Example
-    The following example shows how to register options in the module interface
-
-    and how it can be set via the inventory
-
-
-    ```nix title="/default.nix"
-    custom-module = ./modules/custom-module;
-    ```
-
-    Since the module is called `custom-module` all of its exposed options should be added to `options.clan.custom-module.*...*`
-
-    ```nix title="custom-module/roles/default.nix"
-    {
-        options = {
-            clan.custom-module.foo = mkOption {
-                type = types.str;
-                default = "bar";
-            };
-        };
-    }
-    ```
-
-    If the module is [registered](#registering-the-module).
-    Configuration can be set as follows.
-
-    ```nix title="flake.nix"
-    # Sometimes this attribute set is defined in clan.nix
-    clan-core.lib.clan {
-        inventory.services = {
-            custom-module.instance_1 = {
-                roles.default.machines = [ "machineA" ];
-                roles.default.config = {
-                    # All configuration here is scoped to `clan.custom-module`
-                    foo = "foobar";
-                };
-            };
-        };
-    }
-    ```
-
-## Organizing the ClanModule
-
-Each `{role}.nix` is included into the machine if the machine is declared to have the role.
-
-For example
-
-```nix
-roles.client.machines = ["MachineA"];
-```
-
-Then `roles/client.nix` will be added to the machine `MachineA`.
-
-This behavior makes it possible to split the interface and common code paths when using multiple roles.
-In the concrete example of `borgbackup` this allows a `server` to declare a different interface than the corresponding `client`.
-
-The client offers configuration option, to exclude certain local directories from being backed up:
-
-```nix title="roles/client.nix"
-# Example client interface
-  options.clan.borgbackup.exclude = ...
-```
-
-The server doesn't offer any configuration option. Because everything is set-up automatically.
-
-```nix title="roles/server.nix"
-# Example server interface
-  options.clan.borgbackup = {};
-```
-
-Assuming that there is a common code path or a common interface between `server` and `client` this can be structured as:
-
-```nix title="roles/server.nix, roles/client.nix"
-{...}: {
-    # ...
-    imports = [ ../common.nix ];
-}
-```

docs/site/guides/backups.md

@@ -1,167 +1,199 @@
-# Introduction to Backups
 
-When you're managing your own services, creating regular backups is crucial to ensure your data's safety.
-This guide introduces you to Clan's built-in backup functionalities.
-Clan supports backing up your data to both local storage devices (like USB drives) and remote servers, using well-known tools like borgbackup and rsnapshot.
-We might add more options in the future, but for now, let's dive into how you can secure your data.
+This guide explains how to set up and manage
+[BorgBackup](https://borgbackup.readthedocs.io/) for secure, efficient backups
+in a clan network. BorgBackup provides:
 
-## Backing Up Locally with Localbackup
+- Space efficient storage of backups with deduplication
+- Secure, authenticated encryption
+- Compression: lz4, zstd, zlib, lzma or none
+- Mountable backups with FUSE
+- Easy installation on multiple platforms: Linux, macOS, BSD, …
+- Free software (BSD license).
+- Backed by a large and active open-source community.
 
-Localbackup lets you backup your data onto physical storage devices connected to your computer,
-such as USB hard drives or network-attached storage. It uses a tool called rsnapshot for this purpose.
-
-### Setting Up Localbackup
-
-1. **Identify Your Backup Device:**
-
-First, figure out which device you'll use for backups. You can see all connected devices by running this command in your terminal:
-
-```bash
-lsblk --output NAME,PTUUID,FSTYPE,SIZE,MOUNTPOINT
-```
-
-Look for the device you intend to use for backups and note its details.
-
-2. **Configure Your Backup Device:**
-
-Once you've identified your device, you'll need to add it to your configuration.
-Here's an example NixOS configuration for a device located at `/dev/sda2` with an `ext4` filesystem:
+## Borgbackup Example
 
 ```nix
-{
-  fileSystems."/mnt/hdd" = {
-    device = "/dev/sda2";
-    fsType = "ext4";
-    options = [ "defaults" "noauto" ];
-  };
-}
-```
-
-Replace `/dev/sda2` with your device and `/mnt/hdd` with your preferred mount point.
-
-3. **Set Backup Targets:** Next, define where on your device you'd like the backups to be stored:
-
-   ```nix
-   {
-     clan.localbackup.targets.hdd = {
-       directory = "/mnt/hdd/backup";
-       mountpoint = "/mnt/hdd";
-     };
-   }
-   ```
-
-   Change `/mnt/hdd` to the actual mount point you're using.
-
-4. **Create Backups:** To create a backup, run:
-
-   ```bash
-   clan backups create mymachine
-   ```
-
-   This command saves snapshots of your data onto the backup device.
-
-5. **Listing Backups:** To see available backups, run:
-
-  ```bash
-  clan backups list mymachine
-  ```
-
-## Remote Backups with Borgbackup
-
-### Overview of Borgbackup
-
-Borgbackup splits the backup process into two parts: a backup client that sends data to a backup server.
-The server stores the backups.
-
-### Setting Up the Borgbackup Client
-
-1. **Specify Backup Server:**
-
-Start by indicating where your backup data should be sent. Replace `hostname` with your server's address:
-
-```nix
-{
-  clan.borgbackup.destinations = {
-    myhostname = {
-      repo = "borg@backuphost:/var/lib/borgbackup/myhostname";
+inventory.instances = {
+  borgbackup = {
+    module = {
+      name = "borgbackup";
+      input = "clan";
     };
+    roles.client.machines."jon".settings = {
+      destinations."storagebox" = {
+        repo = "username@$hostname:/./borgbackup";
+        rsh = ''ssh -oPort=23 -i /run/secrets/vars/borgbackup/borgbackup.ssh'';
+      };
+    };
+    roles.server.machines = { };
   };
-}
+};
 ```
 
-2. **Select Folders to Backup:**
+The input should be named according to your flake input. Jon is configured as a
+client machine with a destination pointing to a Hetzner Storage Box.
 
-Decide which folders you want to back up. For example, to backup your home and root directories:
+To see a list of all possible options go to [borgbackup clan service](../reference/clanServices/borgbackup.md)
+
+## Roles
+
+A Clan Service can have multiple roles, each role applies different nix config to the machine.
+
+### 1. Client
+
+Clients are machines that create and send backups to various destinations. Each
+client can have multiple backup destinations configured.
+
+### 2. Server
+
+Servers act as backup repositories, receiving and storing backups from client
+machines. They can be dedicated backup servers within your clan network.
+
+## Backup destinations
+
+This service allows you to perform backups to multiple `destinations`.
+Destinations can be:
+
+- **Local**: Local disk storage
+- **Server**: Your own borgbackup server (using the `server` role)
+- **Third-party services**: Such as Hetzner's Storage Box
+
+## State management
+
+Backups are based on [states](../reference/clan.core/state.md). A state
+defines which files should be backed up and how these files are obtained through
+pre/post backup and restore scripts.
+
+Here's an example for a user application `linkding`:
+
+In this example:
+
+- `/data/podman/linkding` is the application's data directory
+- `/var/backup/linkding` is the staging directory where data is copied for
+  backup
 
 ```nix
-{ clan.core.state.userdata.folders = [ "/home" "/root" ]; }
+clan.core.state.linkding = {
+  folders = [ "/var/backup/linkding" ];
+
+  preBackupScript = ''
+    export PATH=${
+      lib.makeBinPath [
+        config.systemd.package
+        pkgs.coreutils
+        pkgs.rsync
+      ]
+    }
+
+    service_status=$(systemctl is-active podman-linkding)
+
+    if [ "$service_status" = "active" ]; then
+      systemctl stop podman-linkding
+      rsync -avH --delete --numeric-ids "/data/podman/linkding/" /var/backup/linkding/
+      systemctl start podman-linkding
+    fi
+  '';
+
+  postRestoreScript = ''
+    export PATH=${
+      lib.makeBinPath [
+        config.systemd.package
+        pkgs.coreutils
+        pkgs.rsync
+      ]
+    }
+
+    service_status="$(systemctl is-active podman-linkding)"
+
+    if [ "$service_status" = "active" ]; then
+      systemctl stop podman-linkding
+
+      # Backup locally current linkding data
+      cp -rp "/data/podman/linkding" "/data/podman/linkding.bak"
+
+      # Restore from borgbackup
+      rsync -avH --delete --numeric-ids /var/backup/linkding/ "/data/podman/linkding/"
+
+      systemctl start podman-linkding
+    fi
+  '';
+};
 ```
 
-3. **Generate Backup Credentials:**
+## Managing backups
 
-Run `clan facts generate <yourmachine>` to prepare your machine for backup, creating necessary SSH keys and credentials.
+In this section we go over how to manage your collection of backups with the clan command.
 
-### Setting Up the Borgbackup Server
+### Listing states
 
-1. **Configure Backup Repository:**
-
-On the server where backups will be stored, enable the SSH daemon and set up a repository for each client:
-
-```nix
-{
-  services.borgbackup.repos.myhostname = {
-    path = "/var/lib/borgbackup/myhostname";
-    authorizedKeys = [
-      (builtins.readFile  (config.clan.core.settings.directory + "/machines/myhostname/facts/borgbackup.ssh.pub"))
-    ];
-  };
-}
-```
-
-Ensure the path to the public key is correct.
-
-2. **Update Your Systems:** Apply your changes by running `clan machines update` to both the server and your client
-
-### Managing Backups
-
-- **Scheduled Backups:**
-
-  Backups are automatically performed nightly. To check the next scheduled backup, use:
-
-  ```bash
-  systemctl list-timers | grep -E 'NEXT|borg'
-  ```
-
-- **Listing Backups:** To see available backups, run:
-
-  ```bash
-  clan backups list mymachine
-  ```
-
-- **Manual Backups:** You can also initiate a backup manually:
-
-  ```bash
-  clan backups create mymachine
-  ```
-
-- **Restoring Backups:** To restore a backup that has been listed by the list command (NAME):
-
-  ```bash
-  clan backups restore [MACHINE] [PROVIDER] [NAME]
-
-  ```
-
-  Example (Restoring a machine called `client` with the backup provider `borgbackup`):
-
-  ```bash
-  clan backups restore client borgbackup [NAME]
-
-  ```
-
-  The `backups` command is service aware and allows optional specification of the `--service` flag.
-
-  To only restore the service called `zerotier` on a machine called `controller` through the backup provider `borgbackup` use the following command:
+To see which files (`states`) will be backed up on a specific machine, use:
 
 ```bash
-  clan backups restore client borgbackup [NAME] --service zerotier
+clan state list jon
 ```
+
+This will show all configured states for the machine `jon`, for example:
+
+```text
+· service: linkding
+  folders:
+  - /var/backup/linkding
+  preBackupCommand: pre-backup-linkding
+  postRestoreCommand: post-restore-linkding
+
+· service: zerotier
+  folders:
+  - /var/lib/zerotier-one
+```
+
+### Creating backups
+
+To create a backup of a machine (e.g., `jon`), run:
+
+```bash
+clan backups create jon
+```
+
+This will backup all configured states (`zerotier` and `linkding` in this
+example) from the machine `jon`.
+
+### Listing available backups
+
+To see all available backups, use:
+
+```bash
+clan backups list
+```
+
+This will display all backups with their timestamps:
+
+```text
+storagebox::username@username.your-storagebox.de:/./borgbackup::jon-jon-2025-07-22T19:40:10
+storagebox::username@username.your-storagebox.de:/./borgbackup::jon-jon-2025-07-23T01:00:00
+storagebox::username@username.your-storagebox.de:/./borgbackup::jon-storagebox-2025-07-24T01:00:00
+storagebox::username@username.your-storagebox.de:/./borgbackup::jon-storagebox-2025-07-24T06:02:35
+```
+
+### Restoring backups
+
+For restoring a backup you have two options. 
+
+#### Full restoration
+
+To restore all services from a backup:
+
+```bash
+clan backups restore jon borgbackup storagebox::u444061@u444061.your-storagebox.de:/./borgbackup::jon-storagebox-2025-07-24T06:02:35
+```
+
+#### Partial restoration
+
+To restore only a specific service (e.g., `linkding`):
+
+```bash
+clan backups restore --service linkding jon borgbackup storagebox::u444061@u444061.your-storagebox.de:/./borgbackup::jon-storagebox-2025-07-24T06:02:35
+```
+
+
+

docs/site/guides/clanServices.md

@@ -138,7 +138,7 @@ You can use services exposed by Clan’s core module library, `clan-core`.
 
 You can also author your own `clanService` modules.
 
-🔗 Learn how to write your own service: [Authoring a clanService](../guides/authoring/clanServices/index.md)
+🔗 Learn how to write your own service: [Authoring a service](../guides/services/community.md)
 
 You might expose your service module from your flake — this makes it easy for other people to also use your module in their clan.
 
@@ -154,6 +154,6 @@ You might expose your service module from your flake — this makes it easy for
 
 ## What’s Next?
 
-* [Author your own clanService →](../guides/authoring/clanServices/index.md)
+* [Author your own clanService →](../guides/services/community.md)
 * [Migrate from clanModules →](../guides/migrations/migrate-inventory-services.md)
 <!-- TODO: * [Understand the architecture →](../explanation/clan-architecture.md) -->

docs/site/guides/flake-parts.md

@@ -27,7 +27,7 @@ inputs = {
 
 ## Import the Clan flake-parts Module
 
-After updating your flake inputs, the next step is to import the Clan flake-parts module. This will make the [Clan options](../reference/nix-api/clan.md) available within `mkFlake`.
+After updating your flake inputs, the next step is to import the Clan flake-parts module. This will make the [Clan options](../options.md) available within `mkFlake`.
 
 ```nix
 {

docs/site/guides/getting-started/add-machines.md

@@ -6,7 +6,7 @@ Machines can be added using the following methods
 - Editing machines/`machine_name`/configuration.nix (automatically included if it exists)
 - `clan machines create` (imperative)
 
-See the complete [list](../../guides/more-machines.md#automatic-registration) of auto-loaded files.
+See the complete [list](../../concepts/autoincludes.md) of auto-loaded files.
 
 ## Create a machine
 

docs/site/guides/getting-started/add-services.md

@@ -41,7 +41,7 @@ To learn more: [Guide about clanService](../clanServices.md)
 ```
 
 1. See [reference/clanServices](../../reference/clanServices/index.md) for all available services and how to configure them.
-   Or read [authoring/clanServices](../authoring/clanServices/index.md) if you want to bring your own
+   Or read [authoring/clanServices](../../guides/services/community.md) if you want to bring your own
 
 2. Replace `__YOUR_CONTROLLER_` with the *name* of your machine.
 

docs/site/guides/getting-started/add-user.md

@@ -57,7 +57,7 @@ For more information see [clanService/users](../../reference/clanServices/users.
 
 Some people like to define a `users` folder in their repository root.
 That allows to bind all user specific logic to a single place (`default.nix`)
-Which can be imported into individual machines to make the user avilable on that machine.
+Which can be imported into individual machines to make the user available on that machine.
 
 ```bash
 .
@@ -107,7 +107,7 @@ We can use this property of clan services to bind a nixosModule to the user, whi
 }
 ```
 
-1. Type `path` or `string`: Must point to a seperate file. Inlining a module is not possible
+1. Type `path` or `string`: Must point to a separate file. Inlining a module is not possible
 
 !!! Note "This is inspiration"
     Our community might come up with better solutions soon.

docs/site/guides/getting-started/deploy.md

@@ -8,7 +8,6 @@ Now that you have created a machines, added some services and setup secrets. Thi
     - [x] RAM > 2GB
     - [x] **Two Computers**: You need one computer that you're getting ready (we'll call this the Target Computer) and another one to set it up from (we'll call this the Setup Computer). Make sure both can talk to each other over the network using SSH.
     - [x] **Machine configuration**: See our basic [adding and configuring machine guide](./add-machines.md)
-    - [x] **Initialized secrets**: See [secrets](secrets.md) for how to initialize your secrets.
 
 ## Physical Hardware
 
@@ -18,7 +17,7 @@ Steps:
 
 - Create a NixOS installer image and transfer it to a bootable USB drive as described in the [installer](./installer.md).
 - Boot the target machine and connect it to a network that makes it reachable from your setup computer.
-- Note down a reachable ip adress (*ipv4*, *ipv6* or *tor*)
+- Note down a reachable ip address (*ipv4*, *ipv6* or *tor*)
 
 ---
 
@@ -169,7 +168,7 @@ Re-run the command with the correct disk:
 clan templates apply disk single-disk jon --set mainDisk "/dev/disk/by-id/nvme-WD_PC_SN740_SDDQNQD-512G-1201_232557804368"
 ```
 
-Should now be succesfull
+Should now be successful
 
 ```shellSession
 Applied disk template 'single-disk' to machine 'jon'

docs/site/guides/getting-started/index.md

@@ -59,7 +59,7 @@ Enter a *name*, confirm with *enter*. A directory with that name will be created
 
 ## Explore the Project Structure
 
-Take a lookg at all project files:
+Take a look at all project files:
 
 ```bash
 cd my-clan
@@ -125,11 +125,10 @@ To change the name of your clan edit `meta.name` in the `clan.nix` or `flake.nix
 You can continue with **any** of the following steps at your own pace:
 
 - [x] [Install Nix & Clan CLI](./index.md)
-- [x] [Initialize Clan](./index.md#initialize-your-project)
+- [x] [Initialize Clan](./index.md#add-clan-cli-to-your-shell)
 - [ ] [Create USB Installer (optional)](./installer.md)
 - [ ] [Add Machines](./add-machines.md)
 - [ ] [Add a User](./add-user.md)
 - [ ] [Add Services](./add-services.md)
-- [ ] [Configure Secrets](./secrets.md)
 - [ ] [Deploy](./deploy.md) - Requires configured secrets
 - [ ] [Setup CI (optional)](./check.md)

docs/site/guides/getting-started/secrets.md

@@ -1,179 +0,0 @@
-
-Setting up secrets is **Required** for any *machine deployments* or *vm runs* - You need to complete the steps: [Create Admin Keypair](#create-your-admin-keypair) and [Add Your Public Key(s)](#add-your-public-keys)
-
----
-
-Clan enables encryption of secrets (such as passwords & keys) ensuring security and ease-of-use among users.
-
-By default, Clan uses the [sops](https://github.com/getsops/sops) format
-and integrates with [sops-nix](https://github.com/Mic92/sops-nix) on NixOS machines.
-Clan can also be configured to be used with other secret store [backends](../../reference/clan.core/vars.md#clan.core.vars.settings.secretStore).
-
-This guide will walk you through:
-
-- **Creating a Keypair for Your User**: Learn how to generate a keypair for `$USER` to securely control all secrets.
-- **Creating Your First Secret**: Step-by-step instructions on creating your initial secret.
-- **Assigning Machine Access to the Secret**: Understand how to grant a machine access to the newly created secret.
-
-## Create Your Admin Keypair
-
-To get started, you'll need to create **your admin keypair**.
-
-!!! info
-    Don't worry — if you've already made one before, this step won't change or overwrite it.
-
-```bash
-clan secrets key generate
-```
-
-**Output**:
-
-```{.console, .no-copy}
-Public key: age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7
-
-Generated age private key at '/home/joerg/.config/sops/age/keys.txt' for your user. Please back it up on a secure location or you will lose access to your secrets.
-Also add your age public key to the repository with 'clan secrets users add YOUR_USER age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7' (replace YOUR_USER with your actual username)
-```
-
-!!! warning
-    Make sure to keep a safe backup of the private key you've just created.
-    If it's lost, you won't be able to get to your secrets anymore because they all need the admin key to be unlocked.
-
-If you already have an [age] secret key and want to use that instead, you can simply edit `~/.config/sops/age/keys.txt`:
-
-```title="~/.config/sops/age/keys.txt"
-AGE-SECRET-KEY-13GWMK0KNNKXPTJ8KQ9LPSQZU7G3KU8LZDW474NX3D956GGVFAZRQTAE3F4
-```
-
-Alternatively, you can provide your [age] secret key as an environment variable `SOPS_AGE_KEY`, or in a different file
-using `SOPS_AGE_KEY_FILE`.
-For more information see the [SOPS] guide on [encrypting with age].
-
-!!! note
-    It's safe to add any secrets created by the clan CLI and placed in your repository to version control systems like `git`.
-
-### Add Your Public Key(s)
-
-```console
-clan secrets users add $USER --age-key <your_public_key>
-```
-
-It's best to choose the same username as on your Setup/Admin Machine that you use to control the deployment with.
-
-Once run this will create the following files:
-
-```{.console, .no-copy}
-sops/
-└── users/
-    └── <your_username>/
-        └── key.json
-```
-If you followed the quickstart tutorial all necessary secrets are initialized at this point.
-
-!!! note
-    You can add multiple age keys for a user by providing multiple `--age-key <your_public_key>` flags:
-
-    ```console
-    clan secrets users add $USER \
-        --age-key <your_public_key_1> \
-        --age-key <your_public_key_2> \
-        ...
-    ```
-
-### Manage Your Public Key(s)
-
-You can list keys for your user with `clan secrets users get $USER`:
-
-```console
-clan secrets users get alice
-
-[
-  {
-    "publickey": "age1hrrcspp645qtlj29krjpq66pqg990ejaq0djcms6y6evnmgglv5sq0gewu",
-    "type": "age",
-    "username": "alice"
-  },
-  {
-    "publickey": "age13kh4083t3g4x3ktr52nav6h7sy8ynrnky2x58pyp96c5s5nvqytqgmrt79",
-    "type": "age",
-    "username": "alice"
-  }
-]
-```
-
-To add a new key to your user:
-
-```console
-clan secrets users add-key $USER --age-key <your_public_key>
-```
-
-To remove a key from your user:
-
-```console
-clan secrets users remove-key $USER --age-key <your_public_key>
-```
-
-[age]: https://github.com/FiloSottile/age
-[age plugin]: https://github.com/FiloSottile/awesome-age?tab=readme-ov-file#plugins
-[sops]: https://github.com/getsops/sops
-[encrypting with age]: https://github.com/getsops/sops?tab=readme-ov-file#encrypting-using-age
-
-## Further: Using Age Plugins
-
-If you wish to use a key generated using an [age plugin] as your admin key, extra care is needed.
-
-You must **precede your secret key with a comment that contains its corresponding recipient**.
-
-This is usually output as part of the generation process
-and is only required because there is no unified mechanism for recovering a recipient from a plugin secret key.
-
-Here is an example:
-
-```title="~/.config/sops/age/keys.txt"
-# public key: age1zdy49ek6z60q9r34vf5mmzkx6u43pr9haqdh5lqdg7fh5tpwlfwqea356l
-AGE-PLUGIN-FIDO2-HMAC-1QQPQZRFR7ZZ2WCV...
-```
-
-!!! note
-    The comment that precedes the plugin secret key need only contain the recipient.
-    Any other text is ignored.
-
-    In the example above, you can specify `# recipient: age1zdy...`, `# public: age1zdy....` or even
-    just `# age1zdy....`
-
-You will need to add an entry into your `flake.nix` to ensure that the necessary `age` plugins
-are loaded when using Clan:
-
-```nix title="flake.nix"
-{
-  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-  inputs.nixpkgs.follows = "clan-core/nixpkgs";
-
-  outputs =
-    { self, clan-core, ... }:
-    let
-      # Sometimes this attribute set is defined in clan.nix
-      clan = clan-core.lib.clan {
-        inherit self;
-
-        meta.name = "myclan";
-
-        # Add Yubikey and FIDO2 HMAC plugins
-        # Note: the plugins listed here must be available in nixpkgs.
-        secrets.age.plugins = [
-          "age-plugin-yubikey"
-          "age-plugin-fido2-hmac"
-        ];
-
-        machines = {
-          # elided for brevity
-        };
-      };
-    in
-    {
-      inherit (clan) nixosConfigurations nixosModules clanInternals;
-
-      # elided for brevity
-    };
-}
-```

docs/site/guides/macos.md

@@ -7,7 +7,7 @@ This guide explains how to manage macOS machines using Clan.
 Currently, Clan supports the following features for macOS:
 
 - `clan machines update` for existing [nix-darwin](https://github.com/nix-darwin/nix-darwin) installations
-- Support for [vars](../guides/vars-backend.md)
+- Support for [vars](../concepts/generators.md)
 
 ## Add Your Machine to Your Clan Flake
 

docs/site/guides/migrations/migrate-inventory-services.md

@@ -1,7 +1,7 @@
 # Migrating from using `clanModules` to `clanServices`
 
 **Audience**: This is a guide for **people using `clanModules`**.
-If you are a **module author** and need to migrate your modules please consult our **new** [clanServices authoring guide](../authoring/clanServices/index.md)
+If you are a **module author** and need to migrate your modules please consult our **new** [clanServices authoring guide](../../guides/services/community.md)
 
 ## What's Changing?
 
@@ -329,6 +329,6 @@ instances = {
 
 ## Further reference
 
-* [Authoring a 'clan.service' module](../authoring/clanServices/index.md)
+* [Inventory Concept](../../concepts/inventory.md)
+* [Authoring a 'clan.service' module](../../guides/services/community.md)
 * [ClanServices](../clanServices.md)
-* [Inventory Reference](../../reference/nix-api/inventory.md)

docs/site/guides/migrations/migration-facts-vars.md

@@ -3,7 +3,7 @@
 For a high level overview about `vars` see our [blog post](https://clan.lol/blog/vars/).
 
 This guide will help you migrate your modules that still use our [`facts`](../../guides/secrets.md) backend
-to the [`vars`](../../guides/vars-backend.md) backend.
+to the [`vars`](../../concepts/generators.md) backend.
 
 The `vars` [module](../../reference/clan.core/vars.md) and the clan [command](../../reference/cli/vars.md) work in tandem, they should ideally be kept in sync.
 

docs/site/guides/more-machines.md

@@ -1,50 +0,0 @@
-
-Clan has two general methods of adding machines:
-
-- **Automatic**: Detects every folder in the `machines` folder.
-- **Declarative**: Explicit declarations in Nix.
-
-## Automatic registration
-
-Every folder `machines/{machineName}` will be registered automatically as a Clan machine.
-
-!!! info "Automatically loaded files"
-
-    The following files are loaded automatically for each Clan machine:
-
-    - [x] `machines/{machineName}/configuration.nix`
-    - [x] `machines/{machineName}/hardware-configuration.nix`
-    - [x] `machines/{machineName}/facter.json` Automatically configured, for further information see [nixos-facter](https://clan.lol/blog/nixos-facter/)
-    - [x] `machines/{machineName}/disko.nix` Automatically loaded, for further information see the [disko docs](https://github.com/nix-community/disko/blob/master/docs/quickstart.md).
-
-## Manual declaration
-
-Machines can be added via [`clan.inventory.machines`](../guides/inventory.md) or in `clan.machines`, which allows for defining NixOS options.
-
-=== "**Individual Machine Configuration**"
-
-    ```{.nix}
-    clan-core.lib.clan {
-        machines = {
-            "jon" = {
-                # Any valid nixos config
-            };
-        };
-    }
-    ```
-
-=== "**Inventory Configuration**"
-
-    ```{.nix}
-    clan-core.lib.clan {
-        inventory = {
-            machines = {
-                "jon" = {
-                    # Inventory can set tags and other metadata
-                    tags = [ "zone1" ];
-                    deploy.targetHost = "root@jon";
-                };
-            };
-        };
-    }
-    ```

docs/site/guides/secrets.md

@@ -1,25 +1,141 @@
-If you want to know more about how to save and share passwords in your clan read further!
+This article provides an overview over the underlying secrets system which is used by [Vars](../concepts/generators.md).
+Under most circumstances you should use [Vars](../concepts/generators.md) directly instead.
 
-### Adding a Secret
+Consider using `clan secrets` only for managing admin users and groups, as well as a debugging tool.
+
+Manually interacting with secrets via `clan secrets [set|remove]`, etc may break the integrity of your `Vars` state.
+
+---
+
+Clan enables encryption of secrets (such as passwords & keys) ensuring security and ease-of-use among users.
+
+By default, Clan uses the [sops](https://github.com/getsops/sops) format
+and integrates with [sops-nix](https://github.com/Mic92/sops-nix) on NixOS machines.
+Clan can also be configured to be used with other secret store [backends](../reference/clan.core/vars.md#clan.core.vars.settings.secretStore).
+
+## Create Your Admin Keypair
+
+To get started, you'll need to create **your admin keypair**.
+
+!!! info
+    Don't worry — if you've already made one before, this step won't change or overwrite it.
+
+```bash
+clan secrets key generate
+```
+
+**Output**:
+
+```{.console, .no-copy}
+Public key: age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7
+
+Generated age private key at '/home/joerg/.config/sops/age/keys.txt' for your user. Please back it up on a secure location or you will lose access to your secrets.
+Also add your age public key to the repository with 'clan secrets users add YOUR_USER age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7' (replace YOUR_USER with your actual username)
+```
+
+!!! warning
+    Make sure to keep a safe backup of the private key you've just created.
+    If it's lost, you won't be able to get to your secrets anymore because they all need the admin key to be unlocked.
+
+If you already have an [age] secret key and want to use that instead, you can simply edit `~/.config/sops/age/keys.txt`:
+
+```title="~/.config/sops/age/keys.txt"
+AGE-SECRET-KEY-13GWMK0KNNKXPTJ8KQ9LPSQZU7G3KU8LZDW474NX3D956GGVFAZRQTAE3F4
+```
+
+Alternatively, you can provide your [age] secret key as an environment variable `SOPS_AGE_KEY`, or in a different file
+using `SOPS_AGE_KEY_FILE`.
+For more information see the [SOPS] guide on [encrypting with age].
+
+!!! note
+    It's safe to add any secrets created by the clan CLI and placed in your repository to version control systems like `git`.
+
+## Add Your Public Key(s)
+
+```console
+clan secrets users add $USER --age-key <your_public_key>
+```
+
+It's best to choose the same username as on your Setup/Admin Machine that you use to control the deployment with.
+
+Once run this will create the following files:
+
+```{.console, .no-copy}
+sops/
+└── users/
+    └── <your_username>/
+        └── key.json
+```
+If you followed the quickstart tutorial all necessary secrets are initialized at this point.
+
+!!! note
+    You can add multiple age keys for a user by providing multiple `--age-key <your_public_key>` flags:
+
+    ```console
+    clan secrets users add $USER \
+        --age-key <your_public_key_1> \
+        --age-key <your_public_key_2> \
+        ...
+    ```
+
+## Manage Your Public Key(s)
+
+You can list keys for your user with `clan secrets users get $USER`:
+
+```console
+clan secrets users get alice
+
+[
+  {
+    "publickey": "age1hrrcspp645qtlj29krjpq66pqg990ejaq0djcms6y6evnmgglv5sq0gewu",
+    "type": "age",
+    "username": "alice"
+  },
+  {
+    "publickey": "age13kh4083t3g4x3ktr52nav6h7sy8ynrnky2x58pyp96c5s5nvqytqgmrt79",
+    "type": "age",
+    "username": "alice"
+  }
+]
+```
+
+To add a new key to your user:
+
+```console
+clan secrets users add-key $USER --age-key <your_public_key>
+```
+
+To remove a key from your user:
+
+```console
+clan secrets users remove-key $USER --age-key <your_public_key>
+```
+
+[age]: https://github.com/FiloSottile/age
+[age plugin]: https://github.com/FiloSottile/awesome-age?tab=readme-ov-file#plugins
+[sops]: https://github.com/getsops/sops
+[encrypting with age]: https://github.com/getsops/sops?tab=readme-ov-file#encrypting-using-age
+
+## Adding a Secret
 
 ```shellSession
 clan secrets set mysecret
 Paste your secret:
 ```
 
-### Retrieving a Stored Secret
+## Retrieving a Stored Secret
 
 ```bash
 clan secrets get mysecret
 ```
 
-### List all Secrets
+## List all Secrets
 
 ```bash
 clan secrets list
 ```
 
-### NixOS integration
+## NixOS integration
 
 A NixOS machine will automatically import all secrets that are encrypted for the
 current machine. At runtime it will use the host key to decrypt all secrets into
@@ -37,7 +153,7 @@ In your nixos configuration you can get a path to secrets like this `config.sops
 }
 ```
 
-### Assigning Access
+## Assigning Access
 
 When using `clan secrets set <secret>` without arguments, secrets are encrypted for the key of the user named like your current $USER.
 

docs/site/guides/services/community.md

@@ -1,16 +1,16 @@
 # Authoring a 'clan.service' module
 
 !!! Tip
-    This is the successor format to the older [clanModules](../clanModules/index.md)
+    This is the successor format to the older [clanModules](../../reference/clanModules/index.md)
 
     While some features might still be missing we recommend to adapt this format early and give feedback.
 
 ## Service Module Specification
 
 This section explains how to author a clan service module.
-We discussed the initial architecture in [01-clan-service-modules](../../../decisions/01-ClanModules.md) and decided to rework the format.
+We discussed the initial architecture in [01-clan-service-modules](../../decisions/01-ClanModules.md) and decided to rework the format.
 
-For the full specification and current state see: **[Service Author Reference](../../../reference/clanServices/clan-service-author-interface.md)**
+For the full specification and current state see: **[Service Author Reference](../../reference/clanServices/clan-service-author-interface.md)**
 
 ### A Minimal module
 
@@ -52,7 +52,7 @@ The imported module file must fulfill at least the following requirements:
 }
 ```
 
-For more attributes see: **[Service Author Reference](../../../reference/clanServices/clan-service-author-interface.md)**
+For more attributes see: **[Service Author Reference](../../reference/clanServices/clan-service-author-interface.md)**
 
 ### Adding functionality to the module
 
@@ -266,6 +266,6 @@ The benefit of this approach is that downstream users can override the value of
 
 ## Further
 
-- [Reference Documentation for Service Authors](../../../reference/clanServices/clan-service-author-interface.md)
-- [Migration Guide from ClanModules to ClanServices](../../migrations/migrate-inventory-services.md)
-- [Decision that lead to ClanServices](../../../decisions/01-ClanModules.md)
+- [Reference Documentation for Service Authors](../../reference/clanServices/clan-service-author-interface.md)
+- [Migration Guide from ClanModules to ClanServices](../../guides/migrations/migrate-inventory-services.md)
+- [Decision that lead to ClanServices](../../decisions/01-ClanModules.md)

docs/site/index.md

@@ -4,87 +4,72 @@ hide:
   - toc
 ---
 
-# :material-home: Welcome to **Clan**'s  documentation
+# :material-home: What is Clan?
+
+[Clan](https://clan.lol/) is a peer-to-peer computer management framework that
+empowers you to **selfhost in a reliable and scalable way**.   
+
+Built on NixOS, Clan provides a **declarative interface for managing machines** with automated [secret management](./guides/secrets.md), easy [mesh VPN
+connectivity](./guides/mesh-vpn.md), and [automated backups](./guides/backups.md). 
+
+Whether you're running a homelab or maintaining critical computing infrastructure,
+Clan will help **reduce maintenance burden** by allowing a **git repository to define your whole network** of computers.
+
+In combination with [sops-nix](https://github.com/Mic92/sops-nix), [nixos-anywhere](https://github.com/nix-community/nixos-anywhere) and [disko](https://github.com/nix-community/disko), Clan makes it possible to have **collaborative infrastructure**.
+
+At the heart of Clan are [Clan Services](./reference/clanServices/index.md) - the core
+concept that enables you to add functionality across multiple machines in your
+network. While Clan ships with essential core services, you can [create custom
+services](./guides/clanServices.md) tailored to your specific needs.
 
-[Getting Started](./guides/getting-started/index.md){ .md-button }
 
 ## :material-book: Guides
 
-**How-to Guides for achieving a certain goal or solving a specific issue.**
+How-to Guides for achieving a certain goal or solving a specific issue.
 
 <div class="grid cards" markdown>
 
--   [Adding more machines](./guides/more-machines.md)
+-   [:material-clock-fast: Getting Started](./guides/getting-started/index.md)
 
     ---
 
-    Learn how Clan automatically includes machines and Nix files.
+    Get started in less than 20 minutes!
 
--   [Vars Backend](./guides/vars-backend.md)
+-   [Mac OS](./guides/macos.md)
 
     ---
 
-    Learn how to manage secrets with vars.
-
--   [Inventory](./guides/inventory.md)
-
-    ---
-
-    Clan's declaration format for running **services** on one or multiple **machines**.
-
--   [Flake-parts](./guides/flake-parts.md)
-
-    ---
-
-    Use Clan with [https://flake.parts/]()
+    How to manage Mac OS machines with Clan
 
 -   [Contribute](./guides/contributing/CONTRIBUTING.md)
 
     ---
 
-    Discover how to set up a development environment to contribute to Clan!
-
--   [macOS machines](./guides/macos.md)
-
-    ---
-
-    Manage macOS machines with nix-darwin
+    How to set up a development environment
 
 </div>
 
-## API Reference
+## Concepts
 
-**Reference API Documentation**
+Explore the underlying principles of Clan
 
 <div class="grid cards" markdown>
 
--   [CLI Reference](./reference/cli/index.md)
+-   [Generators](./concepts/generators.md)
 
     ---
 
-    The `clan` CLI command
+    Learn about Generators, our way to secret management
 
--   [Service Modules](./reference/clanServices/index.md)
+-   [Inventory](./concepts/inventory.md)
 
     ---
 
-    An overview of available service modules
-
--   [Core](./reference/clan.core/index.md)
-
-    ---
-
-    The clan core nix module.
-    This is imported when using clan and is the basis of the extra functionality
-    that can be provided.
-
--   [(Legacy) Modules](./reference/clanModules/index.md)
-
-    ---
-
-    An overview of available clanModules
-
-    !!! Example "These will be deprecated soon"
-
+    Learn about the Inventory, a multi machine Nix interface
 
 </div>
+
+
+## Blog
+
+Visit our [Clan Blog](https://clan.lol/blog/) for the latest updates, tutorials, and community stories.

docs/site/intern/index.md

@@ -1,25 +0,0 @@
-# Developer Documentation
-
-!!! Danger
-    This documentation is **not** intended for external users. It may contain low-level details and internal-only interfaces.*
-
-Welcome to the internal developer documentation.
-
-This section is intended for contributors, engineers, and internal stakeholders working directly with our system, tooling, and APIs. It provides a technical overview of core components, internal APIs, conventions, and patterns that support the platform.
-
-Our goal is to make the internal workings of the system **transparent, discoverable, and consistent** — helping you contribute confidently, troubleshoot effectively, and build faster.
-
-## What's Here?
-
-!!! note "docs migration ongoing"
-
-- [ ] **API Reference**: 🚧🚧🚧 Detailed documentation of internal API functions, inputs, and expected outputs. 🚧🚧🚧
-- [ ] **System Concepts**: Architectural overviews and domain-specific guides.
-- [ ] **Development Guides**: How to test, extend, or integrate with key components.
-- [ ] **Design Notes**: Rationales behind major design decisions or patterns.
-
-## Who is This For?
-
-* Developers contributing to the platform
-* Engineers debugging or extending internal systems
-* Anyone needing to understand **how** and **why** things work under the hood

docs/site/reference/index.md

@@ -4,10 +4,10 @@ This section of the site provides an overview of available options and commands
 
 ---
 
+- [Clan Configuration Option](../options.md) - for defining a Clan
 - Learn how to use the [Clan CLI](./cli/index.md)
 - Explore available services and application [modules](./clanModules/index.md)
-- Discover [configuration options](./clan.core/index.md) that manage essential features
-- Find descriptions of the [Nix interfaces](./nix-api/clan.md) for defining a Clan
+- [NixOS Configuration Options](./clan.core/index.md) - Additional options avilable on a NixOS machine.
 
 ---
 

docs/site/static/extra.css

@@ -2,6 +2,7 @@
   font-family: "Roboto";
   src: url(./Roboto-Regular.ttf) format("truetype");
 }
+
 @font-face {
   font-family: "Fira Code";
   src: url(./FiraCode-VF.ttf) format("truetype");
@@ -20,3 +21,9 @@
 .md-nav__item.md-nav__item--section > label > span {
   color: var(--md-typeset-a-color);
 }
+
+.md-typeset h4 {
+  margin: 3em 0 0.5em;
+  font-weight: bold;
+  color: #7ebae4;
+}

flake.lock

@@ -31,11 +31,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1752718651,
-        "narHash": "sha256-PkaR0qmyP9q/MDN3uYa+RLeBA0PjvEQiM0rTDDBXkL8=",
+        "lastModified": 1753140376,
+        "narHash": "sha256-7lrVrE0jSvZHrxEzvnfHFE/Wkk9DDqb+mYCodI5uuB8=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "d5ad4485e6f2edcc06751df65c5e16572877db88",
+        "rev": "545aba02960caa78a31bd9a8709a0ad4b6320a5c",
         "type": "github"
       },
       "original": {
@@ -51,11 +51,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1751413152,
-        "narHash": "sha256-Tyw1RjYEsp5scoigs1384gIg6e0GoBVjms4aXFfRssQ=",
+        "lastModified": 1753121425,
+        "narHash": "sha256-TVcTNvOeWWk1DXljFxVRp+E0tzG1LhrVjOGGoMHuXio=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "77826244401ea9de6e3bac47c2db46005e1f30b5",
+        "rev": "644e0fc48951a860279da645ba77fe4a6e814c5e",
         "type": "github"
       },
       "original": {

lib/modules/clan/interface.nix

@@ -229,8 +229,8 @@ in
     };
 
     inventory = lib.mkOption {
-      type = types.submodule {
-        imports = [
+      type = types.submoduleWith {
+        modules = [
           {
             _module.args = { inherit clanLib; };
             _file = "clan interface";

lib/modules/clan/module.nix

@@ -247,7 +247,7 @@ in
               {
                 distributedServices = clanLib.inventory.mapInstances {
                   inherit (clanConfig) inventory exportsModule;
-                  inherit flakeInputs;
+                  inherit flakeInputs directory;
                   clanCoreModules = clan-core.clan.modules;
                   prefix = [ "distributedServices" ];
                 };

lib/modules/inventory/distributed-service/all-services-wrapper.nix

@@ -1,4 +1,8 @@
 # Wraps all services in one fixed point module
+{
+  # TODO: consume directly from clan.config
+  directory,
+}:
 {
   lib,
   config,
@@ -29,6 +33,8 @@ in
               {
                 _module.args._ctx = [ name ];
                 _module.args.exports' = config.exports;
+                _module.args.directory = directory;
+
               }
             )
             ./service-module.nix
@@ -71,8 +77,5 @@ in
       };
       default = { };
     };
-    debug = mkOption {
-      default = lib.mapAttrsToList (_: service: service.exports) config.mappedServices;
-    };
   };
 }

lib/modules/inventory/distributed-service/inventory-adapter.nix

@@ -24,6 +24,7 @@ in
       flakeInputs,
       # The clan inventory
       inventory,
+      directory,
       clanCoreModules,
       prefix ? [ ],
       exportsModule,
@@ -128,7 +129,7 @@ in
             _ctx = prefix;
           };
           modules = [
-            ./all-services-wrapper.nix
+            (import ./all-services-wrapper.nix { inherit directory; })
           ] ++ modules;
         };
 

lib/modules/inventory/distributed-service/service-module.nix

@@ -2,6 +2,7 @@
   lib,
   config,
   _ctx,
+  directory,
   ...
 }:
 let
@@ -212,7 +213,7 @@ in
 
                           options.extraModules = lib.mkOption {
                             default = [ ];
-                            type = types.listOf (types.deferredModule);
+                            type = types.listOf (types.either types.deferredModule types.str);
                           };
                         })
                       ];
@@ -755,10 +756,14 @@ in
             instanceRes
             // {
               nixosModule = {
-                imports = [
-                  # Result of the applied 'perInstance = {...}: {  nixosModule = { ... }; }'
-                  instanceRes.nixosModule
-                ] ++ instanceCfg.roles.${roleName}.extraModules;
+                imports =
+                  [
+                    # Result of the applied 'perInstance = {...}: {  nixosModule = { ... }; }'
+                    instanceRes.nixosModule
+                  ]
+                  ++ (map (
+                    s: if builtins.typeOf s == "string" then "${directory}/${s}" else s
+                  ) instanceCfg.roles.${roleName}.extraModules);
               };
             }
 

lib/modules/inventory/distributed-service/tests/default.nix

@@ -45,6 +45,7 @@ let
       };
     in
     clanLib.inventory.mapInstances {
+      directory = ./.;
       clanCoreModules = { };
       flakeInputs = flakeInputsFixture;
       inherit inventory;
@@ -52,6 +53,7 @@ let
     };
 in
 {
+  extraModules = import ./extraModules.nix { inherit clanLib; };
   exports = import ./exports.nix { inherit lib clanLib; };
   resolve_module_spec = import ./import_module_spec.nix { inherit lib callInventoryAdapter; };
   test_simple =

lib/modules/inventory/distributed-service/tests/extraModules.nix

@@ -0,0 +1,33 @@
+{ clanLib }:
+let
+  clan = clanLib.clan {
+    self = { };
+    directory = ./.;
+
+    machines.jon = {
+      nixpkgs.hostPlatform = "x86_64-linux";
+
+    };
+    # A module that adds exports perMachine
+    modules.A = {
+      manifest.name = "A";
+      roles.peer = { };
+    };
+
+    inventory = {
+      instances.A = {
+        module.input = "self";
+        roles.peer.tags.all = { };
+
+        roles.peer.extraModules = [ ./oneOption.nix ];
+      };
+    };
+  };
+in
+{
+  test_1 = {
+    inherit clan;
+    expr = clan.config.nixosConfigurations.jon.config.testDebug;
+    expected = 42;
+  };
+}

lib/modules/inventory/distributed-service/tests/oneOption.nix

@@ -0,0 +1,6 @@
+{ lib, ... }:
+{
+  options.testDebug = lib.mkOption {
+    default = 42;
+  };
+}

lib/modules/inventoryClass/interface.nix

@@ -142,7 +142,7 @@ in
             - The module MUST have at least `features = [ "inventory" ]` in the frontmatter section.
             - The module MUST have a subfolder `roles` with at least one `{roleName}.nix` file.
 
-            For further information see: [Module Authoring Guide](../../guides/authoring/clanServices/index.md).
+            For further information see: [Module Authoring Guide](../../guides/services/community.md).
 
         ???+ example
             ```nix
@@ -179,8 +179,7 @@ in
               map (m: "'${m}'") (lib.attrNames (lib.filterAttrs (n: _v: !builtins.elem n allowedNames) moduleSet))
             )}
 
-            See: https://docs.clan.lol/guides/clanServices/
-            And: https://docs.clan.lol/guides/authoring/clanServices/
+            See: https://docs.clan.lol/guides/services/community/
           '' moduleSet;
     };
 

lib/test/container-test-driver/test_driver/__init__.py

@@ -313,6 +313,18 @@ class Machine:
         command = f"nc -z {shlex.quote(addr)} {port}"
         self.wait_until_succeeds(command, timeout=timeout)
 
+    def wait_for_file(self, filename: str, timeout: int = 30) -> None:
+        """
+        Waits until the file exists in the machine's file system.
+        """
+
+        def check_file(_last_try: bool) -> bool:
+            result = self.execute(f"test -e {filename}")
+            return result.returncode == 0
+
+        with self.nested(f"waiting for file '{filename}'"):
+            retry(check_file, timeout)
+
     def wait_for_unit(self, unit: str, timeout: int = 900) -> None:
         """
         Wait for a systemd unit to get into "active" state.
@@ -407,6 +419,14 @@ def setup_filesystems(container: ContainerInfo) -> None:
     Path("/etc/os-release").touch()
     Path("/etc/machine-id").write_text("a5ea3f98dedc0278b6f3cc8c37eeaeac")
     container.nix_store_dir.mkdir(parents=True)
+
+    # Recreate symlinks
+    for file in Path("/nix/store").iterdir():
+        if file.is_symlink():
+            target = file.readlink()
+            sym = container.nix_store_dir / file.name
+            os.symlink(target, sym)
+
     # Read /proc/mounts and replicate every bind mount
     with Path("/proc/self/mounts").open() as f:
         for line in f:

nixosModules/clanCore/outputs.nix

@@ -31,6 +31,7 @@
             The deployment data is now accessed directly from the configuration
             instead of being written to a separate JSON file.
           '';
+          defaultText = "error: deployment.json file generation has been removed in favor of direct selectors.";
         };
         deployment.buildHost = lib.mkOption {
           type = lib.types.nullOr lib.types.str;
@@ -54,10 +55,10 @@
         deployment.nixosMobileWorkaround = lib.mkOption {
           type = lib.types.bool;
           description = ''
-            if true, the deployment will first do a nixos-rebuild switch 
+            if true, the deployment will first do a nixos-rebuild switch
             to register the boot profile the command will fail applying it to the running system
-            which is why afterwards we execute a nixos-rebuild test to apply 
-            the new config without having to reboot. 
+            which is why afterwards we execute a nixos-rebuild test to apply
+            the new config without having to reboot.
             This is a nixos-mobile deployment bug and will be removed in the future
           '';
           default = false;

pkgs/clan-app/clan_app/api/api_bridge.py

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any
 
 from clan_lib.api import ApiResponse
 from clan_lib.api.tasks import WebThread
-from clan_lib.async_run import set_should_cancel
+from clan_lib.async_run import set_current_thread_opkey, set_should_cancel
 
 if TYPE_CHECKING:
     from .middleware import Middleware
@@ -98,7 +98,7 @@ class ApiBridge(ABC):
         *,
         thread_name: str = "ApiBridgeThread",
         wait_for_completion: bool = False,
-        timeout: float = 60.0,
+        timeout: float = 60.0 * 60,  # 1 hour default timeout
     ) -> None:
         """Process an API request in a separate thread with cancellation support.
 
@@ -112,6 +112,7 @@ class ApiBridge(ABC):
 
         def thread_task(stop_event: threading.Event) -> None:
             set_should_cancel(lambda: stop_event.is_set())
+            set_current_thread_opkey(op_key)
             try:
                 log.debug(
                     f"Processing {request.method_name} with args {request.args} "

pkgs/clan-app/clan_app/api/file_gtk.py

@@ -9,6 +9,7 @@ gi.require_version("Gtk", "4.0")
 
 from clan_lib.api import ApiError, ErrorDataClass, SuccessDataClass
 from clan_lib.api.directory import FileRequest
+from clan_lib.async_run import get_current_thread_opkey
 from clan_lib.clan.check import check_clan_valid
 from clan_lib.flake import Flake
 from gi.repository import Gio, GLib, Gtk
@@ -24,7 +25,7 @@ def remove_none(_list: list) -> list:
 RESULT: dict[str, SuccessDataClass[list[str] | None] | ErrorDataClass] = {}
 
 
-def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass:
+def get_clan_folder() -> SuccessDataClass[Flake] | ErrorDataClass:
     """
     Opens the clan folder using the GTK file dialog.
     Returns the path to the clan folder or an error if it fails.
@@ -34,7 +35,10 @@ def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass:
         title="Select Clan Folder",
         initial_folder=str(Path.home()),
     )
-    response = get_system_file(file_request, op_key=op_key)
+
+    response = get_system_file(file_request)
+
+    op_key = response.op_key
 
     if isinstance(response, ErrorDataClass):
         return response
@@ -70,8 +74,13 @@ def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass:
 
 
 def get_system_file(
-    file_request: FileRequest, *, op_key: str
+    file_request: FileRequest,
 ) -> SuccessDataClass[list[str] | None] | ErrorDataClass:
+    op_key = get_current_thread_opkey()
+
+    if not op_key:
+        msg = "No operation key found in the current thread context."
+        raise RuntimeError(msg)
     GLib.idle_add(gtk_open_file, file_request, op_key)
 
     while RESULT.get(op_key) is None:

pkgs/clan-app/clan_app/api/middleware/argument_parsing.py

@@ -21,18 +21,12 @@ class ArgumentParsingMiddleware(Middleware):
             # Convert dictionary arguments to dataclass instances
             reconciled_arguments = {}
             for k, v in context.request.args.items():
-                if k == "op_key":
-                    continue
-
                 # Get the expected argument type from the API
                 arg_class = self.api.get_method_argtype(context.request.method_name, k)
 
                 # Convert dictionary to dataclass instance
                 reconciled_arguments[k] = from_dict(arg_class, v)
 
-            # Add op_key to arguments
-            reconciled_arguments["op_key"] = context.request.op_key
-
             # Create a new request with reconciled arguments
 
             updated_request = BackendRequest(

pkgs/clan-app/clan_app/deps/http/http_bridge.py

@@ -1,13 +1,22 @@
 import json
 import logging
+import threading
 import uuid
 from http.server import BaseHTTPRequestHandler
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 from urllib.parse import urlparse
 
-from clan_lib.api import MethodRegistry, SuccessDataClass, dataclass_to_dict
+from clan_lib.api import (
+    MethodRegistry,
+    SuccessDataClass,
+    dataclass_to_dict,
+)
 from clan_lib.api.tasks import WebThread
+from clan_lib.async_run import (
+    set_current_thread_opkey,
+    set_should_cancel,
+)
 
 from clan_app.api.api_bridge import ApiBridge, BackendRequest, BackendResponse
 
@@ -324,17 +333,34 @@ class HttpBridge(ApiBridge, BaseHTTPRequestHandler):
             msg = f"Operation key '{op_key}' is already in use. Please try again."
             raise ValueError(msg)
 
+    def process_request_in_thread(
+        self,
+        request: BackendRequest,
+        *,
+        thread_name: str = "ApiBridgeThread",
+        wait_for_completion: bool = False,
+        timeout: float = 60.0 * 60,  # 1 hour default timeout
+    ) -> None:
+        pass
+
     def _process_api_request_in_thread(
         self, api_request: BackendRequest, method_name: str
     ) -> None:
         """Process the API request in a separate thread."""
-        # Use the inherited thread processing method
-        self.process_request_in_thread(
-            api_request,
-            thread_name="HttpThread",
-            wait_for_completion=True,
-            timeout=60.0,
+        stop_event = threading.Event()
+        request = api_request
+        op_key = request.op_key or "unknown"
+        set_should_cancel(lambda: stop_event.is_set())
+        set_current_thread_opkey(op_key)
+
+        curr_thread = threading.current_thread()
+        self.threads[op_key] = WebThread(thread=curr_thread, stop_event=stop_event)
+
+        log.debug(
+            f"Processing {request.method_name} with args {request.args} "
+            f"and header {request.header}"
         )
+        self.process_request(request)
 
     def log_message(self, format: str, *args: Any) -> None:  # noqa: A002
         """Override default logging to use our logger."""

pkgs/clan-app/process-compose-2d.yaml

@@ -1,39 +0,0 @@
-version: "0.5"
-
-processes:
-  # App Dev
-
-  clan-app-ui:
-    namespace: "app"
-    command: |
-      cd $(git rev-parse --show-toplevel)/pkgs/clan-app/ui-2d
-      npm install
-      vite
-    ready_log_line: "VITE"
-
-  clan-app:
-    namespace: "app"
-    command: |
-      cd $(git rev-parse --show-toplevel)/pkgs/clan-app
-      ./bin/clan-app --debug --content-uri http://localhost:3000
-    depends_on:
-      clan-app-ui:
-        condition: "process_log_ready"
-    is_foreground: true
-    ready_log_line: "Debug mode enabled"
-
-  # Storybook Dev
-
-  storybook:
-    namespace: "storybook"
-    command: |
-      cd $(git rev-parse --show-toplevel)/pkgs/clan-app/ui-2d
-      npm run storybook-dev -- --ci
-    ready_log_line: "started"
-
-  luakit:
-    namespace: "storybook"
-    command: "luakit http://localhost:6006"
-    depends_on:
-      storybook:
-        condition: "process_log_ready"

pkgs/clan-app/ui/package-lock.json

@@ -17,6 +17,7 @@
         "@solidjs/router": "^0.15.3",
         "@tanstack/eslint-plugin-query": "^5.51.12",
         "@tanstack/solid-query": "^5.76.0",
+        "@tanstack/solid-query-devtools": "^5.83.0",
         "solid-js": "^1.9.7",
         "solid-toast": "^0.5.0",
         "three": "^0.176.0",
@@ -53,7 +54,6 @@
         "postcss": "^8.4.38",
         "postcss-url": "^10.1.3",
         "prettier": "^3.2.5",
-        "solid-devtools": "^0.34.0",
         "storybook": "^9.0.8",
         "swagger-ui-dist": "^5.26.2",
         "tailwindcss": "^3.4.3",
@@ -360,22 +360,6 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-syntax-typescript": {
-      "version": "7.27.1",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.27.1.tgz",
-      "integrity": "sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@babel/helper-plugin-utils": "^7.27.1"
-      },
-      "engines": {
-        "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
-      }
-    },
     "node_modules/@babel/runtime": {
       "version": "7.27.6",
       "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.27.6.tgz",
@@ -1552,13 +1536,6 @@
         "node": ">= 8"
       }
     },
-    "node_modules/@nothing-but/utils": {
-      "version": "0.17.0",
-      "resolved": "https://registry.npmjs.org/@nothing-but/utils/-/utils-0.17.0.tgz",
-      "integrity": "sha512-TuCHcHLOqDL0SnaAxACfuRHBNRgNJcNn9X0GiH5H3YSDBVquCr3qEIG3FOQAuMyZCbu9w8nk2CHhOsn7IvhIwQ==",
-      "dev": true,
-      "license": "MIT"
-    },
     "node_modules/@oxc-resolver/binding-darwin-arm64": {
       "version": "11.5.0",
       "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-darwin-arm64/-/binding-darwin-arm64-11.5.0.tgz",
@@ -1813,64 +1790,6 @@
         "@sinonjs/commons": "^3.0.1"
       }
     },
-    "node_modules/@solid-devtools/debugger": {
-      "version": "0.28.1",
-      "resolved": "https://registry.npmjs.org/@solid-devtools/debugger/-/debugger-0.28.1.tgz",
-      "integrity": "sha512-6qIUI6VYkXoRnL8oF5bvh2KgH71qlJ18hNw/mwSyY6v48eb80ZR48/5PDXufUa3q+MBSuYa1uqTMwLewpay9eg==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@nothing-but/utils": "~0.17.0",
-        "@solid-devtools/shared": "^0.20.0",
-        "@solid-primitives/bounds": "^0.1.1",
-        "@solid-primitives/event-listener": "^2.4.1",
-        "@solid-primitives/keyboard": "^1.3.1",
-        "@solid-primitives/rootless": "^1.5.1",
-        "@solid-primitives/scheduled": "^1.5.1",
-        "@solid-primitives/static-store": "^0.1.1",
-        "@solid-primitives/utils": "^6.3.1"
-      },
-      "peerDependencies": {
-        "solid-js": "^1.9.0"
-      }
-    },
-    "node_modules/@solid-devtools/shared": {
-      "version": "0.20.0",
-      "resolved": "https://registry.npmjs.org/@solid-devtools/shared/-/shared-0.20.0.tgz",
-      "integrity": "sha512-o5TACmUOQsxpzpOKCjbQqGk8wL8PMi+frXG9WNu4Lh3PQVUB6hs95Kl/S8xc++zwcMguUKZJn8h5URUiMOca6Q==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@nothing-but/utils": "~0.17.0",
-        "@solid-primitives/event-listener": "^2.4.1",
-        "@solid-primitives/media": "^2.3.1",
-        "@solid-primitives/refs": "^1.1.1",
-        "@solid-primitives/rootless": "^1.5.1",
-        "@solid-primitives/scheduled": "^1.5.1",
-        "@solid-primitives/static-store": "^0.1.1",
-        "@solid-primitives/styles": "^0.1.1",
-        "@solid-primitives/utils": "^6.3.1"
-      },
-      "peerDependencies": {
-        "solid-js": "^1.9.0"
-      }
-    },
-    "node_modules/@solid-primitives/bounds": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/@solid-primitives/bounds/-/bounds-0.1.3.tgz",
-      "integrity": "sha512-UbiyKMdSPmtijcEDnYLQL3zzaejpwWDAJJ4Gt5P0hgVs6A72piov0GyNw7V2SroH7NZFwxlYS22YmOr8A5xc1Q==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@solid-primitives/event-listener": "^2.4.3",
-        "@solid-primitives/resize-observer": "^2.1.3",
-        "@solid-primitives/static-store": "^0.1.2",
-        "@solid-primitives/utils": "^6.3.2"
-      },
-      "peerDependencies": {
-        "solid-js": "^1.6.12"
-      }
-    },
     "node_modules/@solid-primitives/event-listener": {
       "version": "2.4.3",
       "resolved": "https://registry.npmjs.org/@solid-primitives/event-listener/-/event-listener-2.4.3.tgz",
@@ -1883,21 +1802,6 @@
         "solid-js": "^1.6.12"
       }
     },
-    "node_modules/@solid-primitives/keyboard": {
-      "version": "1.3.3",
-      "resolved": "https://registry.npmjs.org/@solid-primitives/keyboard/-/keyboard-1.3.3.tgz",
-      "integrity": "sha512-9dQHTTgLBqyAI7aavtO+HnpTVJgWQA1ghBSrmLtMu1SMxLPDuLfuNr+Tk5udb4AL4Ojg7h9JrKOGEEDqsJXWJA==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@solid-primitives/event-listener": "^2.4.3",
-        "@solid-primitives/rootless": "^1.5.2",
-        "@solid-primitives/utils": "^6.3.2"
-      },
-      "peerDependencies": {
-        "solid-js": "^1.6.12"
-      }
-    },
     "node_modules/@solid-primitives/keyed": {
       "version": "1.5.2",
       "resolved": "https://registry.npmjs.org/@solid-primitives/keyed/-/keyed-1.5.2.tgz",
@@ -1985,16 +1889,6 @@
         "solid-js": "^1.6.12"
       }
     },
-    "node_modules/@solid-primitives/scheduled": {
-      "version": "1.5.2",
-      "resolved": "https://registry.npmjs.org/@solid-primitives/scheduled/-/scheduled-1.5.2.tgz",
-      "integrity": "sha512-/j2igE0xyNaHhj6kMfcUQn5rAVSTLbAX+CDEBm25hSNBmNiHLu2lM7Usj2kJJ5j36D67bE8wR1hBNA8hjtvsQA==",
-      "dev": true,
-      "license": "MIT",
-      "peerDependencies": {
-        "solid-js": "^1.6.12"
-      }
-    },
     "node_modules/@solid-primitives/static-store": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/@solid-primitives/static-store/-/static-store-0.1.2.tgz",
@@ -2028,20 +1922,6 @@
         }
       }
     },
-    "node_modules/@solid-primitives/styles": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/@solid-primitives/styles/-/styles-0.1.2.tgz",
-      "integrity": "sha512-7iX5K+J5b1PRrbgw3Ki92uvU2LgQ0Kd/QMsrAZxDg5dpUBwMyTijZkA3bbs1ikZsT1oQhS41bTyKbjrXeU0Awg==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@solid-primitives/rootless": "^1.5.2",
-        "@solid-primitives/utils": "^6.3.2"
-      },
-      "peerDependencies": {
-        "solid-js": "^1.6.12"
-      }
-    },
     "node_modules/@solid-primitives/trigger": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/@solid-primitives/trigger/-/trigger-1.2.2.tgz",
@@ -2281,9 +2161,19 @@
       }
     },
     "node_modules/@tanstack/query-core": {
-      "version": "5.81.5",
-      "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.81.5.tgz",
-      "integrity": "sha512-ZJOgCy/z2qpZXWaj/oxvodDx07XcQa9BF92c0oINjHkoqUPsmm3uG08HpTaviviZ/N9eP1f9CM7mKSEkIo7O1Q==",
+      "version": "5.83.0",
+      "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.83.0.tgz",
+      "integrity": "sha512-0M8dA+amXUkyz5cVUm/B+zSk3xkQAcuXuz5/Q/LveT4ots2rBpPTZOzd7yJa2Utsf8D2Upl5KyjhHRY+9lB/XA==",
+      "license": "MIT",
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/tannerlinsley"
+      }
+    },
+    "node_modules/@tanstack/query-devtools": {
+      "version": "5.81.2",
+      "resolved": "https://registry.npmjs.org/@tanstack/query-devtools/-/query-devtools-5.81.2.tgz",
+      "integrity": "sha512-jCeJcDCwKfoyyBXjXe9+Lo8aTkavygHHsUHAlxQKKaDeyT0qyQNLKl7+UyqYH2dDF6UN/14873IPBHchcsU+Zg==",
       "license": "MIT",
       "funding": {
         "type": "github",
@@ -2291,12 +2181,12 @@
       }
     },
     "node_modules/@tanstack/solid-query": {
-      "version": "5.81.5",
-      "resolved": "https://registry.npmjs.org/@tanstack/solid-query/-/solid-query-5.81.5.tgz",
-      "integrity": "sha512-VqVXaxiJIsKA6B45uApF+RUD3g8Roj/vdAuGpHMjR+RyHqlyQ+hOwgmALkzlbkbIaWCQi8CJOvrbU6WOBuMOxA==",
+      "version": "5.83.0",
+      "resolved": "https://registry.npmjs.org/@tanstack/solid-query/-/solid-query-5.83.0.tgz",
+      "integrity": "sha512-RF8Tv9+6+Kmzj+EafbTzvzzPq+J5SzHtc1Tz3D2MZ/EvlZTH+GL5q4HNnWK3emg7CB6WzyGnTuERmmWJaZs8/w==",
       "license": "MIT",
       "dependencies": {
-        "@tanstack/query-core": "5.81.5"
+        "@tanstack/query-core": "5.83.0"
       },
       "funding": {
         "type": "github",
@@ -2306,6 +2196,23 @@
         "solid-js": "^1.6.0"
       }
     },
+    "node_modules/@tanstack/solid-query-devtools": {
+      "version": "5.83.0",
+      "resolved": "https://registry.npmjs.org/@tanstack/solid-query-devtools/-/solid-query-devtools-5.83.0.tgz",
+      "integrity": "sha512-Z0wQlAWXz/U2bJ/paMRBTDhMoPnB9Te6GmA21sXnI+nDnAAPZRcPxFBiCgYJS3eFsvbkdRGJwoUSQrdIgy0shg==",
+      "license": "MIT",
+      "dependencies": {
+        "@tanstack/query-devtools": "5.81.2"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/tannerlinsley"
+      },
+      "peerDependencies": {
+        "@tanstack/solid-query": "^5.83.0",
+        "solid-js": "^1.6.0"
+      }
+    },
     "node_modules/@testing-library/dom": {
       "version": "10.4.0",
       "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.0.tgz",
@@ -6996,29 +6903,6 @@
         "url": "https://github.com/sponsors/cyyynthia"
       }
     },
-    "node_modules/solid-devtools": {
-      "version": "0.34.3",
-      "resolved": "https://registry.npmjs.org/solid-devtools/-/solid-devtools-0.34.3.tgz",
-      "integrity": "sha512-ZQua959n+Zu3sLbm9g0IRjYUb1YYlYbu83PWLRoKbSsq0a3ItQNhnS2OBU7rQNmOKZiMexNo9Z3izas9BcOKDg==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@babel/core": "^7.27.4",
-        "@babel/plugin-syntax-typescript": "^7.27.1",
-        "@babel/types": "^7.27.6",
-        "@solid-devtools/debugger": "^0.28.1",
-        "@solid-devtools/shared": "^0.20.0"
-      },
-      "peerDependencies": {
-        "solid-js": "^1.9.0",
-        "vite": "^2.2.3 || ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0"
-      },
-      "peerDependenciesMeta": {
-        "vite": {
-          "optional": true
-        }
-      }
-    },
     "node_modules/solid-js": {
       "version": "1.9.7",
       "resolved": "https://registry.npmjs.org/solid-js/-/solid-js-1.9.7.tgz",

pkgs/clan-app/ui/package.json

@@ -52,7 +52,6 @@
     "postcss": "^8.4.38",
     "postcss-url": "^10.1.3",
     "prettier": "^3.2.5",
-    "solid-devtools": "^0.34.0",
     "storybook": "^9.0.8",
     "swagger-ui-dist": "^5.26.2",
     "tailwindcss": "^3.4.3",
@@ -73,6 +72,7 @@
     "@solidjs/router": "^0.15.3",
     "@tanstack/eslint-plugin-query": "^5.51.12",
     "@tanstack/solid-query": "^5.76.0",
+    "@tanstack/solid-query-devtools": "^5.83.0",
     "solid-js": "^1.9.7",
     "solid-toast": "^0.5.0",
     "three": "^0.176.0",

pkgs/clan-app/ui/src/components/Sidebar/Sidebar.css

@@ -1,5 +1,5 @@
 div.sidebar {
-  @apply h-full w-auto max-w-60 border-none;
+  @apply w-60 border-none z-10;
 
   & > div.header {
   }

pkgs/clan-app/ui/src/components/Sidebar/Sidebar.stories.tsx

@@ -0,0 +1,157 @@
+import type { Meta, StoryObj } from "@kachurun/storybook-solid";
+import {
+  createMemoryHistory,
+  MemoryRouter,
+  Route,
+  RouteSectionProps,
+} from "@solidjs/router";
+import { Sidebar } from "@/src/components/Sidebar/Sidebar";
+import { Suspense } from "solid-js";
+import { QueryClient, QueryClientProvider } from "@tanstack/solid-query";
+import { addClanURI, resetStore } from "@/src/stores/clan";
+import { SolidQueryDevtools } from "@tanstack/solid-query-devtools";
+import { encodeBase64 } from "@/src/hooks/clan";
+
+const defaultClanURI = "/home/brian/clans/my-clan";
+
+const queryData = {
+  "/home/brian/clans/my-clan": {
+    details: {
+      name: "Brian's Clan",
+      uri: "/home/brian/clans/my-clan",
+    },
+    machines: {
+      europa: {
+        name: "Europa",
+        machineClass: "nixos",
+      },
+      ganymede: {
+        name: "Ganymede",
+        machineClass: "nixos",
+      },
+    },
+  },
+  "/home/brian/clans/davhau": {
+    details: {
+      name: "Dave's Clan",
+      uri: "/home/brian/clans/davhau",
+    },
+    machines: {
+      callisto: {
+        name: "Callisto",
+        machineClass: "nixos",
+      },
+      amalthea: {
+        name: "Amalthea",
+        machineClass: "nixos",
+      },
+    },
+  },
+  "/home/brian/clans/mic92": {
+    details: {
+      name: "Mic92's Clan",
+      uri: "/home/brian/clans/mic92",
+    },
+    machines: {
+      thebe: {
+        name: "Thebe",
+        machineClass: "nixos",
+      },
+      sponde: {
+        name: "Sponde",
+        machineClass: "nixos",
+      },
+    },
+  },
+};
+
+const staticSections = [
+  {
+    title: "Links",
+    links: [
+      { label: "GitHub", path: "https://github.com/brian-the-dev" },
+      { label: "Twitter", path: "https://twitter.com/brian_the_dev" },
+      {
+        label: "LinkedIn",
+        path: "https://www.linkedin.com/in/brian-the-dev/",
+      },
+      {
+        label: "Instagram",
+        path: "https://www.instagram.com/brian_the_dev/",
+      },
+    ],
+  },
+];
+
+const meta: Meta<RouteSectionProps> = {
+  title: "Components/Sidebar",
+  component: Sidebar,
+  render: () => {
+    // set history to point to our test clan
+    const history = createMemoryHistory();
+    history.set({ value: `/clans/${encodeBase64(defaultClanURI)}` });
+
+    // reset local storage and then add each clan
+    resetStore();
+
+    Object.keys(queryData).forEach((uri) => addClanURI(uri));
+
+    return (
+      <div style="height: 670px;">
+        <MemoryRouter
+          history={history}
+          root={(props) => <Suspense>{props.children}</Suspense>}
+        >
+          <Route
+            path="/clans/:clanURI"
+            component={() => <Sidebar staticSections={staticSections} />}
+          >
+            <Route path="/" />
+            <Route
+              path="/machines/:machineID"
+              component={() => <h1>Machine</h1>}
+            />
+          </Route>
+        </MemoryRouter>
+        <SolidQueryDevtools initialIsOpen={true} />
+      </div>
+    );
+  },
+};
+
+export default meta;
+
+type Story = StoryObj<RouteSectionProps>;
+
+export const Default: Story = {
+  args: {},
+  decorators: [
+    (Story: StoryObj) => {
+      const queryClient = new QueryClient({
+        defaultOptions: {
+          queries: {
+            retry: false,
+            staleTime: Infinity,
+          },
+        },
+      });
+
+      Object.entries(queryData).forEach(([clanURI, clan]) => {
+        queryClient.setQueryData(
+          ["clans", encodeBase64(clanURI), "details"],
+          clan.details,
+        );
+        queryClient.setQueryData(
+          ["clans", encodeBase64(clanURI), "machines"],
+          clan.machines || {},
+        );
+      });
+
+      return (
+        <QueryClientProvider client={queryClient}>
+          <Story />
+        </QueryClientProvider>
+      );
+    },
+  ],
+};

pkgs/clan-app/ui/src/components/Sidebar/Sidebar.tsx

@@ -0,0 +1,28 @@
+import "./Sidebar.css";
+import { SidebarHeader } from "@/src/components/Sidebar/SidebarHeader";
+import { SidebarBody } from "@/src/components/Sidebar/SidebarBody";
+
+export interface LinkProps {
+  path: string;
+  label?: string;
+}
+
+export interface SectionProps {
+  title: string;
+  links: LinkProps[];
+}
+
+export interface SidebarProps {
+  staticSections?: SectionProps[];
+}
+
+export const Sidebar = (props: SidebarProps) => {
+  return (
+    <>
+      <div class="sidebar">
+        <SidebarHeader />
+        <SidebarBody {...props} />
+      </div>
+    </>
+  );
+};

pkgs/clan-app/ui/src/components/Sidebar/SidebarBody.tsx

@@ -1,46 +1,60 @@
-import "./SidebarNavBody.css";
+import "./SidebarBody.css";
 import { A } from "@solidjs/router";
 import { Accordion } from "@kobalte/core/accordion";
 import Icon from "../Icon/Icon";
 import { Typography } from "@/src/components/Typography/Typography";
-import {
-  MachineProps,
-  SidebarNavProps,
-} from "@/src/components/Sidebar/SidebarNav";
 import { For } from "solid-js";
 import { MachineStatus } from "@/src/components/MachineStatus/MachineStatus";
+import { buildMachinePath, useClanURI } from "@/src/hooks/clan";
+import { useMachinesQuery } from "@/src/queries/queries";
+import { SidebarProps } from "./Sidebar";
+
+interface MachineProps {
+  clanURI: string;
+  machineID: string;
+  name: string;
+  status: MachineStatus;
+  serviceCount: number;
+}
 
 const MachineRoute = (props: MachineProps) => (
-  <div class="flex w-full flex-col gap-2">
-    <div class="flex flex-row items-center justify-between">
-      <Typography
-        hierarchy="label"
-        size="xs"
-        weight="bold"
-        color="primary"
-        inverted={true}
-      >
-        {props.label}
-      </Typography>
-      <MachineStatus status={props.status} />
+  <A href={buildMachinePath(props.clanURI, props.machineID)}>
+    <div class="flex w-full flex-col gap-2">
+      <div class="flex flex-row items-center justify-between">
+        <Typography
+          hierarchy="label"
+          size="xs"
+          weight="bold"
+          color="primary"
+          inverted={true}
+        >
+          {props.name}
+        </Typography>
+        <MachineStatus status={props.status} />
+      </div>
+      <div class="flex w-full flex-row items-center gap-1">
+        <Icon icon="Flash" size="0.75rem" inverted={true} color="tertiary" />
+        <Typography
+          hierarchy="label"
+          family="mono"
+          size="s"
+          inverted={true}
+          color="primary"
+        >
+          {props.serviceCount}
+        </Typography>
+      </div>
     </div>
-    <div class="flex w-full flex-row items-center gap-1">
-      <Icon icon="Flash" size="0.75rem" inverted={true} color="tertiary" />
-      <Typography
-        hierarchy="label"
-        family="mono"
-        size="s"
-        inverted={true}
-        color="primary"
-      >
-        {props.serviceCount}
-      </Typography>
-    </div>
-  </div>
+  </A>
 );
 
-export const SidebarNavBody = (props: SidebarNavProps) => {
-  const sectionLabels = props.extraSections.map((section) => section.label);
+export const SidebarBody = (props: SidebarProps) => {
+  const clanURI = useClanURI();
+  const machineList = useMachinesQuery(clanURI);
+
+  const sectionLabels = (props.staticSections || []).map(
+    (section) => section.title,
+  );
 
   // controls which sections are open by default
   // we want them all to be open by default
@@ -76,20 +90,24 @@ export const SidebarNavBody = (props: SidebarNavProps) => {
           </Accordion.Header>
           <Accordion.Content class="content">
             <nav>
-              <For each={props.clanDetail.machines}>
-                {(machine) => (
-                  <A href={machine.path}>
-                    <MachineRoute {...machine} />
-                  </A>
+              <For each={Object.entries(machineList.data || {})}>
+                {([id, machine]) => (
+                  <MachineRoute
+                    clanURI={clanURI}
+                    machineID={id}
+                    name={machine.name || id}
+                    status="Not Installed"
+                    serviceCount={0}
+                  />
                 )}
               </For>
             </nav>
           </Accordion.Content>
         </Accordion.Item>
 
-        <For each={props.extraSections}>
+        <For each={props.staticSections}>
           {(section) => (
-            <Accordion.Item class="item" value={section.label}>
+            <Accordion.Item class="item" value={section.title}>
               <Accordion.Header class="header">
                 <Accordion.Trigger class="trigger">
                   <Typography
@@ -100,7 +118,7 @@ export const SidebarNavBody = (props: SidebarNavProps) => {
                     inverted={true}
                     color="tertiary"
                   >
-                    {section.label}
+                    {section.title}
                   </Typography>
                   <Icon
                     icon="CaretDown"

pkgs/clan-app/ui/src/components/Sidebar/SidebarHeader.css

@@ -15,10 +15,11 @@ div.sidebar-header {
 
     transition: all 250ms ease-in-out;
 
-    div.title {
+    div.clan-label {
       @apply flex items-center gap-2 justify-start;
 
       & > .clan-icon {
+        @apply flex justify-center items-center;
         @apply rounded-full bg-inv-4 w-7 h-7;
       }
     }
@@ -38,7 +39,7 @@ div.sidebar-header {
 }
 
 .sidebar-dropdown-content {
-  @apply flex flex-col w-full px-2 py-1.5;
+  @apply flex flex-col w-full px-2 py-1.5 z-10;
   @apply bg-def-1 rounded-bl-md rounded-br-md;
   @apply border border-def-2;
 

pkgs/clan-app/ui/src/components/Sidebar/SidebarHeader.tsx

@@ -0,0 +1,107 @@
+import "./SidebarHeader.css";
+import Icon from "@/src/components/Icon/Icon";
+import { DropdownMenu } from "@kobalte/core/dropdown-menu";
+import { useNavigate } from "@solidjs/router";
+import { Typography } from "../Typography/Typography";
+import { createSignal, For, Suspense } from "solid-js";
+import { useClanListQuery } from "@/src/queries/queries";
+import { navigateToClan, useClanURI } from "@/src/hooks/clan";
+import { clanURIs } from "@/src/stores/clan";
+
+export const SidebarHeader = () => {
+  const navigate = useNavigate();
+
+  const [open, setOpen] = createSignal(false);
+
+  // get information about the current active clan
+  const clanURI = useClanURI();
+  const allClans = useClanListQuery(clanURIs());
+
+  const activeClan = () => allClans.find(({ data }) => data?.uri === clanURI);
+
+  return (
+    <div class="sidebar-header">
+      <Suspense fallback={"Loading..."}>
+        <DropdownMenu open={open()} onOpenChange={setOpen} sameWidth={true}>
+          <DropdownMenu.Trigger class="dropdown-trigger">
+            <div class="clan-label">
+              <div class="clan-icon">
+                <Typography
+                  hierarchy="label"
+                  size="s"
+                  weight="bold"
+                  inverted={true}
+                >
+                  {activeClan()?.data?.name.charAt(0).toUpperCase()}
+                </Typography>
+              </div>
+              <Typography
+                hierarchy="label"
+                size="s"
+                weight="bold"
+                inverted={!open()}
+              >
+                {activeClan()?.data?.name}
+              </Typography>
+            </div>
+            <DropdownMenu.Icon>
+              <Icon icon={"CaretDown"} inverted={!open()} size="0.75rem" />
+            </DropdownMenu.Icon>
+          </DropdownMenu.Trigger>
+          <DropdownMenu.Portal>
+            <DropdownMenu.Content class="sidebar-dropdown-content">
+              <DropdownMenu.Item
+                class="dropdown-item"
+                onSelect={() => navigateToClan(navigate, clanURI)}
+              >
+                <Icon
+                  icon="Settings"
+                  size="0.75rem"
+                  inverted={true}
+                  color="tertiary"
+                />
+                <Typography hierarchy="label" size="xs" weight="medium">
+                  Settings
+                </Typography>
+              </DropdownMenu.Item>
+              <DropdownMenu.Group class="dropdown-group">
+                <DropdownMenu.GroupLabel class="dropdown-group-label">
+                  <Typography
+                    hierarchy="label"
+                    family="mono"
+                    size="xs"
+                    color="tertiary"
+                  >
+                    YOUR CLANS
+                  </Typography>
+                </DropdownMenu.GroupLabel>
+                <div class="dropdown-group-items">
+                  <For each={allClans}>
+                    {(clan) => (
+                      <Suspense fallback={"Loading..."}>
+                        <DropdownMenu.Item
+                          class="dropdown-item"
+                          onSelect={() =>
+                            navigateToClan(navigate, clan.data!.uri)
+                          }
+                        >
+                          <Typography
+                            hierarchy="label"
+                            size="xs"
+                            weight="medium"
+                          >
+                            {clan.data?.name}
+                          </Typography>
+                        </DropdownMenu.Item>
+                      </Suspense>
+                    )}
+                  </For>
+                </div>
+              </DropdownMenu.Group>
+            </DropdownMenu.Content>
+          </DropdownMenu.Portal>
+        </DropdownMenu>
+      </Suspense>
+    </div>
+  );
+};

pkgs/clan-app/ui/src/components/Sidebar/SidebarNav.stories.tsx

@@ -1,109 +0,0 @@
-import type { Meta, StoryObj } from "@kachurun/storybook-solid";
-import {
-  createMemoryHistory,
-  MemoryRouter,
-  Route,
-  RouteSectionProps,
-} from "@solidjs/router";
-import {
-  SidebarNav,
-  SidebarNavProps,
-} from "@/src/components/Sidebar/SidebarNav";
-import { Suspense } from "solid-js";
-import { StoryContext } from "@kachurun/storybook-solid-vite";
-
-const sidebarNavProps: SidebarNavProps = {
-  clanLinks: [
-    { label: "Brian's Clan", path: "/clans/1" },
-    { label: "Dave's Clan", path: "/clans/2" },
-    { label: "Mic92's Clan", path: "/clans/3" },
-  ],
-  clanDetail: {
-    label: "Brian's Clan",
-    settingsPath: "/clans/1/settings",
-    machines: [
-      {
-        label: "Backup & Home",
-        path: "/clans/1/machine/backup",
-        serviceCount: 3,
-        status: "Online",
-      },
-      {
-        label: "Raspberry Pi",
-        path: "/clans/1/machine/pi",
-        serviceCount: 1,
-        status: "Offline",
-      },
-      {
-        label: "Mom's Laptop",
-        path: "/clans/1/machine/moms-laptop",
-        serviceCount: 2,
-        status: "Installed",
-      },
-      {
-        label: "Dad's Laptop",
-        path: "/clans/1/machine/dads-laptop",
-        serviceCount: 4,
-        status: "Not Installed",
-      },
-    ],
-  },
-  extraSections: [
-    {
-      label: "Tools",
-      links: [
-        { label: "Borgbackup", path: "/clans/1/service/borgbackup" },
-        { label: "Syncthing", path: "/clans/1/service/syncthing" },
-        { label: "Mumble", path: "/clans/1/service/mumble" },
-        { label: "Minecraft", path: "/clans/1/service/minecraft" },
-      ],
-    },
-    {
-      label: "Links",
-      links: [
-        { label: "GitHub", path: "https://github.com/brian-the-dev" },
-        { label: "Twitter", path: "https://twitter.com/brian_the_dev" },
-        {
-          label: "LinkedIn",
-          path: "https://www.linkedin.com/in/brian-the-dev/",
-        },
-        {
-          label: "Instagram",
-          path: "https://www.instagram.com/brian_the_dev/",
-        },
-      ],
-    },
-  ],
-};
-
-const meta: Meta<RouteSectionProps> = {
-  title: "Components/Sidebar/Nav",
-  component: SidebarNav,
-  render: (_: never, context: StoryContext<SidebarNavProps>) => {
-    const history = createMemoryHistory();
-    history.set({ value: "/clans/1/machine/backup" });
-
-    return (
-      <div style="height: 670px;">
-        <MemoryRouter
-          history={history}
-          root={(props) => (
-            <Suspense>
-              <SidebarNav {...sidebarNavProps} />
-            </Suspense>
-          )}
-        >
-          <Route path="/clans/1/machine/backup" component={(props) => <></>} />
-        </MemoryRouter>
-      </div>
-    );
-  },
-};
-
-export default meta;
-
-type Story = StoryObj<RouteSectionProps>;
-
-export const Default: Story = {
-  args: {},
-};

pkgs/clan-app/ui/src/components/Sidebar/SidebarNav.tsx

@@ -1,47 +0,0 @@
-import "./SidebarNav.css";
-import { SidebarNavHeader } from "@/src/components/Sidebar/SidebarNavHeader";
-import { SidebarNavBody } from "@/src/components/Sidebar/SidebarNavBody";
-import { MachineStatus } from "@/src/components/MachineStatus/MachineStatus";
-
-export interface LinkProps {
-  path: string;
-  label?: string;
-}
-
-export interface SectionProps {
-  label: string;
-  links: LinkProps[];
-}
-
-export interface MachineProps {
-  label: string;
-  path: string;
-  status: MachineStatus;
-  serviceCount: number;
-}
-
-export interface ClanLinkProps {
-  label: string;
-  path: string;
-}
-
-export interface ClanProps {
-  label: string;
-  settingsPath: string;
-  machines: MachineProps[];
-}
-
-export interface SidebarNavProps {
-  clanDetail: ClanProps;
-  clanLinks: ClanLinkProps[];
-  extraSections: SectionProps[];
-}
-
-export const SidebarNav = (props: SidebarNavProps) => {
-  return (
-    <div class="sidebar">
-      <SidebarNavHeader {...props} />
-      <SidebarNavBody {...props} />
-    </div>
-  );
-};

pkgs/clan-app/ui/src/components/Sidebar/SidebarNavHeader.tsx

@@ -1,96 +0,0 @@
-import "./SidebarNavHeader.css";
-import Icon from "@/src/components/Icon/Icon";
-import { DropdownMenu } from "@kobalte/core/dropdown-menu";
-import { useNavigate } from "@solidjs/router";
-import { Typography } from "../Typography/Typography";
-import { createSignal, For } from "solid-js";
-import { ClanLinkProps, ClanProps } from "@/src/components/Sidebar/SidebarNav";
-
-export interface SidebarHeaderProps {
-  clanDetail: ClanProps;
-  clanLinks: ClanLinkProps[];
-}
-
-export const SidebarNavHeader = (props: SidebarHeaderProps) => {
-  const navigate = useNavigate();
-
-  const [open, setOpen] = createSignal(false);
-
-  const firstChar = props.clanDetail.label.charAt(0);
-
-  return (
-    <div class="sidebar-header">
-      <DropdownMenu open={open()} onOpenChange={setOpen} sameWidth={true}>
-        <DropdownMenu.Trigger class="dropdown-trigger">
-          <div class="title">
-            <div class="clan-icon">
-              <Typography
-                hierarchy="label"
-                size="s"
-                weight="bold"
-                inverted={true}
-              >
-                {firstChar.toUpperCase()}
-              </Typography>
-            </div>
-            <Typography
-              hierarchy="label"
-              size="s"
-              weight="bold"
-              inverted={!open()}
-            >
-              {props.clanDetail.label}
-            </Typography>
-          </div>
-          <DropdownMenu.Icon>
-            <Icon icon={"CaretDown"} inverted={!open()} size="0.75rem" />
-          </DropdownMenu.Icon>
-        </DropdownMenu.Trigger>
-        <DropdownMenu.Portal>
-          <DropdownMenu.Content class="sidebar-dropdown-content">
-            <DropdownMenu.Item
-              class="dropdown-item"
-              onSelect={() => navigate(props.clanDetail.settingsPath)}
-            >
-              <Icon
-                icon="Settings"
-                size="0.75rem"
-                inverted={true}
-                color="tertiary"
-              />
-              <Typography hierarchy="label" size="xs" weight="medium">
-                Settings
-              </Typography>
-            </DropdownMenu.Item>
-            <DropdownMenu.Group class="dropdown-group">
-              <DropdownMenu.GroupLabel class="dropdown-group-label">
-                <Typography
-                  hierarchy="label"
-                  family="mono"
-                  size="xs"
-                  color="tertiary"
-                >
-                  YOUR CLANS
-                </Typography>
-              </DropdownMenu.GroupLabel>
-              <div class="dropdown-group-items">
-                <For each={props.clanLinks}>
-                  {(clan) => (
-                    <DropdownMenu.Item
-                      class="dropdown-item"
-                      onSelect={() => navigate(clan.path)}
-                    >
-                      <Typography hierarchy="label" size="xs" weight="medium">
-                        {clan.label}
-                      </Typography>
-                    </DropdownMenu.Item>
-                  )}
-                </For>
-              </div>
-            </DropdownMenu.Group>
-          </DropdownMenu.Content>
-        </DropdownMenu.Portal>
-      </DropdownMenu>
-    </div>
-  );
-};

pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.css

@@ -1,5 +1,16 @@
 div.sidebar-pane {
-  @apply h-full w-auto max-w-60 border-none;
+  @apply border-none z-10;
+
+  animation: sidebarPaneShow 250ms ease-in forwards;
+
+  &.closing {
+    animation: sidebarPaneHide 250ms ease-out 300ms forwards;
+
+    & > div.header > *,
+    & > div.body > * {
+      animation: sidebarFadeOut 250ms ease-out forwards;
+    }
+  }
 
   & > div.header {
     @apply flex items-center justify-between px-3 py-2 rounded-t-[0.5rem];
@@ -7,11 +18,17 @@ div.sidebar-pane {
         border-r-[1px] border-r-bg-inv-3
         border-b-2 border-b-bg-inv-4
         border-l-[1px] border-l-bg-inv-3;
+
     background: linear-gradient(
       90deg,
       theme(colors.bg.inv.3) 0%,
       theme(colors.bg.inv.4) 100%
     );
+
+    & > * {
+      @apply opacity-0;
+      animation: sidebarFadeIn 250ms ease-in 250ms forwards;
+    }
   }
 
   & > div.body {
@@ -29,5 +46,54 @@ div.sidebar-pane {
         theme(colors.bg.inv.2) 0%,
         theme(colors.bg.inv.3) 100%
       );
+
+    & > * {
+      @apply opacity-0;
+      animation: sidebarFadeIn 250ms ease-in 350ms forwards;
+    }
+  }
+}
+
+@keyframes sidebarPaneShow {
+  0% {
+    @apply w-0;
+    @apply opacity-0;
+  }
+  10% {
+    @apply w-8;
+  }
+  30% {
+    @apply opacity-100;
+  }
+  100% {
+    @apply w-60;
+  }
+}
+
+@keyframes sidebarPaneHide {
+  90% {
+    @apply w-8;
+  }
+  100% {
+    @apply w-0;
+    @apply opacity-0;
+  }
+}
+
+@keyframes sidebarFadeIn {
+  from {
+    opacity: 0;
+  }
+  to {
+    opacity: 1;
+  }
+}
+
+@keyframes sidebarFadeOut {
+  from {
+    opacity: 1;
+  }
+  to {
+    opacity: 0;
   }
 }

pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.stories.tsx

@@ -11,7 +11,7 @@ import { Checkbox } from "@/src/components/Form/Checkbox";
 import { Combobox } from "../Form/Combobox";
 
 const meta: Meta<SidebarPaneProps> = {
-  title: "Components/Sidebar/Pane",
+  title: "Components/SidebarPane",
   component: SidebarPane,
 };
 

pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.tsx

@@ -1,8 +1,9 @@
-import { JSX } from "solid-js";
+import { createSignal, JSX } from "solid-js";
 import "./SidebarPane.css";
 import { Typography } from "@/src/components/Typography/Typography";
 import Icon from "../Icon/Icon";
 import { Button as KButton } from "@kobalte/core/button";
+import cx from "classnames";
 
 export interface SidebarPaneProps {
   title: string;
@@ -11,13 +12,20 @@ export interface SidebarPaneProps {
 }
 
 export const SidebarPane = (props: SidebarPaneProps) => {
+  const [closing, setClosing] = createSignal(false);
+
+  const onClose = () => {
+    setClosing(true);
+    setTimeout(() => props.onClose(), 550);
+  };
+
   return (
-    <div class="sidebar-pane">
+    <div class={cx("sidebar-pane", { closing: closing() })}>
       <div class="header">
         <Typography hierarchy="body" size="s" weight="bold" inverted={true}>
           {props.title}
         </Typography>
-        <KButton onClick={props.onClose}>
+        <KButton onClick={onClose}>
           <Icon icon="Close" color="primary" size="0.75rem" inverted={true} />
         </KButton>
       </div>

pkgs/clan-app/ui/src/hooks/api.ts

@@ -20,6 +20,7 @@ export type SuccessData<T extends OperationNames> = SuccessQuery<T>["data"];
 
 interface SendHeaderType {
   logging?: { group_path: string[] };
+  op_key?: string;
 }
 interface BackendSendType<K extends OperationNames> {
   body: OperationArgs<K>;
@@ -64,9 +65,14 @@ export const callApi = <K extends OperationNames>(
     };
   }
 
+  const op_key = backendOpts?.op_key ?? crypto.randomUUID();
+
   const req: BackendSendType<OperationNames> = {
     body: args,
-    header: backendOpts,
+    header: {
+      ...backendOpts,
+      op_key,
+    },
   };
 
   const result = (
@@ -78,9 +84,6 @@ export const callApi = <K extends OperationNames>(
     >
   )[method](req) as Promise<BackendReturnType<K>>;
 
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  const op_key = (result as any)._webviewMessageId as string;
-
   return {
     uuid: op_key,
     result: result.then(({ body }) => body),

pkgs/clan-app/ui/src/hooks/clan.ts

@@ -2,6 +2,9 @@ import { callApi } from "@/src/hooks/api";
 import { addClanURI, setActiveClanURI } from "@/src/stores/clan";
 import { Params, Navigator, useParams } from "@solidjs/router";
 
+export const encodeBase64 = (value: string) => window.btoa(value);
+export const decodeBase64 = (value: string) => window.atob(value);
+
 export const selectClanFolder = async () => {
   const req = callApi("get_clan_folder", {});
   const res = await req.result;
@@ -20,38 +23,51 @@ export const selectClanFolder = async () => {
   throw new Error("Illegal state exception");
 };
 
-export const navigateToClan = (navigate: Navigator, uri: string) => {
-  navigate("/clans/" + window.btoa(uri));
+export const buildClanPath = (clanURI: string) => {
+  return "/clans/" + encodeBase64(clanURI);
+};
+
+export const buildMachinePath = (clanURI: string, machineID: string) => {
+  return (
+    "/clans/" + encodeBase64(clanURI) + "/machines/" + encodeBase64(machineID)
+  );
+};
+
+export const navigateToClan = (navigate: Navigator, clanURI: string) => {
+  const path = buildClanPath(clanURI);
+  console.log("Navigating to clan", clanURI, path);
+  navigate(path);
+};
+
+export const navigateToMachine = (
+  navigate: Navigator,
+  clanURI: string,
+  machineID: string,
+) => {
+  const path = buildMachinePath(clanURI, machineID);
+  console.log("Navigating to machine", clanURI, machineID, path);
+  navigate(path);
 };
 
 export const clanURIParam = (params: Params) => {
-  return window.atob(params.clanURI);
+  return decodeBase64(params.clanURI);
 };
 
-export function useClanURI(opts: { force: true }): string;
-export function useClanURI(opts: { force: boolean }): string | null;
-export function useClanURI(
-  opts: { force: boolean } = { force: false },
-): string | null {
-  const maybe = () => {
-    const params = useParams();
-    if (!params.clanURI) {
-      return null;
-    }
-    const clanURI = clanURIParam(params);
-    if (!clanURI) {
-      throw new Error(
-        "Could not decode clan URI from params: " + params.clanURI,
-      );
-    }
-    return clanURI;
-  };
+export const useClanURI = () => clanURIParam(useParams());
 
-  const uri = maybe();
-  if (!uri && opts.force) {
-    throw new Error(
-      "ClanURI is not set. Use this function only within contexts, where clanURI is guaranteed to have been set.",
-    );
+export const machineIDParam = (params: Params) => {
+  return decodeBase64(params.machineID);
+};
+
+export const useMachineID = (): string => {
+  const params = useParams();
+  return machineIDParam(params);
+};
+
+export const maybeUseMachineID = (): string | null => {
+  const params = useParams();
+  if (params.machineID === undefined) {
+    return null;
   }
-  return uri;
-}
+  return machineIDParam(params);
+};

pkgs/clan-app/ui/src/index.tsx

@@ -6,6 +6,7 @@ import { QueryClient, QueryClientProvider } from "@tanstack/solid-query";
 import { Routes } from "@/src/routes";
 import { Router } from "@solidjs/router";
 import { Layout } from "@/src/routes/Layout";
+import { SolidQueryDevtools } from "@tanstack/solid-query-devtools";
 
 export const client = new QueryClient();
 
@@ -18,13 +19,12 @@ if (import.meta.env.DEV && !(root instanceof HTMLElement)) {
 }
 if (import.meta.env.DEV) {
   console.log("Development mode");
-  // Load the debugger in development mode
-  await import("solid-devtools");
 }
 
 render(
   () => (
     <QueryClientProvider client={client}>
+      {import.meta.env.DEV && <SolidQueryDevtools initialIsOpen={true} />}
       <Router root={Layout}>{Routes}</Router>
     </QueryClientProvider>
   ),

pkgs/clan-app/ui/src/queries/queries.ts

@@ -1,24 +1,22 @@
-import { useQuery, UseQueryResult } from "@tanstack/solid-query";
+import { useQueries, useQuery, UseQueryResult } from "@tanstack/solid-query";
 import { callApi, SuccessData } from "../hooks/api";
+import { encodeBase64 } from "@/src/hooks/clan";
+
+export type ClanDetails = SuccessData<"get_clan_details">;
+export type ClanDetailsWithURI = ClanDetails & { uri: string };
 
 export type ListMachines = SuccessData<"list_machines">;
 
 export type MachinesQueryResult = UseQueryResult<ListMachines>;
+export type ClanListQueryResult = UseQueryResult<ClanDetailsWithURI>[];
 
-interface MachinesQueryParams {
-  clanURI: string | null;
-}
-export const useMachinesQuery = (props: MachinesQueryParams) =>
+export const useMachinesQuery = (clanURI: string) =>
   useQuery<ListMachines>(() => ({
-    queryKey: ["clans", props.clanURI, "machines"],
-    enabled: !!props.clanURI,
+    queryKey: ["clans", encodeBase64(clanURI), "machines"],
     queryFn: async () => {
-      if (!props.clanURI) {
-        return {};
-      }
       const api = callApi("list_machines", {
         flake: {
-          identifier: props.clanURI,
+          identifier: clanURI,
         },
       });
       const result = await api.result;
@@ -29,3 +27,54 @@ export const useMachinesQuery = (props: MachinesQueryParams) =>
       return result.data;
     },
   }));
+
+export const useClanDetailsQuery = (clanURI: string) =>
+  useQuery<ClanDetails>(() => ({
+    queryKey: ["clans", encodeBase64(clanURI), "details"],
+    queryFn: async () => {
+      const call = callApi("get_clan_details", {
+        flake: {
+          identifier: clanURI,
+        },
+      });
+      const result = await call.result;
+
+      if (result.status === "error") {
+        // todo should we create some specific error types?
+        console.error("Error fetching clan details:", result.errors);
+        throw new Error(result.errors[0].message);
+      }
+
+      return {
+        uri: clanURI,
+        ...result.data,
+      };
+    },
+  }));
+
+export const useClanListQuery = (clanURIs: string[]): ClanListQueryResult =>
+  useQueries(() => ({
+    queries: clanURIs.map((clanURI) => ({
+      queryKey: ["clans", encodeBase64(clanURI), "details"],
+      enabled: !!clanURI,
+      queryFn: async () => {
+        const call = callApi("get_clan_details", {
+          flake: {
+            identifier: clanURI,
+          },
+        });
+        const result = await call.result;
+
+        if (result.status === "error") {
+          // todo should we create some specific error types?
+          console.error("Error fetching clan details:", result.errors);
+          throw new Error(result.errors[0].message);
+        }
+
+        return {
+          uri: clanURI,
+          ...result.data,
+        };
+      },
+    })),
+  }));

pkgs/clan-app/ui/src/routes/Clan/Clan.css

@@ -11,3 +11,14 @@
 .create-modal {
   @apply min-w-96;
 }
+
+.sidebar-container {
+}
+
+div.sidebar {
+  @apply absolute top-10 bottom-20 left-4 w-60;
+}
+
+div.sidebar-pane {
+  @apply absolute top-12 bottom-20 left-[16.5rem] w-64;
+}

pkgs/clan-app/ui/src/routes/Clan/Clan.tsx

@@ -1,10 +1,28 @@
 import { RouteSectionProps } from "@solidjs/router";
-import { Component, JSX, Show, createSignal } from "solid-js";
-import { useClanURI } from "@/src/hooks/clan";
+import {
+  Component,
+  JSX,
+  Show,
+  createEffect,
+  createMemo,
+  createSignal,
+  on,
+  onMount,
+} from "solid-js";
+import {
+  buildMachinePath,
+  maybeUseMachineID,
+  useClanURI,
+} from "@/src/hooks/clan";
 import { CubeScene } from "@/src/scene/cubes";
-import { MachinesQueryResult, useMachinesQuery } from "@/src/queries/queries";
+import {
+  ClanListQueryResult,
+  MachinesQueryResult,
+  useClanListQuery,
+  useMachinesQuery,
+} from "@/src/queries/queries";
 import { callApi } from "@/src/hooks/api";
-import { store, setStore } from "@/src/stores/clan";
+import { store, setStore, clanURIs } from "@/src/stores/clan";
 import { produce } from "solid-js/store";
 import { Button } from "@/src/components/Button/Button";
 import { Splash } from "@/src/scene/splash";
@@ -13,19 +31,15 @@ import "./Clan.css";
 import { Modal } from "@/src/components/Modal/Modal";
 import { TextInput } from "@/src/components/Form/TextInput";
 import { createForm, FieldValues, reset } from "@modular-forms/solid";
+import { Sidebar } from "@/src/components/Sidebar/Sidebar";
+import { useNavigate } from "@solidjs/router";
 
 export const Clan: Component<RouteSectionProps> = (props) => {
   return (
     <>
-      <div
-        style={{
-          position: "absolute",
-          top: 0,
-        }}
-      >
-        {props.children}
-      </div>
-      <ClanSceneController />
+      <Sidebar />
+      {props.children}
+      <ClanSceneController {...props} />
     </>
   );
 };
@@ -33,10 +47,12 @@ export const Clan: Component<RouteSectionProps> = (props) => {
 interface CreateFormValues extends FieldValues {
   name: string;
 }
+
 interface MockProps {
   onClose: () => void;
   onSubmit: (formValues: CreateFormValues) => void;
 }
+
 const MockCreateMachine = (props: MockProps) => {
   let container: Node;
 
@@ -69,7 +85,7 @@ const MockCreateMachine = (props: MockProps) => {
               )}
             </Field>
 
-            <div class="flex w-full items-center justify-end gap-4">
+            <div class="mt-4 flex w-full items-center justify-end gap-4">
               <Button size="s" hierarchy="secondary" onClick={props.onClose}>
                 Cancel
               </Button>
@@ -89,8 +105,9 @@ const MockCreateMachine = (props: MockProps) => {
   );
 };
 
-const ClanSceneController = () => {
-  const clanURI = useClanURI({ force: true });
+const ClanSceneController = (props: RouteSectionProps) => {
+  const clanURI = useClanURI();
+  const navigate = useNavigate();
 
   const [dialogHandlers, setDialogHandlers] = createSignal<{
     resolve: ({ id }: { id: string }) => void;
@@ -128,9 +145,61 @@ const ClanSceneController = () => {
 
   const [showModal, setShowModal] = createSignal(false);
 
+  const [loadingCooldown, setLoadingCooldown] = createSignal(false);
+  onMount(() => {
+    setTimeout(() => {
+      setLoadingCooldown(true);
+    }, 1500);
+  });
+
+  const [selectedIds, setSelectedIds] = createSignal<Set<string>>(new Set());
+
+  const onMachineSelect = (ids: Set<string>) => {
+    // Get the first selected ID and navigate to its machine details
+    const selected = ids.values().next().value;
+    if (selected) {
+      navigate(buildMachinePath(clanURI, selected));
+    }
+  };
+
+  const machine = createMemo(() => maybeUseMachineID());
+
+  createEffect(
+    on(machine, (machineId) => {
+      if (machineId) {
+        setSelectedIds(() => {
+          const res = new Set<string>();
+          res.add(machineId);
+          return res;
+        });
+      } else {
+        setSelectedIds(new Set<string>());
+      }
+    }),
+  );
+
   return (
     <SceneDataProvider clanURI={clanURI}>
-      {({ query }) => {
+      {({ clansQuery, machinesQuery }) => {
+        // a combination of the individual clan details query status and the machines query status
+        // the cube scene needs the machines query, the sidebar needs the clans query and machines query results
+        // so we wait on both before removing the loader to avoid any loading artefacts
+        const isLoading = (): boolean => {
+          // check the machines query first
+          if (machinesQuery.isLoading) {
+            return true;
+          }
+
+          // otherwise iterate the clans query and return early if we find a queries that is still loading
+          for (const query of clansQuery) {
+            if (query.isLoading) {
+              return true;
+            }
+          }
+
+          return false;
+        };
+
         return (
           <>
             <Show when={showModal()}>
@@ -162,8 +231,7 @@ const ClanSceneController = () => {
                     produce((s) => {
                       for (const machineId in s.sceneData[clanURI]) {
                         // Reset the position of each machine to [0, 0]
-                        s.sceneData[clanURI] = {}; // Clear the entire object
-                        // delete s.sceneData[clanURI][machineId];
+                        s.sceneData[clanURI] = {};
                       }
                     }),
                   );
@@ -175,23 +243,29 @@ const ClanSceneController = () => {
                 ghost
                 onClick={() => {
                   console.log("Refetching API");
-                  query.refetch();
+                  machinesQuery.refetch();
                 }}
               >
                 Refetch API
               </Button>
             </div>
             {/* TODO: Add minimal display time */}
-            <div class={cx({ "fade-out": !query.isLoading })}>
+            <div
+              class={cx({
+                "fade-out": !machinesQuery.isLoading && loadingCooldown(),
+              })}
+            >
               <Splash />
             </div>
 
             <CubeScene
-              isLoading={query.isLoading}
-              cubesQuery={query}
+              selectedIds={selectedIds}
+              onSelect={onMachineSelect}
+              isLoading={isLoading()}
+              cubesQuery={machinesQuery}
               onCreate={onCreate}
               sceneStore={() => {
-                const clanURI = useClanURI({ force: true });
+                const clanURI = useClanURI();
                 return store.sceneData?.[clanURI];
               }}
               setMachinePos={(machineId: string, pos: [number, number]) => {
@@ -221,11 +295,15 @@ const ClanSceneController = () => {
 };
 
 const SceneDataProvider = (props: {
-  clanURI: string | null;
-  children: (sceneData: { query: MachinesQueryResult }) => JSX.Element;
+  clanURI: string;
+  children: (sceneData: {
+    clansQuery: ClanListQueryResult;
+    machinesQuery: MachinesQueryResult;
+  }) => JSX.Element;
 }) => {
-  const machinesQuery = useMachinesQuery({ clanURI: props.clanURI });
+  const clansQuery = useClanListQuery(clanURIs());
+  const machinesQuery = useMachinesQuery(props.clanURI);
 
   // This component can be used to provide scene data or context if needed
-  return props.children({ query: machinesQuery });
+  return props.children({ clansQuery, machinesQuery });
 };

pkgs/clan-app/ui/src/routes/Machine/Machine.tsx

@@ -0,0 +1,19 @@
+import { RouteSectionProps, useNavigate } from "@solidjs/router";
+import { SidebarPane } from "@/src/components/Sidebar/SidebarPane";
+import { navigateToClan, useClanURI, useMachineID } from "@/src/hooks/clan";
+
+export const Machine = (props: RouteSectionProps) => {
+  const navigate = useNavigate();
+  const clanURI = useClanURI();
+
+  const onClose = () => {
+    // go back to clan route
+    navigateToClan(navigate, clanURI);
+  };
+
+  return (
+    <SidebarPane title={useMachineID()} onClose={onClose}>
+      <h1>Hello world</h1>
+    </SidebarPane>
+  );
+};

pkgs/clan-app/ui/src/routes/Onboarding/Onboarding.css

@@ -57,7 +57,7 @@ main#welcome {
     @apply flex flex-col items-center justify-evenly gap-y-20 size-fit;
 
     & > div.welcome {
-      @apply flex flex-col min-w-80 gap-y-6;
+      @apply flex flex-col w-80 gap-y-6;
 
       & > div.separator {
         @apply grid grid-cols-3 grid-rows-1 gap-x-4 items-center;

pkgs/clan-app/ui/src/routes/Onboarding/Onboarding.tsx

@@ -220,16 +220,8 @@ export const Onboarding: Component<RouteSectionProps> = (props) => {
         // todo allow users to select a template
         template: "minimal",
         initial: {
-          meta: {
-            name: name,
-            description: description,
-            // todo it tries to 'delete' icon if it's not provided
-            // this logic is unexpected, and needs reviewed.
-            icon: null,
-          },
-          machines: {},
-          instances: {},
-          services: {},
+          name,
+          description,
         },
       },
     });

pkgs/clan-app/ui/src/routes/index.tsx

@@ -1,6 +1,7 @@
 import type { RouteDefinition } from "@solidjs/router/dist/types";
 import { Onboarding } from "@/src/routes/Onboarding/Onboarding";
 import { Clan } from "@/src/routes/Clan/Clan";
+import { Machine } from "@/src/routes/Machine/Machine";
 
 export const Routes: RouteDefinition[] = [
   {
@@ -21,25 +22,14 @@ export const Routes: RouteDefinition[] = [
       },
       {
         path: "/:clanURI",
+        component: Clan,
         children: [
           {
             path: "/",
-            component: Clan,
           },
           {
-            path: "/machines",
-            children: [
-              {
-                path: "/",
-                component: () => <h1>Machines (Index)</h1>,
-              },
-              {
-                path: "/:machineID",
-                component: (props) => (
-                  <h1>Machine ID: {props.params.machineID}</h1>
-                ),
-              },
-            ],
+            path: "/machines/:machineID",
+            component: Machine,
           },
         ],
       },

pkgs/clan-app/ui/src/scene/MachineManager.ts

@@ -0,0 +1,154 @@
+import { Accessor, createEffect, createRoot } from "solid-js";
+import { MachineRepr } from "./MachineRepr";
+import * as THREE from "three";
+import { SceneData } from "../stores/clan";
+import { MachinesQueryResult } from "../queries/queries";
+import { ObjectRegistry } from "./ObjectRegistry";
+import { renderLoop } from "./RenderLoop";
+
+function keyFromPos(pos: [number, number]): string {
+  return `${pos[0]},${pos[1]}`;
+}
+
+const CUBE_SPACING = 2;
+
+export class MachineManager {
+  public machines = new Map<string, MachineRepr>();
+
+  private disposeRoot: () => void;
+
+  private machinePositionsSignal: Accessor<SceneData>;
+
+  constructor(
+    scene: THREE.Scene,
+    registry: ObjectRegistry,
+    machinePositionsSignal: Accessor<SceneData>,
+    machinesQueryResult: MachinesQueryResult,
+    selectedIds: Accessor<Set<string>>,
+    setMachinePos: (id: string, position: [number, number]) => void,
+  ) {
+    this.machinePositionsSignal = machinePositionsSignal;
+
+    this.disposeRoot = createRoot((disposeEffects) => {
+      createEffect(() => {
+        const machines = machinePositionsSignal();
+
+        Object.entries(machines).forEach(([id, data]) => {
+          const machineRepr = new MachineRepr(
+            scene,
+            registry,
+            new THREE.Vector2(data.position[0], data.position[1]),
+            id,
+            selectedIds,
+          );
+          this.machines.set(id, machineRepr);
+          scene.add(machineRepr.group);
+        });
+        renderLoop.requestRender();
+      });
+
+      // Push positions of previously existing machines to the scene
+      // TODO: Maybe we should do this in some post query hook?
+      createEffect(() => {
+        if (!machinesQueryResult.data) return;
+
+        const actualMachines = Object.keys(machinesQueryResult.data);
+        const machinePositions = machinePositionsSignal();
+        const placed: Set<string> = machinePositions
+          ? new Set(Object.keys(machinePositions))
+          : new Set();
+
+        const nonPlaced = actualMachines.filter((m) => !placed.has(m));
+
+        // Push not explizitly placed machines to the scene
+        // TODO: Make the user place them manually
+        // We just calculate some next free position
+        for (const id of nonPlaced) {
+          console.log("adding", id);
+          const position = this.nextGridPos();
+
+          setMachinePos(id, position);
+        }
+      });
+
+      return disposeEffects;
+    });
+  }
+
+  nextGridPos(): [number, number] {
+    const occupiedPositions = new Set(
+      Object.values(this.machinePositionsSignal()).map((data) =>
+        keyFromPos(data.position),
+      ),
+    );
+
+    let x = 0,
+      z = 0;
+    let layer = 1;
+
+    while (layer < 100) {
+      // right
+      for (let i = 0; i < layer; i++) {
+        const pos = [x * CUBE_SPACING, z * CUBE_SPACING] as [number, number];
+        const key = keyFromPos(pos);
+        if (!occupiedPositions.has(key)) {
+          return pos;
+        }
+        x += 1;
+      }
+      // down
+      for (let i = 0; i < layer; i++) {
+        const pos = [x * CUBE_SPACING, z * CUBE_SPACING] as [number, number];
+        const key = keyFromPos(pos);
+        if (!occupiedPositions.has(key)) {
+          return pos;
+        }
+        z += 1;
+      }
+      layer++;
+      // left
+      for (let i = 0; i < layer; i++) {
+        const pos = [x * CUBE_SPACING, z * CUBE_SPACING] as [number, number];
+        const key = keyFromPos(pos);
+        if (!occupiedPositions.has(key)) {
+          return pos;
+        }
+        x -= 1;
+      }
+      // up
+      for (let i = 0; i < layer; i++) {
+        const pos = [x * CUBE_SPACING, z * CUBE_SPACING] as [number, number];
+        const key = keyFromPos(pos);
+        if (!occupiedPositions.has(key)) {
+          return pos;
+        }
+        z -= 1;
+      }
+      layer++;
+    }
+    console.warn("No free grid positions available, returning [0, 0]");
+    // Fallback if no position was found
+    return [0, 0] as [number, number];
+  }
+
+  dispose(scene: THREE.Scene) {
+    for (const machine of this.machines.values()) {
+      machine.dispose(scene);
+    }
+    // Stop SolidJS effects
+    this.disposeRoot?.();
+    // Clear references
+    this.machines?.clear();
+  }
+}
+
+// TODO: For service focus
+// const getCirclePosition =
+//   (center: [number, number, number]) =>
+//   (_id: string, index: number, total: number): [number, number, number] => {
+//     const r = total === 1 ? 0 : Math.sqrt(total) * CUBE_SPACING; // Radius based on total cubes
+//     const x = Math.cos((index / total) * 2 * Math.PI) * r + center[0];
+//     const z = Math.sin((index / total) * 2 * Math.PI) * r + center[2];
+//     // Position cubes at y = 0.5 to float above the ground
+//     return [x, CUBE_Y, z];
+//   };

pkgs/clan-app/ui/src/scene/MachineRepr.ts

@@ -0,0 +1,141 @@
+import * as THREE from "three";
+import { ObjectRegistry } from "./ObjectRegistry";
+import { CSS2DObject } from "three/examples/jsm/renderers/CSS2DRenderer.js";
+import { Accessor, createEffect, createRoot, on } from "solid-js";
+import { renderLoop } from "./RenderLoop";
+
+// Constants
+const BASE_SIZE = 0.9;
+const CUBE_SIZE = BASE_SIZE / 1.5;
+const CUBE_HEIGHT = CUBE_SIZE;
+const BASE_HEIGHT = 0.05;
+const CUBE_COLOR = 0xd7e0e1;
+const CUBE_EMISSIVE = 0x303030;
+
+const CUBE_SELECTED_COLOR = 0x4b6767;
+
+const BASE_COLOR = 0xecfdff;
+const BASE_EMISSIVE = 0x0c0c0c;
+const BASE_SELECTED_COLOR = 0x69b0e3;
+const BASE_SELECTED_EMISSIVE = 0x666666; // Emissive color for selected bases
+
+export class MachineRepr {
+  public id: string;
+  public group: THREE.Group;
+
+  private cubeMesh: THREE.Mesh;
+  private baseMesh: THREE.Mesh;
+  private geometry: THREE.BoxGeometry;
+  private material: THREE.MeshPhongMaterial;
+
+  private disposeRoot: () => void;
+
+  constructor(
+    scene: THREE.Scene,
+    registry: ObjectRegistry,
+    position: THREE.Vector2,
+    id: string,
+    selectedSignal: Accessor<Set<string>>,
+  ) {
+    this.id = id;
+    this.geometry = new THREE.BoxGeometry(CUBE_SIZE, CUBE_SIZE, CUBE_SIZE);
+    this.material = new THREE.MeshPhongMaterial({
+      color: CUBE_COLOR,
+      emissive: CUBE_EMISSIVE,
+      shininess: 100,
+    });
+
+    this.cubeMesh = new THREE.Mesh(this.geometry, this.material);
+    this.cubeMesh.castShadow = true;
+    this.cubeMesh.receiveShadow = true;
+    this.cubeMesh.userData = { id };
+    this.cubeMesh.name = "cube";
+    this.cubeMesh.position.set(0, CUBE_HEIGHT / 2, 0);
+
+    this.baseMesh = this.createCubeBase(
+      BASE_COLOR,
+      BASE_EMISSIVE,
+      new THREE.BoxGeometry(BASE_SIZE, BASE_HEIGHT, BASE_SIZE),
+    );
+    this.baseMesh.name = "base";
+
+    const label = this.createLabel(id);
+    this.cubeMesh.add(label);
+
+    this.group = new THREE.Group();
+    this.group.add(this.cubeMesh);
+    this.group.add(this.baseMesh);
+
+    this.group.position.set(position.x, 0, position.y);
+    this.group.userData.id = id;
+
+    this.disposeRoot = createRoot((disposeEffects) => {
+      createEffect(
+        on(selectedSignal, (selectedIds) => {
+          const isSelected = selectedIds.has(this.id);
+          // Update cube
+          (this.cubeMesh.material as THREE.MeshPhongMaterial).color.set(
+            isSelected ? CUBE_SELECTED_COLOR : CUBE_COLOR,
+          );
+
+          // Update base
+          (this.baseMesh.material as THREE.MeshPhongMaterial).color.set(
+            isSelected ? BASE_SELECTED_COLOR : BASE_COLOR,
+          );
+          (this.baseMesh.material as THREE.MeshPhongMaterial).emissive.set(
+            isSelected ? BASE_SELECTED_EMISSIVE : BASE_EMISSIVE,
+          );
+
+          renderLoop.requestRender();
+        }),
+      );
+
+      return disposeEffects;
+    });
+
+    scene.add(this.group);
+
+    registry.add({
+      object: this.group,
+      id,
+      type: "machine",
+      dispose: () => this.dispose(scene),
+    });
+  }
+
+  private createCubeBase(
+    color: THREE.ColorRepresentation,
+    emissive: THREE.ColorRepresentation,
+    geometry: THREE.BoxGeometry,
+  ) {
+    const baseMaterial = new THREE.MeshPhongMaterial({
+      color,
+      emissive,
+      transparent: true,
+      opacity: 1,
+    });
+    const base = new THREE.Mesh(geometry, baseMaterial);
+    base.position.set(0, BASE_HEIGHT / 2, 0);
+    base.receiveShadow = true;
+    return base;
+  }
+
+  private createLabel(id: string) {
+    const div = document.createElement("div");
+    div.className = "machine-label";
+    div.textContent = id;
+    const label = new CSS2DObject(div);
+    label.position.set(0, CUBE_SIZE + 0.1, 0);
+    return label;
+  }
+
+  dispose(scene: THREE.Scene) {
+    this.disposeRoot?.(); // Stop SolidJS effects
+
+    scene.remove(this.group);
+
+    this.geometry.dispose();
+    this.material.dispose();
+    (this.baseMesh.material as THREE.Material).dispose();
+  }
+}

pkgs/clan-app/ui/src/scene/ObjectRegistry.ts

@@ -0,0 +1,43 @@
+import * as THREE from "three";
+
+interface ObjectEntry {
+  object: THREE.Object3D;
+  type: string;
+  id: string;
+  dispose?: () => void;
+}
+
+export class ObjectRegistry {
+  #objects = new Map<string, ObjectEntry>();
+
+  add(entry: ObjectEntry) {
+    const key = `${entry.type}:${entry.id}`;
+    this.#objects.set(key, entry);
+  }
+
+  getById(type: string, id: string) {
+    return this.#objects.get(`${type}:${id}`);
+  }
+
+  getAllByType(type: string) {
+    return [...this.#objects.values()].filter((obj) => obj.type === type);
+  }
+
+  removeById(type: string, id: string, scene: THREE.Scene) {
+    const key = `${type}:${id}`;
+    const entry = this.#objects.get(key);
+    if (entry) {
+      scene.remove(entry.object);
+      entry.dispose?.();
+      this.#objects.delete(key);
+    }
+  }
+
+  disposeAll(scene: THREE.Scene) {
+    for (const entry of this.#objects.values()) {
+      scene.remove(entry.object);
+      entry.dispose?.();
+    }
+    this.#objects.clear();
+  }
+}

pkgs/clan-app/ui/src/scene/RenderLoop.ts

@@ -0,0 +1,133 @@
+import { Scene, Camera, WebGLRenderer } from "three";
+import { MapControls } from "three/examples/jsm/controls/MapControls.js";
+import { CSS2DRenderer } from "three/examples/jsm/renderers/CSS2DRenderer.js";
+
+/**
+ * Private class to manage the render loop
+ * @internal
+ */
+class RenderLoop {
+  // Track if a render is already requested
+  // This prevents multiple requests in the same frame
+  // and ensures only one render per frame
+  // This is important for performance and to avoid flickering
+  private renderRequested = false;
+
+  // References to the scene, camera, renderer, controls, and label renderer
+  // These will be set during initialization
+  private scene!: Scene;
+  private bgScene!: Scene;
+  private camera!: Camera;
+  private bgCamera!: Camera;
+  private renderer!: WebGLRenderer;
+  private controls!: MapControls;
+  private labelRenderer!: CSS2DRenderer;
+
+  // Flag to prevent multiple initializations
+  private initialized = false;
+
+  init(
+    scene: Scene,
+    camera: Camera,
+    renderer: WebGLRenderer,
+    labelRenderer: CSS2DRenderer,
+    controls: MapControls,
+    bgScene: Scene,
+    bgCamera: Camera,
+  ) {
+    if (this.initialized) {
+      console.error("RenderLoop already initialized.");
+      return;
+    }
+    this.scene = scene;
+    this.camera = camera;
+    this.renderer = renderer;
+    this.controls = controls;
+    this.bgScene = bgScene;
+    this.bgCamera = bgCamera;
+    this.labelRenderer = labelRenderer;
+    this.initialized = true;
+  }
+
+  requestRender() {
+    // If not initialized, log an error and return
+    if (!this.initialized) {
+      console.error(
+        "RenderLoop not initialized yet. Make sure to call init() once before usage.",
+      );
+      return;
+    }
+    // If a render is already requested, do nothing
+    if (this.renderRequested) return;
+
+    this.renderRequested = true;
+    requestAnimationFrame(() => {
+      this.updateTweens();
+
+      const needsUpdate = this.controls.update(); // returns true if damping is ongoing
+
+      this.render();
+      this.renderRequested = false;
+
+      if (needsUpdate) {
+        this.requestRender();
+      }
+    });
+  }
+
+  private updateTweens() {
+    // TODO: TWEEN.update() for tween animations in the future
+  }
+
+  private render() {
+    // TODO: Disable console.debug in production
+    // console.debug("Rendering scene...", this);
+
+    this.renderer.clear();
+
+    this.renderer.render(this.bgScene, this.bgCamera);
+    this.renderer.render(this.scene, this.camera);
+    this.labelRenderer.render(this.scene, this.camera);
+  }
+
+  dispose() {
+    // Dispose controls, renderer, remove listeners if any
+    this.controls.dispose();
+    this.renderer.dispose();
+    // clear refs, this prevents memory leaks by allowing garbage collection
+    this.scene = null!;
+    this.bgScene = null!;
+    this.camera = null!;
+    this.bgCamera = null!;
+    this.renderer = null!;
+    this.controls = null!;
+    this.labelRenderer = null!;
+    this.initialized = false;
+  }
+}
+
+/**
+ * Singleton instance of RenderLoop
+ * This is used to manage the re-rendering
+ *
+ * It can only be initialized once then passed to individual components
+ * they can use the renderLoop to request re-renders as needed.
+ *
+ *
+ * Usage:
+ * ```typescript
+ * import { renderLoop } from "./RenderLoop";
+ *
+ * // Somewhere initialize the render loop:
+ * renderLoop.init(scene, camera, renderer, labelRenderer, controls, bgScene, bgCamera);
+ *
+ * // To request a render:
+ * renderLoop.requestRender();
+ *
+ * // To dispose:
+ * onCleanup(() => {
+ *  renderLoop.dispose();
+ * })
+ *
+ */
+export const renderLoop = new RenderLoop();

pkgs/clan-app/ui/src/scene/cubes.css

@@ -5,11 +5,22 @@
 }
 
 .toolbar-container {
-  position: absolute;
-  bottom: 10%;
-  width: 100%;
-  z-index: 10;
-  display: flex;
-  justify-content: center;
-  align-items: center;
+  @apply absolute bottom-8 z-10 w-full;
+  @apply flex justify-center items-center;
+}
+
+.machine-label {
+  @apply text-white bg-inv-4 py-1 px-2 rounded-sm;
+  font-size: 0.75rem;
+}
+
+.machine-label::after {
+  content: "";
+  position: absolute;
+  top: 100%;
+  left: 50%;
+  margin-left: -5px;
+  border-width: 5px;
+  border-style: solid;
+  border-color: #203637 transparent transparent transparent;
 }

pkgs/clan-app/ui/src/scene/cubes.tsx

@@ -1,23 +1,19 @@
-import {
-  createSignal,
-  createEffect,
-  onCleanup,
-  onMount,
-  createMemo,
-  on,
-} from "solid-js";
+import { createSignal, createEffect, onCleanup, onMount, on } from "solid-js";
 import "./cubes.css";
 
 import * as THREE from "three";
 import { MapControls } from "three/examples/jsm/controls/MapControls.js";
+import { CSS2DRenderer } from "three/examples/jsm/renderers/CSS2DRenderer.js";
 
 import { Toolbar } from "../components/Toolbar/Toolbar";
 import { ToolbarButton } from "../components/Toolbar/ToolbarButton";
 import { Divider } from "../components/Divider/Divider";
 import { MachinesQueryResult } from "../queries/queries";
 import { SceneData } from "../stores/clan";
-import { unwrap } from "solid-js/store";
 import { Accessor } from "solid-js";
+import { renderLoop } from "./RenderLoop";
+import { ObjectRegistry } from "./ObjectRegistry";
+import { MachineManager } from "./MachineManager";
 
 function garbageCollectGroup(group: THREE.Group) {
   for (const child of group.children) {
@@ -35,72 +31,35 @@ function garbageCollectGroup(group: THREE.Group) {
   group.clear(); // Clear the group
 }
 
-function getFloorPosition(
-  camera: THREE.PerspectiveCamera,
-  floor: THREE.Object3D,
-): [number, number, number] {
-  const cameraPosition = camera.position.clone();
-
-  // Get camera's direction
-  const direction = new THREE.Vector3();
-  camera.getWorldDirection(direction);
-
-  // Define floor plane (XZ-plane at y=0)
-  const floorPlane = new THREE.Plane(new THREE.Vector3(0, 1, 0), 0); // Normal = up, constant = 0
-
-  // Create ray from camera
-  const ray = new THREE.Ray(cameraPosition, direction);
-
-  // Get intersection point
-  const intersection = new THREE.Vector3();
-  ray.intersectPlane(floorPlane, intersection);
-
-  return intersection.toArray() as [number, number, number];
-}
-
-function keyFromPos(pos: [number, number]): string {
-  return `${pos[0]},${pos[1]}`;
-}
-
-// type SceneDataUpdater = (sceneData: SceneData) => void;
-
 export function CubeScene(props: {
   cubesQuery: MachinesQueryResult;
   onCreate: () => Promise<{ id: string }>;
+  selectedIds: Accessor<Set<string>>;
+  onSelect: (v: Set<string>) => void;
   sceneStore: Accessor<SceneData>;
   setMachinePos: (machineId: string, pos: [number, number]) => void;
   isLoading: boolean;
 }) {
   let container: HTMLDivElement;
   let scene: THREE.Scene;
-  let camera: THREE.PerspectiveCamera;
+  let camera: THREE.OrthographicCamera;
   let renderer: THREE.WebGLRenderer;
+  let labelRenderer: CSS2DRenderer;
   let floor: THREE.Mesh;
   let controls: MapControls;
   // Raycaster for clicking
   const raycaster = new THREE.Raycaster();
   let initBase: THREE.Mesh | undefined;
 
-  let needsRender = false; // Flag to control rendering
-
   // Create background scene
   const bgScene = new THREE.Scene();
   const bgCamera = new THREE.OrthographicCamera(-1, 1, 1, -1, 0, 1);
 
   const groupMap = new Map<string, THREE.Group>();
 
-  const occupiedPositions = new Set<string>();
-
   let sharedCubeGeometry: THREE.BoxGeometry;
   let sharedBaseGeometry: THREE.BoxGeometry;
 
-  // Used for development purposes
-  // Vite does hot-reload but we need to ensure the animation loop doesn't run multiple times
-  // This flag prevents multiple animation loops from running simultaneously
-  // It is set to true when the component mounts and false when it unmounts
-  let isAnimating = false; // Flag to prevent multiple loops
-  let frameCount = 0;
-
   const [positionMode, setPositionMode] = createSignal<"grid" | "circle">(
     "grid",
   );
@@ -108,21 +67,14 @@ export function CubeScene(props: {
   const [worldMode, setWorldMode] = createSignal<"view" | "create">("view");
 
   const [cursorPosition, setCursorPosition] = createSignal<[number, number]>();
-  const [selectedIds, setSelectedIds] = createSignal<Set<string>>(new Set());
 
   const [cameraInfo, setCameraInfo] = createSignal({
     position: { x: 0, y: 0, z: 0 },
     spherical: { radius: 0, theta: 0, phi: 0 },
   });
 
-  // Animation configuration
-  const ANIMATION_DURATION = 800; // milliseconds
-  const DELETE_ANIMATION_DURATION = 400; // milliseconds
-  const CREATE_ANIMATION_DURATION = 600; // milliseconds
-
   // Grid configuration
   const GRID_SIZE = 2;
-  const CUBE_SPACING = 2;
 
   const BASE_SIZE = 0.9; // Height of the cube above the ground
   const CUBE_SIZE = BASE_SIZE / 1.5; //
@@ -132,186 +84,12 @@ export function CubeScene(props: {
 
   const FLOOR_COLOR = 0xcdd8d9;
 
-  const CUBE_COLOR = 0xd7e0e1;
-  const CUBE_EMISSIVE = 0x303030; // Emissive color for cubes
-
-  const CUBE_SELECTED_COLOR = 0x4b6767;
-
   const BASE_COLOR = 0xecfdff;
   const BASE_EMISSIVE = 0x0c0c0c;
-  const BASE_SELECTED_COLOR = 0x69b0e3;
-  const BASE_SELECTED_EMISSIVE = 0x666666; // Emissive color for selected bases
 
   const CREATE_BASE_COLOR = 0x636363;
   const CREATE_BASE_EMISSIVE = 0xc5fad7;
 
-  createEffect(() => {
-    console.log("Direct query data hook");
-    // Update when API updates.
-    if (props.cubesQuery.data) {
-      const actualMachines = Object.keys(props.cubesQuery.data);
-      const rawStored = unwrap(props.sceneStore());
-      const placed: Set<string> = rawStored
-        ? new Set(Object.keys(rawStored))
-        : new Set();
-      const nonPlaced = actualMachines.filter((m) => !placed.has(m));
-
-      // Initialize occupied positions from previously placed cubes
-      for (const id of placed) {
-        occupiedPositions.add(keyFromPos(rawStored[id].position));
-      }
-
-      // Push not explizitly placed machines to the scene
-      // TODO: Make the user place them manually
-      // We just calculate some next free position
-      for (const id of nonPlaced) {
-        console.log("adding", id);
-        const position = nextGridPos();
-        console.log("Got pos", position);
-
-        // Add the machine to the store
-        // Adding it triggers a reactive update
-        props.setMachinePos(id, position);
-        occupiedPositions.add(keyFromPos(position));
-      }
-    }
-  });
-
-  function requestRenderIfNotRequested() {
-    if (!needsRender) {
-      needsRender = true;
-      requestAnimationFrame(renderScene);
-    }
-  }
-
-  function renderScene() {
-    if (!isAnimating) {
-      console.warn("Not animating!");
-      return;
-    }
-    console.log("Rendering scene...", initBase?.position);
-
-    needsRender = false;
-
-    frameCount++;
-
-    renderer.autoClear = false;
-    renderer.render(bgScene, bgCamera);
-    controls.update(); // optional; see note below
-    renderer.render(scene, camera);
-
-    if (frameCount % 30 === 0) logMemoryUsage();
-  }
-
-  function getGridPosition(id: string): [number, number, number] {
-    // TODO: Detect collision with other cubes
-    const machine = props.sceneStore()[id];
-    console.log("getGridPosition", id, machine);
-    if (machine) {
-      return [machine.position[0], 0, machine.position[1]];
-    }
-    // Some fallback to get the next free position
-    // If the position wasn't avilable in the store
-    console.warn(`Position for ${id} not set`);
-    return [0, 0, 0];
-  }
-
-  function nextGridPos(): [number, number] {
-    // Scales up to 10*10 grid = 100 positions
-    // TODO: Make this more scalable and nicer
-    const maxRows = 10; // or dynamic limit if needed
-    const maxCols = 10;
-
-    for (let y = 0; y < maxRows; y++) {
-      for (let x = 0; x < maxCols; x++) {
-        const pos = [x * CUBE_SPACING, y * CUBE_SPACING] as [number, number];
-        const key = keyFromPos(pos);
-
-        if (!occupiedPositions.has(key)) {
-          return pos;
-        }
-      }
-    }
-
-    throw new Error("No free grid positions available.");
-  }
-
-  // Circle IDEA:
-  // Need to talk with timo and W about this
-  const getCirclePosition =
-    (center: [number, number, number]) =>
-    (_id: string, index: number, total: number): [number, number, number] => {
-      const r = total === 1 ? 0 : Math.sqrt(total) * CUBE_SPACING; // Radius based on total cubes
-      const x = Math.cos((index / total) * 2 * Math.PI) * r + center[0];
-      const z = Math.sin((index / total) * 2 * Math.PI) * r + center[2];
-      // Position cubes at y = 0.5 to float above the ground
-      return [x, CUBE_Y, z];
-    };
-
-  // Reactive cubes memo - this recalculates whenever data changes
-  const cubes = createMemo(() => {
-    console.log("Calculating cubes...");
-    const sceneData = props.sceneStore(); // keep it reactive
-    if (!sceneData) return [];
-
-    const currentIds = Object.keys(sceneData);
-    console.log("Current IDs:", currentIds);
-
-    let cameraTarget = [0, 0, 0] as [number, number, number];
-    if (camera && floor) {
-      cameraTarget = getFloorPosition(camera, floor);
-    }
-    const getCubePosition =
-      positionMode() === "grid"
-        ? getGridPosition
-        : getCirclePosition(cameraTarget);
-
-    return currentIds.map((id, index) => {
-      const activeIndex = currentIds.indexOf(id);
-
-      const position = getCubePosition(id, index, currentIds.length);
-
-      const targetPosition =
-        activeIndex >= 0
-          ? getCubePosition(id, activeIndex, currentIds.length)
-          : getCubePosition(id, index, currentIds.length);
-
-      return {
-        id,
-        position,
-        targetPosition,
-      };
-    });
-  });
-
-  // Animation helper function
-  function animateToPosition(
-    thing: THREE.Object3D,
-    targetPosition: [number, number, number],
-    duration: number = ANIMATION_DURATION,
-  ) {
-    const startPosition = thing.position.clone();
-    const endPosition = new THREE.Vector3(...targetPosition);
-    const startTime = Date.now();
-
-    function animate() {
-      const elapsed = Date.now() - startTime;
-      const progress = Math.min(elapsed / duration, 1);
-
-      // Smooth easing function
-      const easeProgress = 1 - Math.pow(1 - progress, 3);
-
-      thing.position.lerpVectors(startPosition, endPosition, easeProgress);
-
-      if (progress < 1) {
-        requestAnimationFrame(animate);
-        requestRenderIfNotRequested();
-      }
-    }
-
-    animate();
-  }
-
   function createCubeBase(
     cube_pos: [number, number, number],
     opacity = 1,
@@ -331,126 +109,13 @@ export function CubeScene(props: {
     return base;
   }
 
-  function deleteSelectedCubes(selectedSet: Set<string>) {
-    if (selectedSet.size === 0) return;
-
-    console.log("Deleting cubes:", selectedSet);
-
-    // Start delete animations
-    selectedSet.forEach((id) => {
-      const group = groupMap.get(id);
-
-      if (group) {
-        groupMap.delete(id); // Remove from group map
-
-        const base = group.children.find((child) => child.name === "base");
-        const cube = group.children.find((child) => child.name === "cube");
-
-        if (!base || !cube) {
-          console.warn(`DELETE: Base mesh not found for id: ${id}`);
-          return;
-        }
-
-        {
-          setSelectedIds(new Set<string>()); // Clear selection after deletion
-
-          garbageCollectGroup(group); // Clean up geometries and materials
-          scene.remove(group); // Remove from scene
-          groupMap.delete(id); // Remove from group map
-        }
-      } else {
-        console.warn(`DELETE: Group not found for id: ${id}`);
-      }
-    });
-  }
-
   function toggleSelection(id: string) {
-    setSelectedIds((curr) => {
-      const next = new Set(curr);
-      if (next.has(id)) {
-        next.delete(id);
-      } else {
-        next.add(id);
-      }
-      return next;
-    });
+    const next = new Set<string>();
+    next.add(id);
+    props.onSelect(next);
   }
 
-  function updateMeshColors(
-    selected: Set<string>,
-    prev: Set<string> | undefined,
-  ) {
-    for (const id of selected) {
-      const group = groupMap.get(id);
-      if (!group) {
-        console.warn(`UPDATE COLORS: Group not found for id: ${id}`);
-        continue;
-      }
-      const base = group.children.find((child) => child.name === "base");
-      if (!base || !(base instanceof THREE.Mesh)) {
-        console.warn(`UPDATE COLORS: Base mesh not found for id: ${id}`);
-        continue;
-      }
-      const cube = group.children.find((child) => child.name === "cube");
-      if (!cube || !(cube instanceof THREE.Mesh)) {
-        console.warn(`UPDATE COLORS: Cube mesh not found for id: ${id}`);
-        continue;
-      }
-
-      const baseMaterial = base.material as THREE.MeshPhongMaterial;
-      const cubeMaterial = cube.material as THREE.MeshPhongMaterial;
-
-      baseMaterial.color.set(BASE_SELECTED_COLOR);
-      baseMaterial.emissive.set(BASE_SELECTED_EMISSIVE);
-
-      cubeMaterial.color.set(CUBE_SELECTED_COLOR);
-    }
-
-    const deselected = Array.from(prev || []).filter((s) => !selected.has(s));
-
-    for (const id of deselected) {
-      const group = groupMap.get(id);
-      if (!group) {
-        console.warn(`UPDATE COLORS: Group not found for id: ${id}`);
-        continue;
-      }
-      const base = group.children.find((child) => child.name === "base");
-      if (!base || !(base instanceof THREE.Mesh)) {
-        console.warn(`UPDATE COLORS: Base mesh not found for id: ${id}`);
-        continue;
-      }
-      const cube = group.children.find((child) => child.name === "cube");
-      if (!cube || !(cube instanceof THREE.Mesh)) {
-        console.warn(`UPDATE COLORS: Cube mesh not found for id: ${id}`);
-        continue;
-      }
-
-      const baseMaterial = base.material as THREE.MeshPhongMaterial;
-      const cubeMaterial = cube.material as THREE.MeshPhongMaterial;
-
-      baseMaterial.color.set(BASE_COLOR);
-      baseMaterial.emissive.set(BASE_EMISSIVE);
-
-      cubeMaterial.color.set(CUBE_COLOR);
-    }
-
-    requestRenderIfNotRequested();
-  }
-
-  function logMemoryUsage() {
-    if (renderer && renderer.info) {
-      console.debug("Three.js Memory:", {
-        frame: renderer.info.render.frame,
-        calls: renderer.info.render.calls,
-        geometries: renderer.info.memory.geometries,
-        textures: renderer.info.memory.textures,
-        programs: renderer.info.programs?.length || 0,
-        triangles: renderer.info.render.triangles,
-      });
-    }
-  }
-
-  const initialCameraPosition = { x: 2.8, y: 4, z: -2 };
+  const initialCameraPosition = { x: 20, y: 20, z: 20 };
   const initialSphericalCameraPosition = new THREE.Spherical();
   initialSphericalCameraPosition.setFromVector3(
     new THREE.Vector3(
@@ -465,7 +130,6 @@ export function CubeScene(props: {
   onMount(() => {
     // Scene setup
     scene = new THREE.Scene();
-    scene.fog = new THREE.Fog(0xffffff, 10, 50); //
     // Transparent background
     scene.background = null;
 
@@ -506,62 +170,104 @@ export function CubeScene(props: {
     bgScene.add(bgMesh);
 
     // Camera setup
-    camera = new THREE.PerspectiveCamera(
-      75,
-      container!.clientWidth / container!.clientHeight,
-      0.1,
+    // /container!.clientWidth / container!.clientHeight,
+    const aspect = window.innerWidth / window.innerHeight;
+    const d = 20;
+    const zoom = 2.5;
+    camera = new THREE.OrthographicCamera(
+      (-d * aspect) / zoom,
+      (d * aspect) / zoom,
+      d / zoom,
+      -d / zoom,
+      0.001,
       1000,
     );
+    camera.zoom = zoom;
+
     camera.position.setFromSpherical(initialSphericalCameraPosition);
     camera.lookAt(0, 0, 0);
+    camera.updateProjectionMatrix();
 
     // Renderer setup
-    renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
+    renderer = new THREE.WebGLRenderer({
+      antialias: true,
+      alpha: true,
+      logarithmicDepthBuffer: true,
+    });
     renderer.setSize(container.clientWidth, container.clientHeight);
     renderer.shadowMap.enabled = true;
     renderer.shadowMap.type = THREE.PCFSoftShadowMap;
+    renderer.autoClear = false;
     container.appendChild(renderer.domElement);
 
+    // Label renderer
+    labelRenderer = new CSS2DRenderer();
+    labelRenderer.setSize(container.clientWidth, container.clientHeight);
+    labelRenderer.domElement.style.position = "absolute";
+    labelRenderer.domElement.style.top = "0px";
+    labelRenderer.domElement.style.pointerEvents = "none";
+    labelRenderer.domElement.style.zIndex = "0";
+    container.appendChild(labelRenderer.domElement);
+
     controls = new MapControls(camera, renderer.domElement);
     controls.enableDamping = true; // an animation loop is required when either damping or auto-rotation are enabled
-    // Enable the context menu,
-    // TODO: disable in production
     controls.mouseButtons.RIGHT = null;
-    controls.addEventListener("change", requestRenderIfNotRequested);
+    controls.enableRotate = false;
+    controls.minZoom = 1.2;
+    controls.maxZoom = 3.5;
+    controls.addEventListener("change", () => {
+      const aspect = container.clientWidth / container.clientHeight;
+      const zoom = camera.zoom;
+      camera.left = (-d * aspect) / zoom;
+      camera.right = (d * aspect) / zoom;
+      camera.top = d / zoom;
+      camera.bottom = -d / zoom;
+      camera.updateProjectionMatrix();
+      renderLoop.requestRender();
+    });
+
+    renderLoop.init(
+      scene,
+      camera,
+      renderer,
+      labelRenderer,
+      controls,
+      bgScene,
+      bgCamera,
+    );
 
     // Lighting
-    const ambientLight = new THREE.AmbientLight(0xffffff, 1.5);
+    const ambientLight = new THREE.AmbientLight(0xffffff, 1);
     scene.add(ambientLight);
 
     const directionalLight = new THREE.DirectionalLight(0xffffff, 2);
 
     // scene.add(new THREE.DirectionalLightHelper(directionalLight));
-    // scene.add(new THREE.CameraHelper(directionalLight.shadow.camera));
     // scene.add(new THREE.CameraHelper(camera));
     const lightPos = new THREE.Spherical(
-      100,
-      initialSphericalCameraPosition.phi,
+      15,
+      initialSphericalCameraPosition.phi - Math.PI / 8,
       initialSphericalCameraPosition.theta - Math.PI / 2,
     );
     directionalLight.position.setFromSpherical(lightPos);
-    directionalLight.target.position.set(0, 0, 0); // Point light at the center
-
+    directionalLight.rotation.set(0, 0, 0);
     // initialSphericalCameraPosition
     directionalLight.castShadow = true;
 
     // Configure shadow camera for hard, crisp shadows
-    directionalLight.shadow.camera.left = -30;
-    directionalLight.shadow.camera.right = 30;
-    directionalLight.shadow.camera.top = 30;
-    directionalLight.shadow.camera.bottom = -30;
+    directionalLight.shadow.camera.left = -20;
+    directionalLight.shadow.camera.right = 20;
+    directionalLight.shadow.camera.top = 20;
+    directionalLight.shadow.camera.bottom = -20;
     directionalLight.shadow.camera.near = 0.1;
-    directionalLight.shadow.camera.far = 200;
+    directionalLight.shadow.camera.far = 30;
     directionalLight.shadow.mapSize.width = 4096; // Higher resolution for sharper shadows
     directionalLight.shadow.mapSize.height = 4096;
     directionalLight.shadow.radius = 1; // Hard shadows (low radius)
     directionalLight.shadow.blurSamples = 4; // Fewer samples for harder edges
     scene.add(directionalLight);
     scene.add(directionalLight.target);
+    // scene.add(new THREE.CameraHelper(directionalLight.shadow.camera));
 
     // Floor/Ground - Make it invisible but keep it for reference
     const floorGeometry = new THREE.PlaneGeometry(1000, 1000);
@@ -631,6 +337,28 @@ export function CubeScene(props: {
     // Initial camera info update
     updateCameraInfo();
 
+    createEffect(
+      on(worldMode, (mode) => {
+        if (mode === "create") {
+          initBase!.visible = true;
+        } else {
+          initBase!.visible = false;
+        }
+        renderLoop.requestRender();
+      }),
+    );
+
+    const registry = new ObjectRegistry();
+
+    const machineManager = new MachineManager(
+      scene,
+      registry,
+      props.sceneStore,
+      props.cubesQuery,
+      props.selectedIds,
+      props.setMachinePos,
+    );
+
     // Click handler:
     // - Select/deselects a cube in "view" mode
     // - Creates a new cube in "create" mode
@@ -665,28 +393,34 @@ export function CubeScene(props: {
 
       raycaster.setFromCamera(mouse, camera);
       const intersects = raycaster.intersectObjects(
-        Array.from(groupMap.values()),
+        Array.from(machineManager.machines.values().map((m) => m.group)),
       );
       console.log("Intersects:", intersects);
       if (intersects.length > 0) {
+        console.log("Clicked on cube:", intersects);
         const id = intersects[0].object.userData.id;
         toggleSelection(id);
       } else {
-        setSelectedIds(new Set<string>()); // Clear selection if clicked outside cubes
+        props.onSelect(new Set<string>()); // Clear selection if clicked outside cubes
       }
     };
 
     renderer.domElement.addEventListener("click", onClick);
 
-    isAnimating = true;
-
-    requestRenderIfNotRequested();
+    renderLoop.requestRender();
 
     // Handle window resize
     const handleResize = () => {
-      camera.aspect = container.clientWidth / container.clientHeight;
+      const aspect = container.clientWidth / container.clientHeight;
+      const zoom = camera.zoom;
+      camera.left = (-d * aspect) / zoom;
+      camera.right = (d * aspect) / zoom;
+      camera.top = d / zoom;
+      camera.bottom = -d / zoom;
       camera.updateProjectionMatrix();
+
       renderer.setSize(container.clientWidth, container.clientHeight);
+      labelRenderer.setSize(container.clientWidth, container.clientHeight);
 
       // Update background shader resolution
       uniforms.resolution.value.set(
@@ -694,8 +428,8 @@ export function CubeScene(props: {
         container.clientHeight,
       );
 
-      renderer.render(bgScene, bgCamera);
-      requestRenderIfNotRequested();
+      // renderer.render(bgScene, bgCamera);
+      renderLoop.requestRender();
     };
 
     renderer.domElement.addEventListener("mousemove", onMouseMove);
@@ -711,10 +445,27 @@ export function CubeScene(props: {
       { capture: true },
     );
 
+    // Initial render
+    renderLoop.requestRender();
+
     // Cleanup function
     onCleanup(() => {
-      // Stop animation loop
-      isAnimating = false;
+      for (const group of groupMap.values()) {
+        garbageCollectGroup(group);
+        scene.remove(group);
+      }
+      groupMap.clear();
+
+      // Dispose shared geometries
+      sharedCubeGeometry?.dispose();
+      sharedBaseGeometry?.dispose();
+
+      renderer?.dispose();
+
+      renderLoop.dispose();
+
+      machineManager.dispose(scene);
+
       renderer.domElement.removeEventListener("click", onClick);
       renderer.domElement.removeEventListener("mousemove", onMouseMove);
       window.removeEventListener("resize", handleResize);
@@ -731,159 +482,13 @@ export function CubeScene(props: {
       if (container) {
         container.innerHTML = "";
       }
-
-      groupMap.forEach((group) => {
-        garbageCollectGroup(group);
-        scene.remove(group);
-      });
-      groupMap.clear();
     });
   });
 
-  // TODO: Move into css
-  // createEffect(
-  //   on(positionMode, (mode) => {
-  //     console.log("Position mode changed:", mode);
-  //     if (mode === "circle") {
-  //       grid.visible = false; // Hide grid when in circle mode
-  //     } else if (mode === "grid") {
-  //       grid.visible = true; // Show grid when in grid mode
-  //     }
-  //   }),
-  // );
-
-  function createCube(
-    gridPosition: [number, number],
-    userData: { id: string },
-  ) {
-    // Creates a cube, base, and other visuals
-    // Groups them together in the scene
-    const cubeMaterial = new THREE.MeshPhongMaterial({
-      color: CUBE_COLOR,
-      emissive: CUBE_EMISSIVE,
-      // specular: 0xffffff,
-      shininess: 100,
-    });
-    const cubeMesh = new THREE.Mesh(sharedCubeGeometry, cubeMaterial);
-    cubeMesh.castShadow = true;
-    cubeMesh.receiveShadow = true;
-    cubeMesh.userData = userData;
-    cubeMesh.name = "cube"; // Name for easy identification
-    cubeMesh.position.set(0, CUBE_Y, 0);
-
-    const baseMesh = createCubeBase([0, BASE_HEIGHT / 2, 0]);
-    baseMesh.name = "base"; // Name for easy identification
-
-    // TODO: Destroy Group in onCleanup
-    const group = new THREE.Group();
-    group.add(cubeMesh);
-    group.add(baseMesh);
-    group.position.set(gridPosition[0], 0, gridPosition[1]); // Position on the grid
-
-    group.userData.id = userData.id;
-    return group;
-  }
-
-  // Effect to manage cube meshes - this runs whenever cubes() changes
-  createEffect(() => {
-    const currentCubes = cubes();
-    console.log("Current cubes:", currentCubes);
-
-    const existing = new Set(groupMap.keys());
-
-    // Update existing cubes and create new ones
-    currentCubes.forEach((cube) => {
-      const existingGroup = groupMap.get(cube.id);
-
-      console.log(
-        "Processing cube:",
-        cube.id,
-        "Existing group:",
-        existingGroup,
-      );
-      if (!existingGroup) {
-        const group = createCube([cube.position[0], cube.position[2]], {
-          id: cube.id,
-        });
-        scene.add(group);
-        groupMap.set(cube.id, group);
-      } else {
-        console.log("Updating existing cube:", cube.id);
-        // Only animate position if not being deleted
-        const targetPosition = cube.targetPosition || cube.position;
-        const currentPosition = existingGroup.position.toArray() as [
-          number,
-          number,
-          number,
-        ];
-        const target = targetPosition;
-        // Check if position actually changed
-        if (
-          Math.abs(currentPosition[0] - target[0]) > 0.01 ||
-          Math.abs(currentPosition[1] - target[1]) > 0.01 ||
-          Math.abs(currentPosition[2] - target[2]) > 0.01
-        ) {
-          animateToPosition(existingGroup, target);
-        }
-      }
-
-      existing.delete(cube.id);
-    });
-
-    // Remove cubes that are no longer in the state and not being deleted
-    existing.forEach((id) => {
-      if (!currentCubes.find((d) => d.id == id)) {
-        const group = groupMap.get(id);
-        if (group) {
-          console.log("Cleaning...", id);
-          garbageCollectGroup(group);
-          scene.remove(group);
-          groupMap.delete(id);
-          const pos = group.position.toArray() as [number, number, number];
-          occupiedPositions.delete(keyFromPos([pos[0], pos[2]]));
-        }
-      }
-    });
-
-    requestRenderIfNotRequested();
-  });
-
-  createEffect(
-    on(selectedIds, (curr, prev) => {
-      console.log("Selected cubes:", curr);
-      // Update colors of selected cubes
-      updateMeshColors(curr, prev);
-    }),
-  );
-
-  onCleanup(() => {
-    for (const group of groupMap.values()) {
-      garbageCollectGroup(group);
-      scene.remove(group);
-    }
-    groupMap.clear();
-
-    // Dispose shared geometries
-    sharedCubeGeometry?.dispose();
-    sharedBaseGeometry?.dispose();
-
-    renderer?.dispose();
-  });
-
-  const onHover = (inside: boolean) => (event: MouseEvent) => {
-    const pos = nextGridPos();
-    if (!initBase) return;
-
-    if (initBase.visible === false && inside) {
-      initBase.position.set(pos[0], BASE_HEIGHT / 2, pos[1]);
-      initBase.visible = true;
-    }
-    requestRenderIfNotRequested();
-  };
-
   const onAddClick = (event: MouseEvent) => {
     setPositionMode("grid");
     setWorldMode("create");
+    renderLoop.requestRender();
   };
   const onMouseMove = (event: MouseEvent) => {
     if (worldMode() !== "create") return;
@@ -914,7 +519,7 @@ export function CubeScene(props: {
         // Only request render if the position actually changed
         initBase.position.set(snapped.x, 0, snapped.z);
         setCursorPosition([snapped.x, snapped.z]); // Update next position for cube creation
-        requestRenderIfNotRequested();
+        renderLoop.requestRender();
       }
     }
   };
@@ -933,8 +538,7 @@ export function CubeScene(props: {
           <ToolbarButton
             name="new-machine"
             icon="NewMachine"
-            onMouseEnter={onHover(true)}
-            onMouseLeave={onHover(false)}
+            disabled={positionMode() === "circle"}
             onClick={onAddClick}
             selected={worldMode() === "create"}
           />
@@ -945,18 +549,16 @@ export function CubeScene(props: {
             onClick={() => {
               if (positionMode() === "grid") {
                 setPositionMode("circle");
+                setWorldMode("view");
                 grid.visible = false;
               } else {
                 setPositionMode("grid");
                 grid.visible = true;
               }
+              renderLoop.requestRender();
             }}
           />
-          <ToolbarButton
-            name="delete"
-            icon="Trash"
-            onClick={() => deleteSelectedCubes(selectedIds())}
-          />
+          <ToolbarButton name="delete" icon="Trash" />
         </Toolbar>
       </div>
     </>

pkgs/clan-app/ui/src/stores/clan.ts

@@ -20,6 +20,14 @@ const [store, setStore] = makePersisted(
   },
 );
 
+const resetStore = () => {
+  setStore({
+    clanURIs: [],
+    activeClanURI: undefined,
+    sceneData: {},
+  });
+};
+
 /**
  * Retrieves the active clan URI from the store.
  *
@@ -92,4 +100,5 @@ export {
   clanURIs,
   addClanURI,
   removeClanURI,
+  resetStore,
 };

pkgs/clan-app/ui/vite.config.ts

@@ -1,7 +1,6 @@
 import { defineConfig } from "vite";
 import solidPlugin from "vite-plugin-solid";
 import solidSvg from "vite-plugin-solid-svg";
-import devtools from "solid-devtools/vite";
 import path from "node:path";
 import { exec } from "child_process";
 
@@ -38,7 +37,6 @@ export default defineConfig({
     Uncomment the following line to enable solid-devtools.
     For more info see https://github.com/thetarnav/solid-devtools/tree/main/packages/extension#readme
     */
-    devtools(),
     solidPlugin(),
     solidSvg(),
     regenPythonApiOnFileChange(),

pkgs/clan-cli/clan_cli/clan/create.py

@@ -6,11 +6,14 @@ from pathlib import Path
 from clan_lib.clan.create import CreateOptions, create_clan
 from clan_lib.errors import ClanError
 
+from clan_cli.completions import add_dynamic_completer, complete_templates_clan
+from clan_cli.vars.keygen import create_secrets_user_auto
+
 log = logging.getLogger(__name__)
 
 
 def register_create_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    template_action = parser.add_argument(
         "--template",
         type=str,
         help="""Reference to the template to use for the clan. default="default". In the format '<flake_ref>#template_name' Where <flake_ref> is a flake reference (e.g. github:org/repo) or a local path (e.g. '.' ).
@@ -18,6 +21,7 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
         """,
         default="default",
     )
+    add_dynamic_completer(template_action, complete_templates_clan)
 
     parser.add_argument(
         "--no-git",
@@ -40,6 +44,12 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
         default=False,
     )
 
+    parser.add_argument(
+        "--user",
+        help="The user to generate the keys for. Default: logged-in OS username (e.g. from $LOGNAME or system)",
+        default=None,
+    )
+
     def create_flake_command(args: argparse.Namespace) -> None:
         # Ask for a path interactively if none provided
         if args.name is None:
@@ -59,5 +69,10 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
                 update_clan=not args.no_update,
             )
         )
+        create_secrets_user_auto(
+            flake_dir=Path(args.name).resolve(),
+            user=args.user,
+            force=True,
+        )
 
     parser.set_defaults(func=create_flake_command)

pkgs/clan-cli/clan_cli/cli.py

@@ -285,7 +285,7 @@ Examples:
   $ clan secrets get [SECRET]
   Will display the content of the specified secret.
 
-For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
+For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/secrets")}
         """
         ),
         formatter_class=argparse.RawTextHelpFormatter,
@@ -324,7 +324,7 @@ Examples:
   This is especially useful for resetting certain passwords while leaving the rest
   of the facts for a machine in place.
 
-For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
+For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/secrets")}
         """
         ),
         formatter_class=argparse.RawTextHelpFormatter,
@@ -362,7 +362,7 @@ Examples:
     This is especially useful for resetting certain passwords while leaving the rest
     of the vars for a machine in place.
 
-For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
+For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/secrets")}
         """
         ),
         formatter_class=argparse.RawTextHelpFormatter,
@@ -482,11 +482,11 @@ For more detailed information, visit: {help_hyperlink("getting-started", "https:
     )
     state.register_parser(parser_state)
 
+    register_common_flags(parser)
+
     if argcomplete:
         argcomplete.autocomplete(parser, exclude=["morph", "network", "net"])
 
-    register_common_flags(parser)
-
     return parser
 
 

pkgs/clan-cli/clan_cli/completions.py

@@ -29,6 +29,8 @@ COMPLETION_TIMEOUT: int = 3
 
 
 def clan_dir(flake: str | None) -> str | None:
+    if flake is not None:
+        return flake
     from clan_lib.dirs import get_clan_flake_toplevel_or_env
 
     path_result = get_clan_flake_toplevel_or_env()
@@ -45,7 +47,9 @@ def complete_machines(
 
     def run_cmd() -> None:
         try:
-            if (clan_dir_result := clan_dir(None)) is not None:
+            if (
+                clan_dir_result := clan_dir(getattr(parsed_args, "flake", None))
+            ) is not None:
                 flake = clan_dir_result
             else:
                 flake = "."
@@ -79,7 +83,9 @@ def complete_services_for_machine(
 
     def run_cmd() -> None:
         try:
-            if (clan_dir_result := clan_dir(None)) is not None:
+            if (
+                clan_dir_result := clan_dir(getattr(parsed_args, "flake", None))
+            ) is not None:
                 flake = clan_dir_result
             else:
                 flake = "."
@@ -121,7 +127,9 @@ def complete_backup_providers_for_machine(
 
     def run_cmd() -> None:
         try:
-            if (clan_dir_result := clan_dir(None)) is not None:
+            if (
+                clan_dir_result := clan_dir(getattr(parsed_args, "flake", None))
+            ) is not None:
                 flake = clan_dir_result
             else:
                 flake = "."
@@ -163,7 +171,9 @@ def complete_state_services_for_machine(
 
     def run_cmd() -> None:
         try:
-            if (clan_dir_result := clan_dir(None)) is not None:
+            if (
+                clan_dir_result := clan_dir(getattr(parsed_args, "flake", None))
+            ) is not None:
                 flake = clan_dir_result
             else:
                 flake = "."
@@ -204,7 +214,12 @@ def complete_secrets(
 
     from .secrets.secrets import list_secrets
 
-    flake = clan_dir_result if (clan_dir_result := clan_dir(None)) is not None else "."
+    flake = (
+        clan_dir_result
+        if (clan_dir_result := clan_dir(getattr(parsed_args, "flake", None)))
+        is not None
+        else "."
+    )
 
     secrets = list_secrets(Flake(flake).path)
 
@@ -222,7 +237,12 @@ def complete_users(
 
     from .secrets.users import list_users
 
-    flake = clan_dir_result if (clan_dir_result := clan_dir(None)) is not None else "."
+    flake = (
+        clan_dir_result
+        if (clan_dir_result := clan_dir(getattr(parsed_args, "flake", None)))
+        is not None
+        else "."
+    )
 
     users = list_users(Path(flake))
 
@@ -240,7 +260,12 @@ def complete_groups(
 
     from .secrets.groups import list_groups
 
-    flake = clan_dir_result if (clan_dir_result := clan_dir(None)) is not None else "."
+    flake = (
+        clan_dir_result
+        if (clan_dir_result := clan_dir(getattr(parsed_args, "flake", None)))
+        is not None
+        else "."
+    )
 
     groups_list = list_groups(Path(flake))
     groups = [group.name for group in groups_list]
@@ -258,7 +283,12 @@ def complete_templates_disko(
 
     from clan_lib.templates import list_templates
 
-    flake = clan_dir_result if (clan_dir_result := clan_dir(None)) is not None else "."
+    flake = (
+        clan_dir_result
+        if (clan_dir_result := clan_dir(getattr(parsed_args, "flake", None)))
+        is not None
+        else "."
+    )
 
     list_all_templates = list_templates(Flake(flake))
     disko_template_list = list_all_templates.builtins.get("disko")
@@ -269,6 +299,74 @@ def complete_templates_disko(
     return []
 
 
+def complete_templates_clan(
+    prefix: str, parsed_args: argparse.Namespace, **kwargs: Any
+) -> Iterable[str]:
+    """
+    Provides completion functionality for clan templates
+    """
+
+    from clan_lib.templates import list_templates
+
+    flake = (
+        clan_dir_result
+        if (clan_dir_result := clan_dir(getattr(parsed_args, "flake", None)))
+        is not None
+        else "."
+    )
+
+    list_all_templates = list_templates(Flake(flake))
+    clan_template_list = list_all_templates.builtins.get("clan")
+    if clan_template_list:
+        clan_templates = list(clan_template_list)
+        clan_dict = dict.fromkeys(clan_templates, "clan")
+        return clan_dict
+    return []
+
+
+def complete_vars_for_machine(
+    prefix: str, parsed_args: argparse.Namespace, **kwargs: Any
+) -> Iterable[str]:
+    """
+    Provides completion functionality for variable names for a specific machine.
+    Only completes vars that already exist in the vars directory on disk.
+    This is fast as it only scans the filesystem without any evaluation.
+    """
+    from pathlib import Path
+
+    machine_name = getattr(parsed_args, "machine", None)
+    if not machine_name:
+        return []
+
+    if (clan_dir_result := clan_dir(getattr(parsed_args, "flake", None))) is not None:
+        flake_path = Path(clan_dir_result)
+    else:
+        flake_path = Path()
+
+    vars_dir = flake_path / "vars" / "per-machine" / machine_name
+    vars_list: list[str] = []
+
+    if vars_dir.exists() and vars_dir.is_dir():
+        try:
+            for generator_dir in vars_dir.iterdir():
+                if not generator_dir.is_dir():
+                    continue
+
+                generator_name = generator_dir.name
+
+                for var_dir in generator_dir.iterdir():
+                    if var_dir.is_dir():
+                        var_name = var_dir.name
+                        var_id = f"{generator_name}/{var_name}"
+                        vars_list.append(var_id)
+
+        except Exception:
+            pass
+
+    vars_dict = dict.fromkeys(vars_list, "var")
+    return vars_dict
+
+
 def complete_target_host(
     prefix: str, parsed_args: argparse.Namespace, **kwargs: Any
 ) -> Iterable[str]:
@@ -280,7 +378,9 @@ def complete_target_host(
 
     def run_cmd() -> None:
         try:
-            if (clan_dir_result := clan_dir(None)) is not None:
+            if (
+                clan_dir_result := clan_dir(getattr(parsed_args, "flake", None))
+            ) is not None:
                 flake = clan_dir_result
             else:
                 flake = "."
@@ -320,7 +420,9 @@ def complete_tags(
 
     def run_computed_tags_cmd() -> None:
         try:
-            if (clan_dir_result := clan_dir(None)) is not None:
+            if (
+                clan_dir_result := clan_dir(getattr(parsed_args, "flake", None))
+            ) is not None:
                 flake = clan_dir_result
             else:
                 flake = "."
@@ -336,7 +438,9 @@ def complete_tags(
     def run_machines_tags_cmd() -> None:
         machine_tags: list[str] = []
         try:
-            if (clan_dir_result := clan_dir(None)) is not None:
+            if (
+                clan_dir_result := clan_dir(getattr(parsed_args, "flake", None))
+            ) is not None:
                 flake = clan_dir_result
             else:
                 flake = "."

pkgs/clan-cli/clan_cli/facts/cli.py

@@ -31,7 +31,7 @@ Examples:
   Will check facts for the specified machine.
 
 
-For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
+For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/secrets")}
         """
         ),
         formatter_class=argparse.RawTextHelpFormatter,
@@ -61,7 +61,7 @@ Examples:
   Will list facts for the specified machine.
 
 
-For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
+For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/secrets")}
         """
         ),
         formatter_class=argparse.RawTextHelpFormatter,
@@ -101,7 +101,7 @@ Examples:
   This is especially useful for resetting certain passwords while leaving the rest
   of the facts for a machine in place.
 
-For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
+For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/secrets")}
         """
         ),
         formatter_class=argparse.RawTextHelpFormatter,
@@ -125,7 +125,7 @@ Examples:
   $ clan facts upload [MACHINE]
   Will upload secrets to a specific machine.
 
-For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
+For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/secrets")}
         """
         ),
         formatter_class=argparse.RawTextHelpFormatter,

pkgs/clan-cli/clan_cli/machines/create.py

@@ -2,7 +2,6 @@ import argparse
 import logging
 import re
 from dataclasses import dataclass
-from typing import TypeVar, cast
 
 from clan_lib.api import API
 from clan_lib.dirs import get_clan_flake_toplevel_or_env
@@ -12,7 +11,7 @@ from clan_lib.git import commit_file
 from clan_lib.nix_models.clan import InventoryMachine
 from clan_lib.nix_models.clan import InventoryMachineDeploy as MachineDeploy
 from clan_lib.persist.inventory_store import InventoryStore
-from clan_lib.persist.util import set_value_by_path
+from clan_lib.persist.util import merge_objects, set_value_by_path
 from clan_lib.templates.handler import machine_template
 
 from clan_cli.completions import add_dynamic_completer, complete_tags
@@ -28,41 +27,6 @@ class CreateOptions:
     target_host: str | None = None
 
 
-T = TypeVar("T")
-
-
-def merge_objects(obj1: T, obj2: T) -> T:
-    """
-    Updates values in obj2 by values of Obj1
-    The output contains values for all keys of Obj1 and Obj2 together
-
-    Lists are deduplicated and appended almost like in the nix module system.
-    """
-    result = {}
-    msg = f"cannot update non-dictionary values: {obj2} by {obj1}"
-    if not isinstance(obj1, dict):
-        raise ClanError(msg)
-    if not isinstance(obj2, dict):
-        raise ClanError(msg)
-
-    all_keys = set(obj1.keys()).union(obj2.keys())
-
-    for key in all_keys:
-        val1 = obj1.get(key)
-        val2 = obj2.get(key)
-
-        if isinstance(val1, dict) and isinstance(val2, dict):
-            result[key] = merge_objects(val1, val2)
-        elif isinstance(val1, list) and isinstance(val2, list):
-            result[key] = list(dict.fromkeys(val2 + val1))  # type: ignore
-        elif key in obj1:
-            result[key] = val1  # type: ignore
-        elif key in obj2:
-            result[key] = val2  # type: ignore
-
-    return cast(T, result)
-
-
 @API.register
 def create_machine(
     opts: CreateOptions,
@@ -122,7 +86,7 @@ def create_machine(
         inventory = inventory_store.read()
 
         curr_machine = inventory.get("machines", {}).get(machine_name, {})
-        new_machine = merge_objects(opts.machine, curr_machine)
+        new_machine = merge_objects(curr_machine, opts.machine)
 
         set_value_by_path(
             inventory,

pkgs/clan-cli/clan_cli/machines/install.py

@@ -35,7 +35,7 @@ def install_command(args: argparse.Namespace) -> None:
         use_tor = False
         if deploy_info:
             host = find_reachable_host(deploy_info)
-            if host is None or host.tor_socks:
+            if host is None or host.socks_port:
                 use_tor = True
                 target_host_str = deploy_info.tor.target
             else:
@@ -63,9 +63,17 @@ def install_command(args: argparse.Namespace) -> None:
             raise ClanError(msg)
 
         if not args.yes:
-            ask = input(f"Install {args.machine} to {target_host.target}? [y/N] ")
-            if ask != "y":
-                return None
+            while True:
+                ask = (
+                    input(f"Install {args.machine} to {target_host.target}? [y/N] ")
+                    .strip()
+                    .lower()
+                )
+                if ask == "y":
+                    break
+                if ask == "n" or ask == "":
+                    return None
+                print(f"Invalid input '{ask}'. Please enter 'y' for yes or 'n' for no.")
 
         if args.identity_file:
             target_host = target_host.override(private_key=args.identity_file)
@@ -74,7 +82,9 @@ def install_command(args: argparse.Namespace) -> None:
             target_host = target_host.override(password=password)
 
         if use_tor:
-            target_host = target_host.override(tor_socks=True)
+            target_host = target_host.override(
+                socks_port=9050, socks_wrapper=["torify"]
+            )
 
         return run_machine_install(
             InstallOptions(

pkgs/clan-cli/clan_cli/machines/list.py

@@ -1,7 +1,7 @@
 import argparse
 import logging
 
-from clan_lib.flake import Flake
+from clan_lib.flake import require_flake
 from clan_lib.machines.actions import list_machines
 
 from clan_cli.completions import add_dynamic_completer, complete_tags
@@ -10,7 +10,7 @@ log = logging.getLogger(__name__)
 
 
 def list_command(args: argparse.Namespace) -> None:
-    flake: Flake = args.flake
+    flake = require_flake(args.flake)
 
     for name in list_machines(flake, opts={"filter": {"tags": args.tags}}):
         print(name)

pkgs/clan-cli/clan_cli/machines/list_test.py

@@ -1,4 +1,5 @@
 import pytest
+from clan_lib.errors import ClanError
 
 from clan_cli.tests import fixtures_flakes
 from clan_cli.tests.helpers import cli
@@ -359,3 +360,12 @@ def list_mixed_tagged_untagged(
     assert "machine-with-tags" not in output.out
     assert "machine-without-tags" not in output.out
     assert output.out.strip() == ""
+
+
+def test_machines_list_require_flake_error() -> None:
+    """Test that machines list command fails when flake is required but not provided."""
+    with pytest.raises(ClanError) as exc_info:
+        cli.run(["machines", "list"])
+
+    error_message = str(exc_info.value)
+    assert "flake" in error_message.lower()