buildClan: remove in favor of lib.clan

README.md

@@ -24,7 +24,7 @@ If you're new to Clan and eager to dive in, start with our quickstart guide and
 
 In the Clan ecosystem, security is paramount. Learn how to handle secrets effectively:
 
-- **Secrets Management**: Securely manage secrets by consulting [Vars](https://docs.clan.lol/concepts/generators/)<!-- [secrets.md](docs/site/concepts/generators.md) -->.
+- **Secrets Management**: Securely manage secrets by consulting [secrets](https://docs.clan.lol/guides/getting-started/secrets/)<!-- [secrets.md](docs/site/guides/getting-started/secrets.md) -->.
 
 ### Contributing to Clan
 

checks/flake-module.nix

@@ -19,29 +19,18 @@ let
   nixosLib = import (self.inputs.nixpkgs + "/nixos/lib") { };
 in
 {
-  imports =
-    let
-      clanCoreModulesDir = ../nixosModules/clanCore;
-      getClanCoreTestModules =
-        let
-          moduleNames = attrNames (builtins.readDir clanCoreModulesDir);
-          testPaths = map (
-            moduleName: clanCoreModulesDir + "/${moduleName}/tests/flake-module.nix"
-          ) moduleNames;
-        in
-        filter pathExists testPaths;
-    in
-    getClanCoreTestModules
-    ++ filter pathExists [
-      ./backups/flake-module.nix
-      ./devshell/flake-module.nix
-      ./flash/flake-module.nix
-      ./impure/flake-module.nix
-      ./installation/flake-module.nix
-      ./morph/flake-module.nix
-      ./nixos-documentation/flake-module.nix
-      ./dont-depend-on-repo-root.nix
-    ];
+  imports = filter pathExists [
+    ./backups/flake-module.nix
+    ../nixosModules/clanCore/machine-id/tests/flake-module.nix
+    ../nixosModules/clanCore/state-version/tests/flake-module.nix
+    ./devshell/flake-module.nix
+    ./flash/flake-module.nix
+    ./impure/flake-module.nix
+    ./installation/flake-module.nix
+    ./morph/flake-module.nix
+    ./nixos-documentation/flake-module.nix
+    ./dont-depend-on-repo-root.nix
+  ];
   flake.check = genAttrs [ "x86_64-linux" "aarch64-darwin" ] (
     system:
     let
@@ -99,6 +88,7 @@ in
             nixos-test-container = self.clanLib.test.containerTest ./container nixosTestArgs;
             nixos-test-zt-tcp-relay = self.clanLib.test.containerTest ./zt-tcp-relay nixosTestArgs;
             nixos-test-matrix-synapse = self.clanLib.test.containerTest ./matrix-synapse nixosTestArgs;
+            nixos-test-postgresql = self.clanLib.test.containerTest ./postgresql nixosTestArgs;
             nixos-test-user-firewall-iptables = self.clanLib.test.containerTest ./user-firewall/iptables.nix nixosTestArgs;
             nixos-test-user-firewall-nftables = self.clanLib.test.containerTest ./user-firewall/nftables.nix nixosTestArgs;
 

checks/postgresql/default.nix

@@ -0,0 +1,73 @@
+({
+  name = "postgresql";
+
+  nodes.machine =
+    { self, config, ... }:
+    {
+      imports = [
+        self.nixosModules.clanCore
+        self.clanModules.postgresql
+        self.clanModules.localbackup
+      ];
+      clan.postgresql.users.test = { };
+      clan.postgresql.databases.test.create.options.OWNER = "test";
+      clan.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ];
+      clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
+      clan.core.settings.directory = ./.;
+
+      systemd.services.sample-service = {
+        wantedBy = [ "multi-user.target" ];
+        script = ''
+          while true; do
+            echo "Hello, world!"
+            sleep 5
+          done
+        '';
+      };
+
+      environment.systemPackages = [ config.services.postgresql.package ];
+    };
+  testScript =
+    { nodes, ... }:
+    ''
+      start_all()
+      machine.wait_for_unit("postgresql")
+      machine.wait_for_unit("sample-service")
+      # Create a test table
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
+
+      machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
+      timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
+
+      machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
+      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+
+      machine.succeed("rm -rf /var/backup/postgres")
+
+      machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
+      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+
+      machine.succeed("""
+      set -x
+      ${nodes.machine.clan.core.state.test.postRestoreCommand}
+      """)
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
+
+      timestamp_after = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
+      assert timestamp_before < timestamp_after, f"{timestamp_before} >= {timestamp_after}: expected sample-service to be restarted after restore"
+
+      # Check that the table is still there
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
+      output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
+      owner = output.split("\n")[1]
+      assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
+
+      # check if restore works if the database does not exist
+      machine.succeed("runuser -u postgres -- dropdb test")
+      machine.succeed("${nodes.machine.clan.core.state.test.postRestoreCommand}")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
+    '';
+})

clanModules/borgbackup-static/README.md

@@ -4,7 +4,7 @@ description = "Statically configure borgbackup with sane defaults."
 !!! Danger "Deprecated"
     Use [borgbackup](borgbackup.md) instead.
 
-    Don't use borgbackup-static through [inventory](../../concepts/inventory.md).
+    Don't use borgbackup-static through [inventory](../../guides/inventory.md).
 
 This module implements the `borgbackup` backend and implements sane defaults
 for backup management through `borgbackup` for members of the clan.

clanModules/matrix-synapse/default.nix

@@ -61,6 +61,7 @@ in
     };
   };
   imports = [
+    ../postgresql
     (lib.mkRemovedOptionModule [
       "clan"
       "matrix-synapse"
@@ -105,16 +106,15 @@ in
       };
     };
 
-    clan.core.postgresql.enable = true;
-    clan.core.postgresql.users.matrix-synapse = { };
-    clan.core.postgresql.databases.matrix-synapse.create.options = {
+    clan.postgresql.users.matrix-synapse = { };
+    clan.postgresql.databases.matrix-synapse.create.options = {
       TEMPLATE = "template0";
       LC_COLLATE = "C";
       LC_CTYPE = "C";
       ENCODING = "UTF8";
       OWNER = "matrix-synapse";
     };
-    clan.core.postgresql.databases.matrix-synapse.restore.stopOnRestore = [ "matrix-synapse" ];
+    clan.postgresql.databases.matrix-synapse.restore.stopOnRestore = [ "matrix-synapse" ];
 
     clan.core.vars.generators =
       {

clanModules/postgresql/default.nix

@@ -1,9 +1,224 @@
-{ lib, ... }:
 {
-  imports = [
-    (lib.mkRemovedOptionModule [
-      "clan"
-      "postgresql"
-    ] "The postgresql module has been migrated to a clan core option. Use clan.core.postgresql instead")
-  ];
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  createDatabaseState =
+    db:
+    let
+      folder = "/var/backup/postgres/${db.name}";
+      current = "${folder}/pg-dump";
+      compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
+    in
+    {
+      folders = [ folder ];
+      preBackupScript = ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+
+        mkdir -p "${folder}"
+        runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
+        mv "${current}.tmp" ${current}
+      '';
+      postRestoreScript = ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+            pkgs.gnugrep
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          (
+            systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
+            trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
+
+            mkdir -p "${folder}"
+            if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
+              runuser -u postgres -- dropdb "${db.name}"
+            fi
+            runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          )
+        else
+          echo No database backup found, skipping restore
+        fi
+      '';
+    };
+
+  createDatabase = db: ''
+    CREATE DATABASE "${db.name}" ${
+      lib.concatStringsSep " " (
+        lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
+      )
+    }
+  '';
+  cfg = config.clan.postgresql;
+
+  userClauses = lib.mapAttrsToList (
+    _: user:
+    ''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
+  ) cfg.users;
+  databaseClauses = lib.mapAttrsToList (
+    name: db:
+    lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
+  ) cfg.databases;
+in
+{
+  options.clan.postgresql = {
+    # we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
+    databases = lib.mkOption {
+      description = "Databases to create";
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options = {
+              name = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "Database name.";
+              };
+              service = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "Service name that we associate with the database.";
+              };
+              # set to false, in case the upstream module uses ensureDatabase option
+              create.enable = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = "Create the database if it does not exist.";
+              };
+              create.options = lib.mkOption {
+                description = "Options to pass to the CREATE DATABASE command.";
+                type = lib.types.lazyAttrsOf lib.types.str;
+                default = { };
+                example = {
+                  TEMPLATE = "template0";
+                  LC_COLLATE = "C";
+                  LC_CTYPE = "C";
+                  ENCODING = "UTF8";
+                  OWNER = "foo";
+                };
+              };
+              restore.stopOnRestore = lib.mkOption {
+                type = lib.types.listOf lib.types.str;
+                default = [ ];
+                description = "List of systemd services to stop before restoring the database.";
+              };
+            };
+          }
+        )
+      );
+    };
+    users = lib.mkOption {
+      description = "Users to create";
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options.name = lib.mkOption {
+              description = "User name";
+              type = lib.types.str;
+              default = name;
+            };
+          }
+        )
+      );
+    };
+  };
+  config = {
+    services.postgresql.settings = {
+      wal_level = "replica";
+      max_wal_senders = 3;
+    };
+
+    services.postgresql.enable = true;
+    # We are duplicating a bit the upstream module but allow to create databases with options
+    systemd.services.postgresql.postStart = ''
+      PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
+
+      while ! $PSQL -d postgres -c "" 2> /dev/null; do
+        if ! kill -0 "$MAINPID"; then exit 1; fi
+        sleep 0.1
+      done
+      ${lib.concatStringsSep "\n" userClauses}
+      ${lib.concatStringsSep "\n" databaseClauses}
+    '';
+
+    clan.core.state = lib.mapAttrs' (
+      _: db: lib.nameValuePair db.service (createDatabaseState db)
+    ) config.clan.postgresql.databases;
+
+    environment.systemPackages = builtins.map (
+      db:
+      let
+        folder = "/var/backup/postgres/${db.name}";
+        current = "${folder}/pg-dump";
+      in
+      pkgs.writeShellScriptBin "postgres-db-restore-command-${db.name}" ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+            pkgs.gnugrep
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          (
+            ${lib.optionalString (db.restore.stopOnRestore != [ ]) ''
+              systemctl stop ${builtins.toString db.restore.stopOnRestore}
+              trap "systemctl start ${builtins.toString db.restore.stopOnRestore}" EXIT
+            ''}
+
+            mkdir -p "${folder}"
+            if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
+              runuser -u postgres -- dropdb "${db.name}"
+            fi
+            runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          )
+        else
+          echo No database backup found, skipping restore
+        fi
+      ''
+    ) (builtins.attrValues config.clan.postgresql.databases);
+  };
 }

clanModules/root-password/README.md

@@ -12,7 +12,7 @@ After the system was installed/deployed the following command can be used to dis
 clan vars get [machine_name] root-password/root-password
 ```
 
-See also: [Vars](../../concepts/generators.md)
+See also: [Vars](../../guides/vars-backend.md)
 
 To regenerate the password run:
 ```

clanModules/user-password/README.md

@@ -16,7 +16,7 @@ After the system was installed/deployed the following command can be used to dis
 clan vars get [machine_name] root-password/root-password
 ```
 
-See also: [Vars](../../concepts/generators.md)
+See also: [Vars](../../guides/vars-backend.md)
 
 To regenerate the password run:
 ```

clanModules/vaultwarden/default.nix

@@ -10,6 +10,7 @@ in
 
 {
   imports = [
+    ../postgresql
     (lib.mkRemovedOptionModule [
       "clan"
       "vaultwarden"
@@ -56,17 +57,15 @@ in
 
   config = {
 
-    clan.core.postgresql.enable = true;
-
-    clan.core.postgresql.users.vaultwarden = { };
-    clan.core.postgresql.databases.vaultwarden.create.options = {
+    clan.postgresql.users.vaultwarden = { };
+    clan.postgresql.databases.vaultwarden.create.options = {
       TEMPLATE = "template0";
       LC_COLLATE = "C";
       LC_CTYPE = "C";
       ENCODING = "UTF8";
       OWNER = "vaultwarden";
     };
-    clan.core.postgresql.databases.vaultwarden.restore.stopOnRestore = [ "vaultwarden" ];
+    clan.postgresql.databases.vaultwarden.restore.stopOnRestore = [ "vaultwarden" ];
 
     services.nginx = {
       enable = true;

clanServices/borgbackup/README.md

@@ -1,59 +1,9 @@
-## Usage
+BorgBackup (short: Borg) gives you:
 
-```nix
-inventory.instances = {
-  borgbackup = {
-    module = {
-      name = "borgbackup";
-      input = "clan";
-    };
-    roles.client.machines."jon".settings = {
-      destinations."storagebox" = {
-        repo = "username@$hostname:/./borgbackup";
-        rsh = ''ssh -oPort=23 -i /run/secrets/vars/borgbackup/borgbackup.ssh'';
-      };
-    };
-    roles.server.machines = { };
-  };
-};
-```
-
-The input should be named according to your flake input. Jon is configured as a
-client machine with a destination pointing to a Hetzner Storage Box.
-
-## Overview
-
-This guide explains how to set up and manage
-[BorgBackup](https://borgbackup.readthedocs.io/) for secure, efficient backups
-in a clan network. BorgBackup provides:
-
-- Space efficient storage of backups with deduplication
-- Secure, authenticated encryption
-- Compression: lz4, zstd, zlib, lzma or none
-- Mountable backups with FUSE
+- Space efficient storage of backups.
+- Secure, authenticated encryption.
+- Compression: lz4, zstd, zlib, lzma or none.
+- Mountable backups with FUSE.
 - Easy installation on multiple platforms: Linux, macOS, BSD, …
 - Free software (BSD license).
 - Backed by a large and active open-source community.
-
-## Roles
-
-### 1. Client
-
-Clients are machines that create and send backups to various destinations. Each
-client can have multiple backup destinations configured.
-
-### 2. Server
-
-Servers act as backup repositories, receiving and storing backups from client
-machines. They can be dedicated backup servers within your clan network.
-
-## Backup destinations
-
-This service allows you to perform backups to multiple `destinations`.
-Destinations can be:
-
-- **Local**: Local disk storage
-- **Server**: Your own borgbackup server (using the `server` role)
-- **Third-party services**: Such as Hetzner's Storage Box
-
-For a more comprehensive guide on backups look into the guide section.

clanServices/zerotier/default.nix

@@ -134,9 +134,9 @@
               systemd.services.zerotier-inventory-autoaccept =
                 let
                   machines = uniqueStrings (
-                    (lib.optionals (roles ? moon) (lib.attrNames roles.moon.machines))
-                    ++ (lib.optionals (roles ? controller) (lib.attrNames roles.controller.machines))
-                    ++ (lib.optionals (roles ? peer) (lib.attrNames roles.peer.machines))
+                    (lib.attrNames roles.moon.machines)
+                    ++ (lib.attrNames roles.controller.machines)
+                    ++ (lib.attrNames roles.peer.machines)
                   );
                   networkIps = builtins.foldl' (
                     ips: name:

clanServices/zerotier/tests/eval-tests.nix

@@ -32,33 +32,6 @@ let
         };
       };
     }).config;
-  testFlakeNoMoon =
-    (clanLib.clan {
-      self = { };
-      directory = ./vm;
-
-      machines.jon = {
-        nixpkgs.hostPlatform = "x86_64-linux";
-      };
-      machines.sara = {
-        nixpkgs.hostPlatform = "x86_64-linux";
-      };
-      machines.bam = {
-        nixpkgs.hostPlatform = "x86_64-linux";
-      };
-
-      modules.zerotier = module;
-
-      inventory.instances = {
-        zerotier = {
-          module.name = "zerotier";
-          module.input = "self";
-
-          roles.peer.tags.all = { };
-          roles.controller.machines.bam = { };
-        };
-      };
-    }).config;
 in
 {
   test_peers = {
@@ -100,30 +73,4 @@ in
       networkName = "zerotier";
     };
   };
-  test_peers_no_moon = {
-    expr = {
-      hasNetworkIds = testFlakeNoMoon.nixosConfigurations.jon.config.services.zerotierone.joinNetworks;
-      isController =
-        testFlakeNoMoon.nixosConfigurations.jon.config.clan.core.networking.zerotier.controller.enable;
-      networkName = testFlakeNoMoon.nixosConfigurations.jon.config.clan.core.networking.zerotier.name;
-    };
-    expected = {
-      hasNetworkIds = [ "0e28cb903344475e" ];
-      isController = false;
-      networkName = "zerotier";
-    };
-  };
-  test_controller_no_moon = {
-    expr = {
-      hasNetworkIds = testFlakeNoMoon.nixosConfigurations.bam.config.services.zerotierone.joinNetworks;
-      isController =
-        testFlakeNoMoon.nixosConfigurations.bam.config.clan.core.networking.zerotier.controller.enable;
-      networkName = testFlakeNoMoon.nixosConfigurations.bam.config.clan.core.networking.zerotier.name;
-    };
-    expected = {
-      hasNetworkIds = [ "0e28cb903344475e" ];
-      isController = true;
-      networkName = "zerotier";
-    };
-  };
 }

docs/mkdocs.yml

@@ -48,81 +48,61 @@ nav:
   - Home: index.md
   - Guides:
       - Getting Started:
-          - Creating Your First Clan: guides/getting-started/index.md
-          - Create USB Installer: guides/getting-started/installer.md
-          - Add Machines: guides/getting-started/add-machines.md
-          - Add User: guides/getting-started/add-user.md
-          - Add Services: guides/getting-started/add-services.md
-          - Deploy Machine: guides/getting-started/deploy.md
-          - Continuous Integration: guides/getting-started/check.md
-      - Using Services: guides/clanServices.md
-      - Backup & Restore: guides/backups.md
+          - 🚀 Creating Your First Clan: guides/getting-started/index.md
+          - 📀 Create USB Installer (optional): guides/getting-started/installer.md
+          - ⚙️ Add Machines: guides/getting-started/add-machines.md
+          - ⚙️ Add User: guides/getting-started/add-user.md
+          - ⚙️ Add Services: guides/getting-started/add-services.md
+          - 🔐 Secrets & Facts: guides/getting-started/secrets.md
+          - 🚢 Deploy Machine: guides/getting-started/deploy.md
+          - 🧪 Continuous Integration: guides/getting-started/check.md
+      - clanServices: guides/clanServices.md
       - Disk Encryption: guides/disk-encryption.md
-      - Age Plugins: guides/age-plugins.md
-      - Secrets management: guides/secrets.md
+      - Mesh VPN: guides/mesh-vpn.md
+      - Backup & Restore: guides/backups.md
+      - Vars Backend: guides/vars-backend.md
+      - Facts Backend: guides/secrets.md
+      - Adding more machines: guides/more-machines.md
       - Target Host: guides/target-host.md
-      - Zerotier VPN: guides/mesh-vpn.md
+      - Inventory:
+          - Inventory: guides/inventory.md
       - Secure Boot: guides/secure-boot.md
       - Flake-parts: guides/flake-parts.md
-      - macOS: guides/macos.md
+      - Authoring:
+          - clanService: guides/authoring/clanServices/index.md
+          - Disk Template: guides/authoring/templates/disk/disko-templates.md
+          - clanModule: guides/authoring/clanModules/index.md
       - Contributing:
-          - Contributing: guides/contributing/CONTRIBUTING.md
+          - Contribute: guides/contributing/CONTRIBUTING.md
           - Debugging: guides/contributing/debugging.md
           - Testing: guides/contributing/testing.md
-
-      - Writing a Service Module: guides/services/community.md
-      - Writing a Disko Template: guides/disko-templates/community.md
       - Migrations:
           - Migrate existing Flakes: guides/migrations/migration-guide.md
           - Migrate inventory Services: guides/migrations/migrate-inventory-services.md
           - Facts Vars Migration: guides/migrations/migration-facts-vars.md
           - Disk id: guides/migrations/disk-id.md
-  - Concepts:
-      - Inventory: concepts/inventory.md
-      - Generators: concepts/generators.md
-      - Autoincludes: concepts/autoincludes.md
-      - Templates: concepts/templates.md
+      - macOS: guides/macos.md
   - Reference:
       - Overview: reference/index.md
-      - Clan Options: options.md
       - Services:
-          - Overview:
-              - reference/clanServices/index.md
-
-              - reference/clanServices/admin.md
-              - reference/clanServices/borgbackup.md
-              - reference/clanServices/data-mesher.md
-              - reference/clanServices/emergency-access.md
-              - reference/clanServices/garage.md
-              - reference/clanServices/hello-world.md
-              - reference/clanServices/importer.md
-              - reference/clanServices/mycelium.md
-              - reference/clanServices/packages.md
-              - reference/clanServices/sshd.md
-              - reference/clanServices/state-version.md
-              - reference/clanServices/trusted-nix-caches.md
-              - reference/clanServices/users.md
-              - reference/clanServices/wifi.md
-              - reference/clanServices/zerotier.md
-          - API: reference/clanServices/clan-service-author-interface.md
-
-      - CLI:
-          - Overview: reference/cli/index.md
-
-          - reference/cli/backups.md
-          - reference/cli/facts.md
-          - reference/cli/flakes.md
-          - reference/cli/flash.md
-          - reference/cli/machines.md
-          - reference/cli/select.md
-          - reference/cli/secrets.md
-          - reference/cli/show.md
-          - reference/cli/ssh.md
-          - reference/cli/state.md
-          - reference/cli/templates.md
-          - reference/cli/vars.md
-          - reference/cli/vms.md
-      - Modules (deprecated):
+          - Overview: reference/clanServices/index.md
+          - reference/clanServices/admin.md
+          - reference/clanServices/borgbackup.md
+          - reference/clanServices/data-mesher.md
+          - reference/clanServices/emergency-access.md
+          - reference/clanServices/garage.md
+          - reference/clanServices/hello-world.md
+          - reference/clanServices/importer.md
+          - reference/clanServices/mycelium.md
+          - reference/clanServices/packages.md
+          - reference/clanServices/sshd.md
+          - reference/clanServices/state-version.md
+          - reference/clanServices/trusted-nix-caches.md
+          - reference/clanServices/users.md
+          - reference/clanServices/wifi.md
+          - reference/clanServices/zerotier.md
+      - Interface for making Services: reference/clanServices/clan-service-author-interface.md
+      - Modules:
           - Overview: reference/clanModules/index.md
           - reference/clanModules/frontmatter/index.md
           # TODO: display the docs of the clan.service modules
@@ -165,21 +145,38 @@ nav:
           - reference/clanModules/zerotier-static-peers.md
           - reference/clanModules/zerotier.md
           - reference/clanModules/zt-tcp-relay.md
+      - CLI:
+          - Overview: reference/cli/index.md
 
-      - clan.core (NixOS Options):
-          - Overview: reference/clan.core/index.md
-          - reference/clan.core/backups.md
-          - reference/clan.core/deployment.md
-          - reference/clan.core/facts.md
-          - reference/clan.core/networking.md
-          - reference/clan.core/postgresql.md
-          - reference/clan.core/settings.md
-          - reference/clan.core/sops.md
-          - reference/clan.core/state.md
-          - reference/clan.core/vars.md
-
-      - Developer-api: api.md
+          - reference/cli/backups.md
+          - reference/cli/facts.md
+          - reference/cli/flakes.md
+          - reference/cli/flash.md
+          - reference/cli/machines.md
+          - reference/cli/select.md
+          - reference/cli/secrets.md
+          - reference/cli/show.md
+          - reference/cli/ssh.md
+          - reference/cli/state.md
+          - reference/cli/templates.md
+          - reference/cli/vars.md
+          - reference/cli/vms.md
+      - NixOS Modules:
+          - clan.core:
+              - Overview: reference/clan.core/index.md
 
+              - reference/clan.core/backups.md
+              - reference/clan.core/deployment.md
+              - reference/clan.core/facts.md
+              - reference/clan.core/networking.md
+              - reference/clan.core/settings.md
+              - reference/clan.core/sops.md
+              - reference/clan.core/state.md
+              - reference/clan.core/vars.md
+      - Nix API:
+          - clan: reference/nix-api/clan.md
+          - Inventory: reference/nix-api/inventory.md
+      - Glossary: reference/glossary.md
       - Decisions:
           - Architecture Decisions: decisions/README.md
           - 01-clanModules: decisions/01-ClanModules.md
@@ -188,7 +185,10 @@ nav:
           - 04-fetching-nix-from-python: decisions/04-fetching-nix-from-python.md
           - 05-deployment-parameters: decisions/05-deployment-parameters.md
           - Template: decisions/_template.md
-      - Glossary: reference/glossary.md
+  - Options: options.md
+  - Developer:
+      - Introduction: intern/index.md
+      - API: intern/api.md
 
 docs_dir: site
 site_dir: out
@@ -199,7 +199,6 @@ theme:
   favicon: https://clan.lol/favicon.svg
   name: material
   features:
-    - navigation.footer
     - navigation.instant
     - navigation.tabs
     - navigation.tabs.sticky
@@ -247,6 +246,3 @@ plugins:
   - search
   - macros
   - redoc-tag
-  - redirects:
-      redirect_maps:
-        guides/getting-started/secrets.md: concepts/generators.md

docs/nix/default.nix

@@ -40,7 +40,6 @@ pkgs.stdenv.mkDerivation {
       mkdocs-material
       mkdocs-macros
       mkdocs-redoc-tag
-      mkdocs-redirects
     ]);
   configurePhase = ''
     pushd docs
@@ -55,7 +54,6 @@ pkgs.stdenv.mkDerivation {
     chmod -R +w ./site/reference
     echo "Generated API documentation in './site/reference/' "
 
-    rm -r ./site/options-page || true
     cp -r ${docs-options} ./site/options-page
     chmod -R +w ./site/options-page
 

docs/nix/flake-module.nix

@@ -11,7 +11,7 @@
       ...
     }:
     let
-      buildClanOptions = self'.legacyPackages.clan-internals-docs;
+      clanOptions = self'.legacyPackages.clan-internals-docs;
       # Simply evaluated options (JSON)
       # { clanCore = «derivation JSON»; clanModules = { ${name} = «derivation JSON» }; }
       jsonDocs = pkgs.callPackage ./get-module-docs.nix {
@@ -99,7 +99,7 @@
             # Frontmatter format for clanModules
             export CLAN_MODULES_FRONTMATTER_DOCS=${clanModulesFrontmatter}/share/doc/nixos/options.json
 
-            export BUILD_CLAN_PATH=${buildClanOptions}/share/doc/nixos/options.json
+            export BUILD_CLAN_PATH=${clanOptions}/share/doc/nixos/options.json
 
             mkdir $out
 

docs/nix/options/flake-module.nix

@@ -114,6 +114,9 @@
         in
         {
           options = {
+            _ = mkOption {
+              type = types.raw;
+            };
             instances.${name} = lib.mkOption {
               inherit description;
               type = types.submodule {
@@ -146,65 +149,20 @@
           };
         };
 
-      docModules = [
-        {
-          inherit self;
-        }
-        self.modules.clan.default
-        {
-          options.inventory = lib.mkOption {
-            type = types.submoduleWith {
-              modules = [
-                { noInstanceOptions = true; }
-              ] ++ mapAttrsToList fakeInstanceOptions serviceModules;
-            };
-          };
-        }
-      ];
-
-      baseModule =
-        # Module
-        { config, ... }:
-        {
-          imports = (import (pkgs.path + "/nixos/modules/module-list.nix"));
-          nixpkgs.pkgs = pkgs;
-          clan.core.name = "dummy";
-          system.stateVersion = config.system.nixos.release;
-          # Set this to work around a bug where `clan.core.settings.machine.name`
-          # is forced due to `networking.interfaces` being forced
-          # somewhere in the nixpkgs options
-          facter.detected.dhcp.enable = lib.mkForce false;
-        };
-
-      evalClanModules =
-        let
-          evaled = lib.evalModules {
-            class = "nixos";
-            modules = [
-              baseModule
-              {
-                clan.core.settings.directory = self;
-              }
-              self.nixosModules.clanCore
-            ];
-          };
-        in
-        evaled;
-
-      coreOptions =
-        (pkgs.nixosOptionsDoc {
-          options = (evalClanModules.options).clan.core or { };
-          warningsAreErrors = true;
-          transformOptions = self.clanLib.docs.stripStorePathsFromDeclarations;
-        }).optionsJSON;
-
+      mkScope = name: modules: {
+        inherit name;
+        modules = [
+          {
+            _module.args = { inherit clanLib; };
+            _file = "docs mkScope";
+          }
+          { noInstanceOptions = true; }
+          ../../../lib/modules/inventoryClass/interface.nix
+        ] ++ mapAttrsToList fakeInstanceOptions modules;
+        urlPrefix = "https://github.com/nix-community/dream2nix/blob/main/";
+      };
     in
     {
-      # Uncomment for debugging
-      # legacyPackages.docModules = lib.evalModules {
-      #   modules = docModules;
-      # };
-
       packages = lib.optionalAttrs ((privateInputs ? nuschtos) || (inputs ? nuschtos)) {
         docs-options =
           (privateInputs.nuschtos or inputs.nuschtos)
@@ -213,19 +171,7 @@
               inherit baseHref;
               title = "Clan Options";
               # scopes = mapAttrsToList mkScope serviceModules;
-              scopes = [
-                {
-                  name = "Clan (Flake Options)";
-                  modules = docModules;
-                  urlPrefix = "https://git.clan.lol/clan/clan-core/src/branch/main/";
-                }
-                {
-                  name = "Clan (Machine Options)";
-                  optionsJSON = "${coreOptions}/share/doc/nixos/options.json";
-                  urlPrefix = "https://git.clan.lol/clan/clan-core/src/branch/main/";
-
-                }
-              ];
+              scopes = [ (mkScope "Clan Inventory" serviceModules) ];
             };
       };
     };

docs/nix/render_options/__init__.py

@@ -193,7 +193,7 @@ def module_header(module_name: str, has_inventory_feature: bool = False) -> str:
 def module_nix_usage(module_name: str) -> str:
     return f"""## Usage via Nix
 
-**This module can be also imported directly in your nixos configuration. Although it is recommended to use the [inventory](../../concepts/inventory.md) interface if available.**
+**This module can be also imported directly in your nixos configuration. Although it is recommended to use the [inventory](../../reference/nix-api/inventory.md) interface if available.**
 
 Some modules are considered 'low-level' or 'expert modules' and are not available via the inventory interface.
 
@@ -373,7 +373,7 @@ This module can be used via predefined roles
             """
 Every role has its own configuration options, which are each listed below.
 
-For more information, see the [inventory guide](../../concepts/inventory.md).
+For more information, see the [inventory guide](../../guides/inventory.md).
 
 ??? Example
     For example the `admin` module adds the following options globally to all machines where it is used.
@@ -402,7 +402,7 @@ certain option types restricted to enable configuration through a graphical
 interface.
 
 !!! note "🔹"
-    Modules with this indicator support the [inventory](../../concepts/inventory.md) feature.
+    Modules with this indicator support the [inventory](../../guides/inventory.md) feature.
 
 """
 
@@ -679,6 +679,86 @@ def build_option_card(module_name: str, frontmatter: Frontmatter) -> str:
     return f"{to_md_li(module_name, frontmatter)}\n\n"
 
 
+def produce_build_clan_docs() -> None:
+    if not BUILD_CLAN_PATH:
+        msg = f"Environment variables are not set correctly: BUILD_CLAN_PATH={BUILD_CLAN_PATH}. Expected a path to the optionsJSON"
+        raise ClanError(msg)
+
+    if not OUT:
+        msg = f"Environment variables are not set correctly: $out={OUT}"
+        raise ClanError(msg)
+
+    output = """# Clan
+
+This provides an overview of the available arguments of the `clan` interface.
+
+Each attribute is documented below
+
+- **clan-core.lib.clan**: A function that takes an attribute set.
+
+    ??? example "clan Example"
+
+        ```nix
+        clan {
+            self = self;
+            machines = {
+                jon = { };
+                sara = { };
+            };
+        };
+        ```
+
+- **clan with flake-parts**: Import the FlakeModule
+
+    After importing the FlakeModule you can define your `clan` as a flake attribute
+
+    All attribute can be defined via `clan.*`
+
+    Further information see: [flake-parts](../../guides/flake-parts.md) guide.
+
+    ??? example "flake-parts Example"
+
+        ```nix
+        flake-parts.lib.mkFlake { inherit inputs; } ({
+            systems = [];
+            imports = [
+                clan-core.flakeModules.default
+            ];
+            clan = {
+                machines = {
+                    jon = { };
+                    sara = { };
+                };
+            };
+        });
+        ```
+
+"""
+    with Path(BUILD_CLAN_PATH).open() as f:
+        options: dict[str, dict[str, Any]] = json.load(f)
+
+        split = split_options_by_root(options)
+        for option_name, options in split.items():
+            # Skip underscore options
+            if option_name.startswith("_"):
+                continue
+            # Skip inventory sub options
+            # Inventory model has its own chapter
+            if option_name.startswith("inventory."):
+                continue
+
+            print(f"[build_clan_docs] Rendering option of {option_name}...")
+            root = options_to_tree(options)
+
+            for option in root.suboptions:
+                output += options_docs_from_tree(option, init_level=2)
+
+    outfile = Path(OUT) / "nix-api/clan.md"
+    outfile.parent.mkdir(parents=True, exist_ok=True)
+    with Path.open(outfile, "w") as of:
+        of.write(output)
+
+
 def split_options_by_root(options: dict[str, Any]) -> dict[str, dict[str, Any]]:
     """
     Split the flat dictionary of options into a dict of which each entry will construct complete option trees.
@@ -725,7 +805,7 @@ Typically needed by module authors to define roles, behavior and metadata for di
 !!! Note
     This is not a user-facing documentation, but rather meant as a reference for *module authors*
 
-    See: [clanService Authoring Guide](../../guides/services/community.md)
+    See: [clanService Authoring Guide](../../guides/authoring/clanServices/index.md)
 """
     # Inventory options are already included under the clan attribute
     # We just omitted them in the clan docs, because we want a separate output for the inventory model
@@ -754,6 +834,48 @@ class Option:
     suboptions: list["Option"] = field(default_factory=list)
 
 
+def produce_inventory_docs() -> None:
+    if not BUILD_CLAN_PATH:
+        msg = f"Environment variables are not set correctly: BUILD_CLAN_PATH={BUILD_CLAN_PATH}. Expected a path to the optionsJSON"
+        raise ClanError(msg)
+
+    if not OUT:
+        msg = f"Environment variables are not set correctly: $out={OUT}"
+        raise ClanError(msg)
+
+    output = """# Inventory
+This provides an overview of the available attributes of the `inventory` model.
+
+It can be set via the `inventory` attribute of the [`clan`](./clan.md#inventory) function, or via the [`clan.inventory`](./clan.md#inventory) attribute of flake-parts.
+
+"""
+    # Inventory options are already included under the clan attribute
+    # We just omitted them in the clan docs, because we want a separate output for the inventory model
+    with Path(BUILD_CLAN_PATH).open() as f:
+        options: dict[str, dict[str, Any]] = json.load(f)
+
+        clan_root_option = options_to_tree(options)
+        # Find the inventory options
+        inventory_opt: None | Option = None
+        for opt in clan_root_option.suboptions:
+            if opt.name == "inventory":
+                inventory_opt = opt
+                break
+
+        if not inventory_opt:
+            print("No inventory options found.")
+            exit(1)
+        # Render the inventory options
+        # This for loop excludes the root node
+        for option in inventory_opt.suboptions:
+            output += options_docs_from_tree(option, init_level=2)
+
+    outfile = Path(OUT) / "nix-api/inventory.md"
+    outfile.parent.mkdir(parents=True, exist_ok=True)
+    with Path.open(outfile, "w") as of:
+        of.write(output)
+
+
 def option_short_name(option_name: str) -> str:
     parts = option_name.split(".")
     short_name = ""
@@ -862,6 +984,9 @@ def options_docs_from_tree(
 if __name__ == "__main__":  #
     produce_clan_core_docs()
 
+    produce_build_clan_docs()
+    produce_inventory_docs()
+
     produce_clan_service_author_docs()
 
     produce_clan_modules_docs()

docs/site/concepts/autoincludes.md

@@ -1,15 +0,0 @@
-
-Clan automatically imports the following files from a directory and registers them.
-
-## Machine registration
-
-Every folder `machines/{machineName}` will be registered automatically as a Clan machine.
-
-!!! info "Automatically loaded files"
-
-    The following files are loaded automatically for each Clan machine:
-
-    - [x] `machines/{machineName}/configuration.nix`
-    - [x] `machines/{machineName}/hardware-configuration.nix`
-    - [x] `machines/{machineName}/facter.json` Automatically configured, for further information see [nixos-facter](https://clan.lol/blog/nixos-facter/)
-    - [x] `machines/{machineName}/disko.nix` Automatically loaded, for further information see the [disko docs](https://github.com/nix-community/disko/blob/master/docs/quickstart.md).

docs/site/concepts/templates.md

@@ -1,69 +0,0 @@
-# How Templates work
-
-Clan offers the ability to use templates for creating different resources.
-It comes with some `<builtin>` templates and discovers all exposed templates from its flake's `inputs`
-
-For example one can list all current templates like this:
-
-```shellSession
-$ clan templates list
-Available 'clan' templates
-├── <builtin>
-│   ├── default: Initialize a new clan flake
-│   ├── flake-parts: Flake-parts
-│   └── minimal: for clans managed via (G)UI
-└── inputs.self:
-    ├── default: Initialize a new clan flake
-    ├── flake-parts: Flake-parts
-    └── minimal: for clans managed via (G)UI
-Available 'disko' templates
-├── <builtin>
-│   └── single-disk: A simple ext4 disk with a single partition
-└── inputs.self:
-    └── single-disk: A simple ext4 disk with a single partition
-Available 'machine' templates
-├── <builtin>
-│   ├── demo-template: Demo machine for the CLAN project
-│   ├── flash-installer: Initialize a new flash-installer machine
-│   ├── new-machine: Initialize a new machine
-│   └── test-morph-template: Morph a machine
-└── inputs.self:
-    ├── demo-template: Demo machine for the CLAN project
-    ├── flash-installer: Initialize a new flash-installer machine
-    ├── new-machine: Initialize a new machine
-    └── test-morph-template: Morph a machine
-```
-
-## Using `<builtin>` Templates
-
-Templates are referenced via the `--template` `selector`
-
-clan-core ships its native/builtin templates. Those are referenced if the selector is a plain string ( without `#` or `./.` )
-
-For example:
-
-`clan flakes create --template=flake-parts`
-
-would use the native `<builtin>.flake-parts` template
-
-## Selectors follow nix flake `reference#attribute` syntax
-
-Selectors follow a very similar pattern as Nix's native attribute selection behavior.
-
-Just like `nix build .` would build `packages.x86-linux.default` of the flake in `./.`
-
-`clan flakes create --template=.` would create a clan from your **local** `default` clan template (`templates.clan.default`).
-
-In fact this command would be equivalent, just make it more explicit
-
-`clan flakes create --template=.#clan.templates.clan.default` (explicit path)
-
-## Remote templates
-
-Just like with Nix you could specify a remote url or path to the flake containing the template
-
-`clan flakes create --template=github:owner/repo#foo`
-
-!!! Note "Implementation Note"
-    Not all features of Nix's attribute selection are currently matched.
-    There are minor differences in case of unexpected behavior please create an [issue](https://git.clan.lol/clan/clan-core/issues/new)

docs/site/decisions/01-ClanModules.md

@@ -6,8 +6,6 @@ Accepted
 
 ## Context
 
-Current state as of writing:
-
 To define a service in Clan, you need to define two things:
 
 - `clanModule` - defined by module authors

docs/site/guides/age-plugins.md

@@ -1,59 +0,0 @@
-## Using Age Plugins
-
-If you wish to use a key generated using an [age plugin] as your admin key, extra care is needed.
-
-You must **precede your secret key with a comment that contains its corresponding recipient**.
-
-This is usually output as part of the generation process
-and is only required because there is no unified mechanism for recovering a recipient from a plugin secret key.
-
-Here is an example:
-
-```title="~/.config/sops/age/keys.txt"
-# public key: age1zdy49ek6z60q9r34vf5mmzkx6u43pr9haqdh5lqdg7fh5tpwlfwqea356l
-AGE-PLUGIN-FIDO2-HMAC-1QQPQZRFR7ZZ2WCV...
-```
-
-!!! note
-    The comment that precedes the plugin secret key need only contain the recipient.
-    Any other text is ignored.
-
-    In the example above, you can specify `# recipient: age1zdy...`, `# public: age1zdy....` or even
-    just `# age1zdy....`
-
-You will need to add an entry into your `flake.nix` to ensure that the necessary `age` plugins
-are loaded when using Clan:
-
-```nix title="flake.nix"
-{
-  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-  inputs.nixpkgs.follows = "clan-core/nixpkgs";
-
-  outputs =
-    { self, clan-core, ... }:
-    let
-      # Sometimes this attribute set is defined in clan.nix
-      clan = clan-core.lib.clan {
-        inherit self;
-
-        meta.name = "myclan";
-
-        # Add Yubikey and FIDO2 HMAC plugins
-        # Note: the plugins listed here must be available in nixpkgs.
-        secrets.age.plugins = [
-          "age-plugin-yubikey"
-          "age-plugin-fido2-hmac"
-        ];
-
-        machines = {
-          # elided for brevity
-        };
-      };
-    in
-    {
-      inherit (clan) nixosConfigurations nixosModules clanInternals;
-
-      # elided for brevity
-    };
-}
-```

docs/site/guides/authoring/clanModules/index.md

@@ -0,0 +1,229 @@
+# Authoring a clanModule
+
+!!! Danger "Will get deprecated soon"
+    Please consider twice creating new modules in this format
+
+    [`clan.service` module](../clanServices/index.md) will be the new standard soon.
+
+This site will guide you through authoring your first module. Explaining which conventions must be followed, such that others will have an enjoyable experience and the module can be used with minimal effort.
+
+
+!!! Tip
+    External ClanModules can be ad-hoc loaded via [`clan.inventory.modules`](../../../reference/nix-api/inventory.md#inventory.modules)
+
+## Bootstrapping the `clanModule`
+
+A ClanModule is a specific subset of a [NixOS Module](https://nix.dev/tutorials/module-system/index.html), but it has some constraints and might be used via the [Inventory](../../../guides/inventory.md) interface.
+In fact a `ClanModule` can be thought of as a layer of abstraction on-top of NixOS and/or other ClanModules. It may configure sane defaults and provide an ergonomic interface that is easy to use and can also be used via a UI that is under development currently.
+
+Because ClanModules should be configurable via `json`/`API` all of its interface (`options`) must be serializable.
+
+!!! Tip
+    ClanModules interface can be checked by running the json schema converter as follows.
+
+    `nix build .#legacyPackages.x86_64-linux.schemas.inventory`
+
+    If the build succeeds the module is compatible.
+
+## Directory structure
+
+Each module SHOULD be a directory of the following format:
+
+```sh
+# Example: borgbackup
+clanModules/borgbackup
+├── README.md
+└── roles
+    ├── client.nix
+    └── server.nix
+```
+
+!!! Tip
+    `README.md` is always required. See section [Readme](#readme) for further details.
+
+    The `roles` folder is strictly required for `features = [ "inventory" ]`.
+
+## Registering the module
+
+=== "User module"
+
+    If the module should be ad-hoc loaded.
+    It can be made available in any project via the [`clan.inventory.modules`](../../../reference/nix-api/inventory.md#inventory.modules) attribute.
+
+    ```nix title="flake.nix"
+    # ...
+    # Sometimes this attribute set is defined in clan.nix
+    clan-core.lib.clan {
+        # 1. Add the module to the available clanModules with inventory support
+        inventory.modules = {
+            custom-module = ./modules/my_module;
+        };
+        # 2. Use the module in the inventory
+        inventory.services = {
+            custom-module.instance_1 = {
+                roles.default.machines = [ "machineA" ];
+            };
+        };
+    };
+    ```
+
+=== "Upstream module"
+
+    If the module will be contributed to [`clan-core`](https://git.clan.lol/clan-core)
+    The clanModule must be registered within the `clanModules` attribute in `clan-core`
+
+    ```nix title="clanModules/flake-module.nix"
+    --8<-- "clanModules/flake-module.nix:0:5"
+        # Register our new module here
+        # ...
+    ```
+
+## Readme
+
+The `README.md` is a required file for all modules. It MUST contain frontmatter in [`toml`](https://toml.io) format.
+
+```markdown
+---
+description = "Module A"
+---
+
+This is the example module that does xyz.
+```
+
+See the [Full Frontmatter reference](../../../reference/clanModules/frontmatter/index.md) further details and all supported attributes.
+
+## Roles
+
+If the module declares to implement `features = [ "inventory" ]` then it MUST contain a roles directory.
+
+Each `.nix` file in the `roles` directory is added as a role to the inventory service.
+
+Other files can also be placed alongside the `.nix` files
+
+```sh
+└── roles
+    ├── client.nix
+    └── server.nix
+```
+
+Adds the roles: `client` and `server`
+
+??? Tip "Good to know"
+    Sometimes a `ClanModule` should be usable via both clan's `inventory` concept but also natively as a NixOS module.
+
+    > In the long term, we want most modules to implement support for the inventory,
+    > but we are also aware that there are certain low-level modules that always serve as a backend for other higher-level `clanModules` with inventory support.
+    > These modules may not want to implement inventory interfaces as they are always used directly by other modules.
+
+    This can be achieved by placing an additional `default.nix` into the root of the ClanModules directory as shown:
+
+    ```sh
+    # ModuleA
+    ├── README.md
+    ├── default.nix
+    └── roles
+        └── default.nix
+    ```
+
+    ```nix title="default.nix"
+    {...}:{
+        imports = [ ./roles/default.nix ];
+    }
+    ```
+
+    By utilizing this pattern the module (`moduleA`) can then be imported into any regular NixOS module via:
+
+    ```nix
+    {...}:{
+        imports  = [ clanModules.moduleA ];
+    }
+    ```
+
+## Adding configuration options
+
+While we recommend to keep the interface as minimal as possible and deriving all required information from the `roles` model it might sometimes be required or convenient to expose customization options beyond `roles`.
+
+The following shows how to add options to your module.
+
+**It is important to understand that every module has its own namespace where it should declare options**
+
+**`clan.{moduleName}`**
+
+???+ Example
+    The following example shows how to register options in the module interface
+
+    and how it can be set via the inventory
+
+
+    ```nix title="/default.nix"
+    custom-module = ./modules/custom-module;
+    ```
+
+    Since the module is called `custom-module` all of its exposed options should be added to `options.clan.custom-module.*...*`
+
+    ```nix title="custom-module/roles/default.nix"
+    {
+        options = {
+            clan.custom-module.foo = mkOption {
+                type = types.str;
+                default = "bar";
+            };
+        };
+    }
+    ```
+
+    If the module is [registered](#registering-the-module).
+    Configuration can be set as follows.
+
+    ```nix title="flake.nix"
+    # Sometimes this attribute set is defined in clan.nix
+    clan-core.lib.clan {
+        inventory.services = {
+            custom-module.instance_1 = {
+                roles.default.machines = [ "machineA" ];
+                roles.default.config = {
+                    # All configuration here is scoped to `clan.custom-module`
+                    foo = "foobar";
+                };
+            };
+        };
+    }
+    ```
+
+## Organizing the ClanModule
+
+Each `{role}.nix` is included into the machine if the machine is declared to have the role.
+
+For example
+
+```nix
+roles.client.machines = ["MachineA"];
+```
+
+Then `roles/client.nix` will be added to the machine `MachineA`.
+
+This behavior makes it possible to split the interface and common code paths when using multiple roles.
+In the concrete example of `borgbackup` this allows a `server` to declare a different interface than the corresponding `client`.
+
+The client offers configuration option, to exclude certain local directories from being backed up:
+
+```nix title="roles/client.nix"
+# Example client interface
+  options.clan.borgbackup.exclude = ...
+```
+
+The server doesn't offer any configuration option. Because everything is set-up automatically.
+
+```nix title="roles/server.nix"
+# Example server interface
+  options.clan.borgbackup = {};
+```
+
+Assuming that there is a common code path or a common interface between `server` and `client` this can be structured as:
+
+```nix title="roles/server.nix, roles/client.nix"
+{...}: {
+    # ...
+    imports = [ ../common.nix ];
+}
+```

docs/site/guides/authoring/clanServices/index.md

@@ -1,16 +1,16 @@
 # Authoring a 'clan.service' module
 
 !!! Tip
-    This is the successor format to the older [clanModules](../../reference/clanModules/index.md)
+    This is the successor format to the older [clanModules](../clanModules/index.md)
 
     While some features might still be missing we recommend to adapt this format early and give feedback.
 
 ## Service Module Specification
 
 This section explains how to author a clan service module.
-We discussed the initial architecture in [01-clan-service-modules](../../decisions/01-ClanModules.md) and decided to rework the format.
+We discussed the initial architecture in [01-clan-service-modules](../../../decisions/01-ClanModules.md) and decided to rework the format.
 
-For the full specification and current state see: **[Service Author Reference](../../reference/clanServices/clan-service-author-interface.md)**
+For the full specification and current state see: **[Service Author Reference](../../../reference/clanServices/clan-service-author-interface.md)**
 
 ### A Minimal module
 
@@ -52,7 +52,7 @@ The imported module file must fulfill at least the following requirements:
 }
 ```
 
-For more attributes see: **[Service Author Reference](../../reference/clanServices/clan-service-author-interface.md)**
+For more attributes see: **[Service Author Reference](../../../reference/clanServices/clan-service-author-interface.md)**
 
 ### Adding functionality to the module
 
@@ -266,6 +266,6 @@ The benefit of this approach is that downstream users can override the value of
 
 ## Further
 
-- [Reference Documentation for Service Authors](../../reference/clanServices/clan-service-author-interface.md)
-- [Migration Guide from ClanModules to ClanServices](../../guides/migrations/migrate-inventory-services.md)
-- [Decision that lead to ClanServices](../../decisions/01-ClanModules.md)
+- [Reference Documentation for Service Authors](../../../reference/clanServices/clan-service-author-interface.md)
+- [Migration Guide from ClanModules to ClanServices](../../migrations/migrate-inventory-services.md)
+- [Decision that lead to ClanServices](../../../decisions/01-ClanModules.md)

docs/site/guides/backups.md

@@ -1,199 +1,167 @@
+# Introduction to Backups
 
-This guide explains how to set up and manage
-[BorgBackup](https://borgbackup.readthedocs.io/) for secure, efficient backups
-in a clan network. BorgBackup provides:
+When you're managing your own services, creating regular backups is crucial to ensure your data's safety.
+This guide introduces you to Clan's built-in backup functionalities.
+Clan supports backing up your data to both local storage devices (like USB drives) and remote servers, using well-known tools like borgbackup and rsnapshot.
+We might add more options in the future, but for now, let's dive into how you can secure your data.
 
-- Space efficient storage of backups with deduplication
-- Secure, authenticated encryption
-- Compression: lz4, zstd, zlib, lzma or none
-- Mountable backups with FUSE
-- Easy installation on multiple platforms: Linux, macOS, BSD, …
-- Free software (BSD license).
-- Backed by a large and active open-source community.
+## Backing Up Locally with Localbackup
 
-## Borgbackup Example
+Localbackup lets you backup your data onto physical storage devices connected to your computer,
+such as USB hard drives or network-attached storage. It uses a tool called rsnapshot for this purpose.
+
+### Setting Up Localbackup
+
+1. **Identify Your Backup Device:**
+
+First, figure out which device you'll use for backups. You can see all connected devices by running this command in your terminal:
+
+```bash
+lsblk --output NAME,PTUUID,FSTYPE,SIZE,MOUNTPOINT
+```
+
+Look for the device you intend to use for backups and note its details.
+
+2. **Configure Your Backup Device:**
+
+Once you've identified your device, you'll need to add it to your configuration.
+Here's an example NixOS configuration for a device located at `/dev/sda2` with an `ext4` filesystem:
 
 ```nix
-inventory.instances = {
-  borgbackup = {
-    module = {
-      name = "borgbackup";
-      input = "clan";
-    };
-    roles.client.machines."jon".settings = {
-      destinations."storagebox" = {
-        repo = "username@$hostname:/./borgbackup";
-        rsh = ''ssh -oPort=23 -i /run/secrets/vars/borgbackup/borgbackup.ssh'';
-      };
-    };
-    roles.server.machines = { };
+{
+  fileSystems."/mnt/hdd" = {
+    device = "/dev/sda2";
+    fsType = "ext4";
+    options = [ "defaults" "noauto" ];
   };
-};
+}
 ```
 
-The input should be named according to your flake input. Jon is configured as a
-client machine with a destination pointing to a Hetzner Storage Box.
+Replace `/dev/sda2` with your device and `/mnt/hdd` with your preferred mount point.
 
-To see a list of all possible options go to [borgbackup clan service](../reference/clanServices/borgbackup.md)
+3. **Set Backup Targets:** Next, define where on your device you'd like the backups to be stored:
 
-## Roles
+   ```nix
+   {
+     clan.localbackup.targets.hdd = {
+       directory = "/mnt/hdd/backup";
+       mountpoint = "/mnt/hdd";
+     };
+   }
+   ```
 
-A Clan Service can have multiple roles, each role applies different nix config to the machine.
+   Change `/mnt/hdd` to the actual mount point you're using.
 
-### 1. Client
+4. **Create Backups:** To create a backup, run:
 
-Clients are machines that create and send backups to various destinations. Each
-client can have multiple backup destinations configured.
+   ```bash
+   clan backups create mymachine
+   ```
 
-### 2. Server
+   This command saves snapshots of your data onto the backup device.
 
-Servers act as backup repositories, receiving and storing backups from client
-machines. They can be dedicated backup servers within your clan network.
+5. **Listing Backups:** To see available backups, run:
 
-## Backup destinations
+  ```bash
+  clan backups list mymachine
+  ```
 
-This service allows you to perform backups to multiple `destinations`.
-Destinations can be:
+## Remote Backups with Borgbackup
 
-- **Local**: Local disk storage
-- **Server**: Your own borgbackup server (using the `server` role)
-- **Third-party services**: Such as Hetzner's Storage Box
+### Overview of Borgbackup
 
-## State management
+Borgbackup splits the backup process into two parts: a backup client that sends data to a backup server.
+The server stores the backups.
 
-Backups are based on [states](../reference/clan.core/state.md). A state
-defines which files should be backed up and how these files are obtained through
-pre/post backup and restore scripts.
+### Setting Up the Borgbackup Client
 
-Here's an example for a user application `linkding`:
+1. **Specify Backup Server:**
 
-In this example:
-
-- `/data/podman/linkding` is the application's data directory
-- `/var/backup/linkding` is the staging directory where data is copied for
-  backup
+Start by indicating where your backup data should be sent. Replace `hostname` with your server's address:
 
 ```nix
-clan.core.state.linkding = {
-  folders = [ "/var/backup/linkding" ];
-
-  preBackupScript = ''
-    export PATH=${
-      lib.makeBinPath [
-        config.systemd.package
-        pkgs.coreutils
-        pkgs.rsync
-      ]
-    }
-
-    service_status=$(systemctl is-active podman-linkding)
-
-    if [ "$service_status" = "active" ]; then
-      systemctl stop podman-linkding
-      rsync -avH --delete --numeric-ids "/data/podman/linkding/" /var/backup/linkding/
-      systemctl start podman-linkding
-    fi
-  '';
-
-  postRestoreScript = ''
-    export PATH=${
-      lib.makeBinPath [
-        config.systemd.package
-        pkgs.coreutils
-        pkgs.rsync
-      ]
-    }
-
-    service_status="$(systemctl is-active podman-linkding)"
-
-    if [ "$service_status" = "active" ]; then
-      systemctl stop podman-linkding
-
-      # Backup locally current linkding data
-      cp -rp "/data/podman/linkding" "/data/podman/linkding.bak"
-
-      # Restore from borgbackup
-      rsync -avH --delete --numeric-ids /var/backup/linkding/ "/data/podman/linkding/"
-
-      systemctl start podman-linkding
-    fi
-  '';
-};
+{
+  clan.borgbackup.destinations = {
+    myhostname = {
+      repo = "borg@backuphost:/var/lib/borgbackup/myhostname";
+    };
+  };
+}
 ```
 
-## Managing backups
+2. **Select Folders to Backup:**
 
-In this section we go over how to manage your collection of backups with the clan command.
+Decide which folders you want to back up. For example, to backup your home and root directories:
 
-### Listing states
+```nix
+{ clan.core.state.userdata.folders = [ "/home" "/root" ]; }
+```
 
-To see which files (`states`) will be backed up on a specific machine, use:
+3. **Generate Backup Credentials:**
+
+Run `clan facts generate <yourmachine>` to prepare your machine for backup, creating necessary SSH keys and credentials.
+
+### Setting Up the Borgbackup Server
+
+1. **Configure Backup Repository:**
+
+On the server where backups will be stored, enable the SSH daemon and set up a repository for each client:
+
+```nix
+{
+  services.borgbackup.repos.myhostname = {
+    path = "/var/lib/borgbackup/myhostname";
+    authorizedKeys = [
+      (builtins.readFile  (config.clan.core.settings.directory + "/machines/myhostname/facts/borgbackup.ssh.pub"))
+    ];
+  };
+}
+```
+
+Ensure the path to the public key is correct.
+
+2. **Update Your Systems:** Apply your changes by running `clan machines update` to both the server and your client
+
+### Managing Backups
+
+- **Scheduled Backups:**
+
+  Backups are automatically performed nightly. To check the next scheduled backup, use:
+
+  ```bash
+  systemctl list-timers | grep -E 'NEXT|borg'
+  ```
+
+- **Listing Backups:** To see available backups, run:
+
+  ```bash
+  clan backups list mymachine
+  ```
+
+- **Manual Backups:** You can also initiate a backup manually:
+
+  ```bash
+  clan backups create mymachine
+  ```
+
+- **Restoring Backups:** To restore a backup that has been listed by the list command (NAME):
+
+  ```bash
+  clan backups restore [MACHINE] [PROVIDER] [NAME]
+
+  ```
+
+  Example (Restoring a machine called `client` with the backup provider `borgbackup`):
+
+  ```bash
+  clan backups restore client borgbackup [NAME]
+
+  ```
+
+  The `backups` command is service aware and allows optional specification of the `--service` flag.
+
+  To only restore the service called `zerotier` on a machine called `controller` through the backup provider `borgbackup` use the following command:
 
 ```bash
-clan state list jon
+  clan backups restore client borgbackup [NAME] --service zerotier
 ```
-
-This will show all configured states for the machine `jon`, for example:
-
-```text
-· service: linkding
-  folders:
-  - /var/backup/linkding
-  preBackupCommand: pre-backup-linkding
-  postRestoreCommand: post-restore-linkding
-
-· service: zerotier
-  folders:
-  - /var/lib/zerotier-one
-```
-
-### Creating backups
-
-To create a backup of a machine (e.g., `jon`), run:
-
-```bash
-clan backups create jon
-```
-
-This will backup all configured states (`zerotier` and `linkding` in this
-example) from the machine `jon`.
-
-### Listing available backups
-
-To see all available backups, use:
-
-```bash
-clan backups list
-```
-
-This will display all backups with their timestamps:
-
-```text
-storagebox::username@username.your-storagebox.de:/./borgbackup::jon-jon-2025-07-22T19:40:10
-storagebox::username@username.your-storagebox.de:/./borgbackup::jon-jon-2025-07-23T01:00:00
-storagebox::username@username.your-storagebox.de:/./borgbackup::jon-storagebox-2025-07-24T01:00:00
-storagebox::username@username.your-storagebox.de:/./borgbackup::jon-storagebox-2025-07-24T06:02:35
-```
-
-### Restoring backups
-
-For restoring a backup you have two options. 
-
-#### Full restoration
-
-To restore all services from a backup:
-
-```bash
-clan backups restore jon borgbackup storagebox::u444061@u444061.your-storagebox.de:/./borgbackup::jon-storagebox-2025-07-24T06:02:35
-```
-
-#### Partial restoration
-
-To restore only a specific service (e.g., `linkding`):
-
-```bash
-clan backups restore --service linkding jon borgbackup storagebox::u444061@u444061.your-storagebox.de:/./borgbackup::jon-storagebox-2025-07-24T06:02:35
-```
-
-
-

docs/site/guides/clanServices.md

@@ -138,7 +138,7 @@ You can use services exposed by Clan’s core module library, `clan-core`.
 
 You can also author your own `clanService` modules.
 
-🔗 Learn how to write your own service: [Authoring a service](../guides/services/community.md)
+🔗 Learn how to write your own service: [Authoring a clanService](../guides/authoring/clanServices/index.md)
 
 You might expose your service module from your flake — this makes it easy for other people to also use your module in their clan.
 
@@ -154,6 +154,6 @@ You might expose your service module from your flake — this makes it easy for
 
 ## What’s Next?
 
-* [Author your own clanService →](../guides/services/community.md)
+* [Author your own clanService →](../guides/authoring/clanServices/index.md)
 * [Migrate from clanModules →](../guides/migrations/migrate-inventory-services.md)
 <!-- TODO: * [Understand the architecture →](../explanation/clan-architecture.md) -->

docs/site/guides/flake-parts.md

@@ -27,7 +27,7 @@ inputs = {
 
 ## Import the Clan flake-parts Module
 
-After updating your flake inputs, the next step is to import the Clan flake-parts module. This will make the [Clan options](../options.md) available within `mkFlake`.
+After updating your flake inputs, the next step is to import the Clan flake-parts module. This will make the [Clan options](../reference/nix-api/clan.md) available within `mkFlake`.
 
 ```nix
 {

docs/site/guides/getting-started/add-machines.md

@@ -6,7 +6,7 @@ Machines can be added using the following methods
 - Editing machines/`machine_name`/configuration.nix (automatically included if it exists)
 - `clan machines create` (imperative)
 
-See the complete [list](../../concepts/autoincludes.md) of auto-loaded files.
+See the complete [list](../../guides/more-machines.md#automatic-registration) of auto-loaded files.
 
 ## Create a machine
 

docs/site/guides/getting-started/add-services.md

@@ -41,7 +41,7 @@ To learn more: [Guide about clanService](../clanServices.md)
 ```
 
 1. See [reference/clanServices](../../reference/clanServices/index.md) for all available services and how to configure them.
-   Or read [authoring/clanServices](../../guides/services/community.md) if you want to bring your own
+   Or read [authoring/clanServices](../authoring/clanServices/index.md) if you want to bring your own
 
 2. Replace `__YOUR_CONTROLLER_` with the *name* of your machine.
 

docs/site/guides/getting-started/add-user.md

@@ -57,7 +57,7 @@ For more information see [clanService/users](../../reference/clanServices/users.
 
 Some people like to define a `users` folder in their repository root.
 That allows to bind all user specific logic to a single place (`default.nix`)
-Which can be imported into individual machines to make the user available on that machine.
+Which can be imported into individual machines to make the user avilable on that machine.
 
 ```bash
 .
@@ -107,7 +107,7 @@ We can use this property of clan services to bind a nixosModule to the user, whi
 }
 ```
 
-1. Type `path` or `string`: Must point to a separate file. Inlining a module is not possible
+1. Type `path` or `string`: Must point to a seperate file. Inlining a module is not possible
 
 !!! Note "This is inspiration"
     Our community might come up with better solutions soon.

docs/site/guides/getting-started/deploy.md

@@ -8,6 +8,7 @@ Now that you have created a machines, added some services and setup secrets. Thi
     - [x] RAM > 2GB
     - [x] **Two Computers**: You need one computer that you're getting ready (we'll call this the Target Computer) and another one to set it up from (we'll call this the Setup Computer). Make sure both can talk to each other over the network using SSH.
     - [x] **Machine configuration**: See our basic [adding and configuring machine guide](./add-machines.md)
+    - [x] **Initialized secrets**: See [secrets](secrets.md) for how to initialize your secrets.
 
 ## Physical Hardware
 
@@ -17,7 +18,7 @@ Steps:
 
 - Create a NixOS installer image and transfer it to a bootable USB drive as described in the [installer](./installer.md).
 - Boot the target machine and connect it to a network that makes it reachable from your setup computer.
-- Note down a reachable ip address (*ipv4*, *ipv6* or *tor*)
+- Note down a reachable ip adress (*ipv4*, *ipv6* or *tor*)
 
 ---
 
@@ -168,7 +169,7 @@ Re-run the command with the correct disk:
 clan templates apply disk single-disk jon --set mainDisk "/dev/disk/by-id/nvme-WD_PC_SN740_SDDQNQD-512G-1201_232557804368"
 ```
 
-Should now be successful
+Should now be succesfull
 
 ```shellSession
 Applied disk template 'single-disk' to machine 'jon'

docs/site/guides/getting-started/index.md

@@ -59,7 +59,7 @@ Enter a *name*, confirm with *enter*. A directory with that name will be created
 
 ## Explore the Project Structure
 
-Take a look at all project files:
+Take a lookg at all project files:
 
 ```bash
 cd my-clan
@@ -125,10 +125,11 @@ To change the name of your clan edit `meta.name` in the `clan.nix` or `flake.nix
 You can continue with **any** of the following steps at your own pace:
 
 - [x] [Install Nix & Clan CLI](./index.md)
-- [x] [Initialize Clan](./index.md#add-clan-cli-to-your-shell)
+- [x] [Initialize Clan](./index.md#initialize-your-project)
 - [ ] [Create USB Installer (optional)](./installer.md)
 - [ ] [Add Machines](./add-machines.md)
 - [ ] [Add a User](./add-user.md)
 - [ ] [Add Services](./add-services.md)
+- [ ] [Configure Secrets](./secrets.md)
 - [ ] [Deploy](./deploy.md) - Requires configured secrets
 - [ ] [Setup CI (optional)](./check.md)

docs/site/guides/getting-started/secrets.md

@@ -0,0 +1,179 @@
+
+Setting up secrets is **Required** for any *machine deployments* or *vm runs* - You need to complete the steps: [Create Admin Keypair](#create-your-admin-keypair) and [Add Your Public Key(s)](#add-your-public-keys)
+
+---
+
+Clan enables encryption of secrets (such as passwords & keys) ensuring security and ease-of-use among users.
+
+By default, Clan uses the [sops](https://github.com/getsops/sops) format
+and integrates with [sops-nix](https://github.com/Mic92/sops-nix) on NixOS machines.
+Clan can also be configured to be used with other secret store [backends](../../reference/clan.core/vars.md#clan.core.vars.settings.secretStore).
+
+This guide will walk you through:
+
+- **Creating a Keypair for Your User**: Learn how to generate a keypair for `$USER` to securely control all secrets.
+- **Creating Your First Secret**: Step-by-step instructions on creating your initial secret.
+- **Assigning Machine Access to the Secret**: Understand how to grant a machine access to the newly created secret.
+
+## Create Your Admin Keypair
+
+To get started, you'll need to create **your admin keypair**.
+
+!!! info
+    Don't worry — if you've already made one before, this step won't change or overwrite it.
+
+```bash
+clan secrets key generate
+```
+
+**Output**:
+
+```{.console, .no-copy}
+Public key: age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7
+
+Generated age private key at '/home/joerg/.config/sops/age/keys.txt' for your user. Please back it up on a secure location or you will lose access to your secrets.
+Also add your age public key to the repository with 'clan secrets users add YOUR_USER age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7' (replace YOUR_USER with your actual username)
+```
+
+!!! warning
+    Make sure to keep a safe backup of the private key you've just created.
+    If it's lost, you won't be able to get to your secrets anymore because they all need the admin key to be unlocked.
+
+If you already have an [age] secret key and want to use that instead, you can simply edit `~/.config/sops/age/keys.txt`:
+
+```title="~/.config/sops/age/keys.txt"
+AGE-SECRET-KEY-13GWMK0KNNKXPTJ8KQ9LPSQZU7G3KU8LZDW474NX3D956GGVFAZRQTAE3F4
+```
+
+Alternatively, you can provide your [age] secret key as an environment variable `SOPS_AGE_KEY`, or in a different file
+using `SOPS_AGE_KEY_FILE`.
+For more information see the [SOPS] guide on [encrypting with age].
+
+!!! note
+    It's safe to add any secrets created by the clan CLI and placed in your repository to version control systems like `git`.
+
+### Add Your Public Key(s)
+
+```console
+clan secrets users add $USER --age-key <your_public_key>
+```
+
+It's best to choose the same username as on your Setup/Admin Machine that you use to control the deployment with.
+
+Once run this will create the following files:
+
+```{.console, .no-copy}
+sops/
+└── users/
+    └── <your_username>/
+        └── key.json
+```
+If you followed the quickstart tutorial all necessary secrets are initialized at this point.
+
+!!! note
+    You can add multiple age keys for a user by providing multiple `--age-key <your_public_key>` flags:
+
+    ```console
+    clan secrets users add $USER \
+        --age-key <your_public_key_1> \
+        --age-key <your_public_key_2> \
+        ...
+    ```
+
+### Manage Your Public Key(s)
+
+You can list keys for your user with `clan secrets users get $USER`:
+
+```console
+clan secrets users get alice
+
+[
+  {
+    "publickey": "age1hrrcspp645qtlj29krjpq66pqg990ejaq0djcms6y6evnmgglv5sq0gewu",
+    "type": "age",
+    "username": "alice"
+  },
+  {
+    "publickey": "age13kh4083t3g4x3ktr52nav6h7sy8ynrnky2x58pyp96c5s5nvqytqgmrt79",
+    "type": "age",
+    "username": "alice"
+  }
+]
+```
+
+To add a new key to your user:
+
+```console
+clan secrets users add-key $USER --age-key <your_public_key>
+```
+
+To remove a key from your user:
+
+```console
+clan secrets users remove-key $USER --age-key <your_public_key>
+```
+
+[age]: https://github.com/FiloSottile/age
+[age plugin]: https://github.com/FiloSottile/awesome-age?tab=readme-ov-file#plugins
+[sops]: https://github.com/getsops/sops
+[encrypting with age]: https://github.com/getsops/sops?tab=readme-ov-file#encrypting-using-age
+
+## Further: Using Age Plugins
+
+If you wish to use a key generated using an [age plugin] as your admin key, extra care is needed.
+
+You must **precede your secret key with a comment that contains its corresponding recipient**.
+
+This is usually output as part of the generation process
+and is only required because there is no unified mechanism for recovering a recipient from a plugin secret key.
+
+Here is an example:
+
+```title="~/.config/sops/age/keys.txt"
+# public key: age1zdy49ek6z60q9r34vf5mmzkx6u43pr9haqdh5lqdg7fh5tpwlfwqea356l
+AGE-PLUGIN-FIDO2-HMAC-1QQPQZRFR7ZZ2WCV...
+```
+
+!!! note
+    The comment that precedes the plugin secret key need only contain the recipient.
+    Any other text is ignored.
+
+    In the example above, you can specify `# recipient: age1zdy...`, `# public: age1zdy....` or even
+    just `# age1zdy....`
+
+You will need to add an entry into your `flake.nix` to ensure that the necessary `age` plugins
+are loaded when using Clan:
+
+```nix title="flake.nix"
+{
+  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
+  inputs.nixpkgs.follows = "clan-core/nixpkgs";
+
+  outputs =
+    { self, clan-core, ... }:
+    let
+      # Sometimes this attribute set is defined in clan.nix
+      clan = clan-core.lib.clan {
+        inherit self;
+
+        meta.name = "myclan";
+
+        # Add Yubikey and FIDO2 HMAC plugins
+        # Note: the plugins listed here must be available in nixpkgs.
+        secrets.age.plugins = [
+          "age-plugin-yubikey"
+          "age-plugin-fido2-hmac"
+        ];
+
+        machines = {
+          # elided for brevity
+        };
+      };
+    in
+    {
+      inherit (clan) nixosConfigurations nixosModules clanInternals;
+
+      # elided for brevity
+    };
+}
+```

docs/site/guides/inventory.md

@@ -9,6 +9,8 @@ The inventory logic will automatically derive the modules and configurations to
 
 The following tutorial will walk through setting up a Backup service where the terms `Service` and `Role` will become more clear.
 
+See also: [Inventory API Documentation](../reference/nix-api/inventory.md)
+
 !!! example "Experimental status"
     The inventory implementation is not considered stable yet.
     We are actively soliciting feedback from users.
@@ -17,7 +19,7 @@ The following tutorial will walk through setting up a Backup service where the t
 
 ## Prerequisites
 
-- [x] [Add some machines](../guides/getting-started/add-machines.md) to your Clan.
+- [x] [Add multiple machines](./more-machines.md) to your Clan.
 
 ## Services
 

docs/site/guides/macos.md

@@ -7,7 +7,7 @@ This guide explains how to manage macOS machines using Clan.
 Currently, Clan supports the following features for macOS:
 
 - `clan machines update` for existing [nix-darwin](https://github.com/nix-darwin/nix-darwin) installations
-- Support for [vars](../concepts/generators.md)
+- Support for [vars](../guides/vars-backend.md)
 
 ## Add Your Machine to Your Clan Flake
 

docs/site/guides/migrations/migrate-inventory-services.md

@@ -1,7 +1,7 @@
 # Migrating from using `clanModules` to `clanServices`
 
 **Audience**: This is a guide for **people using `clanModules`**.
-If you are a **module author** and need to migrate your modules please consult our **new** [clanServices authoring guide](../../guides/services/community.md)
+If you are a **module author** and need to migrate your modules please consult our **new** [clanServices authoring guide](../authoring/clanServices/index.md)
 
 ## What's Changing?
 
@@ -329,6 +329,6 @@ instances = {
 
 ## Further reference
 
-* [Inventory Concept](../../concepts/inventory.md)
-* [Authoring a 'clan.service' module](../../guides/services/community.md)
+* [Authoring a 'clan.service' module](../authoring/clanServices/index.md)
 * [ClanServices](../clanServices.md)
+* [Inventory Reference](../../reference/nix-api/inventory.md)

docs/site/guides/migrations/migration-facts-vars.md

@@ -3,7 +3,7 @@
 For a high level overview about `vars` see our [blog post](https://clan.lol/blog/vars/).
 
 This guide will help you migrate your modules that still use our [`facts`](../../guides/secrets.md) backend
-to the [`vars`](../../concepts/generators.md) backend.
+to the [`vars`](../../guides/vars-backend.md) backend.
 
 The `vars` [module](../../reference/clan.core/vars.md) and the clan [command](../../reference/cli/vars.md) work in tandem, they should ideally be kept in sync.
 

docs/site/guides/more-machines.md

@@ -0,0 +1,50 @@
+
+Clan has two general methods of adding machines:
+
+- **Automatic**: Detects every folder in the `machines` folder.
+- **Declarative**: Explicit declarations in Nix.
+
+## Automatic registration
+
+Every folder `machines/{machineName}` will be registered automatically as a Clan machine.
+
+!!! info "Automatically loaded files"
+
+    The following files are loaded automatically for each Clan machine:
+
+    - [x] `machines/{machineName}/configuration.nix`
+    - [x] `machines/{machineName}/hardware-configuration.nix`
+    - [x] `machines/{machineName}/facter.json` Automatically configured, for further information see [nixos-facter](https://clan.lol/blog/nixos-facter/)
+    - [x] `machines/{machineName}/disko.nix` Automatically loaded, for further information see the [disko docs](https://github.com/nix-community/disko/blob/master/docs/quickstart.md).
+
+## Manual declaration
+
+Machines can be added via [`clan.inventory.machines`](../guides/inventory.md) or in `clan.machines`, which allows for defining NixOS options.
+
+=== "**Individual Machine Configuration**"
+
+    ```{.nix}
+    clan-core.lib.clan {
+        machines = {
+            "jon" = {
+                # Any valid nixos config
+            };
+        };
+    }
+    ```
+
+=== "**Inventory Configuration**"
+
+    ```{.nix}
+    clan-core.lib.clan {
+        inventory = {
+            machines = {
+                "jon" = {
+                    # Inventory can set tags and other metadata
+                    tags = [ "zone1" ];
+                    deploy.targetHost = "root@jon";
+                };
+            };
+        };
+    }
+    ```

docs/site/guides/secrets.md

@@ -1,141 +1,25 @@
-This article provides an overview over the underlying secrets system which is used by [Vars](../concepts/generators.md).
-Under most circumstances you should use [Vars](../concepts/generators.md) directly instead.
+If you want to know more about how to save and share passwords in your clan read further!
 
-Consider using `clan secrets` only for managing admin users and groups, as well as a debugging tool.
-
-Manually interacting with secrets via `clan secrets [set|remove]`, etc may break the integrity of your `Vars` state.
-
----
-
-Clan enables encryption of secrets (such as passwords & keys) ensuring security and ease-of-use among users.
-
-By default, Clan uses the [sops](https://github.com/getsops/sops) format
-and integrates with [sops-nix](https://github.com/Mic92/sops-nix) on NixOS machines.
-Clan can also be configured to be used with other secret store [backends](../reference/clan.core/vars.md#clan.core.vars.settings.secretStore).
-
-## Create Your Admin Keypair
-
-To get started, you'll need to create **your admin keypair**.
-
-!!! info
-    Don't worry — if you've already made one before, this step won't change or overwrite it.
-
-```bash
-clan secrets key generate
-```
-
-**Output**:
-
-```{.console, .no-copy}
-Public key: age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7
-
-Generated age private key at '/home/joerg/.config/sops/age/keys.txt' for your user. Please back it up on a secure location or you will lose access to your secrets.
-Also add your age public key to the repository with 'clan secrets users add YOUR_USER age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7' (replace YOUR_USER with your actual username)
-```
-
-!!! warning
-    Make sure to keep a safe backup of the private key you've just created.
-    If it's lost, you won't be able to get to your secrets anymore because they all need the admin key to be unlocked.
-
-If you already have an [age] secret key and want to use that instead, you can simply edit `~/.config/sops/age/keys.txt`:
-
-```title="~/.config/sops/age/keys.txt"
-AGE-SECRET-KEY-13GWMK0KNNKXPTJ8KQ9LPSQZU7G3KU8LZDW474NX3D956GGVFAZRQTAE3F4
-```
-
-Alternatively, you can provide your [age] secret key as an environment variable `SOPS_AGE_KEY`, or in a different file
-using `SOPS_AGE_KEY_FILE`.
-For more information see the [SOPS] guide on [encrypting with age].
-
-!!! note
-    It's safe to add any secrets created by the clan CLI and placed in your repository to version control systems like `git`.
-
-## Add Your Public Key(s)
-
-```console
-clan secrets users add $USER --age-key <your_public_key>
-```
-
-It's best to choose the same username as on your Setup/Admin Machine that you use to control the deployment with.
-
-Once run this will create the following files:
-
-```{.console, .no-copy}
-sops/
-└── users/
-    └── <your_username>/
-        └── key.json
-```
-If you followed the quickstart tutorial all necessary secrets are initialized at this point.
-
-!!! note
-    You can add multiple age keys for a user by providing multiple `--age-key <your_public_key>` flags:
-
-    ```console
-    clan secrets users add $USER \
-        --age-key <your_public_key_1> \
-        --age-key <your_public_key_2> \
-        ...
-    ```
-
-## Manage Your Public Key(s)
-
-You can list keys for your user with `clan secrets users get $USER`:
-
-```console
-clan secrets users get alice
-
-[
-  {
-    "publickey": "age1hrrcspp645qtlj29krjpq66pqg990ejaq0djcms6y6evnmgglv5sq0gewu",
-    "type": "age",
-    "username": "alice"
-  },
-  {
-    "publickey": "age13kh4083t3g4x3ktr52nav6h7sy8ynrnky2x58pyp96c5s5nvqytqgmrt79",
-    "type": "age",
-    "username": "alice"
-  }
-]
-```
-
-To add a new key to your user:
-
-```console
-clan secrets users add-key $USER --age-key <your_public_key>
-```
-
-To remove a key from your user:
-
-```console
-clan secrets users remove-key $USER --age-key <your_public_key>
-```
-
-[age]: https://github.com/FiloSottile/age
-[age plugin]: https://github.com/FiloSottile/awesome-age?tab=readme-ov-file#plugins
-[sops]: https://github.com/getsops/sops
-[encrypting with age]: https://github.com/getsops/sops?tab=readme-ov-file#encrypting-using-age
-
-## Adding a Secret
+### Adding a Secret
 
 ```shellSession
 clan secrets set mysecret
 Paste your secret:
 ```
 
-## Retrieving a Stored Secret
+### Retrieving a Stored Secret
 
 ```bash
 clan secrets get mysecret
 ```
 
-## List all Secrets
+### List all Secrets
 
 ```bash
 clan secrets list
 ```
 
-## NixOS integration
+### NixOS integration
 
 A NixOS machine will automatically import all secrets that are encrypted for the
 current machine. At runtime it will use the host key to decrypt all secrets into
@@ -153,7 +37,7 @@ In your nixos configuration you can get a path to secrets like this `config.sops
 }
 ```
 
-## Assigning Access
+### Assigning Access
 
 When using `clan secrets set <secret>` without arguments, secrets are encrypted for the key of the user named like your current $USER.
 

docs/site/guides/vars-backend.md

@@ -1,4 +1,7 @@
-# Generators
+
+!!! Note
+    Vars is the new secret backend that will soon replace the Facts backend
+
 
 Defining a linux user's password via the nixos configuration previously required running `mkpasswd ...` and then copying the hash back into the nix configuration.
 
@@ -8,7 +11,7 @@ For a more general explanation of what clan vars are and how it works, see the i
 
 This guide assumes
 - Clan is set up already (see [Getting Started](../guides/getting-started/index.md))
-- a machine has been added to the clan (see [Adding Machines](../guides/getting-started/add-machines.md))
+- a machine has been added to the clan (see [Adding Machines](./more-machines.md))
 
 This section will walk you through the following steps:
 
@@ -20,7 +23,7 @@ This section will walk you through the following steps:
 6. share the root password between machines
 7. change the password
 
-## Declare a generator
+## Declare the generator
 
 In this example, a `vars` `generator` is used to:
 

docs/site/index.md

@@ -4,72 +4,87 @@ hide:
   - toc
 ---
 
-# :material-home: What is Clan?
-
-[Clan](https://clan.lol/) is a peer-to-peer computer management framework that
-empowers you to **selfhost in a reliable and scalable way**.   
-
-Built on NixOS, Clan provides a **declarative interface for managing machines** with automated [secret management](./guides/secrets.md), easy [mesh VPN
-connectivity](./guides/mesh-vpn.md), and [automated backups](./guides/backups.md). 
-
-Whether you're running a homelab or maintaining critical computing infrastructure,
-Clan will help **reduce maintenance burden** by allowing a **git repository to define your whole network** of computers.
-
-In combination with [sops-nix](https://github.com/Mic92/sops-nix), [nixos-anywhere](https://github.com/nix-community/nixos-anywhere) and [disko](https://github.com/nix-community/disko), Clan makes it possible to have **collaborative infrastructure**.
-
-At the heart of Clan are [Clan Services](./reference/clanServices/index.md) - the core
-concept that enables you to add functionality across multiple machines in your
-network. While Clan ships with essential core services, you can [create custom
-services](./guides/clanServices.md) tailored to your specific needs.
+# :material-home: Welcome to **Clan**'s  documentation
 
+[Getting Started](./guides/getting-started/index.md){ .md-button }
 
 ## :material-book: Guides
 
-How-to Guides for achieving a certain goal or solving a specific issue.
+**How-to Guides for achieving a certain goal or solving a specific issue.**
 
 <div class="grid cards" markdown>
 
--   [:material-clock-fast: Getting Started](./guides/getting-started/index.md)
+-   [Adding more machines](./guides/more-machines.md)
 
     ---
 
-    Get started in less than 20 minutes!
+    Learn how Clan automatically includes machines and Nix files.
 
--   [Mac OS](./guides/macos.md)
+-   [Vars Backend](./guides/vars-backend.md)
 
     ---
 
-    How to manage Mac OS machines with Clan
+    Learn how to manage secrets with vars.
+
+-   [Inventory](./guides/inventory.md)
+
+    ---
+
+    Clan's declaration format for running **services** on one or multiple **machines**.
+
+-   [Flake-parts](./guides/flake-parts.md)
+
+    ---
+
+    Use Clan with [https://flake.parts/]()
 
 -   [Contribute](./guides/contributing/CONTRIBUTING.md)
 
     ---
 
-    How to set up a development environment
+    Discover how to set up a development environment to contribute to Clan!
+
+-   [macOS machines](./guides/macos.md)
+
+    ---
+
+    Manage macOS machines with nix-darwin
 
 </div>
 
-## Concepts
+## API Reference
 
-Explore the underlying principles of Clan
+**Reference API Documentation**
 
 <div class="grid cards" markdown>
 
--   [Generators](./concepts/generators.md)
+-   [CLI Reference](./reference/cli/index.md)
 
     ---
 
-    Learn about Generators, our way to secret management
+    The `clan` CLI command
 
--   [Inventory](./concepts/inventory.md)
+-   [Service Modules](./reference/clanServices/index.md)
 
     ---
 
-    Learn about the Inventory, a multi machine Nix interface
+    An overview of available service modules
+
+-   [Core](./reference/clan.core/index.md)
+
+    ---
+
+    The clan core nix module.
+    This is imported when using clan and is the basis of the extra functionality
+    that can be provided.
+
+-   [(Legacy) Modules](./reference/clanModules/index.md)
+
+    ---
+
+    An overview of available clanModules
+
+    !!! Example "These will be deprecated soon"
+
 
 </div>
-
-
-## Blog
-
-Visit our [Clan Blog](https://clan.lol/blog/) for the latest updates, tutorials, and community stories.

docs/site/intern/index.md

@@ -0,0 +1,25 @@
+# Developer Documentation
+
+!!! Danger
+    This documentation is **not** intended for external users. It may contain low-level details and internal-only interfaces.*
+
+Welcome to the internal developer documentation.
+
+This section is intended for contributors, engineers, and internal stakeholders working directly with our system, tooling, and APIs. It provides a technical overview of core components, internal APIs, conventions, and patterns that support the platform.
+
+Our goal is to make the internal workings of the system **transparent, discoverable, and consistent** — helping you contribute confidently, troubleshoot effectively, and build faster.
+
+## What's Here?
+
+!!! note "docs migration ongoing"
+
+- [ ] **API Reference**: 🚧🚧🚧 Detailed documentation of internal API functions, inputs, and expected outputs. 🚧🚧🚧
+- [ ] **System Concepts**: Architectural overviews and domain-specific guides.
+- [ ] **Development Guides**: How to test, extend, or integrate with key components.
+- [ ] **Design Notes**: Rationales behind major design decisions or patterns.
+
+## Who is This For?
+
+* Developers contributing to the platform
+* Engineers debugging or extending internal systems
+* Anyone needing to understand **how** and **why** things work under the hood

docs/site/reference/index.md

@@ -4,10 +4,10 @@ This section of the site provides an overview of available options and commands
 
 ---
 
-- [Clan Configuration Option](../options.md) - for defining a Clan
 - Learn how to use the [Clan CLI](./cli/index.md)
 - Explore available services and application [modules](./clanModules/index.md)
-- [NixOS Configuration Options](./clan.core/index.md) - Additional options avilable on a NixOS machine.
+- Discover [configuration options](./clan.core/index.md) that manage essential features
+- Find descriptions of the [Nix interfaces](./nix-api/clan.md) for defining a Clan
 
 ---
 

docs/site/static/extra.css

@@ -2,7 +2,6 @@
   font-family: "Roboto";
   src: url(./Roboto-Regular.ttf) format("truetype");
 }
-
 @font-face {
   font-family: "Fira Code";
   src: url(./FiraCode-VF.ttf) format("truetype");
@@ -21,9 +20,3 @@
 .md-nav__item.md-nav__item--section > label > span {
   color: var(--md-typeset-a-color);
 }
-
-.md-typeset h4 {
-  margin: 3em 0 0.5em;
-  font-weight: bold;
-  color: #7ebae4;
-}

flake.lock

@@ -31,11 +31,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1753140376,
-        "narHash": "sha256-7lrVrE0jSvZHrxEzvnfHFE/Wkk9DDqb+mYCodI5uuB8=",
+        "lastModified": 1752718651,
+        "narHash": "sha256-PkaR0qmyP9q/MDN3uYa+RLeBA0PjvEQiM0rTDDBXkL8=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "545aba02960caa78a31bd9a8709a0ad4b6320a5c",
+        "rev": "d5ad4485e6f2edcc06751df65c5e16572877db88",
         "type": "github"
       },
       "original": {
@@ -51,11 +51,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1753121425,
-        "narHash": "sha256-TVcTNvOeWWk1DXljFxVRp+E0tzG1LhrVjOGGoMHuXio=",
+        "lastModified": 1751413152,
+        "narHash": "sha256-Tyw1RjYEsp5scoigs1384gIg6e0GoBVjms4aXFfRssQ=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "644e0fc48951a860279da645ba77fe4a6e814c5e",
+        "rev": "77826244401ea9de6e3bac47c2db46005e1f30b5",
         "type": "github"
       },
       "original": {

lib/default.nix

@@ -21,7 +21,6 @@ lib.fix (
   {
 
     inherit (buildClanLib)
-      buildClan
       clan
       ;
     /**

lib/modules/clan/interface.nix

@@ -229,8 +229,8 @@ in
     };
 
     inventory = lib.mkOption {
-      type = types.submoduleWith {
-        modules = [
+      type = types.submodule {
+        imports = [
           {
             _module.args = { inherit clanLib; };
             _file = "clan interface";

lib/modules/clan/module.nix

@@ -247,7 +247,7 @@ in
               {
                 distributedServices = clanLib.inventory.mapInstances {
                   inherit (clanConfig) inventory exportsModule;
-                  inherit flakeInputs directory;
+                  inherit flakeInputs;
                   clanCoreModules = clan-core.clan.modules;
                   prefix = [ "distributedServices" ];
                 };

lib/modules/default.nix

@@ -5,32 +5,7 @@
   clan-core,
   ...
 }:
-rec {
-  buildClan =
-    module:
-    lib.warn ''
-      ==================== DEPRECATION NOTICE ====================
-      Please migrate 
-      from: 'clan = inputs.<clan-core>.lib.buildClan' 
-      to  : 'clan = inputs.<clan-core>.lib.clan'
-      in your flake.nix.
-
-      Please also migrate
-      from: 'inherit (clan) nixosConfigurations clanInternals; '
-      to  : "
-              inherit (clan.config) nixosConfigurations clanInternals;
-              clan = clan.config;
-            "
-      in your flake.nix.
-
-      Reason: 
-      - Improves consistency between flake-parts and non-flake-parts users.
-              
-      - It also allows us to use the top level attribute 'clan' to expose
-        attributes that can be used for cross-clan functionality.
-      ============================================================
-    '' (clan module).config;
-
+{
   clan =
     {
       self ? lib.warn "Argument: 'self' must be set" null, # Reference to the current flake

lib/modules/inventory/distributed-service/all-services-wrapper.nix

@@ -1,8 +1,4 @@
 # Wraps all services in one fixed point module
-{
-  # TODO: consume directly from clan.config
-  directory,
-}:
 {
   lib,
   config,
@@ -33,8 +29,6 @@ in
               {
                 _module.args._ctx = [ name ];
                 _module.args.exports' = config.exports;
-                _module.args.directory = directory;
-
               }
             )
             ./service-module.nix
@@ -77,5 +71,8 @@ in
       };
       default = { };
     };
+    debug = mkOption {
+      default = lib.mapAttrsToList (_: service: service.exports) config.mappedServices;
+    };
   };
 }

lib/modules/inventory/distributed-service/inventory-adapter.nix

@@ -24,7 +24,6 @@ in
       flakeInputs,
       # The clan inventory
       inventory,
-      directory,
       clanCoreModules,
       prefix ? [ ],
       exportsModule,
@@ -129,7 +128,7 @@ in
             _ctx = prefix;
           };
           modules = [
-            (import ./all-services-wrapper.nix { inherit directory; })
+            ./all-services-wrapper.nix
           ] ++ modules;
         };
 

lib/modules/inventory/distributed-service/service-module.nix

@@ -2,7 +2,6 @@
   lib,
   config,
   _ctx,
-  directory,
   ...
 }:
 let
@@ -213,7 +212,7 @@ in
 
                           options.extraModules = lib.mkOption {
                             default = [ ];
-                            type = types.listOf (types.either types.deferredModule types.str);
+                            type = types.listOf (types.deferredModule);
                           };
                         })
                       ];
@@ -756,14 +755,10 @@ in
             instanceRes
             // {
               nixosModule = {
-                imports =
-                  [
-                    # Result of the applied 'perInstance = {...}: {  nixosModule = { ... }; }'
-                    instanceRes.nixosModule
-                  ]
-                  ++ (map (
-                    s: if builtins.typeOf s == "string" then "${directory}/${s}" else s
-                  ) instanceCfg.roles.${roleName}.extraModules);
+                imports = [
+                  # Result of the applied 'perInstance = {...}: {  nixosModule = { ... }; }'
+                  instanceRes.nixosModule
+                ] ++ instanceCfg.roles.${roleName}.extraModules;
               };
             }
 

lib/modules/inventory/distributed-service/tests/default.nix

@@ -45,7 +45,6 @@ let
       };
     in
     clanLib.inventory.mapInstances {
-      directory = ./.;
       clanCoreModules = { };
       flakeInputs = flakeInputsFixture;
       inherit inventory;
@@ -53,7 +52,6 @@ let
     };
 in
 {
-  extraModules = import ./extraModules.nix { inherit clanLib; };
   exports = import ./exports.nix { inherit lib clanLib; };
   resolve_module_spec = import ./import_module_spec.nix { inherit lib callInventoryAdapter; };
   test_simple =

lib/modules/inventory/distributed-service/tests/extraModules.nix

@@ -1,33 +0,0 @@
-{ clanLib }:
-let
-  clan = clanLib.clan {
-    self = { };
-    directory = ./.;
-
-    machines.jon = {
-      nixpkgs.hostPlatform = "x86_64-linux";
-
-    };
-    # A module that adds exports perMachine
-    modules.A = {
-      manifest.name = "A";
-      roles.peer = { };
-    };
-
-    inventory = {
-      instances.A = {
-        module.input = "self";
-        roles.peer.tags.all = { };
-
-        roles.peer.extraModules = [ ./oneOption.nix ];
-      };
-    };
-  };
-in
-{
-  test_1 = {
-    inherit clan;
-    expr = clan.config.nixosConfigurations.jon.config.testDebug;
-    expected = 42;
-  };
-}

lib/modules/inventory/distributed-service/tests/oneOption.nix

@@ -1,6 +0,0 @@
-{ lib, ... }:
-{
-  options.testDebug = lib.mkOption {
-    default = 42;
-  };
-}

lib/modules/inventoryClass/interface.nix

@@ -142,7 +142,7 @@ in
             - The module MUST have at least `features = [ "inventory" ]` in the frontmatter section.
             - The module MUST have a subfolder `roles` with at least one `{roleName}.nix` file.
 
-            For further information see: [Module Authoring Guide](../../guides/services/community.md).
+            For further information see: [Module Authoring Guide](../../guides/authoring/clanServices/index.md).
 
         ???+ example
             ```nix
@@ -179,7 +179,8 @@ in
               map (m: "'${m}'") (lib.attrNames (lib.filterAttrs (n: _v: !builtins.elem n allowedNames) moduleSet))
             )}
 
-            See: https://docs.clan.lol/guides/services/community/
+            See: https://docs.clan.lol/guides/clanServices/
+            And: https://docs.clan.lol/guides/authoring/clanServices/
           '' moduleSet;
     };
 

lib/test/container-test-driver/test_driver/__init__.py

@@ -313,18 +313,6 @@ class Machine:
         command = f"nc -z {shlex.quote(addr)} {port}"
         self.wait_until_succeeds(command, timeout=timeout)
 
-    def wait_for_file(self, filename: str, timeout: int = 30) -> None:
-        """
-        Waits until the file exists in the machine's file system.
-        """
-
-        def check_file(_last_try: bool) -> bool:
-            result = self.execute(f"test -e {filename}")
-            return result.returncode == 0
-
-        with self.nested(f"waiting for file '{filename}'"):
-            retry(check_file, timeout)
-
     def wait_for_unit(self, unit: str, timeout: int = 900) -> None:
         """
         Wait for a systemd unit to get into "active" state.
@@ -419,14 +407,6 @@ def setup_filesystems(container: ContainerInfo) -> None:
     Path("/etc/os-release").touch()
     Path("/etc/machine-id").write_text("a5ea3f98dedc0278b6f3cc8c37eeaeac")
     container.nix_store_dir.mkdir(parents=True)
-
-    # Recreate symlinks
-    for file in Path("/nix/store").iterdir():
-        if file.is_symlink():
-            target = file.readlink()
-            sym = container.nix_store_dir / file.name
-            os.symlink(target, sym)
-
     # Read /proc/mounts and replicate every bind mount
     with Path("/proc/self/mounts").open() as f:
         for line in f:

nixosModules/clanCore/default.nix

@@ -18,7 +18,6 @@
     ++ lib.optionals (_class == "nixos") [
       ./nixos-facter.nix
       ./vm.nix
-      ./postgresql
       ./machine-id
       ./state-version
       ./wayland-proxy-virtwl.nix

nixosModules/clanCore/outputs.nix

@@ -31,7 +31,6 @@
             The deployment data is now accessed directly from the configuration
             instead of being written to a separate JSON file.
           '';
-          defaultText = "error: deployment.json file generation has been removed in favor of direct selectors.";
         };
         deployment.buildHost = lib.mkOption {
           type = lib.types.nullOr lib.types.str;
@@ -55,10 +54,10 @@
         deployment.nixosMobileWorkaround = lib.mkOption {
           type = lib.types.bool;
           description = ''
-            if true, the deployment will first do a nixos-rebuild switch
+            if true, the deployment will first do a nixos-rebuild switch 
             to register the boot profile the command will fail applying it to the running system
-            which is why afterwards we execute a nixos-rebuild test to apply
-            the new config without having to reboot.
+            which is why afterwards we execute a nixos-rebuild test to apply 
+            the new config without having to reboot. 
             This is a nixos-mobile deployment bug and will be removed in the future
           '';
           default = false;

nixosModules/clanCore/postgresql/default.nix

@@ -1,236 +0,0 @@
-{
-  lib,
-  config,
-  pkgs,
-  ...
-}:
-let
-
-  cfg = config.clan.core.postgresql;
-
-  createDatabaseState =
-    db:
-    let
-      folder = "/var/backup/postgres/${db.name}";
-      current = "${folder}/pg-dump";
-      compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
-    in
-    {
-      folders = [ folder ];
-      preBackupScript = ''
-        export PATH=${
-          lib.makeBinPath [
-            config.services.postgresql.package
-            config.systemd.package
-            pkgs.coreutils
-            pkgs.util-linux
-            pkgs.zstd
-          ]
-        }
-        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
-          sleep 1
-        done
-
-        mkdir -p "${folder}"
-        runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
-        mv "${current}.tmp" ${current}
-      '';
-      postRestoreScript = ''
-        export PATH=${
-          lib.makeBinPath [
-            config.services.postgresql.package
-            config.systemd.package
-            pkgs.coreutils
-            pkgs.util-linux
-            pkgs.zstd
-            pkgs.gnugrep
-          ]
-        }
-        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
-          sleep 1
-        done
-        echo "Waiting for postgres to be ready..."
-        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
-          if ! systemctl is-active postgresql; then exit 1; fi
-          sleep 0.1
-        done
-
-        if [[ -e "${current}" ]]; then
-          (
-            systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
-            trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
-
-            mkdir -p "${folder}"
-            if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
-              runuser -u postgres -- dropdb "${db.name}"
-            fi
-            runuser -u postgres -- pg_restore -C -d postgres "${current}"
-          )
-        else
-          echo No database backup found, skipping restore
-        fi
-      '';
-    };
-
-  createDatabase = db: ''
-    CREATE DATABASE "${db.name}" ${
-      lib.concatStringsSep " " (
-        lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
-      )
-    }
-  '';
-
-  userClauses = lib.mapAttrsToList (
-    _: user:
-    ''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
-  ) cfg.users;
-  databaseClauses = lib.mapAttrsToList (
-    name: db:
-    lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
-  ) cfg.databases;
-in
-{
-  options.clan.core.postgresql = {
-
-    enable = lib.mkEnableOption "Whether to enable PostgreSQL Server";
-
-    # we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
-    databases = lib.mkOption {
-      description = "Databases to create";
-      default = { };
-      type = lib.types.attrsOf (
-        lib.types.submodule (
-          { name, ... }:
-          {
-            options = {
-              name = lib.mkOption {
-                type = lib.types.str;
-                default = name;
-                description = "Database name.";
-              };
-              service = lib.mkOption {
-                type = lib.types.str;
-                default = name;
-                description = "Service name that we associate with the database.";
-              };
-              # set to false, in case the upstream module uses ensureDatabase option
-              create.enable = lib.mkOption {
-                type = lib.types.bool;
-                default = true;
-                description = "Create the database if it does not exist.";
-              };
-              create.options = lib.mkOption {
-                description = "Options to pass to the CREATE DATABASE command.";
-                type = lib.types.lazyAttrsOf lib.types.str;
-                default = { };
-                example = {
-                  TEMPLATE = "template0";
-                  LC_COLLATE = "C";
-                  LC_CTYPE = "C";
-                  ENCODING = "UTF8";
-                  OWNER = "foo";
-                };
-              };
-              restore.stopOnRestore = lib.mkOption {
-                type = lib.types.listOf lib.types.str;
-                default = [ ];
-                description = "List of systemd services to stop before restoring the database.";
-              };
-            };
-          }
-        )
-      );
-    };
-    users = lib.mkOption {
-      description = "Users to create";
-      default = { };
-      type = lib.types.attrsOf (
-        lib.types.submodule (
-          { name, ... }:
-          {
-            options.name = lib.mkOption {
-              description = "User name";
-              type = lib.types.str;
-              default = name;
-            };
-          }
-        )
-      );
-    };
-  };
-
-  config = lib.mkIf (config.clan.core.postgresql.enable) {
-
-    clan.core.settings.state-version.enable = true;
-
-    # services.postgresql.package = lib.mkDefault pkgs.postgresql_16;
-
-    services.postgresql.enable = true;
-
-    services.postgresql.settings = {
-      wal_level = "replica";
-      max_wal_senders = 3;
-    };
-
-    # We are duplicating a bit the upstream module but allow to create databases with options
-    systemd.services.postgresql.postStart = ''
-      PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
-
-      while ! $PSQL -d postgres -c "" 2> /dev/null; do
-        if ! kill -0 "$MAINPID"; then exit 1; fi
-        sleep 0.1
-      done
-      ${lib.concatStringsSep "\n" userClauses}
-      ${lib.concatStringsSep "\n" databaseClauses}
-    '';
-
-    clan.core.state = lib.mapAttrs' (
-      _: db: lib.nameValuePair db.service (createDatabaseState db)
-    ) config.clan.core.postgresql.databases;
-
-    environment.systemPackages = builtins.map (
-      db:
-      let
-        folder = "/var/backup/postgres/${db.name}";
-        current = "${folder}/pg-dump";
-      in
-      pkgs.writeShellScriptBin "postgres-db-restore-command-${db.name}" ''
-        export PATH=${
-          lib.makeBinPath [
-            config.services.postgresql.package
-            config.systemd.package
-            pkgs.coreutils
-            pkgs.util-linux
-            pkgs.zstd
-            pkgs.gnugrep
-          ]
-        }
-        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
-          sleep 1
-        done
-        echo "Waiting for postgres to be ready..."
-        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
-          if ! systemctl is-active postgresql; then exit 1; fi
-          sleep 0.1
-        done
-
-        if [[ -e "${current}" ]]; then
-          (
-            ${lib.optionalString (db.restore.stopOnRestore != [ ]) ''
-              systemctl stop ${builtins.toString db.restore.stopOnRestore}
-              trap "systemctl start ${builtins.toString db.restore.stopOnRestore}" EXIT
-            ''}
-
-            mkdir -p "${folder}"
-            if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
-              runuser -u postgres -- dropdb "${db.name}"
-            fi
-            runuser -u postgres -- pg_restore -C -d postgres "${current}"
-          )
-        else
-          echo No database backup found, skipping restore
-        fi
-      ''
-    ) (builtins.attrValues config.clan.core.postgresql.databases);
-  };
-}

nixosModules/clanCore/postgresql/tests/flake-module.nix

@@ -1,106 +0,0 @@
-{ self, ... }:
-{
-  perSystem =
-    { ... }:
-    {
-      clan.nixosTests.postgresql = {
-
-        name = "service-postgresql";
-
-        clan = {
-          directory = ./.;
-
-          # Workaround until we can use nodes.machine = { };
-          modules."@clan/importer" = ../../../../clanServices/importer;
-
-          inventory = {
-            machines.machine = { };
-            instances.importer = {
-              module.name = "@clan/importer";
-              module.input = "self";
-              roles.default.tags.all = { };
-              roles.default.extraModules = [
-                {
-
-                  imports = [
-                    #   self.nixosModules.clanCore
-                    self.clanModules.localbackup
-                  ];
-
-                  clan.core.postgresql.enable = true;
-                  clan.core.postgresql.users.test = { };
-                  clan.core.postgresql.databases.test.create.options.OWNER = "test";
-                  clan.core.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ];
-                  clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
-                  clan.core.settings.directory = ./.;
-
-                  systemd.services.sample-service = {
-                    wantedBy = [ "multi-user.target" ];
-                    script = ''
-                      while true; do
-                        echo "Hello, world!"
-                        sleep 5
-                      done
-                    '';
-                  };
-
-                }
-              ];
-            };
-          };
-        };
-
-        # TODO: Broken. Use instead of importer after fixing.
-        # nodes.machine = { };
-
-        testScript =
-
-          { nodes, ... }:
-
-          ''
-            start_all()
-            machine.wait_for_unit("postgresql")
-            machine.wait_for_unit("sample-service")
-            # Create a test table
-            machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
-
-            machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
-            timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
-
-            # import time
-            # time.sleep(5400000)
-
-            machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
-            machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
-            machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
-            machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
-
-            machine.succeed("rm -rf /var/backup/postgres")
-
-            machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
-            machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
-
-            machine.succeed("""
-            set -x
-            ${nodes.machine.clan.core.state.test.postRestoreCommand}
-            """)
-            machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
-            machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
-
-            timestamp_after = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
-            assert timestamp_before < timestamp_after, f"{timestamp_before} >= {timestamp_after}: expected sample-service to be restarted after restore"
-
-            # Check that the table is still there
-            machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
-            output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
-            owner = output.split("\n")[1]
-            assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
-
-            # check if restore works if the database does not exist
-            machine.succeed("runuser -u postgres -- dropdb test")
-            machine.succeed("${nodes.machine.clan.core.state.test.postRestoreCommand}")
-            machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
-          '';
-      };
-    };
-}

pkgs/clan-app/clan_app/api/api_bridge.py

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any
 
 from clan_lib.api import ApiResponse
 from clan_lib.api.tasks import WebThread
-from clan_lib.async_run import set_current_thread_opkey, set_should_cancel
+from clan_lib.async_run import set_should_cancel
 
 if TYPE_CHECKING:
     from .middleware import Middleware
@@ -98,7 +98,7 @@ class ApiBridge(ABC):
         *,
         thread_name: str = "ApiBridgeThread",
         wait_for_completion: bool = False,
-        timeout: float = 60.0 * 60,  # 1 hour default timeout
+        timeout: float = 60.0,
     ) -> None:
         """Process an API request in a separate thread with cancellation support.
 
@@ -112,7 +112,6 @@ class ApiBridge(ABC):
 
         def thread_task(stop_event: threading.Event) -> None:
             set_should_cancel(lambda: stop_event.is_set())
-            set_current_thread_opkey(op_key)
             try:
                 log.debug(
                     f"Processing {request.method_name} with args {request.args} "

pkgs/clan-app/clan_app/api/file_gtk.py

@@ -9,7 +9,6 @@ gi.require_version("Gtk", "4.0")
 
 from clan_lib.api import ApiError, ErrorDataClass, SuccessDataClass
 from clan_lib.api.directory import FileRequest
-from clan_lib.async_run import get_current_thread_opkey
 from clan_lib.clan.check import check_clan_valid
 from clan_lib.flake import Flake
 from gi.repository import Gio, GLib, Gtk
@@ -25,7 +24,7 @@ def remove_none(_list: list) -> list:
 RESULT: dict[str, SuccessDataClass[list[str] | None] | ErrorDataClass] = {}
 
 
-def get_clan_folder() -> SuccessDataClass[Flake] | ErrorDataClass:
+def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass:
     """
     Opens the clan folder using the GTK file dialog.
     Returns the path to the clan folder or an error if it fails.
@@ -35,10 +34,7 @@ def get_clan_folder() -> SuccessDataClass[Flake] | ErrorDataClass:
         title="Select Clan Folder",
         initial_folder=str(Path.home()),
     )
-
-    response = get_system_file(file_request)
-
-    op_key = response.op_key
+    response = get_system_file(file_request, op_key=op_key)
 
     if isinstance(response, ErrorDataClass):
         return response
@@ -74,13 +70,8 @@ def get_clan_folder() -> SuccessDataClass[Flake] | ErrorDataClass:
 
 
 def get_system_file(
-    file_request: FileRequest,
+    file_request: FileRequest, *, op_key: str
 ) -> SuccessDataClass[list[str] | None] | ErrorDataClass:
-    op_key = get_current_thread_opkey()
-
-    if not op_key:
-        msg = "No operation key found in the current thread context."
-        raise RuntimeError(msg)
     GLib.idle_add(gtk_open_file, file_request, op_key)
 
     while RESULT.get(op_key) is None:

pkgs/clan-app/clan_app/api/middleware/argument_parsing.py

@@ -21,12 +21,18 @@ class ArgumentParsingMiddleware(Middleware):
             # Convert dictionary arguments to dataclass instances
             reconciled_arguments = {}
             for k, v in context.request.args.items():
+                if k == "op_key":
+                    continue
+
                 # Get the expected argument type from the API
                 arg_class = self.api.get_method_argtype(context.request.method_name, k)
 
                 # Convert dictionary to dataclass instance
                 reconciled_arguments[k] = from_dict(arg_class, v)
 
+            # Add op_key to arguments
+            reconciled_arguments["op_key"] = context.request.op_key
+
             # Create a new request with reconciled arguments
 
             updated_request = BackendRequest(

pkgs/clan-app/clan_app/deps/http/http_bridge.py

@@ -1,22 +1,13 @@
 import json
 import logging
-import threading
 import uuid
 from http.server import BaseHTTPRequestHandler
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 from urllib.parse import urlparse
 
-from clan_lib.api import (
-    MethodRegistry,
-    SuccessDataClass,
-    dataclass_to_dict,
-)
+from clan_lib.api import MethodRegistry, SuccessDataClass, dataclass_to_dict
 from clan_lib.api.tasks import WebThread
-from clan_lib.async_run import (
-    set_current_thread_opkey,
-    set_should_cancel,
-)
 
 from clan_app.api.api_bridge import ApiBridge, BackendRequest, BackendResponse
 
@@ -333,34 +324,17 @@ class HttpBridge(ApiBridge, BaseHTTPRequestHandler):
             msg = f"Operation key '{op_key}' is already in use. Please try again."
             raise ValueError(msg)
 
-    def process_request_in_thread(
-        self,
-        request: BackendRequest,
-        *,
-        thread_name: str = "ApiBridgeThread",
-        wait_for_completion: bool = False,
-        timeout: float = 60.0 * 60,  # 1 hour default timeout
-    ) -> None:
-        pass
-
     def _process_api_request_in_thread(
         self, api_request: BackendRequest, method_name: str
     ) -> None:
         """Process the API request in a separate thread."""
-        stop_event = threading.Event()
-        request = api_request
-        op_key = request.op_key or "unknown"
-        set_should_cancel(lambda: stop_event.is_set())
-        set_current_thread_opkey(op_key)
-
-        curr_thread = threading.current_thread()
-        self.threads[op_key] = WebThread(thread=curr_thread, stop_event=stop_event)
-
-        log.debug(
-            f"Processing {request.method_name} with args {request.args} "
-            f"and header {request.header}"
+        # Use the inherited thread processing method
+        self.process_request_in_thread(
+            api_request,
+            thread_name="HttpThread",
+            wait_for_completion=True,
+            timeout=60.0,
         )
-        self.process_request(request)
 
     def log_message(self, format: str, *args: Any) -> None:  # noqa: A002
         """Override default logging to use our logger."""

pkgs/clan-app/process-compose-2d.yaml

@@ -0,0 +1,39 @@
+version: "0.5"
+
+processes:
+  # App Dev
+
+  clan-app-ui:
+    namespace: "app"
+    command: |
+      cd $(git rev-parse --show-toplevel)/pkgs/clan-app/ui-2d
+      npm install
+      vite
+    ready_log_line: "VITE"
+
+  clan-app:
+    namespace: "app"
+    command: |
+      cd $(git rev-parse --show-toplevel)/pkgs/clan-app
+      ./bin/clan-app --debug --content-uri http://localhost:3000
+    depends_on:
+      clan-app-ui:
+        condition: "process_log_ready"
+    is_foreground: true
+    ready_log_line: "Debug mode enabled"
+
+  # Storybook Dev
+
+  storybook:
+    namespace: "storybook"
+    command: |
+      cd $(git rev-parse --show-toplevel)/pkgs/clan-app/ui-2d
+      npm run storybook-dev -- --ci
+    ready_log_line: "started"
+
+  luakit:
+    namespace: "storybook"
+    command: "luakit http://localhost:6006"
+    depends_on:
+      storybook:
+        condition: "process_log_ready"

pkgs/clan-app/ui/package-lock.json

@@ -17,7 +17,6 @@
         "@solidjs/router": "^0.15.3",
         "@tanstack/eslint-plugin-query": "^5.51.12",
         "@tanstack/solid-query": "^5.76.0",
-        "@tanstack/solid-query-devtools": "^5.83.0",
         "solid-js": "^1.9.7",
         "solid-toast": "^0.5.0",
         "three": "^0.176.0",
@@ -54,6 +53,7 @@
         "postcss": "^8.4.38",
         "postcss-url": "^10.1.3",
         "prettier": "^3.2.5",
+        "solid-devtools": "^0.34.0",
         "storybook": "^9.0.8",
         "swagger-ui-dist": "^5.26.2",
         "tailwindcss": "^3.4.3",
@@ -360,6 +360,22 @@
         "@babel/core": "^7.0.0-0"
       }
     },
+    "node_modules/@babel/plugin-syntax-typescript": {
+      "version": "7.27.1",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.27.1.tgz",
+      "integrity": "sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@babel/helper-plugin-utils": "^7.27.1"
+      },
+      "engines": {
+        "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
+      }
+    },
     "node_modules/@babel/runtime": {
       "version": "7.27.6",
       "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.27.6.tgz",
@@ -1536,6 +1552,13 @@
         "node": ">= 8"
       }
     },
+    "node_modules/@nothing-but/utils": {
+      "version": "0.17.0",
+      "resolved": "https://registry.npmjs.org/@nothing-but/utils/-/utils-0.17.0.tgz",
+      "integrity": "sha512-TuCHcHLOqDL0SnaAxACfuRHBNRgNJcNn9X0GiH5H3YSDBVquCr3qEIG3FOQAuMyZCbu9w8nk2CHhOsn7IvhIwQ==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/@oxc-resolver/binding-darwin-arm64": {
       "version": "11.5.0",
       "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-darwin-arm64/-/binding-darwin-arm64-11.5.0.tgz",
@@ -1790,6 +1813,64 @@
         "@sinonjs/commons": "^3.0.1"
       }
     },
+    "node_modules/@solid-devtools/debugger": {
+      "version": "0.28.1",
+      "resolved": "https://registry.npmjs.org/@solid-devtools/debugger/-/debugger-0.28.1.tgz",
+      "integrity": "sha512-6qIUI6VYkXoRnL8oF5bvh2KgH71qlJ18hNw/mwSyY6v48eb80ZR48/5PDXufUa3q+MBSuYa1uqTMwLewpay9eg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@nothing-but/utils": "~0.17.0",
+        "@solid-devtools/shared": "^0.20.0",
+        "@solid-primitives/bounds": "^0.1.1",
+        "@solid-primitives/event-listener": "^2.4.1",
+        "@solid-primitives/keyboard": "^1.3.1",
+        "@solid-primitives/rootless": "^1.5.1",
+        "@solid-primitives/scheduled": "^1.5.1",
+        "@solid-primitives/static-store": "^0.1.1",
+        "@solid-primitives/utils": "^6.3.1"
+      },
+      "peerDependencies": {
+        "solid-js": "^1.9.0"
+      }
+    },
+    "node_modules/@solid-devtools/shared": {
+      "version": "0.20.0",
+      "resolved": "https://registry.npmjs.org/@solid-devtools/shared/-/shared-0.20.0.tgz",
+      "integrity": "sha512-o5TACmUOQsxpzpOKCjbQqGk8wL8PMi+frXG9WNu4Lh3PQVUB6hs95Kl/S8xc++zwcMguUKZJn8h5URUiMOca6Q==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@nothing-but/utils": "~0.17.0",
+        "@solid-primitives/event-listener": "^2.4.1",
+        "@solid-primitives/media": "^2.3.1",
+        "@solid-primitives/refs": "^1.1.1",
+        "@solid-primitives/rootless": "^1.5.1",
+        "@solid-primitives/scheduled": "^1.5.1",
+        "@solid-primitives/static-store": "^0.1.1",
+        "@solid-primitives/styles": "^0.1.1",
+        "@solid-primitives/utils": "^6.3.1"
+      },
+      "peerDependencies": {
+        "solid-js": "^1.9.0"
+      }
+    },
+    "node_modules/@solid-primitives/bounds": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/@solid-primitives/bounds/-/bounds-0.1.3.tgz",
+      "integrity": "sha512-UbiyKMdSPmtijcEDnYLQL3zzaejpwWDAJJ4Gt5P0hgVs6A72piov0GyNw7V2SroH7NZFwxlYS22YmOr8A5xc1Q==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@solid-primitives/event-listener": "^2.4.3",
+        "@solid-primitives/resize-observer": "^2.1.3",
+        "@solid-primitives/static-store": "^0.1.2",
+        "@solid-primitives/utils": "^6.3.2"
+      },
+      "peerDependencies": {
+        "solid-js": "^1.6.12"
+      }
+    },
     "node_modules/@solid-primitives/event-listener": {
       "version": "2.4.3",
       "resolved": "https://registry.npmjs.org/@solid-primitives/event-listener/-/event-listener-2.4.3.tgz",
@@ -1802,6 +1883,21 @@
         "solid-js": "^1.6.12"
       }
     },
+    "node_modules/@solid-primitives/keyboard": {
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/@solid-primitives/keyboard/-/keyboard-1.3.3.tgz",
+      "integrity": "sha512-9dQHTTgLBqyAI7aavtO+HnpTVJgWQA1ghBSrmLtMu1SMxLPDuLfuNr+Tk5udb4AL4Ojg7h9JrKOGEEDqsJXWJA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@solid-primitives/event-listener": "^2.4.3",
+        "@solid-primitives/rootless": "^1.5.2",
+        "@solid-primitives/utils": "^6.3.2"
+      },
+      "peerDependencies": {
+        "solid-js": "^1.6.12"
+      }
+    },
     "node_modules/@solid-primitives/keyed": {
       "version": "1.5.2",
       "resolved": "https://registry.npmjs.org/@solid-primitives/keyed/-/keyed-1.5.2.tgz",
@@ -1889,6 +1985,16 @@
         "solid-js": "^1.6.12"
       }
     },
+    "node_modules/@solid-primitives/scheduled": {
+      "version": "1.5.2",
+      "resolved": "https://registry.npmjs.org/@solid-primitives/scheduled/-/scheduled-1.5.2.tgz",
+      "integrity": "sha512-/j2igE0xyNaHhj6kMfcUQn5rAVSTLbAX+CDEBm25hSNBmNiHLu2lM7Usj2kJJ5j36D67bE8wR1hBNA8hjtvsQA==",
+      "dev": true,
+      "license": "MIT",
+      "peerDependencies": {
+        "solid-js": "^1.6.12"
+      }
+    },
     "node_modules/@solid-primitives/static-store": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/@solid-primitives/static-store/-/static-store-0.1.2.tgz",
@@ -1922,6 +2028,20 @@
         }
       }
     },
+    "node_modules/@solid-primitives/styles": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/@solid-primitives/styles/-/styles-0.1.2.tgz",
+      "integrity": "sha512-7iX5K+J5b1PRrbgw3Ki92uvU2LgQ0Kd/QMsrAZxDg5dpUBwMyTijZkA3bbs1ikZsT1oQhS41bTyKbjrXeU0Awg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@solid-primitives/rootless": "^1.5.2",
+        "@solid-primitives/utils": "^6.3.2"
+      },
+      "peerDependencies": {
+        "solid-js": "^1.6.12"
+      }
+    },
     "node_modules/@solid-primitives/trigger": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/@solid-primitives/trigger/-/trigger-1.2.2.tgz",
@@ -2161,19 +2281,9 @@
       }
     },
     "node_modules/@tanstack/query-core": {
-      "version": "5.83.0",
-      "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.83.0.tgz",
-      "integrity": "sha512-0M8dA+amXUkyz5cVUm/B+zSk3xkQAcuXuz5/Q/LveT4ots2rBpPTZOzd7yJa2Utsf8D2Upl5KyjhHRY+9lB/XA==",
-      "license": "MIT",
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/tannerlinsley"
-      }
-    },
-    "node_modules/@tanstack/query-devtools": {
-      "version": "5.81.2",
-      "resolved": "https://registry.npmjs.org/@tanstack/query-devtools/-/query-devtools-5.81.2.tgz",
-      "integrity": "sha512-jCeJcDCwKfoyyBXjXe9+Lo8aTkavygHHsUHAlxQKKaDeyT0qyQNLKl7+UyqYH2dDF6UN/14873IPBHchcsU+Zg==",
+      "version": "5.81.5",
+      "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.81.5.tgz",
+      "integrity": "sha512-ZJOgCy/z2qpZXWaj/oxvodDx07XcQa9BF92c0oINjHkoqUPsmm3uG08HpTaviviZ/N9eP1f9CM7mKSEkIo7O1Q==",
       "license": "MIT",
       "funding": {
         "type": "github",
@@ -2181,12 +2291,12 @@
       }
     },
     "node_modules/@tanstack/solid-query": {
-      "version": "5.83.0",
-      "resolved": "https://registry.npmjs.org/@tanstack/solid-query/-/solid-query-5.83.0.tgz",
-      "integrity": "sha512-RF8Tv9+6+Kmzj+EafbTzvzzPq+J5SzHtc1Tz3D2MZ/EvlZTH+GL5q4HNnWK3emg7CB6WzyGnTuERmmWJaZs8/w==",
+      "version": "5.81.5",
+      "resolved": "https://registry.npmjs.org/@tanstack/solid-query/-/solid-query-5.81.5.tgz",
+      "integrity": "sha512-VqVXaxiJIsKA6B45uApF+RUD3g8Roj/vdAuGpHMjR+RyHqlyQ+hOwgmALkzlbkbIaWCQi8CJOvrbU6WOBuMOxA==",
       "license": "MIT",
       "dependencies": {
-        "@tanstack/query-core": "5.83.0"
+        "@tanstack/query-core": "5.81.5"
       },
       "funding": {
         "type": "github",
@@ -2196,23 +2306,6 @@
         "solid-js": "^1.6.0"
       }
     },
-    "node_modules/@tanstack/solid-query-devtools": {
-      "version": "5.83.0",
-      "resolved": "https://registry.npmjs.org/@tanstack/solid-query-devtools/-/solid-query-devtools-5.83.0.tgz",
-      "integrity": "sha512-Z0wQlAWXz/U2bJ/paMRBTDhMoPnB9Te6GmA21sXnI+nDnAAPZRcPxFBiCgYJS3eFsvbkdRGJwoUSQrdIgy0shg==",
-      "license": "MIT",
-      "dependencies": {
-        "@tanstack/query-devtools": "5.81.2"
-      },
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/tannerlinsley"
-      },
-      "peerDependencies": {
-        "@tanstack/solid-query": "^5.83.0",
-        "solid-js": "^1.6.0"
-      }
-    },
     "node_modules/@testing-library/dom": {
       "version": "10.4.0",
       "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.0.tgz",
@@ -6903,6 +6996,29 @@
         "url": "https://github.com/sponsors/cyyynthia"
       }
     },
+    "node_modules/solid-devtools": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/solid-devtools/-/solid-devtools-0.34.3.tgz",
+      "integrity": "sha512-ZQua959n+Zu3sLbm9g0IRjYUb1YYlYbu83PWLRoKbSsq0a3ItQNhnS2OBU7rQNmOKZiMexNo9Z3izas9BcOKDg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@babel/core": "^7.27.4",
+        "@babel/plugin-syntax-typescript": "^7.27.1",
+        "@babel/types": "^7.27.6",
+        "@solid-devtools/debugger": "^0.28.1",
+        "@solid-devtools/shared": "^0.20.0"
+      },
+      "peerDependencies": {
+        "solid-js": "^1.9.0",
+        "vite": "^2.2.3 || ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0"
+      },
+      "peerDependenciesMeta": {
+        "vite": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/solid-js": {
       "version": "1.9.7",
       "resolved": "https://registry.npmjs.org/solid-js/-/solid-js-1.9.7.tgz",

pkgs/clan-app/ui/package.json

@@ -52,6 +52,7 @@
     "postcss": "^8.4.38",
     "postcss-url": "^10.1.3",
     "prettier": "^3.2.5",
+    "solid-devtools": "^0.34.0",
     "storybook": "^9.0.8",
     "swagger-ui-dist": "^5.26.2",
     "tailwindcss": "^3.4.3",
@@ -72,7 +73,6 @@
     "@solidjs/router": "^0.15.3",
     "@tanstack/eslint-plugin-query": "^5.51.12",
     "@tanstack/solid-query": "^5.76.0",
-    "@tanstack/solid-query-devtools": "^5.83.0",
     "solid-js": "^1.9.7",
     "solid-toast": "^0.5.0",
     "three": "^0.176.0",

pkgs/clan-app/ui/src/components/Sidebar/Sidebar.stories.tsx

@@ -1,157 +0,0 @@
-import type { Meta, StoryObj } from "@kachurun/storybook-solid";
-import {
-  createMemoryHistory,
-  MemoryRouter,
-  Route,
-  RouteSectionProps,
-} from "@solidjs/router";
-import { Sidebar } from "@/src/components/Sidebar/Sidebar";
-import { Suspense } from "solid-js";
-import { QueryClient, QueryClientProvider } from "@tanstack/solid-query";
-import { addClanURI, resetStore } from "@/src/stores/clan";
-import { SolidQueryDevtools } from "@tanstack/solid-query-devtools";
-import { encodeBase64 } from "@/src/hooks/clan";
-
-const defaultClanURI = "/home/brian/clans/my-clan";
-
-const queryData = {
-  "/home/brian/clans/my-clan": {
-    details: {
-      name: "Brian's Clan",
-      uri: "/home/brian/clans/my-clan",
-    },
-    machines: {
-      europa: {
-        name: "Europa",
-        machineClass: "nixos",
-      },
-      ganymede: {
-        name: "Ganymede",
-        machineClass: "nixos",
-      },
-    },
-  },
-  "/home/brian/clans/davhau": {
-    details: {
-      name: "Dave's Clan",
-      uri: "/home/brian/clans/davhau",
-    },
-    machines: {
-      callisto: {
-        name: "Callisto",
-        machineClass: "nixos",
-      },
-      amalthea: {
-        name: "Amalthea",
-        machineClass: "nixos",
-      },
-    },
-  },
-  "/home/brian/clans/mic92": {
-    details: {
-      name: "Mic92's Clan",
-      uri: "/home/brian/clans/mic92",
-    },
-    machines: {
-      thebe: {
-        name: "Thebe",
-        machineClass: "nixos",
-      },
-      sponde: {
-        name: "Sponde",
-        machineClass: "nixos",
-      },
-    },
-  },
-};
-
-const staticSections = [
-  {
-    title: "Links",
-    links: [
-      { label: "GitHub", path: "https://github.com/brian-the-dev" },
-      { label: "Twitter", path: "https://twitter.com/brian_the_dev" },
-      {
-        label: "LinkedIn",
-        path: "https://www.linkedin.com/in/brian-the-dev/",
-      },
-      {
-        label: "Instagram",
-        path: "https://www.instagram.com/brian_the_dev/",
-      },
-    ],
-  },
-];
-
-const meta: Meta<RouteSectionProps> = {
-  title: "Components/Sidebar",
-  component: Sidebar,
-  render: () => {
-    // set history to point to our test clan
-    const history = createMemoryHistory();
-    history.set({ value: `/clans/${encodeBase64(defaultClanURI)}` });
-
-    // reset local storage and then add each clan
-    resetStore();
-
-    Object.keys(queryData).forEach((uri) => addClanURI(uri));
-
-    return (
-      <div style="height: 670px;">
-        <MemoryRouter
-          history={history}
-          root={(props) => <Suspense>{props.children}</Suspense>}
-        >
-          <Route
-            path="/clans/:clanURI"
-            component={() => <Sidebar staticSections={staticSections} />}
-          >
-            <Route path="/" />
-            <Route
-              path="/machines/:machineID"
-              component={() => <h1>Machine</h1>}
-            />
-          </Route>
-        </MemoryRouter>
-        <SolidQueryDevtools initialIsOpen={true} />
-      </div>
-    );
-  },
-};
-
-export default meta;
-
-type Story = StoryObj<RouteSectionProps>;
-
-export const Default: Story = {
-  args: {},
-  decorators: [
-    (Story: StoryObj) => {
-      const queryClient = new QueryClient({
-        defaultOptions: {
-          queries: {
-            retry: false,
-            staleTime: Infinity,
-          },
-        },
-      });
-
-      Object.entries(queryData).forEach(([clanURI, clan]) => {
-        queryClient.setQueryData(
-          ["clans", encodeBase64(clanURI), "details"],
-          clan.details,
-        );
-        queryClient.setQueryData(
-          ["clans", encodeBase64(clanURI), "machines"],
-          clan.machines || {},
-        );
-      });
-
-      return (
-        <QueryClientProvider client={queryClient}>
-          <Story />
-        </QueryClientProvider>
-      );
-    },
-  ],
-};

pkgs/clan-app/ui/src/components/Sidebar/Sidebar.tsx

@@ -1,28 +0,0 @@
-import "./Sidebar.css";
-import { SidebarHeader } from "@/src/components/Sidebar/SidebarHeader";
-import { SidebarBody } from "@/src/components/Sidebar/SidebarBody";
-
-export interface LinkProps {
-  path: string;
-  label?: string;
-}
-
-export interface SectionProps {
-  title: string;
-  links: LinkProps[];
-}
-
-export interface SidebarProps {
-  staticSections?: SectionProps[];
-}
-
-export const Sidebar = (props: SidebarProps) => {
-  return (
-    <>
-      <div class="sidebar">
-        <SidebarHeader />
-        <SidebarBody {...props} />
-      </div>
-    </>
-  );
-};

pkgs/clan-app/ui/src/components/Sidebar/SidebarHeader.tsx

@@ -1,107 +0,0 @@
-import "./SidebarHeader.css";
-import Icon from "@/src/components/Icon/Icon";
-import { DropdownMenu } from "@kobalte/core/dropdown-menu";
-import { useNavigate } from "@solidjs/router";
-import { Typography } from "../Typography/Typography";
-import { createSignal, For, Suspense } from "solid-js";
-import { useClanListQuery } from "@/src/queries/queries";
-import { navigateToClan, useClanURI } from "@/src/hooks/clan";
-import { clanURIs } from "@/src/stores/clan";
-
-export const SidebarHeader = () => {
-  const navigate = useNavigate();
-
-  const [open, setOpen] = createSignal(false);
-
-  // get information about the current active clan
-  const clanURI = useClanURI();
-  const allClans = useClanListQuery(clanURIs());
-
-  const activeClan = () => allClans.find(({ data }) => data?.uri === clanURI);
-
-  return (
-    <div class="sidebar-header">
-      <Suspense fallback={"Loading..."}>
-        <DropdownMenu open={open()} onOpenChange={setOpen} sameWidth={true}>
-          <DropdownMenu.Trigger class="dropdown-trigger">
-            <div class="clan-label">
-              <div class="clan-icon">
-                <Typography
-                  hierarchy="label"
-                  size="s"
-                  weight="bold"
-                  inverted={true}
-                >
-                  {activeClan()?.data?.name.charAt(0).toUpperCase()}
-                </Typography>
-              </div>
-              <Typography
-                hierarchy="label"
-                size="s"
-                weight="bold"
-                inverted={!open()}
-              >
-                {activeClan()?.data?.name}
-              </Typography>
-            </div>
-            <DropdownMenu.Icon>
-              <Icon icon={"CaretDown"} inverted={!open()} size="0.75rem" />
-            </DropdownMenu.Icon>
-          </DropdownMenu.Trigger>
-          <DropdownMenu.Portal>
-            <DropdownMenu.Content class="sidebar-dropdown-content">
-              <DropdownMenu.Item
-                class="dropdown-item"
-                onSelect={() => navigateToClan(navigate, clanURI)}
-              >
-                <Icon
-                  icon="Settings"
-                  size="0.75rem"
-                  inverted={true}
-                  color="tertiary"
-                />
-                <Typography hierarchy="label" size="xs" weight="medium">
-                  Settings
-                </Typography>
-              </DropdownMenu.Item>
-              <DropdownMenu.Group class="dropdown-group">
-                <DropdownMenu.GroupLabel class="dropdown-group-label">
-                  <Typography
-                    hierarchy="label"
-                    family="mono"
-                    size="xs"
-                    color="tertiary"
-                  >
-                    YOUR CLANS
-                  </Typography>
-                </DropdownMenu.GroupLabel>
-                <div class="dropdown-group-items">
-                  <For each={allClans}>
-                    {(clan) => (
-                      <Suspense fallback={"Loading..."}>
-                        <DropdownMenu.Item
-                          class="dropdown-item"
-                          onSelect={() =>
-                            navigateToClan(navigate, clan.data!.uri)
-                          }
-                        >
-                          <Typography
-                            hierarchy="label"
-                            size="xs"
-                            weight="medium"
-                          >
-                            {clan.data?.name}
-                          </Typography>
-                        </DropdownMenu.Item>
-                      </Suspense>
-                    )}
-                  </For>
-                </div>
-              </DropdownMenu.Group>
-            </DropdownMenu.Content>
-          </DropdownMenu.Portal>
-        </DropdownMenu>
-      </Suspense>
-    </div>
-  );
-};

pkgs/clan-app/ui/src/components/Sidebar/SidebarNav.css

@@ -1,5 +1,5 @@
 div.sidebar {
-  @apply w-60 border-none z-10;
+  @apply h-full w-auto max-w-60 border-none;
 
   & > div.header {
   }

pkgs/clan-app/ui/src/components/Sidebar/SidebarNav.stories.tsx

@@ -0,0 +1,109 @@
+import type { Meta, StoryObj } from "@kachurun/storybook-solid";
+import {
+  createMemoryHistory,
+  MemoryRouter,
+  Route,
+  RouteSectionProps,
+} from "@solidjs/router";
+import {
+  SidebarNav,
+  SidebarNavProps,
+} from "@/src/components/Sidebar/SidebarNav";
+import { Suspense } from "solid-js";
+import { StoryContext } from "@kachurun/storybook-solid-vite";
+
+const sidebarNavProps: SidebarNavProps = {
+  clanLinks: [
+    { label: "Brian's Clan", path: "/clans/1" },
+    { label: "Dave's Clan", path: "/clans/2" },
+    { label: "Mic92's Clan", path: "/clans/3" },
+  ],
+  clanDetail: {
+    label: "Brian's Clan",
+    settingsPath: "/clans/1/settings",
+    machines: [
+      {
+        label: "Backup & Home",
+        path: "/clans/1/machine/backup",
+        serviceCount: 3,
+        status: "Online",
+      },
+      {
+        label: "Raspberry Pi",
+        path: "/clans/1/machine/pi",
+        serviceCount: 1,
+        status: "Offline",
+      },
+      {
+        label: "Mom's Laptop",
+        path: "/clans/1/machine/moms-laptop",
+        serviceCount: 2,
+        status: "Installed",
+      },
+      {
+        label: "Dad's Laptop",
+        path: "/clans/1/machine/dads-laptop",
+        serviceCount: 4,
+        status: "Not Installed",
+      },
+    ],
+  },
+  extraSections: [
+    {
+      label: "Tools",
+      links: [
+        { label: "Borgbackup", path: "/clans/1/service/borgbackup" },
+        { label: "Syncthing", path: "/clans/1/service/syncthing" },
+        { label: "Mumble", path: "/clans/1/service/mumble" },
+        { label: "Minecraft", path: "/clans/1/service/minecraft" },
+      ],
+    },
+    {
+      label: "Links",
+      links: [
+        { label: "GitHub", path: "https://github.com/brian-the-dev" },
+        { label: "Twitter", path: "https://twitter.com/brian_the_dev" },
+        {
+          label: "LinkedIn",
+          path: "https://www.linkedin.com/in/brian-the-dev/",
+        },
+        {
+          label: "Instagram",
+          path: "https://www.instagram.com/brian_the_dev/",
+        },
+      ],
+    },
+  ],
+};
+
+const meta: Meta<RouteSectionProps> = {
+  title: "Components/Sidebar/Nav",
+  component: SidebarNav,
+  render: (_: never, context: StoryContext<SidebarNavProps>) => {
+    const history = createMemoryHistory();
+    history.set({ value: "/clans/1/machine/backup" });
+
+    return (
+      <div style="height: 670px;">
+        <MemoryRouter
+          history={history}
+          root={(props) => (
+            <Suspense>
+              <SidebarNav {...sidebarNavProps} />
+            </Suspense>
+          )}
+        >
+          <Route path="/clans/1/machine/backup" component={(props) => <></>} />
+        </MemoryRouter>
+      </div>
+    );
+  },
+};
+
+export default meta;
+
+type Story = StoryObj<RouteSectionProps>;
+
+export const Default: Story = {
+  args: {},
+};

pkgs/clan-app/ui/src/components/Sidebar/SidebarNav.tsx

@@ -0,0 +1,47 @@
+import "./SidebarNav.css";
+import { SidebarNavHeader } from "@/src/components/Sidebar/SidebarNavHeader";
+import { SidebarNavBody } from "@/src/components/Sidebar/SidebarNavBody";
+import { MachineStatus } from "@/src/components/MachineStatus/MachineStatus";
+
+export interface LinkProps {
+  path: string;
+  label?: string;
+}
+
+export interface SectionProps {
+  label: string;
+  links: LinkProps[];
+}
+
+export interface MachineProps {
+  label: string;
+  path: string;
+  status: MachineStatus;
+  serviceCount: number;
+}
+
+export interface ClanLinkProps {
+  label: string;
+  path: string;
+}
+
+export interface ClanProps {
+  label: string;
+  settingsPath: string;
+  machines: MachineProps[];
+}
+
+export interface SidebarNavProps {
+  clanDetail: ClanProps;
+  clanLinks: ClanLinkProps[];
+  extraSections: SectionProps[];
+}
+
+export const SidebarNav = (props: SidebarNavProps) => {
+  return (
+    <div class="sidebar">
+      <SidebarNavHeader {...props} />
+      <SidebarNavBody {...props} />
+    </div>
+  );
+};

pkgs/clan-app/ui/src/components/Sidebar/SidebarNavBody.tsx

@@ -1,60 +1,46 @@
-import "./SidebarBody.css";
+import "./SidebarNavBody.css";
 import { A } from "@solidjs/router";
 import { Accordion } from "@kobalte/core/accordion";
 import Icon from "../Icon/Icon";
 import { Typography } from "@/src/components/Typography/Typography";
+import {
+  MachineProps,
+  SidebarNavProps,
+} from "@/src/components/Sidebar/SidebarNav";
 import { For } from "solid-js";
 import { MachineStatus } from "@/src/components/MachineStatus/MachineStatus";
-import { buildMachinePath, useClanURI } from "@/src/hooks/clan";
-import { useMachinesQuery } from "@/src/queries/queries";
-import { SidebarProps } from "./Sidebar";
-
-interface MachineProps {
-  clanURI: string;
-  machineID: string;
-  name: string;
-  status: MachineStatus;
-  serviceCount: number;
-}
 
 const MachineRoute = (props: MachineProps) => (
-  <A href={buildMachinePath(props.clanURI, props.machineID)}>
-    <div class="flex w-full flex-col gap-2">
-      <div class="flex flex-row items-center justify-between">
-        <Typography
-          hierarchy="label"
-          size="xs"
-          weight="bold"
-          color="primary"
-          inverted={true}
-        >
-          {props.name}
-        </Typography>
-        <MachineStatus status={props.status} />
-      </div>
-      <div class="flex w-full flex-row items-center gap-1">
-        <Icon icon="Flash" size="0.75rem" inverted={true} color="tertiary" />
-        <Typography
-          hierarchy="label"
-          family="mono"
-          size="s"
-          inverted={true}
-          color="primary"
-        >
-          {props.serviceCount}
-        </Typography>
-      </div>
+  <div class="flex w-full flex-col gap-2">
+    <div class="flex flex-row items-center justify-between">
+      <Typography
+        hierarchy="label"
+        size="xs"
+        weight="bold"
+        color="primary"
+        inverted={true}
+      >
+        {props.label}
+      </Typography>
+      <MachineStatus status={props.status} />
     </div>
-  </A>
+    <div class="flex w-full flex-row items-center gap-1">
+      <Icon icon="Flash" size="0.75rem" inverted={true} color="tertiary" />
+      <Typography
+        hierarchy="label"
+        family="mono"
+        size="s"
+        inverted={true}
+        color="primary"
+      >
+        {props.serviceCount}
+      </Typography>
+    </div>
+  </div>
 );
 
-export const SidebarBody = (props: SidebarProps) => {
-  const clanURI = useClanURI();
-  const machineList = useMachinesQuery(clanURI);
-
-  const sectionLabels = (props.staticSections || []).map(
-    (section) => section.title,
-  );
+export const SidebarNavBody = (props: SidebarNavProps) => {
+  const sectionLabels = props.extraSections.map((section) => section.label);
 
   // controls which sections are open by default
   // we want them all to be open by default
@@ -90,24 +76,20 @@ export const SidebarBody = (props: SidebarProps) => {
           </Accordion.Header>
           <Accordion.Content class="content">
             <nav>
-              <For each={Object.entries(machineList.data || {})}>
-                {([id, machine]) => (
-                  <MachineRoute
-                    clanURI={clanURI}
-                    machineID={id}
-                    name={machine.name || id}
-                    status="Not Installed"
-                    serviceCount={0}
-                  />
+              <For each={props.clanDetail.machines}>
+                {(machine) => (
+                  <A href={machine.path}>
+                    <MachineRoute {...machine} />
+                  </A>
                 )}
               </For>
             </nav>
           </Accordion.Content>
         </Accordion.Item>
 
-        <For each={props.staticSections}>
+        <For each={props.extraSections}>
           {(section) => (
-            <Accordion.Item class="item" value={section.title}>
+            <Accordion.Item class="item" value={section.label}>
               <Accordion.Header class="header">
                 <Accordion.Trigger class="trigger">
                   <Typography
@@ -118,7 +100,7 @@ export const SidebarBody = (props: SidebarProps) => {
                     inverted={true}
                     color="tertiary"
                   >
-                    {section.title}
+                    {section.label}
                   </Typography>
                   <Icon
                     icon="CaretDown"

pkgs/clan-app/ui/src/components/Sidebar/SidebarNavHeader.css

@@ -15,11 +15,10 @@ div.sidebar-header {
 
     transition: all 250ms ease-in-out;
 
-    div.clan-label {
+    div.title {
       @apply flex items-center gap-2 justify-start;
 
       & > .clan-icon {
-        @apply flex justify-center items-center;
         @apply rounded-full bg-inv-4 w-7 h-7;
       }
     }
@@ -39,7 +38,7 @@ div.sidebar-header {
 }
 
 .sidebar-dropdown-content {
-  @apply flex flex-col w-full px-2 py-1.5 z-10;
+  @apply flex flex-col w-full px-2 py-1.5;
   @apply bg-def-1 rounded-bl-md rounded-br-md;
   @apply border border-def-2;
 

pkgs/clan-app/ui/src/components/Sidebar/SidebarNavHeader.tsx

@@ -0,0 +1,96 @@
+import "./SidebarNavHeader.css";
+import Icon from "@/src/components/Icon/Icon";
+import { DropdownMenu } from "@kobalte/core/dropdown-menu";
+import { useNavigate } from "@solidjs/router";
+import { Typography } from "../Typography/Typography";
+import { createSignal, For } from "solid-js";
+import { ClanLinkProps, ClanProps } from "@/src/components/Sidebar/SidebarNav";
+
+export interface SidebarHeaderProps {
+  clanDetail: ClanProps;
+  clanLinks: ClanLinkProps[];
+}
+
+export const SidebarNavHeader = (props: SidebarHeaderProps) => {
+  const navigate = useNavigate();
+
+  const [open, setOpen] = createSignal(false);
+
+  const firstChar = props.clanDetail.label.charAt(0);
+
+  return (
+    <div class="sidebar-header">
+      <DropdownMenu open={open()} onOpenChange={setOpen} sameWidth={true}>
+        <DropdownMenu.Trigger class="dropdown-trigger">
+          <div class="title">
+            <div class="clan-icon">
+              <Typography
+                hierarchy="label"
+                size="s"
+                weight="bold"
+                inverted={true}
+              >
+                {firstChar.toUpperCase()}
+              </Typography>
+            </div>
+            <Typography
+              hierarchy="label"
+              size="s"
+              weight="bold"
+              inverted={!open()}
+            >
+              {props.clanDetail.label}
+            </Typography>
+          </div>
+          <DropdownMenu.Icon>
+            <Icon icon={"CaretDown"} inverted={!open()} size="0.75rem" />
+          </DropdownMenu.Icon>
+        </DropdownMenu.Trigger>
+        <DropdownMenu.Portal>
+          <DropdownMenu.Content class="sidebar-dropdown-content">
+            <DropdownMenu.Item
+              class="dropdown-item"
+              onSelect={() => navigate(props.clanDetail.settingsPath)}
+            >
+              <Icon
+                icon="Settings"
+                size="0.75rem"
+                inverted={true}
+                color="tertiary"
+              />
+              <Typography hierarchy="label" size="xs" weight="medium">
+                Settings
+              </Typography>
+            </DropdownMenu.Item>
+            <DropdownMenu.Group class="dropdown-group">
+              <DropdownMenu.GroupLabel class="dropdown-group-label">
+                <Typography
+                  hierarchy="label"
+                  family="mono"
+                  size="xs"
+                  color="tertiary"
+                >
+                  YOUR CLANS
+                </Typography>
+              </DropdownMenu.GroupLabel>
+              <div class="dropdown-group-items">
+                <For each={props.clanLinks}>
+                  {(clan) => (
+                    <DropdownMenu.Item
+                      class="dropdown-item"
+                      onSelect={() => navigate(clan.path)}
+                    >
+                      <Typography hierarchy="label" size="xs" weight="medium">
+                        {clan.label}
+                      </Typography>
+                    </DropdownMenu.Item>
+                  )}
+                </For>
+              </div>
+            </DropdownMenu.Group>
+          </DropdownMenu.Content>
+        </DropdownMenu.Portal>
+      </DropdownMenu>
+    </div>
+  );
+};

pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.css

@@ -1,20 +1,5 @@
 div.sidebar-pane {
-  @apply border-none z-10;
-
-  animation: sidebarPaneShow 250ms ease-in forwards;
-
-  &.open {
-    @apply w-60;
-  }
-
-  &.closing {
-    animation: sidebarPaneHide 250ms ease-out 300ms forwards;
-
-    & > div.header > *,
-    & > div.body > * {
-      animation: sidebarFadeOut 250ms ease-out forwards;
-    }
-  }
+  @apply h-full w-auto max-w-60 border-none;
 
   & > div.header {
     @apply flex items-center justify-between px-3 py-2 rounded-t-[0.5rem];
@@ -22,17 +7,11 @@ div.sidebar-pane {
         border-r-[1px] border-r-bg-inv-3
         border-b-2 border-b-bg-inv-4
         border-l-[1px] border-l-bg-inv-3;
-
     background: linear-gradient(
       90deg,
       theme(colors.bg.inv.3) 0%,
       theme(colors.bg.inv.4) 100%
     );
-
-    & > * {
-      @apply opacity-0;
-      animation: sidebarFadeIn 250ms ease-in 250ms forwards;
-    }
   }
 
   & > div.body {
@@ -50,54 +29,5 @@ div.sidebar-pane {
         theme(colors.bg.inv.2) 0%,
         theme(colors.bg.inv.3) 100%
       );
-
-    & > * {
-      @apply opacity-0;
-      animation: sidebarFadeIn 250ms ease-in 350ms forwards;
-    }
-  }
-}
-
-@keyframes sidebarPaneShow {
-  0% {
-    @apply w-0;
-    @apply opacity-0;
-  }
-  10% {
-    @apply w-8;
-  }
-  30% {
-    @apply opacity-100;
-  }
-  100% {
-    @apply w-60;
-  }
-}
-
-@keyframes sidebarPaneHide {
-  90% {
-    @apply w-8;
-  }
-  100% {
-    @apply w-0;
-    @apply opacity-0;
-  }
-}
-
-@keyframes sidebarFadeIn {
-  from {
-    opacity: 0;
-  }
-  to {
-    opacity: 1;
-  }
-}
-
-@keyframes sidebarFadeOut {
-  from {
-    opacity: 1;
-  }
-  to {
-    opacity: 0;
   }
 }

pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.stories.tsx

@@ -11,7 +11,7 @@ import { Checkbox } from "@/src/components/Form/Checkbox";
 import { Combobox } from "../Form/Combobox";
 
 const meta: Meta<SidebarPaneProps> = {
-  title: "Components/SidebarPane",
+  title: "Components/Sidebar/Pane",
   component: SidebarPane,
 };
 

pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.tsx

@@ -1,9 +1,8 @@
-import { createSignal, JSX, onMount } from "solid-js";
+import { JSX } from "solid-js";
 import "./SidebarPane.css";
 import { Typography } from "@/src/components/Typography/Typography";
 import Icon from "../Icon/Icon";
 import { Button as KButton } from "@kobalte/core/button";
-import cx from "classnames";
 
 export interface SidebarPaneProps {
   title: string;
@@ -12,26 +11,13 @@ export interface SidebarPaneProps {
 }
 
 export const SidebarPane = (props: SidebarPaneProps) => {
-  const [closing, setClosing] = createSignal(false);
-  const [open, setOpened] = createSignal(true);
-
-  const onClose = () => {
-    setClosing(true);
-    setTimeout(() => props.onClose(), 550);
-  };
-  onMount(() => {
-    setTimeout(() => {
-      setOpened(true);
-    }, 250);
-  });
-
   return (
-    <div class={cx("sidebar-pane", { closing: closing(), open: open() })}>
+    <div class="sidebar-pane">
       <div class="header">
         <Typography hierarchy="body" size="s" weight="bold" inverted={true}>
           {props.title}
         </Typography>
-        <KButton onClick={onClose}>
+        <KButton onClick={props.onClose}>
           <Icon icon="Close" color="primary" size="0.75rem" inverted={true} />
         </KButton>
       </div>

pkgs/clan-app/ui/src/components/Toolbar/Toolbar.css

@@ -1,4 +1,4 @@
-.toolbar {
+div.toolbar {
   @apply flex flex-row items-center justify-center gap-1.5 p-1 size-fit rounded-md self-stretch;
 
   border: 0.0625rem solid theme(colors.off.toolbar_border);
@@ -11,20 +11,20 @@
   & > hr {
     @apply h-full min-h-8;
   }
-}
 
-.toolbar_button {
-  @apply w-6 h-6 p-1 rounded-[0.25rem];
+  & > button.toolbar-button {
+    @apply w-6 h-6 p-1 rounded-[0.25rem];
 
-  &:hover {
-    @apply bg-inv-1;
-  }
+    &:hover {
+      @apply bg-inv-1;
+    }
 
-  &:active {
-    @apply bg-inv-4;
-  }
+    &:active {
+      @apply bg-inv-4;
+    }
 
-  &.selected {
-    @apply bg-bg-semantic-success-4;
+    &.selected {
+      @apply bg-bg-semantic-success-4;
+    }
   }
 }

pkgs/clan-app/ui/src/components/Toolbar/Toolbar.stories.tsx

@@ -2,8 +2,6 @@ import { Meta, StoryObj } from "@kachurun/storybook-solid";
 import { Toolbar, ToolbarProps } from "@/src/components/Toolbar/Toolbar";
 import { Divider } from "@/src/components/Divider/Divider";
 import { ToolbarButton } from "./ToolbarButton";
-import { Tooltip } from "../Tooltip/Tooltip";
-import { Typography } from "../Typography/Typography";
 
 const meta: Meta<ToolbarProps> = {
   title: "Components/Toolbar",
@@ -27,63 +25,3 @@ export const Default: Story = {
     ),
   },
 };
-
-export const WithTooltip: Story = {
-  // @ts-expect-error: args in storybook is not typed correctly. This is a storybook issue.
-  render: (args) => (
-    <div class="flex h-[80vh]">
-      <div class="mt-auto">
-        <Toolbar {...args} />
-      </div>
-    </div>
-  ),
-  args: {
-    children: (
-      <>
-        <Tooltip
-          trigger={<ToolbarButton name="select" icon="Cursor" />}
-          placement="top"
-        >
-          <div class="mb-1 p-1 text-fg-inv-1">
-            <Typography hierarchy="label" size="s" color="inherit">
-              Select an object
-            </Typography>
-          </div>
-        </Tooltip>
-        <Divider orientation="vertical" />
-        <Tooltip
-          trigger={<ToolbarButton name="new-machine" icon="NewMachine" />}
-          placement="top"
-        >
-          <div class="mb-1 p-1 text-fg-inv-1">
-            <Typography hierarchy="label" size="s" color="inherit">
-              Create a new machine
-            </Typography>
-          </div>
-        </Tooltip>
-        <Tooltip
-          trigger={
-            <ToolbarButton name="modules" icon="Modules" selected={true} />
-          }
-          placement="top"
-        >
-          <div class="mb-1 p-1 text-fg-inv-1">
-            <Typography hierarchy="label" size="s" color="inherit">
-              Manage Services
-            </Typography>
-          </div>
-        </Tooltip>
-        <Tooltip
-          trigger={<ToolbarButton name="ai" icon="AI" />}
-          placement="top"
-        >
-          <div class="mb-1 p-1 text-fg-inv-1">
-            <Typography hierarchy="label" size="s" color="inherit">
-              Chat with AI
-            </Typography>
-          </div>
-        </Tooltip>
-      </>
-    ),
-  },
-};

pkgs/clan-app/ui/src/components/Toolbar/Toolbar.tsx

@@ -1,5 +1,5 @@
+import "./Toolbar.css";
 import { JSX } from "solid-js";
-import styles from "./Toolbar.module.css";
 
 export interface ToolbarProps {
   children: JSX.Element;
@@ -7,7 +7,7 @@ export interface ToolbarProps {
 
 export const Toolbar = (props: ToolbarProps) => {
   return (
-    <div class={styles.toolbar} role="toolbar" aria-orientation="horizontal">
+    <div class="toolbar" role="toolbar" aria-orientation="horizontal">
       {props.children}
     </div>
   );

pkgs/clan-app/ui/src/components/Toolbar/ToolbarButton.tsx

@@ -1,4 +1,4 @@
-import styles from "./Toolbar.module.css";
+import "./Toolbar.css";
 import cx from "classnames";
 import { Button } from "@kobalte/core/button";
 import Icon, { IconVariant } from "@/src/components/Icon/Icon";
@@ -13,7 +13,7 @@ export interface ToolbarButtonProps
 export const ToolbarButton = (props: ToolbarButtonProps) => {
   return (
     <Button
-      class={cx(styles.toolbar_button, { selected: props.selected })}
+      class={cx("toolbar-button", { selected: props.selected })}
       {...props}
     >
       <Icon icon={props.icon} inverted={!props.selected} />

pkgs/clan-app/ui/src/hooks/api.ts

@@ -20,7 +20,6 @@ export type SuccessData<T extends OperationNames> = SuccessQuery<T>["data"];
 
 interface SendHeaderType {
   logging?: { group_path: string[] };
-  op_key?: string;
 }
 interface BackendSendType<K extends OperationNames> {
   body: OperationArgs<K>;
@@ -65,14 +64,9 @@ export const callApi = <K extends OperationNames>(
     };
   }
 
-  const op_key = backendOpts?.op_key ?? crypto.randomUUID();
-
   const req: BackendSendType<OperationNames> = {
     body: args,
-    header: {
-      ...backendOpts,
-      op_key,
-    },
+    header: backendOpts,
   };
 
   const result = (
@@ -84,6 +78,9 @@ export const callApi = <K extends OperationNames>(
     >
   )[method](req) as Promise<BackendReturnType<K>>;
 
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const op_key = (result as any)._webviewMessageId as string;
+
   return {
     uuid: op_key,
     result: result.then(({ body }) => body),

pkgs/clan-app/ui/src/hooks/clan.ts

@@ -2,9 +2,6 @@ import { callApi } from "@/src/hooks/api";
 import { addClanURI, setActiveClanURI } from "@/src/stores/clan";
 import { Params, Navigator, useParams } from "@solidjs/router";
 
-export const encodeBase64 = (value: string) => window.btoa(value);
-export const decodeBase64 = (value: string) => window.atob(value);
-
 export const selectClanFolder = async () => {
   const req = callApi("get_clan_folder", {});
   const res = await req.result;
@@ -23,51 +20,38 @@ export const selectClanFolder = async () => {
   throw new Error("Illegal state exception");
 };
 
-export const buildClanPath = (clanURI: string) => {
-  return "/clans/" + encodeBase64(clanURI);
-};
-
-export const buildMachinePath = (clanURI: string, machineID: string) => {
-  return (
-    "/clans/" + encodeBase64(clanURI) + "/machines/" + encodeBase64(machineID)
-  );
-};
-
-export const navigateToClan = (navigate: Navigator, clanURI: string) => {
-  const path = buildClanPath(clanURI);
-  console.log("Navigating to clan", clanURI, path);
-  navigate(path);
-};
-
-export const navigateToMachine = (
-  navigate: Navigator,
-  clanURI: string,
-  machineID: string,
-) => {
-  const path = buildMachinePath(clanURI, machineID);
-  console.log("Navigating to machine", clanURI, machineID, path);
-  navigate(path);
+export const navigateToClan = (navigate: Navigator, uri: string) => {
+  navigate("/clans/" + window.btoa(uri));
 };
 
 export const clanURIParam = (params: Params) => {
-  return decodeBase64(params.clanURI);
+  return window.atob(params.clanURI);
 };
 
-export const useClanURI = () => clanURIParam(useParams());
+export function useClanURI(opts: { force: true }): string;
+export function useClanURI(opts: { force: boolean }): string | null;
+export function useClanURI(
+  opts: { force: boolean } = { force: false },
+): string | null {
+  const maybe = () => {
+    const params = useParams();
+    if (!params.clanURI) {
+      return null;
+    }
+    const clanURI = clanURIParam(params);
+    if (!clanURI) {
+      throw new Error(
+        "Could not decode clan URI from params: " + params.clanURI,
+      );
+    }
+    return clanURI;
+  };
 
-export const machineIDParam = (params: Params) => {
-  return decodeBase64(params.machineID);
-};
-
-export const useMachineID = (): string => {
-  const params = useParams();
-  return machineIDParam(params);
-};
-
-export const maybeUseMachineID = (): string | null => {
-  const params = useParams();
-  if (params.machineID === undefined) {
-    return null;
+  const uri = maybe();
+  if (!uri && opts.force) {
+    throw new Error(
+      "ClanURI is not set. Use this function only within contexts, where clanURI is guaranteed to have been set.",
+    );
   }
-  return machineIDParam(params);
-};
+  return uri;
+}

pkgs/clan-app/ui/src/index.tsx

@@ -6,7 +6,6 @@ import { QueryClient, QueryClientProvider } from "@tanstack/solid-query";
 import { Routes } from "@/src/routes";
 import { Router } from "@solidjs/router";
 import { Layout } from "@/src/routes/Layout";
-import { SolidQueryDevtools } from "@tanstack/solid-query-devtools";
 
 export const client = new QueryClient();
 
@@ -19,12 +18,13 @@ if (import.meta.env.DEV && !(root instanceof HTMLElement)) {
 }
 if (import.meta.env.DEV) {
   console.log("Development mode");
+  // Load the debugger in development mode
+  await import("solid-devtools");
 }
 
 render(
   () => (
     <QueryClientProvider client={client}>
-      {import.meta.env.DEV && <SolidQueryDevtools initialIsOpen={true} />}
       <Router root={Layout}>{Routes}</Router>
     </QueryClientProvider>
   ),

pkgs/clan-app/ui/src/queries/queries.ts

@@ -1,22 +1,24 @@
-import { useQueries, useQuery, UseQueryResult } from "@tanstack/solid-query";
+import { useQuery, UseQueryResult } from "@tanstack/solid-query";
 import { callApi, SuccessData } from "../hooks/api";
-import { encodeBase64 } from "@/src/hooks/clan";
-
-export type ClanDetails = SuccessData<"get_clan_details">;
-export type ClanDetailsWithURI = ClanDetails & { uri: string };
 
 export type ListMachines = SuccessData<"list_machines">;
 
 export type MachinesQueryResult = UseQueryResult<ListMachines>;
-export type ClanListQueryResult = UseQueryResult<ClanDetailsWithURI>[];
 
-export const useMachinesQuery = (clanURI: string) =>
+interface MachinesQueryParams {
+  clanURI: string | null;
+}
+export const useMachinesQuery = (props: MachinesQueryParams) =>
   useQuery<ListMachines>(() => ({
-    queryKey: ["clans", encodeBase64(clanURI), "machines"],
+    queryKey: ["clans", props.clanURI, "machines"],
+    enabled: !!props.clanURI,
     queryFn: async () => {
+      if (!props.clanURI) {
+        return {};
+      }
       const api = callApi("list_machines", {
         flake: {
-          identifier: clanURI,
+          identifier: props.clanURI,
         },
       });
       const result = await api.result;
@@ -27,54 +29,3 @@ export const useMachinesQuery = (clanURI: string) =>
       return result.data;
     },
   }));
-
-export const useClanDetailsQuery = (clanURI: string) =>
-  useQuery<ClanDetails>(() => ({
-    queryKey: ["clans", encodeBase64(clanURI), "details"],
-    queryFn: async () => {
-      const call = callApi("get_clan_details", {
-        flake: {
-          identifier: clanURI,
-        },
-      });
-      const result = await call.result;
-
-      if (result.status === "error") {
-        // todo should we create some specific error types?
-        console.error("Error fetching clan details:", result.errors);
-        throw new Error(result.errors[0].message);
-      }
-
-      return {
-        uri: clanURI,
-        ...result.data,
-      };
-    },
-  }));
-
-export const useClanListQuery = (clanURIs: string[]): ClanListQueryResult =>
-  useQueries(() => ({
-    queries: clanURIs.map((clanURI) => ({
-      queryKey: ["clans", encodeBase64(clanURI), "details"],
-      enabled: !!clanURI,
-      queryFn: async () => {
-        const call = callApi("get_clan_details", {
-          flake: {
-            identifier: clanURI,
-          },
-        });
-        const result = await call.result;
-
-        if (result.status === "error") {
-          // todo should we create some specific error types?
-          console.error("Error fetching clan details:", result.errors);
-          throw new Error(result.errors[0].message);
-        }
-
-        return {
-          uri: clanURI,
-          ...result.data,
-        };
-      },
-    })),
-  }));

pkgs/clan-app/ui/src/routes/Clan/Clan.css

@@ -11,14 +11,3 @@
 .create-modal {
   @apply min-w-96;
 }
-
-.sidebar-container {
-}
-
-div.sidebar {
-  @apply absolute top-10 bottom-20 left-4 w-60;
-}
-
-div.sidebar-pane {
-  @apply absolute top-12 bottom-20 left-[16.5rem] w-64;
-}

pkgs/clan-app/ui/src/routes/Clan/Clan.tsx

@@ -1,28 +1,10 @@
 import { RouteSectionProps } from "@solidjs/router";
-import {
-  Component,
-  JSX,
-  Show,
-  createEffect,
-  createMemo,
-  createSignal,
-  on,
-  onMount,
-} from "solid-js";
-import {
-  buildMachinePath,
-  maybeUseMachineID,
-  useClanURI,
-} from "@/src/hooks/clan";
+import { Component, JSX, Show, createSignal } from "solid-js";
+import { useClanURI } from "@/src/hooks/clan";
 import { CubeScene } from "@/src/scene/cubes";
-import {
-  ClanListQueryResult,
-  MachinesQueryResult,
-  useClanListQuery,
-  useMachinesQuery,
-} from "@/src/queries/queries";
+import { MachinesQueryResult, useMachinesQuery } from "@/src/queries/queries";
 import { callApi } from "@/src/hooks/api";
-import { store, setStore, clanURIs } from "@/src/stores/clan";
+import { store, setStore } from "@/src/stores/clan";
 import { produce } from "solid-js/store";
 import { Button } from "@/src/components/Button/Button";
 import { Splash } from "@/src/scene/splash";
@@ -31,15 +13,19 @@ import "./Clan.css";
 import { Modal } from "@/src/components/Modal/Modal";
 import { TextInput } from "@/src/components/Form/TextInput";
 import { createForm, FieldValues, reset } from "@modular-forms/solid";
-import { Sidebar } from "@/src/components/Sidebar/Sidebar";
-import { useNavigate } from "@solidjs/router";
 
 export const Clan: Component<RouteSectionProps> = (props) => {
   return (
     <>
-      <Sidebar />
-      {props.children}
-      <ClanSceneController {...props} />
+      <div
+        style={{
+          position: "absolute",
+          top: 0,
+        }}
+      >
+        {props.children}
+      </div>
+      <ClanSceneController />
     </>
   );
 };
@@ -47,12 +33,10 @@ export const Clan: Component<RouteSectionProps> = (props) => {
 interface CreateFormValues extends FieldValues {
   name: string;
 }
-
 interface MockProps {
   onClose: () => void;
   onSubmit: (formValues: CreateFormValues) => void;
 }
-
 const MockCreateMachine = (props: MockProps) => {
   let container: Node;
 
@@ -85,7 +69,7 @@ const MockCreateMachine = (props: MockProps) => {
               )}
             </Field>
 
-            <div class="mt-4 flex w-full items-center justify-end gap-4">
+            <div class="flex w-full items-center justify-end gap-4">
               <Button size="s" hierarchy="secondary" onClick={props.onClose}>
                 Cancel
               </Button>
@@ -105,9 +89,8 @@ const MockCreateMachine = (props: MockProps) => {
   );
 };
 
-const ClanSceneController = (props: RouteSectionProps) => {
-  const clanURI = useClanURI();
-  const navigate = useNavigate();
+const ClanSceneController = () => {
+  const clanURI = useClanURI({ force: true });
 
   const [dialogHandlers, setDialogHandlers] = createSignal<{
     resolve: ({ id }: { id: string }) => void;
@@ -145,61 +128,9 @@ const ClanSceneController = (props: RouteSectionProps) => {
 
   const [showModal, setShowModal] = createSignal(false);
 
-  const [loadingCooldown, setLoadingCooldown] = createSignal(false);
-  onMount(() => {
-    setTimeout(() => {
-      setLoadingCooldown(true);
-    }, 1500);
-  });
-
-  const [selectedIds, setSelectedIds] = createSignal<Set<string>>(new Set());
-
-  const onMachineSelect = (ids: Set<string>) => {
-    // Get the first selected ID and navigate to its machine details
-    const selected = ids.values().next().value;
-    if (selected) {
-      navigate(buildMachinePath(clanURI, selected));
-    }
-  };
-
-  const machine = createMemo(() => maybeUseMachineID());
-
-  createEffect(
-    on(machine, (machineId) => {
-      if (machineId) {
-        setSelectedIds(() => {
-          const res = new Set<string>();
-          res.add(machineId);
-          return res;
-        });
-      } else {
-        setSelectedIds(new Set<string>());
-      }
-    }),
-  );
-
   return (
     <SceneDataProvider clanURI={clanURI}>
-      {({ clansQuery, machinesQuery }) => {
-        // a combination of the individual clan details query status and the machines query status
-        // the cube scene needs the machines query, the sidebar needs the clans query and machines query results
-        // so we wait on both before removing the loader to avoid any loading artefacts
-        const isLoading = (): boolean => {
-          // check the machines query first
-          if (machinesQuery.isLoading) {
-            return true;
-          }
-
-          // otherwise iterate the clans query and return early if we find a queries that is still loading
-          for (const query of clansQuery) {
-            if (query.isLoading) {
-              return true;
-            }
-          }
-
-          return false;
-        };
-
+      {({ query }) => {
         return (
           <>
             <Show when={showModal()}>
@@ -231,7 +162,8 @@ const ClanSceneController = (props: RouteSectionProps) => {
                     produce((s) => {
                       for (const machineId in s.sceneData[clanURI]) {
                         // Reset the position of each machine to [0, 0]
-                        s.sceneData[clanURI] = {};
+                        s.sceneData[clanURI] = {}; // Clear the entire object
+                        // delete s.sceneData[clanURI][machineId];
                       }
                     }),
                   );
@@ -243,29 +175,23 @@ const ClanSceneController = (props: RouteSectionProps) => {
                 ghost
                 onClick={() => {
                   console.log("Refetching API");
-                  machinesQuery.refetch();
+                  query.refetch();
                 }}
               >
                 Refetch API
               </Button>
             </div>
             {/* TODO: Add minimal display time */}
-            <div
-              class={cx({
-                "fade-out": !machinesQuery.isLoading && loadingCooldown(),
-              })}
-            >
+            <div class={cx({ "fade-out": !query.isLoading })}>
               <Splash />
             </div>
 
             <CubeScene
-              selectedIds={selectedIds}
-              onSelect={onMachineSelect}
-              isLoading={isLoading()}
-              cubesQuery={machinesQuery}
+              isLoading={query.isLoading}
+              cubesQuery={query}
               onCreate={onCreate}
               sceneStore={() => {
-                const clanURI = useClanURI();
+                const clanURI = useClanURI({ force: true });
                 return store.sceneData?.[clanURI];
               }}
               setMachinePos={(machineId: string, pos: [number, number]) => {
@@ -295,15 +221,11 @@ const ClanSceneController = (props: RouteSectionProps) => {
 };
 
 const SceneDataProvider = (props: {
-  clanURI: string;
-  children: (sceneData: {
-    clansQuery: ClanListQueryResult;
-    machinesQuery: MachinesQueryResult;
-  }) => JSX.Element;
+  clanURI: string | null;
+  children: (sceneData: { query: MachinesQueryResult }) => JSX.Element;
 }) => {
-  const clansQuery = useClanListQuery(clanURIs());
-  const machinesQuery = useMachinesQuery(props.clanURI);
+  const machinesQuery = useMachinesQuery({ clanURI: props.clanURI });
 
   // This component can be used to provide scene data or context if needed
-  return props.children({ clansQuery, machinesQuery });
+  return props.children({ query: machinesQuery });
 };

pkgs/clan-app/ui/src/routes/Machine/Machine.tsx

@@ -1,19 +0,0 @@
-import { RouteSectionProps, useNavigate } from "@solidjs/router";
-import { SidebarPane } from "@/src/components/Sidebar/SidebarPane";
-import { navigateToClan, useClanURI, useMachineID } from "@/src/hooks/clan";
-
-export const Machine = (props: RouteSectionProps) => {
-  const navigate = useNavigate();
-  const clanURI = useClanURI();
-
-  const onClose = () => {
-    // go back to clan route
-    navigateToClan(navigate, clanURI);
-  };
-
-  return (
-    <SidebarPane title={useMachineID()} onClose={onClose}>
-      <h1>Hello world</h1>
-    </SidebarPane>
-  );
-};

pkgs/clan-app/ui/src/routes/Onboarding/Onboarding.css

@@ -57,7 +57,7 @@ main#welcome {
     @apply flex flex-col items-center justify-evenly gap-y-20 size-fit;
 
     & > div.welcome {
-      @apply flex flex-col w-80 gap-y-6;
+      @apply flex flex-col min-w-80 gap-y-6;
 
       & > div.separator {
         @apply grid grid-cols-3 grid-rows-1 gap-x-4 items-center;

pkgs/clan-app/ui/src/routes/Onboarding/Onboarding.tsx

@@ -220,8 +220,16 @@ export const Onboarding: Component<RouteSectionProps> = (props) => {
         // todo allow users to select a template
         template: "minimal",
         initial: {
-          name,
-          description,
+          meta: {
+            name: name,
+            description: description,
+            // todo it tries to 'delete' icon if it's not provided
+            // this logic is unexpected, and needs reviewed.
+            icon: null,
+          },
+          machines: {},
+          instances: {},
+          services: {},
         },
       },
     });

pkgs/clan-app/ui/src/routes/index.tsx

@@ -1,7 +1,6 @@
 import type { RouteDefinition } from "@solidjs/router/dist/types";
 import { Onboarding } from "@/src/routes/Onboarding/Onboarding";
 import { Clan } from "@/src/routes/Clan/Clan";
-import { Machine } from "@/src/routes/Machine/Machine";
 
 export const Routes: RouteDefinition[] = [
   {
@@ -22,14 +21,25 @@ export const Routes: RouteDefinition[] = [
       },
       {
         path: "/:clanURI",
-        component: Clan,
         children: [
           {
             path: "/",
+            component: Clan,
           },
           {
-            path: "/machines/:machineID",
-            component: Machine,
+            path: "/machines",
+            children: [
+              {
+                path: "/",
+                component: () => <h1>Machines (Index)</h1>,
+              },
+              {
+                path: "/:machineID",
+                component: (props) => (
+                  <h1>Machine ID: {props.params.machineID}</h1>
+                ),
+              },
+            ],
           },
         ],
       },

pkgs/clan-app/ui/src/scene/MachineManager.ts

@@ -1,154 +0,0 @@
-import { Accessor, createEffect, createRoot } from "solid-js";
-import { MachineRepr } from "./MachineRepr";
-import * as THREE from "three";
-import { SceneData } from "../stores/clan";
-import { MachinesQueryResult } from "../queries/queries";
-import { ObjectRegistry } from "./ObjectRegistry";
-import { renderLoop } from "./RenderLoop";
-
-function keyFromPos(pos: [number, number]): string {
-  return `${pos[0]},${pos[1]}`;
-}
-
-const CUBE_SPACING = 2;
-
-export class MachineManager {
-  public machines = new Map<string, MachineRepr>();
-
-  private disposeRoot: () => void;
-
-  private machinePositionsSignal: Accessor<SceneData>;
-
-  constructor(
-    scene: THREE.Scene,
-    registry: ObjectRegistry,
-    machinePositionsSignal: Accessor<SceneData>,
-    machinesQueryResult: MachinesQueryResult,
-    selectedIds: Accessor<Set<string>>,
-    setMachinePos: (id: string, position: [number, number]) => void,
-  ) {
-    this.machinePositionsSignal = machinePositionsSignal;
-
-    this.disposeRoot = createRoot((disposeEffects) => {
-      createEffect(() => {
-        const machines = machinePositionsSignal();
-
-        Object.entries(machines).forEach(([id, data]) => {
-          const machineRepr = new MachineRepr(
-            scene,
-            registry,
-            new THREE.Vector2(data.position[0], data.position[1]),
-            id,
-            selectedIds,
-          );
-          this.machines.set(id, machineRepr);
-          scene.add(machineRepr.group);
-        });
-        renderLoop.requestRender();
-      });
-
-      // Push positions of previously existing machines to the scene
-      // TODO: Maybe we should do this in some post query hook?
-      createEffect(() => {
-        if (!machinesQueryResult.data) return;
-
-        const actualMachines = Object.keys(machinesQueryResult.data);
-        const machinePositions = machinePositionsSignal();
-        const placed: Set<string> = machinePositions
-          ? new Set(Object.keys(machinePositions))
-          : new Set();
-
-        const nonPlaced = actualMachines.filter((m) => !placed.has(m));
-
-        // Push not explizitly placed machines to the scene
-        // TODO: Make the user place them manually
-        // We just calculate some next free position
-        for (const id of nonPlaced) {
-          console.log("adding", id);
-          const position = this.nextGridPos();
-
-          setMachinePos(id, position);
-        }
-      });
-
-      return disposeEffects;
-    });
-  }
-
-  nextGridPos(): [number, number] {
-    const occupiedPositions = new Set(
-      Object.values(this.machinePositionsSignal()).map((data) =>
-        keyFromPos(data.position),
-      ),
-    );
-
-    let x = 0,
-      z = 0;
-    let layer = 1;
-
-    while (layer < 100) {
-      // right
-      for (let i = 0; i < layer; i++) {
-        const pos = [x * CUBE_SPACING, z * CUBE_SPACING] as [number, number];
-        const key = keyFromPos(pos);
-        if (!occupiedPositions.has(key)) {
-          return pos;
-        }
-        x += 1;
-      }
-      // down
-      for (let i = 0; i < layer; i++) {
-        const pos = [x * CUBE_SPACING, z * CUBE_SPACING] as [number, number];
-        const key = keyFromPos(pos);
-        if (!occupiedPositions.has(key)) {
-          return pos;
-        }
-        z += 1;
-      }
-      layer++;
-      // left
-      for (let i = 0; i < layer; i++) {
-        const pos = [x * CUBE_SPACING, z * CUBE_SPACING] as [number, number];
-        const key = keyFromPos(pos);
-        if (!occupiedPositions.has(key)) {
-          return pos;
-        }
-        x -= 1;
-      }
-      // up
-      for (let i = 0; i < layer; i++) {
-        const pos = [x * CUBE_SPACING, z * CUBE_SPACING] as [number, number];
-        const key = keyFromPos(pos);
-        if (!occupiedPositions.has(key)) {
-          return pos;
-        }
-        z -= 1;
-      }
-      layer++;
-    }
-    console.warn("No free grid positions available, returning [0, 0]");
-    // Fallback if no position was found
-    return [0, 0] as [number, number];
-  }
-
-  dispose(scene: THREE.Scene) {
-    for (const machine of this.machines.values()) {
-      machine.dispose(scene);
-    }
-    // Stop SolidJS effects
-    this.disposeRoot?.();
-    // Clear references
-    this.machines?.clear();
-  }
-}
-
-// TODO: For service focus
-// const getCirclePosition =
-//   (center: [number, number, number]) =>
-//   (_id: string, index: number, total: number): [number, number, number] => {
-//     const r = total === 1 ? 0 : Math.sqrt(total) * CUBE_SPACING; // Radius based on total cubes
-//     const x = Math.cos((index / total) * 2 * Math.PI) * r + center[0];
-//     const z = Math.sin((index / total) * 2 * Math.PI) * r + center[2];
-//     // Position cubes at y = 0.5 to float above the ground
-//     return [x, CUBE_Y, z];
-//   };

pkgs/clan-app/ui/src/scene/MachineRepr.ts

@@ -1,141 +0,0 @@
-import * as THREE from "three";
-import { ObjectRegistry } from "./ObjectRegistry";
-import { CSS2DObject } from "three/examples/jsm/renderers/CSS2DRenderer.js";
-import { Accessor, createEffect, createRoot, on } from "solid-js";
-import { renderLoop } from "./RenderLoop";
-
-// Constants
-const BASE_SIZE = 0.9;
-const CUBE_SIZE = BASE_SIZE / 1.5;
-const CUBE_HEIGHT = CUBE_SIZE;
-const BASE_HEIGHT = 0.05;
-const CUBE_COLOR = 0xd7e0e1;
-const CUBE_EMISSIVE = 0x303030;
-
-const CUBE_SELECTED_COLOR = 0x4b6767;
-
-const BASE_COLOR = 0xecfdff;
-const BASE_EMISSIVE = 0x0c0c0c;
-const BASE_SELECTED_COLOR = 0x69b0e3;
-const BASE_SELECTED_EMISSIVE = 0x666666; // Emissive color for selected bases
-
-export class MachineRepr {
-  public id: string;
-  public group: THREE.Group;
-
-  private cubeMesh: THREE.Mesh;
-  private baseMesh: THREE.Mesh;
-  private geometry: THREE.BoxGeometry;
-  private material: THREE.MeshPhongMaterial;
-
-  private disposeRoot: () => void;
-
-  constructor(
-    scene: THREE.Scene,
-    registry: ObjectRegistry,
-    position: THREE.Vector2,
-    id: string,
-    selectedSignal: Accessor<Set<string>>,
-  ) {
-    this.id = id;
-    this.geometry = new THREE.BoxGeometry(CUBE_SIZE, CUBE_SIZE, CUBE_SIZE);
-    this.material = new THREE.MeshPhongMaterial({
-      color: CUBE_COLOR,
-      emissive: CUBE_EMISSIVE,
-      shininess: 100,
-    });
-
-    this.cubeMesh = new THREE.Mesh(this.geometry, this.material);
-    this.cubeMesh.castShadow = true;
-    this.cubeMesh.receiveShadow = true;
-    this.cubeMesh.userData = { id };
-    this.cubeMesh.name = "cube";
-    this.cubeMesh.position.set(0, CUBE_HEIGHT / 2, 0);
-
-    this.baseMesh = this.createCubeBase(
-      BASE_COLOR,
-      BASE_EMISSIVE,
-      new THREE.BoxGeometry(BASE_SIZE, BASE_HEIGHT, BASE_SIZE),
-    );
-    this.baseMesh.name = "base";
-
-    const label = this.createLabel(id);
-    this.cubeMesh.add(label);
-
-    this.group = new THREE.Group();
-    this.group.add(this.cubeMesh);
-    this.group.add(this.baseMesh);
-
-    this.group.position.set(position.x, 0, position.y);
-    this.group.userData.id = id;
-
-    this.disposeRoot = createRoot((disposeEffects) => {
-      createEffect(
-        on(selectedSignal, (selectedIds) => {
-          const isSelected = selectedIds.has(this.id);
-          // Update cube
-          (this.cubeMesh.material as THREE.MeshPhongMaterial).color.set(
-            isSelected ? CUBE_SELECTED_COLOR : CUBE_COLOR,
-          );
-
-          // Update base
-          (this.baseMesh.material as THREE.MeshPhongMaterial).color.set(
-            isSelected ? BASE_SELECTED_COLOR : BASE_COLOR,
-          );
-          (this.baseMesh.material as THREE.MeshPhongMaterial).emissive.set(
-            isSelected ? BASE_SELECTED_EMISSIVE : BASE_EMISSIVE,
-          );
-
-          renderLoop.requestRender();
-        }),
-      );
-
-      return disposeEffects;
-    });
-
-    scene.add(this.group);
-
-    registry.add({
-      object: this.group,
-      id,
-      type: "machine",
-      dispose: () => this.dispose(scene),
-    });
-  }
-
-  private createCubeBase(
-    color: THREE.ColorRepresentation,
-    emissive: THREE.ColorRepresentation,
-    geometry: THREE.BoxGeometry,
-  ) {
-    const baseMaterial = new THREE.MeshPhongMaterial({
-      color,
-      emissive,
-      transparent: true,
-      opacity: 1,
-    });
-    const base = new THREE.Mesh(geometry, baseMaterial);
-    base.position.set(0, BASE_HEIGHT / 2, 0);
-    base.receiveShadow = true;
-    return base;
-  }
-
-  private createLabel(id: string) {
-    const div = document.createElement("div");
-    div.className = "machine-label";
-    div.textContent = id;
-    const label = new CSS2DObject(div);
-    label.position.set(0, CUBE_SIZE + 0.1, 0);
-    return label;
-  }
-
-  dispose(scene: THREE.Scene) {
-    this.disposeRoot?.(); // Stop SolidJS effects
-
-    scene.remove(this.group);
-
-    this.geometry.dispose();
-    this.material.dispose();
-    (this.baseMesh.material as THREE.Material).dispose();
-  }
-}

pkgs/clan-app/ui/src/scene/ObjectRegistry.ts

@@ -1,43 +0,0 @@
-import * as THREE from "three";
-
-interface ObjectEntry {
-  object: THREE.Object3D;
-  type: string;
-  id: string;
-  dispose?: () => void;
-}
-
-export class ObjectRegistry {
-  #objects = new Map<string, ObjectEntry>();
-
-  add(entry: ObjectEntry) {
-    const key = `${entry.type}:${entry.id}`;
-    this.#objects.set(key, entry);
-  }
-
-  getById(type: string, id: string) {
-    return this.#objects.get(`${type}:${id}`);
-  }
-
-  getAllByType(type: string) {
-    return [...this.#objects.values()].filter((obj) => obj.type === type);
-  }
-
-  removeById(type: string, id: string, scene: THREE.Scene) {
-    const key = `${type}:${id}`;
-    const entry = this.#objects.get(key);
-    if (entry) {
-      scene.remove(entry.object);
-      entry.dispose?.();
-      this.#objects.delete(key);
-    }
-  }
-
-  disposeAll(scene: THREE.Scene) {
-    for (const entry of this.#objects.values()) {
-      scene.remove(entry.object);
-      entry.dispose?.();
-    }
-    this.#objects.clear();
-  }
-}

pkgs/clan-app/ui/src/scene/RenderLoop.ts

@@ -1,133 +0,0 @@
-import { Scene, Camera, WebGLRenderer } from "three";
-import { MapControls } from "three/examples/jsm/controls/MapControls.js";
-import { CSS2DRenderer } from "three/examples/jsm/renderers/CSS2DRenderer.js";
-
-/**
- * Private class to manage the render loop
- * @internal
- */
-class RenderLoop {
-  // Track if a render is already requested
-  // This prevents multiple requests in the same frame
-  // and ensures only one render per frame
-  // This is important for performance and to avoid flickering
-  private renderRequested = false;
-
-  // References to the scene, camera, renderer, controls, and label renderer
-  // These will be set during initialization
-  private scene!: Scene;
-  private bgScene!: Scene;
-  private camera!: Camera;
-  private bgCamera!: Camera;
-  private renderer!: WebGLRenderer;
-  private controls!: MapControls;
-  private labelRenderer!: CSS2DRenderer;
-
-  // Flag to prevent multiple initializations
-  private initialized = false;
-
-  init(
-    scene: Scene,
-    camera: Camera,
-    renderer: WebGLRenderer,
-    labelRenderer: CSS2DRenderer,
-    controls: MapControls,
-    bgScene: Scene,
-    bgCamera: Camera,
-  ) {
-    if (this.initialized) {
-      console.error("RenderLoop already initialized.");
-      return;
-    }
-    this.scene = scene;
-    this.camera = camera;
-    this.renderer = renderer;
-    this.controls = controls;
-    this.bgScene = bgScene;
-    this.bgCamera = bgCamera;
-    this.labelRenderer = labelRenderer;
-    this.initialized = true;
-  }
-
-  requestRender() {
-    // If not initialized, log an error and return
-    if (!this.initialized) {
-      console.error(
-        "RenderLoop not initialized yet. Make sure to call init() once before usage.",
-      );
-      return;
-    }
-    // If a render is already requested, do nothing
-    if (this.renderRequested) return;
-
-    this.renderRequested = true;
-    requestAnimationFrame(() => {
-      this.updateTweens();
-
-      const needsUpdate = this.controls.update(); // returns true if damping is ongoing
-
-      this.render();
-      this.renderRequested = false;
-
-      if (needsUpdate) {
-        this.requestRender();
-      }
-    });
-  }
-
-  private updateTweens() {
-    // TODO: TWEEN.update() for tween animations in the future
-  }
-
-  private render() {
-    // TODO: Disable console.debug in production
-    // console.debug("Rendering scene...", this);
-
-    this.renderer.clear();
-
-    this.renderer.render(this.bgScene, this.bgCamera);
-    this.renderer.render(this.scene, this.camera);
-    this.labelRenderer.render(this.scene, this.camera);
-  }
-
-  dispose() {
-    // Dispose controls, renderer, remove listeners if any
-    this.controls.dispose();
-    this.renderer.dispose();
-    // clear refs, this prevents memory leaks by allowing garbage collection
-    this.scene = null!;
-    this.bgScene = null!;
-    this.camera = null!;
-    this.bgCamera = null!;
-    this.renderer = null!;
-    this.controls = null!;
-    this.labelRenderer = null!;
-    this.initialized = false;
-  }
-}
-
-/**
- * Singleton instance of RenderLoop
- * This is used to manage the re-rendering
- *
- * It can only be initialized once then passed to individual components
- * they can use the renderLoop to request re-renders as needed.
- *
- *
- * Usage:
- * ```typescript
- * import { renderLoop } from "./RenderLoop";
- *
- * // Somewhere initialize the render loop:
- * renderLoop.init(scene, camera, renderer, labelRenderer, controls, bgScene, bgCamera);
- *
- * // To request a render:
- * renderLoop.requestRender();
- *
- * // To dispose:
- * onCleanup(() => {
- *  renderLoop.dispose();
- * })
- *
- */
-export const renderLoop = new RenderLoop();