wip

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4506

.gitea/workflows/build-clan-app-darwin.yml

@@ -0,0 +1,20 @@
+name: Build Clan App (Darwin)
+
+on:
+  schedule:
+    # Run every 4 hours
+    - cron: "0 */4 * * *"
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  build-clan-app-darwin:
+    runs-on: nix
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build clan-app for x86_64-darwin
+        run: |
+          nix build .#packages.x86_64-darwin.clan-app --system x86_64-darwin --log-format bar-with-logs

.gitea/workflows/create-pr.sh

@@ -1,6 +1,7 @@
-#!/usr/bin/env bash
+#!/bin/sh
+
 # Shared script for creating pull requests in Gitea workflows
-set -euo pipefail
+set -eu
 
 # Required environment variables:
 # - CI_BOT_TOKEN: Gitea bot token for authentication
@@ -8,22 +9,22 @@ set -euo pipefail
 # - PR_TITLE: Title of the pull request
 # - PR_BODY: Body/description of the pull request
 
-if [[ -z "${CI_BOT_TOKEN:-}" ]]; then
+if [ -z "${CI_BOT_TOKEN:-}" ]; then
   echo "Error: CI_BOT_TOKEN is not set" >&2
   exit 1
 fi
 
-if [[ -z "${PR_BRANCH:-}" ]]; then
+if [ -z "${PR_BRANCH:-}" ]; then
   echo "Error: PR_BRANCH is not set" >&2
   exit 1
 fi
 
-if [[ -z "${PR_TITLE:-}" ]]; then
+if [ -z "${PR_TITLE:-}" ]; then
   echo "Error: PR_TITLE is not set" >&2
   exit 1
 fi
 
-if [[ -z "${PR_BODY:-}" ]]; then
+if [ -z "${PR_BODY:-}" ]; then
   echo "Error: PR_BODY is not set" >&2
   exit 1
 fi
@@ -43,9 +44,12 @@ resp=$(nix run --inputs-from . nixpkgs#curl -- -X POST \
   }" \
   "https://git.clan.lol/api/v1/repos/clan/clan-core/pulls")
 
-pr_number=$(echo "$resp" | jq -r '.number')
+if ! pr_number=$(echo "$resp" | jq -r '.number'); then
+  echo "Error parsing response from pull request creation" >&2
+  exit 1
+fi
 
-if [[ "$pr_number" == "null" ]]; then
+if [ "$pr_number" = "null" ]; then
   echo "Error creating pull request:" >&2
   echo "$resp" | jq . >&2
   exit 1
@@ -64,12 +68,15 @@ while true; do
       "delete_branch_after_merge": true
     }' \
     "https://git.clan.lol/api/v1/repos/clan/clan-core/pulls/$pr_number/merge")
-  msg=$(echo "$resp" | jq -r '.message')
-  if [[ "$msg" != "Please try again later" ]]; then
+  if ! msg=$(echo "$resp" | jq -r '.message'); then
+    echo "Error parsing merge response" >&2
+    exit 1
+  fi
+  if [ "$msg" != "Please try again later" ]; then
     break
   fi
   echo "Retrying in 2 seconds..."
   sleep 2
 done
 
-echo "Pull request #$pr_number merge initiated"
+echo "Pull request #$pr_number merge initiated"

README.md

@@ -24,7 +24,7 @@ If you're new to Clan and eager to dive in, start with our quickstart guide and
 
 In the Clan ecosystem, security is paramount. Learn how to handle secrets effectively:
 
-- **Secrets Management**: Securely manage secrets by consulting [secrets](https://docs.clan.lol/guides/getting-started/secrets/)<!-- [secrets.md](docs/site/guides/getting-started/secrets.md) -->.
+- **Secrets Management**: Securely manage secrets by consulting [Vars](https://docs.clan.lol/concepts/generators/)<!-- [secrets.md](docs/site/concepts/generators.md) -->.
 
 ### Contributing to Clan
 

checks/clan-core-for-checks.nix

@@ -1,6 +1,6 @@
 { fetchgit }:
 fetchgit {
   url = "https://git.clan.lol/clan/clan-core.git";
-  rev = "eea93ea22c9818da67e148ba586277bab9e73cea";
-  sha256 = "sha256-PV0Z+97QuxQbkYSVuNIJwUNXMbHZG/vhsA9M4cDTCOE=";
+  rev = "ba8a80eccf091fc7f99aef3895e31617d3813d20";
+  sha256 = "189srg4mc5y3prapm8day0x0wpibbqc72hrnl61agsmiq7cfmbkd";
 }

checks/flake-module.nix

@@ -19,18 +19,30 @@ let
   nixosLib = import (self.inputs.nixpkgs + "/nixos/lib") { };
 in
 {
-  imports = filter pathExists [
-    ./backups/flake-module.nix
-    ../nixosModules/clanCore/machine-id/tests/flake-module.nix
-    ../nixosModules/clanCore/state-version/tests/flake-module.nix
-    ./devshell/flake-module.nix
-    ./flash/flake-module.nix
-    ./impure/flake-module.nix
-    ./installation/flake-module.nix
-    ./morph/flake-module.nix
-    ./nixos-documentation/flake-module.nix
-    ./dont-depend-on-repo-root.nix
-  ];
+  imports =
+    let
+      clanCoreModulesDir = ../nixosModules/clanCore;
+      getClanCoreTestModules =
+        let
+          moduleNames = attrNames (builtins.readDir clanCoreModulesDir);
+          testPaths = map (
+            moduleName: clanCoreModulesDir + "/${moduleName}/tests/flake-module.nix"
+          ) moduleNames;
+        in
+        filter pathExists testPaths;
+    in
+    getClanCoreTestModules
+    ++ filter pathExists [
+      ./backups/flake-module.nix
+      ./devshell/flake-module.nix
+      ./flash/flake-module.nix
+      ./impure/flake-module.nix
+      ./installation/flake-module.nix
+      ./update/flake-module.nix
+      ./morph/flake-module.nix
+      ./nixos-documentation/flake-module.nix
+      ./dont-depend-on-repo-root.nix
+    ];
   flake.check = genAttrs [ "x86_64-linux" "aarch64-darwin" ] (
     system:
     let
@@ -88,7 +100,6 @@ in
             nixos-test-container = self.clanLib.test.containerTest ./container nixosTestArgs;
             nixos-test-zt-tcp-relay = self.clanLib.test.containerTest ./zt-tcp-relay nixosTestArgs;
             nixos-test-matrix-synapse = self.clanLib.test.containerTest ./matrix-synapse nixosTestArgs;
-            nixos-test-postgresql = self.clanLib.test.containerTest ./postgresql nixosTestArgs;
             nixos-test-user-firewall-iptables = self.clanLib.test.containerTest ./user-firewall/iptables.nix nixosTestArgs;
             nixos-test-user-firewall-nftables = self.clanLib.test.containerTest ./user-firewall/nftables.nix nixosTestArgs;
 
@@ -147,8 +158,11 @@ in
 
           clan-core-for-checks = pkgs.runCommand "clan-core-for-checks" { } ''
             cp -r ${pkgs.callPackage ./clan-core-for-checks.nix { }} $out
-            chmod +w $out/flake.lock
+            chmod -R +w $out
             cp ${../flake.lock} $out/flake.lock
+
+            # Create marker file to disable private flake loading in tests
+            touch $out/.skip-private-inputs
           '';
         };
       packages = lib.optionalAttrs (pkgs.stdenv.isLinux) {

checks/installation/flake-module.nix

@@ -149,7 +149,6 @@
       # vm-test-run-test-installation-> target:   To debug, enter the VM and run 'systemctl status backdoor.service'.
       checks =
         let
-          # Custom Python package for port management utilities
           closureInfo = pkgs.closureInfo {
             rootPaths = [
               self.checks.x86_64-linux.clan-core-for-checks
@@ -225,7 +224,7 @@
                       "install",
                       "--phases", "disko,install",
                       "--debug",
-                      "--flake", flake_dir,
+                      "--flake", str(flake_dir),
                       "--yes", "test-install-machine-without-system",
                       "--target-host", f"nonrootuser@localhost:{ssh_conn.host_port}",
                       "-i", ssh_conn.ssh_key,
@@ -289,9 +288,6 @@
                   assert not os.path.exists(hw_config_file), "hardware-configuration.nix should not exist initially"
                   assert not os.path.exists(facter_file), "facter.json should not exist initially"
 
-                  # Set CLAN_FLAKE for the commands
-                  os.environ["CLAN_FLAKE"] = flake_dir
-
                   # Test facter backend
                   clan_cmd = [
                       "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan",

checks/postgresql/default.nix

@@ -1,73 +0,0 @@
-({
-  name = "postgresql";
-
-  nodes.machine =
-    { self, config, ... }:
-    {
-      imports = [
-        self.nixosModules.clanCore
-        self.clanModules.postgresql
-        self.clanModules.localbackup
-      ];
-      clan.postgresql.users.test = { };
-      clan.postgresql.databases.test.create.options.OWNER = "test";
-      clan.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ];
-      clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
-      clan.core.settings.directory = ./.;
-
-      systemd.services.sample-service = {
-        wantedBy = [ "multi-user.target" ];
-        script = ''
-          while true; do
-            echo "Hello, world!"
-            sleep 5
-          done
-        '';
-      };
-
-      environment.systemPackages = [ config.services.postgresql.package ];
-    };
-  testScript =
-    { nodes, ... }:
-    ''
-      start_all()
-      machine.wait_for_unit("postgresql")
-      machine.wait_for_unit("sample-service")
-      # Create a test table
-      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
-
-      machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
-      timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
-
-      machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
-      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
-      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
-      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
-
-      machine.succeed("rm -rf /var/backup/postgres")
-
-      machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
-      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
-
-      machine.succeed("""
-      set -x
-      ${nodes.machine.clan.core.state.test.postRestoreCommand}
-      """)
-      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
-      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
-
-      timestamp_after = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
-      assert timestamp_before < timestamp_after, f"{timestamp_before} >= {timestamp_after}: expected sample-service to be restarted after restore"
-
-      # Check that the table is still there
-      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
-      output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
-      owner = output.split("\n")[1]
-      assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
-
-      # check if restore works if the database does not exist
-      machine.succeed("runuser -u postgres -- dropdb test")
-      machine.succeed("${nodes.machine.clan.core.state.test.postRestoreCommand}")
-      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
-    '';
-})

checks/service-dummy-test-from-flake/default.nix

@@ -29,18 +29,10 @@ nixosLib.runTest (
     testScript =
       { nodes, ... }:
       ''
+        import subprocess
         from nixos_test_lib.nix_setup import setup_nix_in_nix # type: ignore[import-untyped]
-        setup_nix_in_nix(None)  # No closure info for this test
 
-        def run_clan(cmd: list[str], **kwargs) -> str:
-            import subprocess
-            clan = "${clan-core.packages.${hostPkgs.system}.clan-cli}/bin/clan"
-            clan_args = ["--flake", "${config.clan.test.flakeForSandbox}"]
-            return subprocess.run(
-                ["${hostPkgs.util-linux}/bin/unshare", "--user", "--map-user", "1000", "--map-group", "1000", clan, *cmd, *clan_args],
-                **kwargs,
-                check=True,
-            ).stdout
+        setup_nix_in_nix(None)  # No closure info for this test
 
         start_all()
         admin1.wait_for_unit("multi-user.target")
@@ -60,7 +52,13 @@ nixosLib.runTest (
         # Check that the file is in the '0644' mode
         assert "-rw-r--r--" in ls_out, f"File is not in the '0644' mode: {ls_out}"
 
-        run_clan(["machines", "list"])
+        # Run clan command
+        result = subprocess.run(
+            ["${
+              clan-core.packages.${hostPkgs.system}.clan-cli
+            }/bin/clan", "machines", "list", "--flake", "${config.clan.test.flakeForSandbox}"],
+            check=True
+        )
       '';
   }
 )

checks/update/flake-module.nix

@@ -0,0 +1,237 @@
+{ self, ... }:
+{
+  # Machine for update test
+  clan.machines.test-update-machine = {
+    imports = [
+      self.nixosModules.test-update-machine
+      # Import the configuration file that will be created/updated during the test
+      ./test-update-machine/configuration.nix
+    ];
+  };
+  flake.nixosModules.test-update-machine =
+    { lib, modulesPath, ... }:
+    {
+      imports = [
+        (modulesPath + "/testing/test-instrumentation.nix")
+        (modulesPath + "/profiles/qemu-guest.nix")
+        self.clanLib.test.minifyModule
+        ../../lib/test/container-test-driver/nixos-module.nix
+      ];
+
+      # Apply patch to fix x-initrd.mount filesystem handling in switch-to-configuration-ng
+      nixpkgs.overlays = [
+        (_final: prev: {
+          switch-to-configuration-ng = prev.switch-to-configuration-ng.overrideAttrs (old: {
+            patches = (old.patches or [ ]) ++ [ ./switch-to-configuration-initrd-mount-fix.patch ];
+          });
+        })
+      ];
+
+      networking.hostName = "update-machine";
+
+      environment.etc."install-successful".text = "ok";
+
+      # Enable SSH and add authorized key for testing
+      services.openssh.enable = true;
+      services.openssh.settings.PasswordAuthentication = false;
+      users.users.root.openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ];
+      security.sudo.wheelNeedsPassword = false;
+
+      boot.consoleLogLevel = lib.mkForce 100;
+      boot.kernelParams = [ "boot.shell_on_fail" ];
+
+      boot.isContainer = true;
+      nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+      # Preserve the IP addresses assigned by the test framework
+      # (based on virtualisation.vlans = [1] and node number 1)
+      networking.interfaces.eth1 = {
+        useDHCP = false;
+        ipv4.addresses = [
+          {
+            address = "192.168.1.1";
+            prefixLength = 24;
+          }
+        ];
+        ipv6.addresses = [
+          {
+            address = "2001:db8:1::1";
+            prefixLength = 64;
+          }
+        ];
+      };
+
+      # Define the mounts that exist in the container to prevent them from being stopped
+      fileSystems = {
+        "/" = {
+          device = "/dev/disk/by-label/nixos";
+          fsType = "ext4";
+          options = [ "x-initrd.mount" ];
+        };
+        "/nix/.rw-store" = {
+          device = "tmpfs";
+          fsType = "tmpfs";
+          options = [
+            "mode=0755"
+          ];
+        };
+        "/nix/store" = {
+          device = "overlay";
+          fsType = "overlay";
+          options = [
+            "lowerdir=/nix/.ro-store"
+            "upperdir=/nix/.rw-store/upper"
+            "workdir=/nix/.rw-store/work"
+          ];
+        };
+      };
+    };
+
+  perSystem =
+    {
+      pkgs,
+      ...
+    }:
+    {
+      checks =
+        pkgs.lib.optionalAttrs (pkgs.stdenv.isLinux && pkgs.stdenv.hostPlatform.system == "x86_64-linux")
+          {
+            nixos-test-update =
+              let
+                closureInfo = pkgs.closureInfo {
+                  rootPaths = [
+                    self.checks.x86_64-linux.clan-core-for-checks
+                    self.clanInternals.machines.${pkgs.hostPlatform.system}.test-update-machine.config.system.build.toplevel
+                    pkgs.stdenv.drvPath
+                    pkgs.bash.drvPath
+                    pkgs.buildPackages.xorg.lndir
+                  ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
+                };
+              in
+              self.clanLib.test.containerTest {
+                name = "update";
+                nodes.machine = {
+                  imports = [ self.nixosModules.test-update-machine ];
+                };
+                extraPythonPackages = _p: [
+                  self.legacyPackages.${pkgs.system}.nixosTestLib
+                ];
+
+                testScript = ''
+                  import tempfile
+                  import os
+                  import subprocess
+                  from nixos_test_lib.ssh import setup_ssh_connection # type: ignore[import-untyped]
+                  from nixos_test_lib.nix_setup import prepare_test_flake # type: ignore[import-untyped]
+
+                  start_all()
+                  machine.wait_for_unit("multi-user.target")
+
+                  # Verify initial state
+                  machine.succeed("test -f /etc/install-successful")
+                  machine.fail("test -f /etc/update-successful")
+
+                  # Set up test environment
+                  with tempfile.TemporaryDirectory() as temp_dir:
+                      # Prepare test flake and Nix store
+                      flake_dir = prepare_test_flake(
+                          temp_dir,
+                          "${self.checks.x86_64-linux.clan-core-for-checks}",
+                          "${closureInfo}"
+                      )
+                      (flake_dir / ".clan-flake").write_text("")  # Ensure .clan-flake exists
+
+                      # Set up SSH connection
+                      ssh_conn = setup_ssh_connection(
+                          machine,
+                          temp_dir,
+                          "${../assets/ssh/privkey}"
+                      )
+
+                      # Update the machine configuration to add a new file
+                      machine_config_path = os.path.join(flake_dir, "machines", "test-update-machine", "configuration.nix")
+                      os.makedirs(os.path.dirname(machine_config_path), exist_ok=True)
+                      
+                      with open(machine_config_path, "w") as f:
+                          f.write("""
+                      {
+                        environment.etc."update-successful".text = "ok";
+                      }
+                      """)
+
+                      # Run clan update command  
+                      # Note: update command doesn't accept -i flag, SSH key must be in ssh-agent
+                      # Start ssh-agent and add the key
+                      agent_output = subprocess.check_output(["${pkgs.openssh}/bin/ssh-agent", "-s"], text=True)
+                      for line in agent_output.splitlines():
+                          if line.startswith("SSH_AUTH_SOCK="):
+                              os.environ["SSH_AUTH_SOCK"] = line.split("=", 1)[1].split(";")[0]
+                          elif line.startswith("SSH_AGENT_PID="):
+                              os.environ["SSH_AGENT_PID"] = line.split("=", 1)[1].split(";")[0]
+                      
+                      # Add the SSH key to the agent
+                      subprocess.run(["${pkgs.openssh}/bin/ssh-add", ssh_conn.ssh_key], check=True)
+
+
+                      # Run clan update command
+                      subprocess.run([
+                          "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan",
+                          "machines",
+                          "update",
+                          "--debug",
+                          "--flake", flake_dir,
+                          "--host-key-check", "none",
+                          "--fetch-local",  # Use local store instead of fetching from network
+                          "test-update-machine",
+                          "--target-host", f"root@192.168.1.1:{ssh_conn.host_port}",
+                      ], check=True)
+
+                      # Verify the update was successful
+                      machine.succeed("test -f /etc/update-successful")
+
+                      # Test update with --build-host
+                      # Update configuration again to test build-host functionality
+                      with open(machine_config_path, "w") as f:
+                          f.write("""
+                      {
+                        environment.etc."build-host-update-successful".text = "ok";
+                      }
+                      """)
+
+                      # Run clan update command with --build-host
+                      subprocess.run([
+                          "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan",
+                          "machines",
+                          "update",
+                          "--debug",
+                          "--flake", flake_dir,
+                          "--host-key-check", "none",
+                          "--fetch-local",  # Use local store instead of fetching from network
+                          "--build-host", f"root@192.168.1.1:{ssh_conn.host_port}",
+                          "test-update-machine",
+                          "--target-host", f"root@192.168.1.1:{ssh_conn.host_port}",
+                      ], check=True)
+
+                      # Verify the second update was successful
+                      machine.succeed("test -f /etc/build-host-update-successful")
+
+                      # Run clan update command with --build-host
+                      subprocess.run([
+                          "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan",
+                          "machines",
+                          "update",
+                          "--debug",
+                          "--flake", flake_dir,
+                          "--host-key-check", "none",
+                          "--fetch-local",  # Use local store instead of fetching from network
+                          "--build-host", f"root@192.168.1.1:{ssh_conn.host_port}",
+                          "test-update-machine",
+                          "--target-host", f"root@192.168.1.1:{ssh_conn.host_port}",
+                      ], check=True)
+
+                      # Verify the second update was successful
+                      machine.succeed("test -f /etc/build-host-update-successful")
+                '';
+              } { inherit pkgs self; };
+          };
+    };
+}

checks/update/switch-to-configuration-initrd-mount-fix.patch

@@ -0,0 +1,17 @@
+diff --git a/src/main.rs b/src/main.rs
+index 8baf5924a7db..1234567890ab 100644
+--- a/src/main.rs
++++ b/src/main.rs
+@@ -1295,6 +1295,12 @@ won't take effect until you reboot the system.
+ 
+     for (mountpoint, current_filesystem) in current_filesystems {
+         // Use current version of systemctl binary before daemon is reexeced.
++        
++        // Skip filesystem comparison if x-initrd.mount is present in options
++        if current_filesystem.options.contains("x-initrd.mount") {
++            continue;
++        }
++        
+         let unit = path_to_unit_name(&current_system_bin, &mountpoint);
+         if let Some(new_filesystem) = new_filesystems.get(&mountpoint) {
+             if current_filesystem.fs_type != new_filesystem.fs_type

checks/update/test-update-machine/configuration.nix

@@ -0,0 +1,3 @@
+{
+  # Initial empty configuration
+}

clanModules/borgbackup-static/README.md

@@ -4,7 +4,7 @@ description = "Statically configure borgbackup with sane defaults."
 !!! Danger "Deprecated"
     Use [borgbackup](borgbackup.md) instead.
 
-    Don't use borgbackup-static through [inventory](../../guides/inventory.md).
+    Don't use borgbackup-static through [inventory](../../concepts/inventory.md).
 
 This module implements the `borgbackup` backend and implements sane defaults
 for backup management through `borgbackup` for members of the clan.

clanModules/matrix-synapse/default.nix

@@ -61,7 +61,6 @@ in
     };
   };
   imports = [
-    ../postgresql
     (lib.mkRemovedOptionModule [
       "clan"
       "matrix-synapse"
@@ -106,15 +105,16 @@ in
       };
     };
 
-    clan.postgresql.users.matrix-synapse = { };
-    clan.postgresql.databases.matrix-synapse.create.options = {
+    clan.core.postgresql.enable = true;
+    clan.core.postgresql.users.matrix-synapse = { };
+    clan.core.postgresql.databases.matrix-synapse.create.options = {
       TEMPLATE = "template0";
       LC_COLLATE = "C";
       LC_CTYPE = "C";
       ENCODING = "UTF8";
       OWNER = "matrix-synapse";
     };
-    clan.postgresql.databases.matrix-synapse.restore.stopOnRestore = [ "matrix-synapse" ];
+    clan.core.postgresql.databases.matrix-synapse.restore.stopOnRestore = [ "matrix-synapse" ];
 
     clan.core.vars.generators =
       {

clanModules/nginx/default.nix

@@ -38,7 +38,6 @@
       recommendedOptimisation = lib.mkDefault true;
       recommendedProxySettings = lib.mkDefault true;
       recommendedTlsSettings = lib.mkDefault true;
-      recommendedZstdSettings = lib.mkDefault true;
 
       # Nginx sends all the access logs to /var/log/nginx/access.log by default.
       # instead of going to the journal!

clanModules/postgresql/default.nix

@@ -1,224 +1,9 @@
+{ lib, ... }:
 {
-  pkgs,
-  lib,
-  config,
-  ...
-}:
-let
-  createDatabaseState =
-    db:
-    let
-      folder = "/var/backup/postgres/${db.name}";
-      current = "${folder}/pg-dump";
-      compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
-    in
-    {
-      folders = [ folder ];
-      preBackupScript = ''
-        export PATH=${
-          lib.makeBinPath [
-            config.services.postgresql.package
-            config.systemd.package
-            pkgs.coreutils
-            pkgs.util-linux
-            pkgs.zstd
-          ]
-        }
-        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
-          sleep 1
-        done
-
-        mkdir -p "${folder}"
-        runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
-        mv "${current}.tmp" ${current}
-      '';
-      postRestoreScript = ''
-        export PATH=${
-          lib.makeBinPath [
-            config.services.postgresql.package
-            config.systemd.package
-            pkgs.coreutils
-            pkgs.util-linux
-            pkgs.zstd
-            pkgs.gnugrep
-          ]
-        }
-        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
-          sleep 1
-        done
-        echo "Waiting for postgres to be ready..."
-        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
-          if ! systemctl is-active postgresql; then exit 1; fi
-          sleep 0.1
-        done
-
-        if [[ -e "${current}" ]]; then
-          (
-            systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
-            trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
-
-            mkdir -p "${folder}"
-            if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
-              runuser -u postgres -- dropdb "${db.name}"
-            fi
-            runuser -u postgres -- pg_restore -C -d postgres "${current}"
-          )
-        else
-          echo No database backup found, skipping restore
-        fi
-      '';
-    };
-
-  createDatabase = db: ''
-    CREATE DATABASE "${db.name}" ${
-      lib.concatStringsSep " " (
-        lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
-      )
-    }
-  '';
-  cfg = config.clan.postgresql;
-
-  userClauses = lib.mapAttrsToList (
-    _: user:
-    ''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
-  ) cfg.users;
-  databaseClauses = lib.mapAttrsToList (
-    name: db:
-    lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
-  ) cfg.databases;
-in
-{
-  options.clan.postgresql = {
-    # we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
-    databases = lib.mkOption {
-      description = "Databases to create";
-      default = { };
-      type = lib.types.attrsOf (
-        lib.types.submodule (
-          { name, ... }:
-          {
-            options = {
-              name = lib.mkOption {
-                type = lib.types.str;
-                default = name;
-                description = "Database name.";
-              };
-              service = lib.mkOption {
-                type = lib.types.str;
-                default = name;
-                description = "Service name that we associate with the database.";
-              };
-              # set to false, in case the upstream module uses ensureDatabase option
-              create.enable = lib.mkOption {
-                type = lib.types.bool;
-                default = true;
-                description = "Create the database if it does not exist.";
-              };
-              create.options = lib.mkOption {
-                description = "Options to pass to the CREATE DATABASE command.";
-                type = lib.types.lazyAttrsOf lib.types.str;
-                default = { };
-                example = {
-                  TEMPLATE = "template0";
-                  LC_COLLATE = "C";
-                  LC_CTYPE = "C";
-                  ENCODING = "UTF8";
-                  OWNER = "foo";
-                };
-              };
-              restore.stopOnRestore = lib.mkOption {
-                type = lib.types.listOf lib.types.str;
-                default = [ ];
-                description = "List of systemd services to stop before restoring the database.";
-              };
-            };
-          }
-        )
-      );
-    };
-    users = lib.mkOption {
-      description = "Users to create";
-      default = { };
-      type = lib.types.attrsOf (
-        lib.types.submodule (
-          { name, ... }:
-          {
-            options.name = lib.mkOption {
-              description = "User name";
-              type = lib.types.str;
-              default = name;
-            };
-          }
-        )
-      );
-    };
-  };
-  config = {
-    services.postgresql.settings = {
-      wal_level = "replica";
-      max_wal_senders = 3;
-    };
-
-    services.postgresql.enable = true;
-    # We are duplicating a bit the upstream module but allow to create databases with options
-    systemd.services.postgresql.postStart = ''
-      PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
-
-      while ! $PSQL -d postgres -c "" 2> /dev/null; do
-        if ! kill -0 "$MAINPID"; then exit 1; fi
-        sleep 0.1
-      done
-      ${lib.concatStringsSep "\n" userClauses}
-      ${lib.concatStringsSep "\n" databaseClauses}
-    '';
-
-    clan.core.state = lib.mapAttrs' (
-      _: db: lib.nameValuePair db.service (createDatabaseState db)
-    ) config.clan.postgresql.databases;
-
-    environment.systemPackages = builtins.map (
-      db:
-      let
-        folder = "/var/backup/postgres/${db.name}";
-        current = "${folder}/pg-dump";
-      in
-      pkgs.writeShellScriptBin "postgres-db-restore-command-${db.name}" ''
-        export PATH=${
-          lib.makeBinPath [
-            config.services.postgresql.package
-            config.systemd.package
-            pkgs.coreutils
-            pkgs.util-linux
-            pkgs.zstd
-            pkgs.gnugrep
-          ]
-        }
-        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
-          sleep 1
-        done
-        echo "Waiting for postgres to be ready..."
-        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
-          if ! systemctl is-active postgresql; then exit 1; fi
-          sleep 0.1
-        done
-
-        if [[ -e "${current}" ]]; then
-          (
-            ${lib.optionalString (db.restore.stopOnRestore != [ ]) ''
-              systemctl stop ${builtins.toString db.restore.stopOnRestore}
-              trap "systemctl start ${builtins.toString db.restore.stopOnRestore}" EXIT
-            ''}
-
-            mkdir -p "${folder}"
-            if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
-              runuser -u postgres -- dropdb "${db.name}"
-            fi
-            runuser -u postgres -- pg_restore -C -d postgres "${current}"
-          )
-        else
-          echo No database backup found, skipping restore
-        fi
-      ''
-    ) (builtins.attrValues config.clan.postgresql.databases);
-  };
+  imports = [
+    (lib.mkRemovedOptionModule [
+      "clan"
+      "postgresql"
+    ] "The postgresql module has been migrated to a clan core option. Use clan.core.postgresql instead")
+  ];
 }

clanModules/root-password/README.md

@@ -12,7 +12,7 @@ After the system was installed/deployed the following command can be used to dis
 clan vars get [machine_name] root-password/root-password
 ```
 
-See also: [Vars](../../guides/vars-backend.md)
+See also: [Vars](../../concepts/generators.md)
 
 To regenerate the password run:
 ```

clanModules/user-password/README.md

@@ -16,7 +16,7 @@ After the system was installed/deployed the following command can be used to dis
 clan vars get [machine_name] root-password/root-password
 ```
 
-See also: [Vars](../../guides/vars-backend.md)
+See also: [Vars](../../concepts/generators.md)
 
 To regenerate the password run:
 ```

clanModules/vaultwarden/default.nix

@@ -10,7 +10,6 @@ in
 
 {
   imports = [
-    ../postgresql
     (lib.mkRemovedOptionModule [
       "clan"
       "vaultwarden"
@@ -57,15 +56,17 @@ in
 
   config = {
 
-    clan.postgresql.users.vaultwarden = { };
-    clan.postgresql.databases.vaultwarden.create.options = {
+    clan.core.postgresql.enable = true;
+
+    clan.core.postgresql.users.vaultwarden = { };
+    clan.core.postgresql.databases.vaultwarden.create.options = {
       TEMPLATE = "template0";
       LC_COLLATE = "C";
       LC_CTYPE = "C";
       ENCODING = "UTF8";
       OWNER = "vaultwarden";
     };
-    clan.postgresql.databases.vaultwarden.restore.stopOnRestore = [ "vaultwarden" ];
+    clan.core.postgresql.databases.vaultwarden.restore.stopOnRestore = [ "vaultwarden" ];
 
     services.nginx = {
       enable = true;

clanServices/borgbackup/README.md

@@ -1,9 +1,59 @@
-BorgBackup (short: Borg) gives you:
+## Usage
 
-- Space efficient storage of backups.
-- Secure, authenticated encryption.
-- Compression: lz4, zstd, zlib, lzma or none.
-- Mountable backups with FUSE.
+```nix
+inventory.instances = {
+  borgbackup = {
+    module = {
+      name = "borgbackup";
+      input = "clan";
+    };
+    roles.client.machines."jon".settings = {
+      destinations."storagebox" = {
+        repo = "username@$hostname:/./borgbackup";
+        rsh = ''ssh -oPort=23 -i /run/secrets/vars/borgbackup/borgbackup.ssh'';
+      };
+    };
+    roles.server.machines = { };
+  };
+};
+```
+
+The input should be named according to your flake input. Jon is configured as a
+client machine with a destination pointing to a Hetzner Storage Box.
+
+## Overview
+
+This guide explains how to set up and manage
+[BorgBackup](https://borgbackup.readthedocs.io/) for secure, efficient backups
+in a clan network. BorgBackup provides:
+
+- Space efficient storage of backups with deduplication
+- Secure, authenticated encryption
+- Compression: lz4, zstd, zlib, lzma or none
+- Mountable backups with FUSE
 - Easy installation on multiple platforms: Linux, macOS, BSD, …
 - Free software (BSD license).
 - Backed by a large and active open-source community.
+
+## Roles
+
+### 1. Client
+
+Clients are machines that create and send backups to various destinations. Each
+client can have multiple backup destinations configured.
+
+### 2. Server
+
+Servers act as backup repositories, receiving and storing backups from client
+machines. They can be dedicated backup servers within your clan network.
+
+## Backup destinations
+
+This service allows you to perform backups to multiple `destinations`.
+Destinations can be:
+
+- **Local**: Local disk storage
+- **Server**: Your own borgbackup server (using the `server` role)
+- **Third-party services**: Such as Hetzner's Storage Box
+
+For a more comprehensive guide on backups look into the guide section.

clanServices/internet/default.nix

@@ -0,0 +1,47 @@
+{ ... }:
+{
+  _class = "clan.service";
+  manifest.name = "clan-core/internet";
+  manifest.description = "direct access (or via ssh jumphost) to machines";
+  manifest.categories = [
+    "System"
+    "Network"
+  ];
+  roles.default = {
+    interface =
+      { lib, ... }:
+      {
+        options = {
+          host = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              ip address or hostname (domain) of the machine
+            '';
+          };
+          jumphosts = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [ ];
+            description = ''
+              optional list of jumphosts to use to connect to the machine
+            '';
+          };
+        };
+      };
+    perInstance =
+      {
+        roles,
+        lib,
+        settings,
+        ...
+      }:
+      {
+        exports.networking = {
+          # TODO add user space network support to clan-cli
+          peers = lib.mapAttrs (_name: machine: {
+            host.plain = machine.settings.host;
+            SSHOptions = map (_x: "-J x") machine.settings.jumphosts;
+          }) roles.default.machines;
+        };
+      };
+  };
+}

clanServices/internet/flake-module.nix

@@ -0,0 +1,9 @@
+{ lib, ... }:
+let
+  module = lib.modules.importApply ./default.nix { };
+in
+{
+  clan.modules = {
+    internet = module;
+  };
+}

clanServices/tor/default.nix

@@ -0,0 +1,110 @@
+{ ... }:
+{
+  _class = "clan.service";
+  manifest.name = "clan-core/tor";
+  manifest.description = "Onion routing, use Hidden services to connect your machines";
+  manifest.categories = [
+    "System"
+    "Network"
+  ];
+
+  roles.client = {
+    perInstance =
+      {
+        ...
+      }:
+      {
+        nixosModule =
+          {
+            ...
+          }:
+          {
+            config = {
+              services.tor = {
+                enable = true;
+                torsocks.enable = true;
+                client.enable = true;
+              };
+            };
+          };
+      };
+  };
+
+  roles.server = {
+    # interface =
+    #   { lib, ... }:
+    #   {
+    #     options = {
+    #       OciSettings = lib.mkOption {
+    #         type = lib.types.raw;
+    #         default = null;
+    #         description = "NixOS settings for virtualisation.oci-container.<name>.settings";
+    #       };
+    #       buildContainer = lib.mkOption {
+    #         type = lib.types.nullOr lib.types.str;
+    #         default = null;
+    #       };
+    #     };
+    #   };
+    perInstance =
+      {
+        instanceName,
+        roles,
+        lib,
+        ...
+      }:
+      {
+        exports.networking = {
+          priority = lib.mkDefault 10;
+          # TODO add user space network support to clan-cli
+          module = "clan_lib.network.tor";
+          peers = lib.mapAttrs (name: machine: {
+            host.var = {
+              machine = name;
+              generator = "tor_${instanceName}";
+              file = "hostname";
+            };
+          }) roles.server.machines;
+        };
+        nixosModule =
+          {
+            pkgs,
+            config,
+            ...
+          }:
+          {
+            config = {
+              services.tor = {
+                enable = true;
+                relay.onionServices."clan_${instanceName}" = {
+                  version = 3;
+                  # TODO get ports from instance machine config
+                  map = [
+                    {
+                      port = 22;
+                      target.port = 22;
+                    }
+                  ];
+                  secretKey = config.clan.core.vars.generators."tor_${instanceName}".files.hs_ed25519_secret_key.path;
+                };
+              };
+              clan.core.vars.generators."tor_${instanceName}" = {
+                files.hs_ed25519_secret_key = { };
+                files.hostname = { };
+                runtimeInputs = with pkgs; [
+                  coreutils
+                  tor
+                ];
+                script = ''
+                  mkdir -p data
+                  echo -e "DataDirectory ./data\nSocksPort 0\nHiddenServiceDir ./hs\nHiddenServicePort 80 127.0.0.1:80" > torrc
+                  timeout 2 tor -f torrc || :
+                  mv hs/hs_ed25519_secret_key $out/hs_ed25519_secret_key
+                  mv hs/hostname $out/hostname
+                '';
+              };
+            };
+          };
+      };
+  };
+}

clanServices/tor/flake-module.nix

@@ -0,0 +1,9 @@
+{ lib, ... }:
+let
+  module = lib.modules.importApply ./default.nix { };
+in
+{
+  clan.modules = {
+    tor = module;
+  };
+}

clanServices/wifi/default.nix

@@ -39,7 +39,7 @@ in
     };
 
     perInstance =
-      { settings, ... }:
+      { instanceName, settings, ... }:
       {
         nixosModule =
           { pkgs, config, ... }:
@@ -86,7 +86,7 @@ in
 
             # service to generate the environment file containing all secrets, as
             #   expected by the nixos NetworkManager-ensure-profile service
-            systemd.services.NetworkManager-setup-secrets = {
+            systemd.services."NetworkManager-setup-secrets-${instanceName}" = {
               description = "Generate wifi secrets for NetworkManager";
               requiredBy = [ "NetworkManager-ensure-profiles.service" ];
               partOf = [ "NetworkManager-ensure-profiles.service" ];

clanServices/wifi/tests/vm/default.nix

@@ -7,8 +7,16 @@
     inventory = {
 
       machines.test = { };
+      machines.second = { };
 
       instances = {
+        wg-test-all = {
+          module.name = "@clan/wifi";
+          module.input = "self";
+          roles.default.tags.all = { };
+          roles.default.settings.networks.all = { };
+        };
+
         wg-test-one = {
           module.name = "@clan/wifi";
           module.input = "self";

clanServices/zerotier/default.nix

@@ -134,9 +134,9 @@
               systemd.services.zerotier-inventory-autoaccept =
                 let
                   machines = uniqueStrings (
-                    (lib.attrNames roles.moon.machines)
-                    ++ (lib.attrNames roles.controller.machines)
-                    ++ (lib.attrNames roles.peer.machines)
+                    (lib.optionals (roles ? moon) (lib.attrNames roles.moon.machines))
+                    ++ (lib.optionals (roles ? controller) (lib.attrNames roles.controller.machines))
+                    ++ (lib.optionals (roles ? peer) (lib.attrNames roles.peer.machines))
                   );
                   networkIps = builtins.foldl' (
                     ips: name:

clanServices/zerotier/tests/eval-tests.nix

@@ -32,6 +32,33 @@ let
         };
       };
     }).config;
+  testFlakeNoMoon =
+    (clanLib.clan {
+      self = { };
+      directory = ./vm;
+
+      machines.jon = {
+        nixpkgs.hostPlatform = "x86_64-linux";
+      };
+      machines.sara = {
+        nixpkgs.hostPlatform = "x86_64-linux";
+      };
+      machines.bam = {
+        nixpkgs.hostPlatform = "x86_64-linux";
+      };
+
+      modules.zerotier = module;
+
+      inventory.instances = {
+        zerotier = {
+          module.name = "zerotier";
+          module.input = "self";
+
+          roles.peer.tags.all = { };
+          roles.controller.machines.bam = { };
+        };
+      };
+    }).config;
 in
 {
   test_peers = {
@@ -73,4 +100,30 @@ in
       networkName = "zerotier";
     };
   };
+  test_peers_no_moon = {
+    expr = {
+      hasNetworkIds = testFlakeNoMoon.nixosConfigurations.jon.config.services.zerotierone.joinNetworks;
+      isController =
+        testFlakeNoMoon.nixosConfigurations.jon.config.clan.core.networking.zerotier.controller.enable;
+      networkName = testFlakeNoMoon.nixosConfigurations.jon.config.clan.core.networking.zerotier.name;
+    };
+    expected = {
+      hasNetworkIds = [ "0e28cb903344475e" ];
+      isController = false;
+      networkName = "zerotier";
+    };
+  };
+  test_controller_no_moon = {
+    expr = {
+      hasNetworkIds = testFlakeNoMoon.nixosConfigurations.bam.config.services.zerotierone.joinNetworks;
+      isController =
+        testFlakeNoMoon.nixosConfigurations.bam.config.clan.core.networking.zerotier.controller.enable;
+      networkName = testFlakeNoMoon.nixosConfigurations.bam.config.clan.core.networking.zerotier.name;
+    };
+    expected = {
+      hasNetworkIds = [ "0e28cb903344475e" ];
+      isController = true;
+      networkName = "zerotier";
+    };
+  };
 }

docs/mkdocs.yml

@@ -48,61 +48,81 @@ nav:
   - Home: index.md
   - Guides:
       - Getting Started:
-          - 🚀 Creating Your First Clan: guides/getting-started/index.md
-          - 📀 Create USB Installer (optional): guides/getting-started/installer.md
-          - ⚙️ Add Machines: guides/getting-started/add-machines.md
-          - ⚙️ Add User: guides/getting-started/add-user.md
-          - ⚙️ Add Services: guides/getting-started/add-services.md
-          - 🔐 Secrets & Facts: guides/getting-started/secrets.md
-          - 🚢 Deploy Machine: guides/getting-started/deploy.md
-          - 🧪 Continuous Integration: guides/getting-started/check.md
-      - clanServices: guides/clanServices.md
-      - Disk Encryption: guides/disk-encryption.md
-      - Mesh VPN: guides/mesh-vpn.md
+          - Creating Your First Clan: guides/getting-started/index.md
+          - Create USB Installer: guides/getting-started/installer.md
+          - Add Machines: guides/getting-started/add-machines.md
+          - Add User: guides/getting-started/add-user.md
+          - Add Services: guides/getting-started/add-services.md
+          - Deploy Machine: guides/getting-started/deploy.md
+          - Continuous Integration: guides/getting-started/check.md
+      - Using Services: guides/clanServices.md
       - Backup & Restore: guides/backups.md
-      - Vars Backend: guides/vars-backend.md
-      - Facts Backend: guides/secrets.md
-      - Adding more machines: guides/more-machines.md
+      - Disk Encryption: guides/disk-encryption.md
+      - Age Plugins: guides/age-plugins.md
+      - Secrets management: guides/secrets.md
       - Target Host: guides/target-host.md
-      - Inventory:
-          - Inventory: guides/inventory.md
+      - Zerotier VPN: guides/mesh-vpn.md
       - Secure Boot: guides/secure-boot.md
       - Flake-parts: guides/flake-parts.md
-      - Authoring:
-          - clanService: guides/authoring/clanServices/index.md
-          - Disk Template: guides/authoring/templates/disk/disko-templates.md
-          - clanModule: guides/authoring/clanModules/index.md
+      - macOS: guides/macos.md
       - Contributing:
-          - Contribute: guides/contributing/CONTRIBUTING.md
+          - Contributing: guides/contributing/CONTRIBUTING.md
           - Debugging: guides/contributing/debugging.md
           - Testing: guides/contributing/testing.md
+
+      - Writing a Service Module: guides/services/community.md
+      - Writing a Disko Template: guides/disko-templates/community.md
       - Migrations:
           - Migrate existing Flakes: guides/migrations/migration-guide.md
           - Migrate inventory Services: guides/migrations/migrate-inventory-services.md
           - Facts Vars Migration: guides/migrations/migration-facts-vars.md
           - Disk id: guides/migrations/disk-id.md
-      - macOS: guides/macos.md
+  - Concepts:
+      - Inventory: concepts/inventory.md
+      - Generators: concepts/generators.md
+      - Autoincludes: concepts/autoincludes.md
+      - Templates: concepts/templates.md
   - Reference:
       - Overview: reference/index.md
+      - Clan Options: options.md
       - Services:
-          - Overview: reference/clanServices/index.md
-          - reference/clanServices/admin.md
-          - reference/clanServices/borgbackup.md
-          - reference/clanServices/data-mesher.md
-          - reference/clanServices/emergency-access.md
-          - reference/clanServices/garage.md
-          - reference/clanServices/hello-world.md
-          - reference/clanServices/importer.md
-          - reference/clanServices/mycelium.md
-          - reference/clanServices/packages.md
-          - reference/clanServices/sshd.md
-          - reference/clanServices/state-version.md
-          - reference/clanServices/trusted-nix-caches.md
-          - reference/clanServices/users.md
-          - reference/clanServices/wifi.md
-          - reference/clanServices/zerotier.md
-      - Interface for making Services: reference/clanServices/clan-service-author-interface.md
-      - Modules:
+          - Overview:
+              - reference/clanServices/index.md
+
+              - reference/clanServices/admin.md
+              - reference/clanServices/borgbackup.md
+              - reference/clanServices/data-mesher.md
+              - reference/clanServices/emergency-access.md
+              - reference/clanServices/garage.md
+              - reference/clanServices/hello-world.md
+              - reference/clanServices/importer.md
+              - reference/clanServices/mycelium.md
+              - reference/clanServices/packages.md
+              - reference/clanServices/sshd.md
+              - reference/clanServices/state-version.md
+              - reference/clanServices/trusted-nix-caches.md
+              - reference/clanServices/users.md
+              - reference/clanServices/wifi.md
+              - reference/clanServices/zerotier.md
+          - API: reference/clanServices/clan-service-author-interface.md
+
+      - CLI:
+          - Overview: reference/cli/index.md
+
+          - reference/cli/backups.md
+          - reference/cli/facts.md
+          - reference/cli/flakes.md
+          - reference/cli/flash.md
+          - reference/cli/machines.md
+          - reference/cli/select.md
+          - reference/cli/secrets.md
+          - reference/cli/show.md
+          - reference/cli/ssh.md
+          - reference/cli/state.md
+          - reference/cli/templates.md
+          - reference/cli/vars.md
+          - reference/cli/vms.md
+      - Modules (deprecated):
           - Overview: reference/clanModules/index.md
           - reference/clanModules/frontmatter/index.md
           # TODO: display the docs of the clan.service modules
@@ -145,38 +165,21 @@ nav:
           - reference/clanModules/zerotier-static-peers.md
           - reference/clanModules/zerotier.md
           - reference/clanModules/zt-tcp-relay.md
-      - CLI:
-          - Overview: reference/cli/index.md
 
-          - reference/cli/backups.md
-          - reference/cli/facts.md
-          - reference/cli/flakes.md
-          - reference/cli/flash.md
-          - reference/cli/machines.md
-          - reference/cli/select.md
-          - reference/cli/secrets.md
-          - reference/cli/show.md
-          - reference/cli/ssh.md
-          - reference/cli/state.md
-          - reference/cli/templates.md
-          - reference/cli/vars.md
-          - reference/cli/vms.md
-      - NixOS Modules:
-          - clan.core:
-              - Overview: reference/clan.core/index.md
+      - clan.core (NixOS Options):
+          - Overview: reference/clan.core/index.md
+          - reference/clan.core/backups.md
+          - reference/clan.core/deployment.md
+          - reference/clan.core/facts.md
+          - reference/clan.core/networking.md
+          - reference/clan.core/postgresql.md
+          - reference/clan.core/settings.md
+          - reference/clan.core/sops.md
+          - reference/clan.core/state.md
+          - reference/clan.core/vars.md
+
+      - Developer-api: api.md
 
-              - reference/clan.core/backups.md
-              - reference/clan.core/deployment.md
-              - reference/clan.core/facts.md
-              - reference/clan.core/networking.md
-              - reference/clan.core/settings.md
-              - reference/clan.core/sops.md
-              - reference/clan.core/state.md
-              - reference/clan.core/vars.md
-      - Nix API:
-          - clan: reference/nix-api/clan.md
-          - Inventory: reference/nix-api/inventory.md
-      - Glossary: reference/glossary.md
       - Decisions:
           - Architecture Decisions: decisions/README.md
           - 01-clanModules: decisions/01-ClanModules.md
@@ -185,10 +188,7 @@ nav:
           - 04-fetching-nix-from-python: decisions/04-fetching-nix-from-python.md
           - 05-deployment-parameters: decisions/05-deployment-parameters.md
           - Template: decisions/_template.md
-  - Options: options.md
-  - Developer:
-      - Introduction: intern/index.md
-      - API: intern/api.md
+      - Glossary: reference/glossary.md
 
 docs_dir: site
 site_dir: out
@@ -199,6 +199,7 @@ theme:
   favicon: https://clan.lol/favicon.svg
   name: material
   features:
+    - navigation.footer
     - navigation.instant
     - navigation.tabs
     - navigation.tabs.sticky
@@ -246,3 +247,6 @@ plugins:
   - search
   - macros
   - redoc-tag
+  - redirects:
+      redirect_maps:
+        guides/getting-started/secrets.md: concepts/generators.md

docs/nix/default.nix

@@ -40,6 +40,7 @@ pkgs.stdenv.mkDerivation {
       mkdocs-material
       mkdocs-macros
       mkdocs-redoc-tag
+      mkdocs-redirects
     ]);
   configurePhase = ''
     pushd docs
@@ -54,6 +55,7 @@ pkgs.stdenv.mkDerivation {
     chmod -R +w ./site/reference
     echo "Generated API documentation in './site/reference/' "
 
+    rm -r ./site/options-page || true
     cp -r ${docs-options} ./site/options-page
     chmod -R +w ./site/options-page
 

docs/nix/options/flake-module.nix

@@ -114,9 +114,6 @@
         in
         {
           options = {
-            _ = mkOption {
-              type = types.raw;
-            };
             instances.${name} = lib.mkOption {
               inherit description;
               type = types.submodule {
@@ -149,20 +146,29 @@
           };
         };
 
-      mkScope = name: modules: {
-        inherit name;
-        modules = [
-          {
-            _module.args = { inherit clanLib; };
-            _file = "docs mkScope";
-          }
-          { noInstanceOptions = true; }
-          ../../../lib/modules/inventoryClass/interface.nix
-        ] ++ mapAttrsToList fakeInstanceOptions modules;
-        urlPrefix = "https://github.com/nix-community/dream2nix/blob/main/";
-      };
+      docModules = [
+        {
+          inherit self;
+        }
+        self.modules.clan.default
+        {
+          options.inventory = lib.mkOption {
+            type = types.submoduleWith {
+              modules = [
+                { noInstanceOptions = true; }
+              ] ++ mapAttrsToList fakeInstanceOptions serviceModules;
+            };
+          };
+        }
+      ];
+
     in
     {
+      # Uncomment for debugging
+      # legacyPackages.docModules = lib.evalModules {
+      #   modules = docModules;
+      # };
+
       packages = lib.optionalAttrs ((privateInputs ? nuschtos) || (inputs ? nuschtos)) {
         docs-options =
           (privateInputs.nuschtos or inputs.nuschtos)
@@ -171,7 +177,13 @@
               inherit baseHref;
               title = "Clan Options";
               # scopes = mapAttrsToList mkScope serviceModules;
-              scopes = [ (mkScope "Clan Inventory" serviceModules) ];
+              scopes = [
+                {
+                  name = "Clan";
+                  modules = docModules;
+                  urlPrefix = "https://git.clan.lol/clan/clan-core/src/branch/main/";
+                }
+              ];
             };
       };
     };

docs/nix/render_options/__init__.py

@@ -193,7 +193,7 @@ def module_header(module_name: str, has_inventory_feature: bool = False) -> str:
 def module_nix_usage(module_name: str) -> str:
     return f"""## Usage via Nix
 
-**This module can be also imported directly in your nixos configuration. Although it is recommended to use the [inventory](../../reference/nix-api/inventory.md) interface if available.**
+**This module can be also imported directly in your nixos configuration. Although it is recommended to use the [inventory](../../concepts/inventory.md) interface if available.**
 
 Some modules are considered 'low-level' or 'expert modules' and are not available via the inventory interface.
 
@@ -373,7 +373,7 @@ This module can be used via predefined roles
             """
 Every role has its own configuration options, which are each listed below.
 
-For more information, see the [inventory guide](../../guides/inventory.md).
+For more information, see the [inventory guide](../../concepts/inventory.md).
 
 ??? Example
     For example the `admin` module adds the following options globally to all machines where it is used.
@@ -402,7 +402,7 @@ certain option types restricted to enable configuration through a graphical
 interface.
 
 !!! note "🔹"
-    Modules with this indicator support the [inventory](../../guides/inventory.md) feature.
+    Modules with this indicator support the [inventory](../../concepts/inventory.md) feature.
 
 """
 
@@ -465,6 +465,10 @@ Learn how to use `clanServices` in practice in the [Using clanServices guide](..
         service_links: dict[str, dict[str, dict[str, Any]]] = json.load(f3)
 
     for module_name, module_info in service_links.items():
+        # Skip specific modules that are not ready for documentation
+        if module_name in ["internet", "tor"]:
+            continue
+
         output = f"# {module_name}\n\n"
         # output += f"`clan.modules.{module_name}`\n"
         output += f"*{module_info['manifest']['description']}*\n"
@@ -675,86 +679,6 @@ def build_option_card(module_name: str, frontmatter: Frontmatter) -> str:
     return f"{to_md_li(module_name, frontmatter)}\n\n"
 
 
-def produce_build_clan_docs() -> None:
-    if not BUILD_CLAN_PATH:
-        msg = f"Environment variables are not set correctly: BUILD_CLAN_PATH={BUILD_CLAN_PATH}. Expected a path to the optionsJSON"
-        raise ClanError(msg)
-
-    if not OUT:
-        msg = f"Environment variables are not set correctly: $out={OUT}"
-        raise ClanError(msg)
-
-    output = """# Clan
-
-This provides an overview of the available arguments of the `clan` interface.
-
-Each attribute is documented below
-
-- **clan-core.lib.clan**: A function that takes an attribute set.
-
-    ??? example "clan Example"
-
-        ```nix
-        clan {
-            self = self;
-            machines = {
-                jon = { };
-                sara = { };
-            };
-        };
-        ```
-
-- **clan with flake-parts**: Import the FlakeModule
-
-    After importing the FlakeModule you can define your `clan` as a flake attribute
-
-    All attribute can be defined via `clan.*`
-
-    Further information see: [flake-parts](../../guides/flake-parts.md) guide.
-
-    ??? example "flake-parts Example"
-
-        ```nix
-        flake-parts.lib.mkFlake { inherit inputs; } ({
-            systems = [];
-            imports = [
-                clan-core.flakeModules.default
-            ];
-            clan = {
-                machines = {
-                    jon = { };
-                    sara = { };
-                };
-            };
-        });
-        ```
-
-"""
-    with Path(BUILD_CLAN_PATH).open() as f:
-        options: dict[str, dict[str, Any]] = json.load(f)
-
-        split = split_options_by_root(options)
-        for option_name, options in split.items():
-            # Skip underscore options
-            if option_name.startswith("_"):
-                continue
-            # Skip inventory sub options
-            # Inventory model has its own chapter
-            if option_name.startswith("inventory."):
-                continue
-
-            print(f"[build_clan_docs] Rendering option of {option_name}...")
-            root = options_to_tree(options)
-
-            for option in root.suboptions:
-                output += options_docs_from_tree(option, init_level=2)
-
-    outfile = Path(OUT) / "nix-api/clan.md"
-    outfile.parent.mkdir(parents=True, exist_ok=True)
-    with Path.open(outfile, "w") as of:
-        of.write(output)
-
-
 def split_options_by_root(options: dict[str, Any]) -> dict[str, dict[str, Any]]:
     """
     Split the flat dictionary of options into a dict of which each entry will construct complete option trees.
@@ -801,7 +725,7 @@ Typically needed by module authors to define roles, behavior and metadata for di
 !!! Note
     This is not a user-facing documentation, but rather meant as a reference for *module authors*
 
-    See: [clanService Authoring Guide](../../guides/authoring/clanServices/index.md)
+    See: [clanService Authoring Guide](../../guides/services/community.md)
 """
     # Inventory options are already included under the clan attribute
     # We just omitted them in the clan docs, because we want a separate output for the inventory model
@@ -830,48 +754,6 @@ class Option:
     suboptions: list["Option"] = field(default_factory=list)
 
 
-def produce_inventory_docs() -> None:
-    if not BUILD_CLAN_PATH:
-        msg = f"Environment variables are not set correctly: BUILD_CLAN_PATH={BUILD_CLAN_PATH}. Expected a path to the optionsJSON"
-        raise ClanError(msg)
-
-    if not OUT:
-        msg = f"Environment variables are not set correctly: $out={OUT}"
-        raise ClanError(msg)
-
-    output = """# Inventory
-This provides an overview of the available attributes of the `inventory` model.
-
-It can be set via the `inventory` attribute of the [`clan`](./clan.md#inventory) function, or via the [`clan.inventory`](./clan.md#inventory) attribute of flake-parts.
-
-"""
-    # Inventory options are already included under the clan attribute
-    # We just omitted them in the clan docs, because we want a separate output for the inventory model
-    with Path(BUILD_CLAN_PATH).open() as f:
-        options: dict[str, dict[str, Any]] = json.load(f)
-
-        clan_root_option = options_to_tree(options)
-        # Find the inventory options
-        inventory_opt: None | Option = None
-        for opt in clan_root_option.suboptions:
-            if opt.name == "inventory":
-                inventory_opt = opt
-                break
-
-        if not inventory_opt:
-            print("No inventory options found.")
-            exit(1)
-        # Render the inventory options
-        # This for loop excludes the root node
-        for option in inventory_opt.suboptions:
-            output += options_docs_from_tree(option, init_level=2)
-
-    outfile = Path(OUT) / "nix-api/inventory.md"
-    outfile.parent.mkdir(parents=True, exist_ok=True)
-    with Path.open(outfile, "w") as of:
-        of.write(output)
-
-
 def option_short_name(option_name: str) -> str:
     parts = option_name.split(".")
     short_name = ""
@@ -980,9 +862,6 @@ def options_docs_from_tree(
 if __name__ == "__main__":  #
     produce_clan_core_docs()
 
-    produce_build_clan_docs()
-    produce_inventory_docs()
-
     produce_clan_service_author_docs()
 
     produce_clan_modules_docs()

docs/site/concepts/autoincludes.md

@@ -0,0 +1,15 @@
+
+Clan automatically imports the following files from a directory and registers them.
+
+## Machine registration
+
+Every folder `machines/{machineName}` will be registered automatically as a Clan machine.
+
+!!! info "Automatically loaded files"
+
+    The following files are loaded automatically for each Clan machine:
+
+    - [x] `machines/{machineName}/configuration.nix`
+    - [x] `machines/{machineName}/hardware-configuration.nix`
+    - [x] `machines/{machineName}/facter.json` Automatically configured, for further information see [nixos-facter](https://clan.lol/blog/nixos-facter/)
+    - [x] `machines/{machineName}/disko.nix` Automatically loaded, for further information see the [disko docs](https://github.com/nix-community/disko/blob/master/docs/quickstart.md).

docs/site/concepts/generators.md

@@ -1,7 +1,4 @@
-
-!!! Note
-    Vars is the new secret backend that will soon replace the Facts backend
-
+# Generators
 
 Defining a linux user's password via the nixos configuration previously required running `mkpasswd ...` and then copying the hash back into the nix configuration.
 
@@ -11,7 +8,7 @@ For a more general explanation of what clan vars are and how it works, see the i
 
 This guide assumes
 - Clan is set up already (see [Getting Started](../guides/getting-started/index.md))
-- a machine has been added to the clan (see [Adding Machines](./more-machines.md))
+- a machine has been added to the clan (see [Adding Machines](../guides/getting-started/add-machines.md))
 
 This section will walk you through the following steps:
 
@@ -23,7 +20,7 @@ This section will walk you through the following steps:
 6. share the root password between machines
 7. change the password
 
-## Declare the generator
+## Declare a generator
 
 In this example, a `vars` `generator` is used to:
 

docs/site/concepts/inventory.md

@@ -9,8 +9,6 @@ The inventory logic will automatically derive the modules and configurations to
 
 The following tutorial will walk through setting up a Backup service where the terms `Service` and `Role` will become more clear.
 
-See also: [Inventory API Documentation](../reference/nix-api/inventory.md)
-
 !!! example "Experimental status"
     The inventory implementation is not considered stable yet.
     We are actively soliciting feedback from users.
@@ -19,7 +17,7 @@ See also: [Inventory API Documentation](../reference/nix-api/inventory.md)
 
 ## Prerequisites
 
-- [x] [Add multiple machines](./more-machines.md) to your Clan.
+- [x] [Add some machines](../guides/getting-started/add-machines.md) to your Clan.
 
 ## Services
 

docs/site/concepts/templates.md

@@ -0,0 +1,69 @@
+# How Templates work
+
+Clan offers the ability to use templates for creating different resources.
+It comes with some `<builtin>` templates and discovers all exposed templates from its flake's `inputs`
+
+For example one can list all current templates like this:
+
+```shellSession
+$ clan templates list
+Available 'clan' templates
+├── <builtin>
+│   ├── default: Initialize a new clan flake
+│   ├── flake-parts: Flake-parts
+│   └── minimal: for clans managed via (G)UI
+└── inputs.self:
+    ├── default: Initialize a new clan flake
+    ├── flake-parts: Flake-parts
+    └── minimal: for clans managed via (G)UI
+Available 'disko' templates
+├── <builtin>
+│   └── single-disk: A simple ext4 disk with a single partition
+└── inputs.self:
+    └── single-disk: A simple ext4 disk with a single partition
+Available 'machine' templates
+├── <builtin>
+│   ├── demo-template: Demo machine for the CLAN project
+│   ├── flash-installer: Initialize a new flash-installer machine
+│   ├── new-machine: Initialize a new machine
+│   └── test-morph-template: Morph a machine
+└── inputs.self:
+    ├── demo-template: Demo machine for the CLAN project
+    ├── flash-installer: Initialize a new flash-installer machine
+    ├── new-machine: Initialize a new machine
+    └── test-morph-template: Morph a machine
+```
+
+## Using `<builtin>` Templates
+
+Templates are referenced via the `--template` `selector`
+
+clan-core ships its native/builtin templates. Those are referenced if the selector is a plain string ( without `#` or `./.` )
+
+For example:
+
+`clan flakes create --template=flake-parts`
+
+would use the native `<builtin>.flake-parts` template
+
+## Selectors follow nix flake `reference#attribute` syntax
+
+Selectors follow a very similar pattern as Nix's native attribute selection behavior.
+
+Just like `nix build .` would build `packages.x86-linux.default` of the flake in `./.`
+
+`clan flakes create --template=.` would create a clan from your **local** `default` clan template (`templates.clan.default`).
+
+In fact this command would be equivalent, just make it more explicit
+
+`clan flakes create --template=.#clan.templates.clan.default` (explicit path)
+
+## Remote templates
+
+Just like with Nix you could specify a remote url or path to the flake containing the template
+
+`clan flakes create --template=github:owner/repo#foo`
+
+!!! Note "Implementation Note"
+    Not all features of Nix's attribute selection are currently matched.
+    There are minor differences in case of unexpected behavior please create an [issue](https://git.clan.lol/clan/clan-core/issues/new)

docs/site/decisions/01-ClanModules.md

@@ -6,6 +6,8 @@ Accepted
 
 ## Context
 
+Current state as of writing:
+
 To define a service in Clan, you need to define two things:
 
 - `clanModule` - defined by module authors

docs/site/guides/age-plugins.md

@@ -0,0 +1,59 @@
+## Using Age Plugins
+
+If you wish to use a key generated using an [age plugin] as your admin key, extra care is needed.
+
+You must **precede your secret key with a comment that contains its corresponding recipient**.
+
+This is usually output as part of the generation process
+and is only required because there is no unified mechanism for recovering a recipient from a plugin secret key.
+
+Here is an example:
+
+```title="~/.config/sops/age/keys.txt"
+# public key: age1zdy49ek6z60q9r34vf5mmzkx6u43pr9haqdh5lqdg7fh5tpwlfwqea356l
+AGE-PLUGIN-FIDO2-HMAC-1QQPQZRFR7ZZ2WCV...
+```
+
+!!! note
+    The comment that precedes the plugin secret key need only contain the recipient.
+    Any other text is ignored.
+
+    In the example above, you can specify `# recipient: age1zdy...`, `# public: age1zdy....` or even
+    just `# age1zdy....`
+
+You will need to add an entry into your `flake.nix` to ensure that the necessary `age` plugins
+are loaded when using Clan:
+
+```nix title="flake.nix"
+{
+  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
+  inputs.nixpkgs.follows = "clan-core/nixpkgs";
+
+  outputs =
+    { self, clan-core, ... }:
+    let
+      # Sometimes this attribute set is defined in clan.nix
+      clan = clan-core.lib.clan {
+        inherit self;
+
+        meta.name = "myclan";
+
+        # Add Yubikey and FIDO2 HMAC plugins
+        # Note: the plugins listed here must be available in nixpkgs.
+        secrets.age.plugins = [
+          "age-plugin-yubikey"
+          "age-plugin-fido2-hmac"
+        ];
+
+        machines = {
+          # elided for brevity
+        };
+      };
+    in
+    {
+      inherit (clan) nixosConfigurations nixosModules clanInternals;
+
+      # elided for brevity
+    };
+}
+```

docs/site/guides/authoring/clanModules/index.md

@@ -1,229 +0,0 @@
-# Authoring a clanModule
-
-!!! Danger "Will get deprecated soon"
-    Please consider twice creating new modules in this format
-
-    [`clan.service` module](../clanServices/index.md) will be the new standard soon.
-
-This site will guide you through authoring your first module. Explaining which conventions must be followed, such that others will have an enjoyable experience and the module can be used with minimal effort.
-
-
-!!! Tip
-    External ClanModules can be ad-hoc loaded via [`clan.inventory.modules`](../../../reference/nix-api/inventory.md#inventory.modules)
-
-## Bootstrapping the `clanModule`
-
-A ClanModule is a specific subset of a [NixOS Module](https://nix.dev/tutorials/module-system/index.html), but it has some constraints and might be used via the [Inventory](../../../guides/inventory.md) interface.
-In fact a `ClanModule` can be thought of as a layer of abstraction on-top of NixOS and/or other ClanModules. It may configure sane defaults and provide an ergonomic interface that is easy to use and can also be used via a UI that is under development currently.
-
-Because ClanModules should be configurable via `json`/`API` all of its interface (`options`) must be serializable.
-
-!!! Tip
-    ClanModules interface can be checked by running the json schema converter as follows.
-
-    `nix build .#legacyPackages.x86_64-linux.schemas.inventory`
-
-    If the build succeeds the module is compatible.
-
-## Directory structure
-
-Each module SHOULD be a directory of the following format:
-
-```sh
-# Example: borgbackup
-clanModules/borgbackup
-├── README.md
-└── roles
-    ├── client.nix
-    └── server.nix
-```
-
-!!! Tip
-    `README.md` is always required. See section [Readme](#readme) for further details.
-
-    The `roles` folder is strictly required for `features = [ "inventory" ]`.
-
-## Registering the module
-
-=== "User module"
-
-    If the module should be ad-hoc loaded.
-    It can be made available in any project via the [`clan.inventory.modules`](../../../reference/nix-api/inventory.md#inventory.modules) attribute.
-
-    ```nix title="flake.nix"
-    # ...
-    # Sometimes this attribute set is defined in clan.nix
-    clan-core.lib.clan {
-        # 1. Add the module to the available clanModules with inventory support
-        inventory.modules = {
-            custom-module = ./modules/my_module;
-        };
-        # 2. Use the module in the inventory
-        inventory.services = {
-            custom-module.instance_1 = {
-                roles.default.machines = [ "machineA" ];
-            };
-        };
-    };
-    ```
-
-=== "Upstream module"
-
-    If the module will be contributed to [`clan-core`](https://git.clan.lol/clan-core)
-    The clanModule must be registered within the `clanModules` attribute in `clan-core`
-
-    ```nix title="clanModules/flake-module.nix"
-    --8<-- "clanModules/flake-module.nix:0:5"
-        # Register our new module here
-        # ...
-    ```
-
-## Readme
-
-The `README.md` is a required file for all modules. It MUST contain frontmatter in [`toml`](https://toml.io) format.
-
-```markdown
----
-description = "Module A"
----
-
-This is the example module that does xyz.
-```
-
-See the [Full Frontmatter reference](../../../reference/clanModules/frontmatter/index.md) further details and all supported attributes.
-
-## Roles
-
-If the module declares to implement `features = [ "inventory" ]` then it MUST contain a roles directory.
-
-Each `.nix` file in the `roles` directory is added as a role to the inventory service.
-
-Other files can also be placed alongside the `.nix` files
-
-```sh
-└── roles
-    ├── client.nix
-    └── server.nix
-```
-
-Adds the roles: `client` and `server`
-
-??? Tip "Good to know"
-    Sometimes a `ClanModule` should be usable via both clan's `inventory` concept but also natively as a NixOS module.
-
-    > In the long term, we want most modules to implement support for the inventory,
-    > but we are also aware that there are certain low-level modules that always serve as a backend for other higher-level `clanModules` with inventory support.
-    > These modules may not want to implement inventory interfaces as they are always used directly by other modules.
-
-    This can be achieved by placing an additional `default.nix` into the root of the ClanModules directory as shown:
-
-    ```sh
-    # ModuleA
-    ├── README.md
-    ├── default.nix
-    └── roles
-        └── default.nix
-    ```
-
-    ```nix title="default.nix"
-    {...}:{
-        imports = [ ./roles/default.nix ];
-    }
-    ```
-
-    By utilizing this pattern the module (`moduleA`) can then be imported into any regular NixOS module via:
-
-    ```nix
-    {...}:{
-        imports  = [ clanModules.moduleA ];
-    }
-    ```
-
-## Adding configuration options
-
-While we recommend to keep the interface as minimal as possible and deriving all required information from the `roles` model it might sometimes be required or convenient to expose customization options beyond `roles`.
-
-The following shows how to add options to your module.
-
-**It is important to understand that every module has its own namespace where it should declare options**
-
-**`clan.{moduleName}`**
-
-???+ Example
-    The following example shows how to register options in the module interface
-
-    and how it can be set via the inventory
-
-
-    ```nix title="/default.nix"
-    custom-module = ./modules/custom-module;
-    ```
-
-    Since the module is called `custom-module` all of its exposed options should be added to `options.clan.custom-module.*...*`
-
-    ```nix title="custom-module/roles/default.nix"
-    {
-        options = {
-            clan.custom-module.foo = mkOption {
-                type = types.str;
-                default = "bar";
-            };
-        };
-    }
-    ```
-
-    If the module is [registered](#registering-the-module).
-    Configuration can be set as follows.
-
-    ```nix title="flake.nix"
-    # Sometimes this attribute set is defined in clan.nix
-    clan-core.lib.clan {
-        inventory.services = {
-            custom-module.instance_1 = {
-                roles.default.machines = [ "machineA" ];
-                roles.default.config = {
-                    # All configuration here is scoped to `clan.custom-module`
-                    foo = "foobar";
-                };
-            };
-        };
-    }
-    ```
-
-## Organizing the ClanModule
-
-Each `{role}.nix` is included into the machine if the machine is declared to have the role.
-
-For example
-
-```nix
-roles.client.machines = ["MachineA"];
-```
-
-Then `roles/client.nix` will be added to the machine `MachineA`.
-
-This behavior makes it possible to split the interface and common code paths when using multiple roles.
-In the concrete example of `borgbackup` this allows a `server` to declare a different interface than the corresponding `client`.
-
-The client offers configuration option, to exclude certain local directories from being backed up:
-
-```nix title="roles/client.nix"
-# Example client interface
-  options.clan.borgbackup.exclude = ...
-```
-
-The server doesn't offer any configuration option. Because everything is set-up automatically.
-
-```nix title="roles/server.nix"
-# Example server interface
-  options.clan.borgbackup = {};
-```
-
-Assuming that there is a common code path or a common interface between `server` and `client` this can be structured as:
-
-```nix title="roles/server.nix, roles/client.nix"
-{...}: {
-    # ...
-    imports = [ ../common.nix ];
-}
-```

docs/site/guides/backups.md

@@ -1,167 +1,199 @@
-# Introduction to Backups
 
-When you're managing your own services, creating regular backups is crucial to ensure your data's safety.
-This guide introduces you to Clan's built-in backup functionalities.
-Clan supports backing up your data to both local storage devices (like USB drives) and remote servers, using well-known tools like borgbackup and rsnapshot.
-We might add more options in the future, but for now, let's dive into how you can secure your data.
+This guide explains how to set up and manage
+[BorgBackup](https://borgbackup.readthedocs.io/) for secure, efficient backups
+in a clan network. BorgBackup provides:
 
-## Backing Up Locally with Localbackup
+- Space efficient storage of backups with deduplication
+- Secure, authenticated encryption
+- Compression: lz4, zstd, zlib, lzma or none
+- Mountable backups with FUSE
+- Easy installation on multiple platforms: Linux, macOS, BSD, …
+- Free software (BSD license).
+- Backed by a large and active open-source community.
 
-Localbackup lets you backup your data onto physical storage devices connected to your computer,
-such as USB hard drives or network-attached storage. It uses a tool called rsnapshot for this purpose.
-
-### Setting Up Localbackup
-
-1. **Identify Your Backup Device:**
-
-First, figure out which device you'll use for backups. You can see all connected devices by running this command in your terminal:
-
-```bash
-lsblk --output NAME,PTUUID,FSTYPE,SIZE,MOUNTPOINT
-```
-
-Look for the device you intend to use for backups and note its details.
-
-2. **Configure Your Backup Device:**
-
-Once you've identified your device, you'll need to add it to your configuration.
-Here's an example NixOS configuration for a device located at `/dev/sda2` with an `ext4` filesystem:
+## Borgbackup Example
 
 ```nix
-{
-  fileSystems."/mnt/hdd" = {
-    device = "/dev/sda2";
-    fsType = "ext4";
-    options = [ "defaults" "noauto" ];
-  };
-}
-```
-
-Replace `/dev/sda2` with your device and `/mnt/hdd` with your preferred mount point.
-
-3. **Set Backup Targets:** Next, define where on your device you'd like the backups to be stored:
-
-   ```nix
-   {
-     clan.localbackup.targets.hdd = {
-       directory = "/mnt/hdd/backup";
-       mountpoint = "/mnt/hdd";
-     };
-   }
-   ```
-
-   Change `/mnt/hdd` to the actual mount point you're using.
-
-4. **Create Backups:** To create a backup, run:
-
-   ```bash
-   clan backups create mymachine
-   ```
-
-   This command saves snapshots of your data onto the backup device.
-
-5. **Listing Backups:** To see available backups, run:
-
-  ```bash
-  clan backups list mymachine
-  ```
-
-## Remote Backups with Borgbackup
-
-### Overview of Borgbackup
-
-Borgbackup splits the backup process into two parts: a backup client that sends data to a backup server.
-The server stores the backups.
-
-### Setting Up the Borgbackup Client
-
-1. **Specify Backup Server:**
-
-Start by indicating where your backup data should be sent. Replace `hostname` with your server's address:
-
-```nix
-{
-  clan.borgbackup.destinations = {
-    myhostname = {
-      repo = "borg@backuphost:/var/lib/borgbackup/myhostname";
+inventory.instances = {
+  borgbackup = {
+    module = {
+      name = "borgbackup";
+      input = "clan";
     };
+    roles.client.machines."jon".settings = {
+      destinations."storagebox" = {
+        repo = "username@$hostname:/./borgbackup";
+        rsh = ''ssh -oPort=23 -i /run/secrets/vars/borgbackup/borgbackup.ssh'';
+      };
+    };
+    roles.server.machines = { };
   };
-}
+};
 ```
 
-2. **Select Folders to Backup:**
+The input should be named according to your flake input. Jon is configured as a
+client machine with a destination pointing to a Hetzner Storage Box.
 
-Decide which folders you want to back up. For example, to backup your home and root directories:
+To see a list of all possible options go to [borgbackup clan service](../reference/clanServices/borgbackup.md)
+
+## Roles
+
+A Clan Service can have multiple roles, each role applies different nix config to the machine.
+
+### 1. Client
+
+Clients are machines that create and send backups to various destinations. Each
+client can have multiple backup destinations configured.
+
+### 2. Server
+
+Servers act as backup repositories, receiving and storing backups from client
+machines. They can be dedicated backup servers within your clan network.
+
+## Backup destinations
+
+This service allows you to perform backups to multiple `destinations`.
+Destinations can be:
+
+- **Local**: Local disk storage
+- **Server**: Your own borgbackup server (using the `server` role)
+- **Third-party services**: Such as Hetzner's Storage Box
+
+## State management
+
+Backups are based on [states](../reference/clan.core/state.md). A state
+defines which files should be backed up and how these files are obtained through
+pre/post backup and restore scripts.
+
+Here's an example for a user application `linkding`:
+
+In this example:
+
+- `/data/podman/linkding` is the application's data directory
+- `/var/backup/linkding` is the staging directory where data is copied for
+  backup
 
 ```nix
-{ clan.core.state.userdata.folders = [ "/home" "/root" ]; }
+clan.core.state.linkding = {
+  folders = [ "/var/backup/linkding" ];
+
+  preBackupScript = ''
+    export PATH=${
+      lib.makeBinPath [
+        config.systemd.package
+        pkgs.coreutils
+        pkgs.rsync
+      ]
+    }
+
+    service_status=$(systemctl is-active podman-linkding)
+
+    if [ "$service_status" = "active" ]; then
+      systemctl stop podman-linkding
+      rsync -avH --delete --numeric-ids "/data/podman/linkding/" /var/backup/linkding/
+      systemctl start podman-linkding
+    fi
+  '';
+
+  postRestoreScript = ''
+    export PATH=${
+      lib.makeBinPath [
+        config.systemd.package
+        pkgs.coreutils
+        pkgs.rsync
+      ]
+    }
+
+    service_status="$(systemctl is-active podman-linkding)"
+
+    if [ "$service_status" = "active" ]; then
+      systemctl stop podman-linkding
+
+      # Backup locally current linkding data
+      cp -rp "/data/podman/linkding" "/data/podman/linkding.bak"
+
+      # Restore from borgbackup
+      rsync -avH --delete --numeric-ids /var/backup/linkding/ "/data/podman/linkding/"
+
+      systemctl start podman-linkding
+    fi
+  '';
+};
 ```
 
-3. **Generate Backup Credentials:**
+## Managing backups
 
-Run `clan facts generate <yourmachine>` to prepare your machine for backup, creating necessary SSH keys and credentials.
+In this section we go over how to manage your collection of backups with the clan command.
 
-### Setting Up the Borgbackup Server
+### Listing states
 
-1. **Configure Backup Repository:**
-
-On the server where backups will be stored, enable the SSH daemon and set up a repository for each client:
-
-```nix
-{
-  services.borgbackup.repos.myhostname = {
-    path = "/var/lib/borgbackup/myhostname";
-    authorizedKeys = [
-      (builtins.readFile  (config.clan.core.settings.directory + "/machines/myhostname/facts/borgbackup.ssh.pub"))
-    ];
-  };
-}
-```
-
-Ensure the path to the public key is correct.
-
-2. **Update Your Systems:** Apply your changes by running `clan machines update` to both the server and your client
-
-### Managing Backups
-
-- **Scheduled Backups:**
-
-  Backups are automatically performed nightly. To check the next scheduled backup, use:
-
-  ```bash
-  systemctl list-timers | grep -E 'NEXT|borg'
-  ```
-
-- **Listing Backups:** To see available backups, run:
-
-  ```bash
-  clan backups list mymachine
-  ```
-
-- **Manual Backups:** You can also initiate a backup manually:
-
-  ```bash
-  clan backups create mymachine
-  ```
-
-- **Restoring Backups:** To restore a backup that has been listed by the list command (NAME):
-
-  ```bash
-  clan backups restore [MACHINE] [PROVIDER] [NAME]
-
-  ```
-
-  Example (Restoring a machine called `client` with the backup provider `borgbackup`):
-
-  ```bash
-  clan backups restore client borgbackup [NAME]
-
-  ```
-
-  The `backups` command is service aware and allows optional specification of the `--service` flag.
-
-  To only restore the service called `zerotier` on a machine called `controller` through the backup provider `borgbackup` use the following command:
+To see which files (`states`) will be backed up on a specific machine, use:
 
 ```bash
-  clan backups restore client borgbackup [NAME] --service zerotier
+clan state list jon
 ```
+
+This will show all configured states for the machine `jon`, for example:
+
+```text
+· service: linkding
+  folders:
+  - /var/backup/linkding
+  preBackupCommand: pre-backup-linkding
+  postRestoreCommand: post-restore-linkding
+
+· service: zerotier
+  folders:
+  - /var/lib/zerotier-one
+```
+
+### Creating backups
+
+To create a backup of a machine (e.g., `jon`), run:
+
+```bash
+clan backups create jon
+```
+
+This will backup all configured states (`zerotier` and `linkding` in this
+example) from the machine `jon`.
+
+### Listing available backups
+
+To see all available backups, use:
+
+```bash
+clan backups list
+```
+
+This will display all backups with their timestamps:
+
+```text
+storagebox::username@username.your-storagebox.de:/./borgbackup::jon-jon-2025-07-22T19:40:10
+storagebox::username@username.your-storagebox.de:/./borgbackup::jon-jon-2025-07-23T01:00:00
+storagebox::username@username.your-storagebox.de:/./borgbackup::jon-storagebox-2025-07-24T01:00:00
+storagebox::username@username.your-storagebox.de:/./borgbackup::jon-storagebox-2025-07-24T06:02:35
+```
+
+### Restoring backups
+
+For restoring a backup you have two options. 
+
+#### Full restoration
+
+To restore all services from a backup:
+
+```bash
+clan backups restore jon borgbackup storagebox::u444061@u444061.your-storagebox.de:/./borgbackup::jon-storagebox-2025-07-24T06:02:35
+```
+
+#### Partial restoration
+
+To restore only a specific service (e.g., `linkding`):
+
+```bash
+clan backups restore --service linkding jon borgbackup storagebox::u444061@u444061.your-storagebox.de:/./borgbackup::jon-storagebox-2025-07-24T06:02:35
+```
+
+
+

docs/site/guides/clanServices.md

@@ -138,7 +138,7 @@ You can use services exposed by Clan’s core module library, `clan-core`.
 
 You can also author your own `clanService` modules.
 
-🔗 Learn how to write your own service: [Authoring a clanService](../guides/authoring/clanServices/index.md)
+🔗 Learn how to write your own service: [Authoring a service](../guides/services/community.md)
 
 You might expose your service module from your flake — this makes it easy for other people to also use your module in their clan.
 
@@ -154,6 +154,6 @@ You might expose your service module from your flake — this makes it easy for
 
 ## What’s Next?
 
-* [Author your own clanService →](../guides/authoring/clanServices/index.md)
+* [Author your own clanService →](../guides/services/community.md)
 * [Migrate from clanModules →](../guides/migrations/migrate-inventory-services.md)
 <!-- TODO: * [Understand the architecture →](../explanation/clan-architecture.md) -->

docs/site/guides/flake-parts.md

@@ -27,7 +27,7 @@ inputs = {
 
 ## Import the Clan flake-parts Module
 
-After updating your flake inputs, the next step is to import the Clan flake-parts module. This will make the [Clan options](../reference/nix-api/clan.md) available within `mkFlake`.
+After updating your flake inputs, the next step is to import the Clan flake-parts module. This will make the [Clan options](../options.md) available within `mkFlake`.
 
 ```nix
 {

docs/site/guides/getting-started/add-machines.md

@@ -6,7 +6,7 @@ Machines can be added using the following methods
 - Editing machines/`machine_name`/configuration.nix (automatically included if it exists)
 - `clan machines create` (imperative)
 
-See the complete [list](../../guides/more-machines.md#automatic-registration) of auto-loaded files.
+See the complete [list](../../concepts/autoincludes.md) of auto-loaded files.
 
 ## Create a machine
 

docs/site/guides/getting-started/add-services.md

@@ -41,7 +41,7 @@ To learn more: [Guide about clanService](../clanServices.md)
 ```
 
 1. See [reference/clanServices](../../reference/clanServices/index.md) for all available services and how to configure them.
-   Or read [authoring/clanServices](../authoring/clanServices/index.md) if you want to bring your own
+   Or read [authoring/clanServices](../../guides/services/community.md) if you want to bring your own
 
 2. Replace `__YOUR_CONTROLLER_` with the *name* of your machine.
 

docs/site/guides/getting-started/add-user.md

@@ -57,7 +57,7 @@ For more information see [clanService/users](../../reference/clanServices/users.
 
 Some people like to define a `users` folder in their repository root.
 That allows to bind all user specific logic to a single place (`default.nix`)
-Which can be imported into individual machines to make the user avilable on that machine.
+Which can be imported into individual machines to make the user available on that machine.
 
 ```bash
 .
@@ -107,7 +107,7 @@ We can use this property of clan services to bind a nixosModule to the user, whi
 }
 ```
 
-1. Type `path` or `string`: Must point to a seperate file. Inlining a module is not possible
+1. Type `path` or `string`: Must point to a separate file. Inlining a module is not possible
 
 !!! Note "This is inspiration"
     Our community might come up with better solutions soon.

docs/site/guides/getting-started/deploy.md

@@ -8,7 +8,6 @@ Now that you have created a machines, added some services and setup secrets. Thi
     - [x] RAM > 2GB
     - [x] **Two Computers**: You need one computer that you're getting ready (we'll call this the Target Computer) and another one to set it up from (we'll call this the Setup Computer). Make sure both can talk to each other over the network using SSH.
     - [x] **Machine configuration**: See our basic [adding and configuring machine guide](./add-machines.md)
-    - [x] **Initialized secrets**: See [secrets](secrets.md) for how to initialize your secrets.
 
 ## Physical Hardware
 
@@ -18,7 +17,7 @@ Steps:
 
 - Create a NixOS installer image and transfer it to a bootable USB drive as described in the [installer](./installer.md).
 - Boot the target machine and connect it to a network that makes it reachable from your setup computer.
-- Note down a reachable ip adress (*ipv4*, *ipv6* or *tor*)
+- Note down a reachable ip address (*ipv4*, *ipv6* or *tor*)
 
 ---
 
@@ -169,7 +168,7 @@ Re-run the command with the correct disk:
 clan templates apply disk single-disk jon --set mainDisk "/dev/disk/by-id/nvme-WD_PC_SN740_SDDQNQD-512G-1201_232557804368"
 ```
 
-Should now be succesfull
+Should now be successful
 
 ```shellSession
 Applied disk template 'single-disk' to machine 'jon'

docs/site/guides/getting-started/index.md

@@ -59,7 +59,7 @@ Enter a *name*, confirm with *enter*. A directory with that name will be created
 
 ## Explore the Project Structure
 
-Take a lookg at all project files:
+Take a look at all project files:
 
 ```bash
 cd my-clan
@@ -125,11 +125,10 @@ To change the name of your clan edit `meta.name` in the `clan.nix` or `flake.nix
 You can continue with **any** of the following steps at your own pace:
 
 - [x] [Install Nix & Clan CLI](./index.md)
-- [x] [Initialize Clan](./index.md#initialize-your-project)
+- [x] [Initialize Clan](./index.md#add-clan-cli-to-your-shell)
 - [ ] [Create USB Installer (optional)](./installer.md)
 - [ ] [Add Machines](./add-machines.md)
 - [ ] [Add a User](./add-user.md)
 - [ ] [Add Services](./add-services.md)
-- [ ] [Configure Secrets](./secrets.md)
 - [ ] [Deploy](./deploy.md) - Requires configured secrets
 - [ ] [Setup CI (optional)](./check.md)

docs/site/guides/getting-started/secrets.md

@@ -1,179 +0,0 @@
-
-Setting up secrets is **Required** for any *machine deployments* or *vm runs* - You need to complete the steps: [Create Admin Keypair](#create-your-admin-keypair) and [Add Your Public Key(s)](#add-your-public-keys)
-
----
-
-Clan enables encryption of secrets (such as passwords & keys) ensuring security and ease-of-use among users.
-
-By default, Clan uses the [sops](https://github.com/getsops/sops) format
-and integrates with [sops-nix](https://github.com/Mic92/sops-nix) on NixOS machines.
-Clan can also be configured to be used with other secret store [backends](../../reference/clan.core/vars.md#clan.core.vars.settings.secretStore).
-
-This guide will walk you through:
-
-- **Creating a Keypair for Your User**: Learn how to generate a keypair for `$USER` to securely control all secrets.
-- **Creating Your First Secret**: Step-by-step instructions on creating your initial secret.
-- **Assigning Machine Access to the Secret**: Understand how to grant a machine access to the newly created secret.
-
-## Create Your Admin Keypair
-
-To get started, you'll need to create **your admin keypair**.
-
-!!! info
-    Don't worry — if you've already made one before, this step won't change or overwrite it.
-
-```bash
-clan secrets key generate
-```
-
-**Output**:
-
-```{.console, .no-copy}
-Public key: age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7
-
-Generated age private key at '/home/joerg/.config/sops/age/keys.txt' for your user. Please back it up on a secure location or you will lose access to your secrets.
-Also add your age public key to the repository with 'clan secrets users add YOUR_USER age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7' (replace YOUR_USER with your actual username)
-```
-
-!!! warning
-    Make sure to keep a safe backup of the private key you've just created.
-    If it's lost, you won't be able to get to your secrets anymore because they all need the admin key to be unlocked.
-
-If you already have an [age] secret key and want to use that instead, you can simply edit `~/.config/sops/age/keys.txt`:
-
-```title="~/.config/sops/age/keys.txt"
-AGE-SECRET-KEY-13GWMK0KNNKXPTJ8KQ9LPSQZU7G3KU8LZDW474NX3D956GGVFAZRQTAE3F4
-```
-
-Alternatively, you can provide your [age] secret key as an environment variable `SOPS_AGE_KEY`, or in a different file
-using `SOPS_AGE_KEY_FILE`.
-For more information see the [SOPS] guide on [encrypting with age].
-
-!!! note
-    It's safe to add any secrets created by the clan CLI and placed in your repository to version control systems like `git`.
-
-### Add Your Public Key(s)
-
-```console
-clan secrets users add $USER --age-key <your_public_key>
-```
-
-It's best to choose the same username as on your Setup/Admin Machine that you use to control the deployment with.
-
-Once run this will create the following files:
-
-```{.console, .no-copy}
-sops/
-└── users/
-    └── <your_username>/
-        └── key.json
-```
-If you followed the quickstart tutorial all necessary secrets are initialized at this point.
-
-!!! note
-    You can add multiple age keys for a user by providing multiple `--age-key <your_public_key>` flags:
-
-    ```console
-    clan secrets users add $USER \
-        --age-key <your_public_key_1> \
-        --age-key <your_public_key_2> \
-        ...
-    ```
-
-### Manage Your Public Key(s)
-
-You can list keys for your user with `clan secrets users get $USER`:
-
-```console
-clan secrets users get alice
-
-[
-  {
-    "publickey": "age1hrrcspp645qtlj29krjpq66pqg990ejaq0djcms6y6evnmgglv5sq0gewu",
-    "type": "age",
-    "username": "alice"
-  },
-  {
-    "publickey": "age13kh4083t3g4x3ktr52nav6h7sy8ynrnky2x58pyp96c5s5nvqytqgmrt79",
-    "type": "age",
-    "username": "alice"
-  }
-]
-```
-
-To add a new key to your user:
-
-```console
-clan secrets users add-key $USER --age-key <your_public_key>
-```
-
-To remove a key from your user:
-
-```console
-clan secrets users remove-key $USER --age-key <your_public_key>
-```
-
-[age]: https://github.com/FiloSottile/age
-[age plugin]: https://github.com/FiloSottile/awesome-age?tab=readme-ov-file#plugins
-[sops]: https://github.com/getsops/sops
-[encrypting with age]: https://github.com/getsops/sops?tab=readme-ov-file#encrypting-using-age
-
-## Further: Using Age Plugins
-
-If you wish to use a key generated using an [age plugin] as your admin key, extra care is needed.
-
-You must **precede your secret key with a comment that contains its corresponding recipient**.
-
-This is usually output as part of the generation process
-and is only required because there is no unified mechanism for recovering a recipient from a plugin secret key.
-
-Here is an example:
-
-```title="~/.config/sops/age/keys.txt"
-# public key: age1zdy49ek6z60q9r34vf5mmzkx6u43pr9haqdh5lqdg7fh5tpwlfwqea356l
-AGE-PLUGIN-FIDO2-HMAC-1QQPQZRFR7ZZ2WCV...
-```
-
-!!! note
-    The comment that precedes the plugin secret key need only contain the recipient.
-    Any other text is ignored.
-
-    In the example above, you can specify `# recipient: age1zdy...`, `# public: age1zdy....` or even
-    just `# age1zdy....`
-
-You will need to add an entry into your `flake.nix` to ensure that the necessary `age` plugins
-are loaded when using Clan:
-
-```nix title="flake.nix"
-{
-  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-  inputs.nixpkgs.follows = "clan-core/nixpkgs";
-
-  outputs =
-    { self, clan-core, ... }:
-    let
-      # Sometimes this attribute set is defined in clan.nix
-      clan = clan-core.lib.clan {
-        inherit self;
-
-        meta.name = "myclan";
-
-        # Add Yubikey and FIDO2 HMAC plugins
-        # Note: the plugins listed here must be available in nixpkgs.
-        secrets.age.plugins = [
-          "age-plugin-yubikey"
-          "age-plugin-fido2-hmac"
-        ];
-
-        machines = {
-          # elided for brevity
-        };
-      };
-    in
-    {
-      inherit (clan) nixosConfigurations nixosModules clanInternals;
-
-      # elided for brevity
-    };
-}
-```

docs/site/guides/macos.md

@@ -7,7 +7,7 @@ This guide explains how to manage macOS machines using Clan.
 Currently, Clan supports the following features for macOS:
 
 - `clan machines update` for existing [nix-darwin](https://github.com/nix-darwin/nix-darwin) installations
-- Support for [vars](../guides/vars-backend.md)
+- Support for [vars](../concepts/generators.md)
 
 ## Add Your Machine to Your Clan Flake
 

docs/site/guides/migrations/migrate-inventory-services.md

@@ -1,7 +1,7 @@
 # Migrating from using `clanModules` to `clanServices`
 
 **Audience**: This is a guide for **people using `clanModules`**.
-If you are a **module author** and need to migrate your modules please consult our **new** [clanServices authoring guide](../authoring/clanServices/index.md)
+If you are a **module author** and need to migrate your modules please consult our **new** [clanServices authoring guide](../../guides/services/community.md)
 
 ## What's Changing?
 
@@ -35,6 +35,37 @@ services = {
 };
 ```
 
+### Complex Example: Multi-service Setup
+
+```nix
+# Old format
+services = {
+    borgbackup.production = {
+        roles.server.machines = [ "backup-server" ];
+        roles.server.config = {
+            directory = "/var/backup/borg";
+        };
+        roles.client.tags = [ "backup" ];
+        roles.client.extraModules = [ "nixosModules/borgbackup.nix" ];
+    };
+
+    zerotier.company-network = {
+        roles.controller.machines = [ "network-controller" ];
+        roles.moon.machines = [ "moon-1" "moon-2" ];
+        roles.peer.tags = [ "nixos" ];
+    };
+
+    sshd.internal = {
+        roles.server.tags = [ "nixos" ];
+        roles.client.tags = [ "nixos" ];
+        config.certificate.searchDomains = [
+            "internal.example.com"
+            "vpn.example.com"
+        ];
+    };
+};
+```
+
 ---
 
 ## ✅ After: New `instances` Definition with `clanServices`
@@ -70,6 +101,56 @@ instances = {
 };
 ```
 
+### Complex Example Migrated
+
+```nix
+# New format
+instances = {
+    borgbackup-production = {
+        module = {
+            name = "borgbackup";
+            input = "clan-core";
+        };
+        roles.server.machines."backup-server" = { };
+        roles.server.settings = {
+            directory = "/var/backup/borg";
+        };
+        roles.client.tags.backup = { };
+        roles.client.extraModules = [ ../nixosModules/borgbackup.nix ];
+    };
+
+    zerotier-company-network = {
+        module = {
+            name = "zerotier";
+            input = "clan-core";
+        };
+        roles.controller.machines."network-controller" = { };
+        roles.moon.machines."moon-1".settings = {
+            stableEndpoints = [ "10.0.0.1" "2001:db8::1" ];
+        };
+        roles.moon.machines."moon-2".settings = {
+            stableEndpoints = [ "10.0.0.2" "2001:db8::2" ];
+        };
+        roles.peer.tags.nixos = { };
+    };
+
+    sshd-internal = {
+        module = {
+            name = "sshd";
+            input = "clan-core";
+        };
+        roles.server.tags.nixos = { };
+        roles.client.tags.nixos = { };
+        roles.client.settings = {
+            certificate.searchDomains = [
+                "internal.example.com"
+                "vpn.example.com"
+            ];
+        };
+    };
+};
+```
+
 ---
 
 ## Steps to Migrate
@@ -131,6 +212,33 @@ roles.default.machines."test-inventory-machine".settings = {
 };
 ```
 
+### Important Type Changes
+
+The new `instances` format uses **attribute sets** instead of **lists** for tags and machines:
+
+```nix
+# ❌ Old format (lists)
+roles.client.tags = [ "backup" ];
+roles.server.machines = [ "blob64" ];
+
+# ✅ New format (attribute sets)
+roles.client.tags.backup = { };
+roles.server.machines.blob64 = { };
+```
+
+### Handling Multiple Machines/Tags
+
+When you need to assign multiple machines or tags to a role:
+
+```nix
+# ❌ Old format
+roles.moon.machines = [ "eva" "eve" ];
+
+# ✅ New format - each machine gets its own attribute
+roles.moon.machines.eva = { };
+roles.moon.machines.eve = { };
+```
+
 ---
 
 !!! Warning
@@ -138,8 +246,89 @@ roles.default.machines."test-inventory-machine".settings = {
     * `inventory.services` is no longer recommended; use `inventory.instances` instead.
     * Module authors should begin exporting service modules under the `clan.modules` attribute of their flake.
 
+## Troubleshooting Common Migration Errors
+
+### Error: "not of type `attribute set of (submodule)`"
+
+This error occurs when using lists instead of attribute sets for tags or machines:
+
+```
+error: A definition for option `flake.clan.inventory.instances.borgbackup-blob64.roles.client.tags' is not of type `attribute set of (submodule)'.
+```
+
+**Solution**: Convert lists to attribute sets as shown in the "Important Type Changes" section above.
+
+### Error: "unsupported attribute `module`"
+
+This error indicates the module structure is incorrect:
+
+```
+error: Module ':anon-4:anon-1' has an unsupported attribute `module'.
+```
+
+**Solution**: Ensure the `module` attribute has exactly two fields: `name` and `input`.
+
+### Error: "attribute 'pkgs' missing"
+
+This suggests the instance configuration is trying to use imports incorrectly:
+
+```
+error: attribute 'pkgs' missing
+```
+
+**Solution**: Use the `module = { name = "..."; input = "..."; }` format instead of `imports`.
+
+### Removed Features
+
+The following features from the old `services` format are no longer supported in `instances`:
+
+- Top-level `config` attribute (use `roles.<role>.settings` instead)
+- Direct module imports (use the `module` declaration instead)
+
+### extraModules Support
+
+The `extraModules` attribute is still supported in the new instances format! The key change is how modules are specified:
+
+**Old format (string paths relative to clan root):**
+```nix
+roles.client.extraModules = [ "nixosModules/borgbackup.nix" ];
+```
+
+**New format (NixOS modules):**
+```nix
+# Direct module reference
+roles.client.extraModules = [ ../nixosModules/borgbackup.nix ];
+
+# Or using self
+roles.client.extraModules = [ self.nixosModules.borgbackup ];
+
+# Or inline module definition
+roles.client.extraModules = [
+  { config, ... }: {
+    # Your module configuration here
+  }
+];
+```
+
+The `extraModules` now expects actual **NixOS modules** rather than string paths. This provides better type checking and more flexibility in how modules are specified.
+
+**Alternative: Using @clan/importer**
+
+For scenarios where you need to import modules with specific tag-based targeting, you can also use the dedicated `@clan/importer` service:
+
+```nix
+instances = {
+  my-importer = {
+    module.name = "@clan/importer";
+    module.input = "clan-core";
+    roles.default.tags.my-tag = { };
+    roles.default.extraModules = [ self.nixosModules.myModule ];
+  };
+};
+```
+
 ## Further reference
 
-* [Authoring a 'clan.service' module](../authoring/clanServices/index.md)
+* [Inventory Concept](../../concepts/inventory.md)
+* [Authoring a 'clan.service' module](../../guides/services/community.md)
 * [ClanServices](../clanServices.md)
-* [Inventory Reference](../../reference/nix-api/inventory.md)

docs/site/guides/migrations/migration-facts-vars.md

@@ -3,7 +3,7 @@
 For a high level overview about `vars` see our [blog post](https://clan.lol/blog/vars/).
 
 This guide will help you migrate your modules that still use our [`facts`](../../guides/secrets.md) backend
-to the [`vars`](../../guides/vars-backend.md) backend.
+to the [`vars`](../../concepts/generators.md) backend.
 
 The `vars` [module](../../reference/clan.core/vars.md) and the clan [command](../../reference/cli/vars.md) work in tandem, they should ideally be kept in sync.
 

docs/site/guides/more-machines.md

@@ -1,50 +0,0 @@
-
-Clan has two general methods of adding machines:
-
-- **Automatic**: Detects every folder in the `machines` folder.
-- **Declarative**: Explicit declarations in Nix.
-
-## Automatic registration
-
-Every folder `machines/{machineName}` will be registered automatically as a Clan machine.
-
-!!! info "Automatically loaded files"
-
-    The following files are loaded automatically for each Clan machine:
-
-    - [x] `machines/{machineName}/configuration.nix`
-    - [x] `machines/{machineName}/hardware-configuration.nix`
-    - [x] `machines/{machineName}/facter.json` Automatically configured, for further information see [nixos-facter](https://clan.lol/blog/nixos-facter/)
-    - [x] `machines/{machineName}/disko.nix` Automatically loaded, for further information see the [disko docs](https://github.com/nix-community/disko/blob/master/docs/quickstart.md).
-
-## Manual declaration
-
-Machines can be added via [`clan.inventory.machines`](../guides/inventory.md) or in `clan.machines`, which allows for defining NixOS options.
-
-=== "**Individual Machine Configuration**"
-
-    ```{.nix}
-    clan-core.lib.clan {
-        machines = {
-            "jon" = {
-                # Any valid nixos config
-            };
-        };
-    }
-    ```
-
-=== "**Inventory Configuration**"
-
-    ```{.nix}
-    clan-core.lib.clan {
-        inventory = {
-            machines = {
-                "jon" = {
-                    # Inventory can set tags and other metadata
-                    tags = [ "zone1" ];
-                    deploy.targetHost = "root@jon";
-                };
-            };
-        };
-    }
-    ```

docs/site/guides/secrets.md

@@ -1,25 +1,141 @@
-If you want to know more about how to save and share passwords in your clan read further!
+This article provides an overview over the underlying secrets system which is used by [Vars](../concepts/generators.md).
+Under most circumstances you should use [Vars](../concepts/generators.md) directly instead.
 
-### Adding a Secret
+Consider using `clan secrets` only for managing admin users and groups, as well as a debugging tool.
+
+Manually interacting with secrets via `clan secrets [set|remove]`, etc may break the integrity of your `Vars` state.
+
+---
+
+Clan enables encryption of secrets (such as passwords & keys) ensuring security and ease-of-use among users.
+
+By default, Clan uses the [sops](https://github.com/getsops/sops) format
+and integrates with [sops-nix](https://github.com/Mic92/sops-nix) on NixOS machines.
+Clan can also be configured to be used with other secret store [backends](../reference/clan.core/vars.md#clan.core.vars.settings.secretStore).
+
+## Create Your Admin Keypair
+
+To get started, you'll need to create **your admin keypair**.
+
+!!! info
+    Don't worry — if you've already made one before, this step won't change or overwrite it.
+
+```bash
+clan secrets key generate
+```
+
+**Output**:
+
+```{.console, .no-copy}
+Public key: age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7
+
+Generated age private key at '/home/joerg/.config/sops/age/keys.txt' for your user. Please back it up on a secure location or you will lose access to your secrets.
+Also add your age public key to the repository with 'clan secrets users add YOUR_USER age1wkth7uhpkl555g40t8hjsysr20drq286netu8zptw50lmqz7j95sw2t3l7' (replace YOUR_USER with your actual username)
+```
+
+!!! warning
+    Make sure to keep a safe backup of the private key you've just created.
+    If it's lost, you won't be able to get to your secrets anymore because they all need the admin key to be unlocked.
+
+If you already have an [age] secret key and want to use that instead, you can simply edit `~/.config/sops/age/keys.txt`:
+
+```title="~/.config/sops/age/keys.txt"
+AGE-SECRET-KEY-13GWMK0KNNKXPTJ8KQ9LPSQZU7G3KU8LZDW474NX3D956GGVFAZRQTAE3F4
+```
+
+Alternatively, you can provide your [age] secret key as an environment variable `SOPS_AGE_KEY`, or in a different file
+using `SOPS_AGE_KEY_FILE`.
+For more information see the [SOPS] guide on [encrypting with age].
+
+!!! note
+    It's safe to add any secrets created by the clan CLI and placed in your repository to version control systems like `git`.
+
+## Add Your Public Key(s)
+
+```console
+clan secrets users add $USER --age-key <your_public_key>
+```
+
+It's best to choose the same username as on your Setup/Admin Machine that you use to control the deployment with.
+
+Once run this will create the following files:
+
+```{.console, .no-copy}
+sops/
+└── users/
+    └── <your_username>/
+        └── key.json
+```
+If you followed the quickstart tutorial all necessary secrets are initialized at this point.
+
+!!! note
+    You can add multiple age keys for a user by providing multiple `--age-key <your_public_key>` flags:
+
+    ```console
+    clan secrets users add $USER \
+        --age-key <your_public_key_1> \
+        --age-key <your_public_key_2> \
+        ...
+    ```
+
+## Manage Your Public Key(s)
+
+You can list keys for your user with `clan secrets users get $USER`:
+
+```console
+clan secrets users get alice
+
+[
+  {
+    "publickey": "age1hrrcspp645qtlj29krjpq66pqg990ejaq0djcms6y6evnmgglv5sq0gewu",
+    "type": "age",
+    "username": "alice"
+  },
+  {
+    "publickey": "age13kh4083t3g4x3ktr52nav6h7sy8ynrnky2x58pyp96c5s5nvqytqgmrt79",
+    "type": "age",
+    "username": "alice"
+  }
+]
+```
+
+To add a new key to your user:
+
+```console
+clan secrets users add-key $USER --age-key <your_public_key>
+```
+
+To remove a key from your user:
+
+```console
+clan secrets users remove-key $USER --age-key <your_public_key>
+```
+
+[age]: https://github.com/FiloSottile/age
+[age plugin]: https://github.com/FiloSottile/awesome-age?tab=readme-ov-file#plugins
+[sops]: https://github.com/getsops/sops
+[encrypting with age]: https://github.com/getsops/sops?tab=readme-ov-file#encrypting-using-age
+
+## Adding a Secret
 
 ```shellSession
 clan secrets set mysecret
 Paste your secret:
 ```
 
-### Retrieving a Stored Secret
+## Retrieving a Stored Secret
 
 ```bash
 clan secrets get mysecret
 ```
 
-### List all Secrets
+## List all Secrets
 
 ```bash
 clan secrets list
 ```
 
-### NixOS integration
+## NixOS integration
 
 A NixOS machine will automatically import all secrets that are encrypted for the
 current machine. At runtime it will use the host key to decrypt all secrets into
@@ -37,7 +153,7 @@ In your nixos configuration you can get a path to secrets like this `config.sops
 }
 ```
 
-### Assigning Access
+## Assigning Access
 
 When using `clan secrets set <secret>` without arguments, secrets are encrypted for the key of the user named like your current $USER.
 

docs/site/guides/services/community.md

@@ -1,16 +1,16 @@
 # Authoring a 'clan.service' module
 
 !!! Tip
-    This is the successor format to the older [clanModules](../clanModules/index.md)
+    This is the successor format to the older [clanModules](../../reference/clanModules/index.md)
 
     While some features might still be missing we recommend to adapt this format early and give feedback.
 
 ## Service Module Specification
 
 This section explains how to author a clan service module.
-We discussed the initial architecture in [01-clan-service-modules](../../../decisions/01-ClanModules.md) and decided to rework the format.
+We discussed the initial architecture in [01-clan-service-modules](../../decisions/01-ClanModules.md) and decided to rework the format.
 
-For the full specification and current state see: **[Service Author Reference](../../../reference/clanServices/clan-service-author-interface.md)**
+For the full specification and current state see: **[Service Author Reference](../../reference/clanServices/clan-service-author-interface.md)**
 
 ### A Minimal module
 
@@ -52,7 +52,7 @@ The imported module file must fulfill at least the following requirements:
 }
 ```
 
-For more attributes see: **[Service Author Reference](../../../reference/clanServices/clan-service-author-interface.md)**
+For more attributes see: **[Service Author Reference](../../reference/clanServices/clan-service-author-interface.md)**
 
 ### Adding functionality to the module
 
@@ -266,6 +266,6 @@ The benefit of this approach is that downstream users can override the value of
 
 ## Further
 
-- [Reference Documentation for Service Authors](../../../reference/clanServices/clan-service-author-interface.md)
-- [Migration Guide from ClanModules to ClanServices](../../migrations/migrate-inventory-services.md)
-- [Decision that lead to ClanServices](../../../decisions/01-ClanModules.md)
+- [Reference Documentation for Service Authors](../../reference/clanServices/clan-service-author-interface.md)
+- [Migration Guide from ClanModules to ClanServices](../../guides/migrations/migrate-inventory-services.md)
+- [Decision that lead to ClanServices](../../decisions/01-ClanModules.md)

docs/site/index.md

@@ -4,87 +4,72 @@ hide:
   - toc
 ---
 
-# :material-home: Welcome to **Clan**'s  documentation
+# :material-home: What is Clan?
+
+[Clan](https://clan.lol/) is a peer-to-peer computer management framework that
+empowers you to **selfhost in a reliable and scalable way**.   
+
+Built on NixOS, Clan provides a **declarative interface for managing machines** with automated [secret management](./guides/secrets.md), easy [mesh VPN
+connectivity](./guides/mesh-vpn.md), and [automated backups](./guides/backups.md). 
+
+Whether you're running a homelab or maintaining critical computing infrastructure,
+Clan will help **reduce maintenance burden** by allowing a **git repository to define your whole network** of computers.
+
+In combination with [sops-nix](https://github.com/Mic92/sops-nix), [nixos-anywhere](https://github.com/nix-community/nixos-anywhere) and [disko](https://github.com/nix-community/disko), Clan makes it possible to have **collaborative infrastructure**.
+
+At the heart of Clan are [Clan Services](./reference/clanServices/index.md) - the core
+concept that enables you to add functionality across multiple machines in your
+network. While Clan ships with essential core services, you can [create custom
+services](./guides/clanServices.md) tailored to your specific needs.
 
-[Getting Started](./guides/getting-started/index.md){ .md-button }
 
 ## :material-book: Guides
 
-**How-to Guides for achieving a certain goal or solving a specific issue.**
+How-to Guides for achieving a certain goal or solving a specific issue.
 
 <div class="grid cards" markdown>
 
--   [Adding more machines](./guides/more-machines.md)
+-   [:material-clock-fast: Getting Started](./guides/getting-started/index.md)
 
     ---
 
-    Learn how Clan automatically includes machines and Nix files.
+    Get started in less than 20 minutes!
 
--   [Vars Backend](./guides/vars-backend.md)
+-   [Mac OS](./guides/macos.md)
 
     ---
 
-    Learn how to manage secrets with vars.
-
--   [Inventory](./guides/inventory.md)
-
-    ---
-
-    Clan's declaration format for running **services** on one or multiple **machines**.
-
--   [Flake-parts](./guides/flake-parts.md)
-
-    ---
-
-    Use Clan with [https://flake.parts/]()
+    How to manage Mac OS machines with Clan
 
 -   [Contribute](./guides/contributing/CONTRIBUTING.md)
 
     ---
 
-    Discover how to set up a development environment to contribute to Clan!
-
--   [macOS machines](./guides/macos.md)
-
-    ---
-
-    Manage macOS machines with nix-darwin
+    How to set up a development environment
 
 </div>
 
-## API Reference
+## Concepts
 
-**Reference API Documentation**
+Explore the underlying principles of Clan
 
 <div class="grid cards" markdown>
 
--   [CLI Reference](./reference/cli/index.md)
+-   [Generators](./concepts/generators.md)
 
     ---
 
-    The `clan` CLI command
+    Learn about Generators, our way to secret management
 
--   [Service Modules](./reference/clanServices/index.md)
+-   [Inventory](./concepts/inventory.md)
 
     ---
 
-    An overview of available service modules
-
--   [Core](./reference/clan.core/index.md)
-
-    ---
-
-    The clan core nix module.
-    This is imported when using clan and is the basis of the extra functionality
-    that can be provided.
-
--   [(Legacy) Modules](./reference/clanModules/index.md)
-
-    ---
-
-    An overview of available clanModules
-
-    !!! Example "These will be deprecated soon"
-
+    Learn about the Inventory, a multi machine Nix interface
 
 </div>
+
+
+## Blog
+
+Visit our [Clan Blog](https://clan.lol/blog/) for the latest updates, tutorials, and community stories.

docs/site/intern/index.md

@@ -1,25 +0,0 @@
-# Developer Documentation
-
-!!! Danger
-    This documentation is **not** intended for external users. It may contain low-level details and internal-only interfaces.*
-
-Welcome to the internal developer documentation.
-
-This section is intended for contributors, engineers, and internal stakeholders working directly with our system, tooling, and APIs. It provides a technical overview of core components, internal APIs, conventions, and patterns that support the platform.
-
-Our goal is to make the internal workings of the system **transparent, discoverable, and consistent** — helping you contribute confidently, troubleshoot effectively, and build faster.
-
-## What's Here?
-
-!!! note "docs migration ongoing"
-
-- [ ] **API Reference**: 🚧🚧🚧 Detailed documentation of internal API functions, inputs, and expected outputs. 🚧🚧🚧
-- [ ] **System Concepts**: Architectural overviews and domain-specific guides.
-- [ ] **Development Guides**: How to test, extend, or integrate with key components.
-- [ ] **Design Notes**: Rationales behind major design decisions or patterns.
-
-## Who is This For?
-
-* Developers contributing to the platform
-* Engineers debugging or extending internal systems
-* Anyone needing to understand **how** and **why** things work under the hood

docs/site/reference/index.md

@@ -4,10 +4,10 @@ This section of the site provides an overview of available options and commands
 
 ---
 
+- [Clan Configuration Option](../options.md) - for defining a Clan
 - Learn how to use the [Clan CLI](./cli/index.md)
 - Explore available services and application [modules](./clanModules/index.md)
-- Discover [configuration options](./clan.core/index.md) that manage essential features
-- Find descriptions of the [Nix interfaces](./nix-api/clan.md) for defining a Clan
+- [NixOS Configuration Options](./clan.core/index.md) - Additional options avilable on a NixOS machine.
 
 ---
 

docs/site/static/extra.css

@@ -2,6 +2,7 @@
   font-family: "Roboto";
   src: url(./Roboto-Regular.ttf) format("truetype");
 }
+
 @font-face {
   font-family: "Fira Code";
   src: url(./FiraCode-VF.ttf) format("truetype");
@@ -20,3 +21,9 @@
 .md-nav__item.md-nav__item--section > label > span {
   color: var(--md-typeset-a-color);
 }
+
+.md-typeset h4 {
+  margin: 3em 0 0.5em;
+  font-weight: bold;
+  color: #7ebae4;
+}

flake.lock

@@ -13,11 +13,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1752589312,
-        "narHash": "sha256-BafZOenlzMYdumG12AzgVLhEVu+GcEa8nYNDSIYe1U0=",
-        "rev": "496bbf05a2aa7b061ef464254db5804d1c6f45b4",
+        "lastModified": 1753067306,
+        "narHash": "sha256-jyoEbaXa8/MwVQ+PajUdT63y3gYhgD9o7snO/SLaikw=",
+        "rev": "18dfd42bdb2cfff510b8c74206005f733e38d8b9",
         "type": "tarball",
-        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/496bbf05a2aa7b061ef464254db5804d1c6f45b4.tar.gz"
+        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/18dfd42bdb2cfff510b8c74206005f733e38d8b9.tar.gz"
       },
       "original": {
         "type": "tarball",
@@ -31,11 +31,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1752541678,
-        "narHash": "sha256-dyhGzkld6jPqnT/UfGV2oqe7tYn7hppAqFvF3GZTyXY=",
+        "lastModified": 1753140376,
+        "narHash": "sha256-7lrVrE0jSvZHrxEzvnfHFE/Wkk9DDqb+mYCodI5uuB8=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "2bf3421f7fed5c84d9392b62dcb9d76ef09796a7",
+        "rev": "545aba02960caa78a31bd9a8709a0ad4b6320a5c",
         "type": "github"
       },
       "original": {
@@ -51,11 +51,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1751413152,
-        "narHash": "sha256-Tyw1RjYEsp5scoigs1384gIg6e0GoBVjms4aXFfRssQ=",
+        "lastModified": 1753121425,
+        "narHash": "sha256-TVcTNvOeWWk1DXljFxVRp+E0tzG1LhrVjOGGoMHuXio=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "77826244401ea9de6e3bac47c2db46005e1f30b5",
+        "rev": "644e0fc48951a860279da645ba77fe4a6e814c5e",
         "type": "github"
       },
       "original": {
@@ -181,11 +181,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1752055615,
-        "narHash": "sha256-19m7P4O/Aw/6+CzncWMAJu89JaKeMh3aMle1CNQSIwM=",
+        "lastModified": 1753772294,
+        "narHash": "sha256-8rkd13WfClfZUBIYpX5dvG3O9V9w3K9FPQ9rY14VtBE=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "c9d477b5d5bd7f26adddd3f96cfd6a904768d4f9",
+        "rev": "6b9214fffbcf3f1e608efa15044431651635ca83",
         "type": "github"
       },
       "original": {

flake.nix

@@ -30,7 +30,6 @@
       inputs = {
         flake-parts.follows = "flake-parts";
         nixpkgs.follows = "nixpkgs";
-        systems.follows = "systems";
         treefmt-nix.follows = "treefmt-nix";
       };
     };
@@ -51,6 +50,7 @@
         pathExists
         ;
 
+      # Load private flake inputs if available
       loadDevFlake =
         path:
         let
@@ -61,7 +61,13 @@
 
       devFlake = builtins.tryEval (loadDevFlake ./devFlake/private);
 
-      privateInputs = if devFlake.success then devFlake.value.inputs else { };
+      privateInputs =
+        if pathExists ./.skip-private-inputs then
+          { }
+        else if devFlake.success then
+          devFlake.value.inputs
+        else
+          { };
     in
     flake-parts.lib.mkFlake { inherit inputs; } (
       { ... }:

lib/modules/clan/interface.nix

@@ -78,7 +78,87 @@ in
       internal = true;
       visible = false;
       type = types.deferredModule;
-      default = { };
+      default = {
+        options.networking = lib.mkOption {
+          default = null;
+          type = lib.types.nullOr (
+            lib.types.submodule {
+              options = {
+                priority = lib.mkOption {
+                  type = lib.types.int;
+                  default = 1000;
+                  description = ''
+                    priority with which this network should be tried.
+                    higher priority means it gets used earlier in the chain
+                  '';
+                };
+                module = lib.mkOption {
+                  # type = lib.types.enum [
+                  #   "clan_lib.network.direct"
+                  #   "clan_lib.network.tor"
+                  # ];
+                  type = lib.types.str;
+                  default = "clan_lib.network.direct";
+                  description = ''
+                    the technology this network uses to connect to the target
+                    This is used for userspace networking with socks proxies.
+                  '';
+                };
+                # should we call this machines? hosts?
+                peers = lib.mkOption {
+                  # <name>
+                  type = lib.types.attrsOf (
+                    lib.types.submodule (
+                      { name, ... }:
+                      {
+                        options = {
+                          name = lib.mkOption {
+                            type = lib.types.str;
+                            default = name;
+                          };
+                          SSHOptions = lib.mkOption {
+                            type = lib.types.listOf lib.types.str;
+                            default = [ ];
+                          };
+                          host = lib.mkOption {
+                            description = '''';
+                            type = lib.types.attrTag {
+                              plain = lib.mkOption {
+                                type = lib.types.str;
+                                description = ''
+                                  a plain value, which can be read directly from the config
+                                '';
+                              };
+                              var = lib.mkOption {
+                                type = lib.types.submodule {
+                                  options = {
+                                    machine = lib.mkOption {
+                                      type = lib.types.str;
+                                      example = "jon";
+                                    };
+                                    generator = lib.mkOption {
+                                      type = lib.types.str;
+                                      example = "tor-ssh";
+                                    };
+                                    file = lib.mkOption {
+                                      type = lib.types.str;
+                                      example = "hostname";
+                                    };
+                                  };
+                                };
+                              };
+                            };
+                          };
+                        };
+                      }
+                    )
+                  );
+                };
+              };
+            }
+          );
+        };
+      };
       description = ''
         A module that is used to define the module of flake level exports -
 
@@ -149,8 +229,8 @@ in
     };
 
     inventory = lib.mkOption {
-      type = types.submodule {
-        imports = [
+      type = types.submoduleWith {
+        modules = [
           {
             _module.args = { inherit clanLib; };
             _file = "clan interface";

lib/modules/clan/module.nix

@@ -247,7 +247,7 @@ in
               {
                 distributedServices = clanLib.inventory.mapInstances {
                   inherit (clanConfig) inventory exportsModule;
-                  inherit flakeInputs;
+                  inherit flakeInputs directory;
                   clanCoreModules = clan-core.clan.modules;
                   prefix = [ "distributedServices" ];
                 };

lib/modules/default.nix

@@ -7,8 +7,29 @@
 }:
 rec {
   buildClan =
-    # TODO: Once all templates and docs are migrated add: lib.warn "'buildClan' is deprecated. Use 'clan-core.lib.clan' instead"
-    module: (clan module).config;
+    module:
+    lib.warn ''
+      ==================== DEPRECATION NOTICE ====================
+      Please migrate 
+      from: 'clan = inputs.<clan-core>.lib.buildClan' 
+      to  : 'clan = inputs.<clan-core>.lib.clan'
+      in your flake.nix.
+
+      Please also migrate
+      from: 'inherit (clan) nixosConfigurations clanInternals; '
+      to  : "
+              inherit (clan.config) nixosConfigurations clanInternals;
+              clan = clan.config;
+            "
+      in your flake.nix.
+
+      Reason: 
+      - Improves consistency between flake-parts and non-flake-parts users.
+              
+      - It also allows us to use the top level attribute 'clan' to expose
+        attributes that can be used for cross-clan functionality.
+      ============================================================
+    '' (clan module).config;
 
   clan =
     {

lib/modules/inventory/distributed-service/all-services-wrapper.nix

@@ -1,4 +1,8 @@
 # Wraps all services in one fixed point module
+{
+  # TODO: consume directly from clan.config
+  directory,
+}:
 {
   lib,
   config,
@@ -29,6 +33,8 @@ in
               {
                 _module.args._ctx = [ name ];
                 _module.args.exports' = config.exports;
+                _module.args.directory = directory;
+
               }
             )
             ./service-module.nix
@@ -48,6 +54,7 @@ in
           {
             options = {
               instances = lib.mkOption {
+                default = { };
                 # instances.<instanceName>...
                 type = types.attrsOf (submoduleWith {
                   modules = [
@@ -57,6 +64,7 @@ in
               };
               # instances.<machineName>...
               machines = lib.mkOption {
+                default = { };
                 type = types.attrsOf (submoduleWith {
                   modules = [
                     config.exportsModule
@@ -69,8 +77,5 @@ in
       };
       default = { };
     };
-    debug = mkOption {
-      default = lib.mapAttrsToList (_: service: service.exports) config.mappedServices;
-    };
   };
 }

lib/modules/inventory/distributed-service/inventory-adapter.nix

@@ -24,6 +24,7 @@ in
       flakeInputs,
       # The clan inventory
       inventory,
+      directory,
       clanCoreModules,
       prefix ? [ ],
       exportsModule,
@@ -128,7 +129,7 @@ in
             _ctx = prefix;
           };
           modules = [
-            ./all-services-wrapper.nix
+            (import ./all-services-wrapper.nix { inherit directory; })
           ] ++ modules;
         };
 

lib/modules/inventory/distributed-service/service-module.nix

@@ -2,6 +2,7 @@
   lib,
   config,
   _ctx,
+  directory,
   ...
 }:
 let
@@ -212,7 +213,7 @@ in
 
                           options.extraModules = lib.mkOption {
                             default = [ ];
-                            type = types.listOf (types.deferredModule);
+                            type = types.listOf (types.either types.deferredModule types.str);
                           };
                         })
                       ];
@@ -755,10 +756,14 @@ in
             instanceRes
             // {
               nixosModule = {
-                imports = [
-                  # Result of the applied 'perInstance = {...}: {  nixosModule = { ... }; }'
-                  instanceRes.nixosModule
-                ] ++ instanceCfg.roles.${roleName}.extraModules;
+                imports =
+                  [
+                    # Result of the applied 'perInstance = {...}: {  nixosModule = { ... }; }'
+                    instanceRes.nixosModule
+                  ]
+                  ++ (map (
+                    s: if builtins.typeOf s == "string" then "${directory}/${s}" else s
+                  ) instanceCfg.roles.${roleName}.extraModules);
               };
             }
 

lib/modules/inventory/distributed-service/tests/default.nix

@@ -45,6 +45,7 @@ let
       };
     in
     clanLib.inventory.mapInstances {
+      directory = ./.;
       clanCoreModules = { };
       flakeInputs = flakeInputsFixture;
       inherit inventory;
@@ -52,6 +53,7 @@ let
     };
 in
 {
+  extraModules = import ./extraModules.nix { inherit clanLib; };
   exports = import ./exports.nix { inherit lib clanLib; };
   resolve_module_spec = import ./import_module_spec.nix { inherit lib callInventoryAdapter; };
   test_simple =

lib/modules/inventory/distributed-service/tests/extraModules.nix

@@ -0,0 +1,33 @@
+{ clanLib }:
+let
+  clan = clanLib.clan {
+    self = { };
+    directory = ./.;
+
+    machines.jon = {
+      nixpkgs.hostPlatform = "x86_64-linux";
+
+    };
+    # A module that adds exports perMachine
+    modules.A = {
+      manifest.name = "A";
+      roles.peer = { };
+    };
+
+    inventory = {
+      instances.A = {
+        module.input = "self";
+        roles.peer.tags.all = { };
+
+        roles.peer.extraModules = [ ./oneOption.nix ];
+      };
+    };
+  };
+in
+{
+  test_1 = {
+    inherit clan;
+    expr = clan.config.nixosConfigurations.jon.config.testDebug;
+    expected = 42;
+  };
+}

lib/modules/inventory/distributed-service/tests/oneOption.nix

@@ -0,0 +1,6 @@
+{ lib, ... }:
+{
+  options.testDebug = lib.mkOption {
+    default = 42;
+  };
+}

lib/modules/inventoryClass/interface.nix

@@ -142,7 +142,7 @@ in
             - The module MUST have at least `features = [ "inventory" ]` in the frontmatter section.
             - The module MUST have a subfolder `roles` with at least one `{roleName}.nix` file.
 
-            For further information see: [Module Authoring Guide](../../guides/authoring/clanServices/index.md).
+            For further information see: [Module Authoring Guide](../../guides/services/community.md).
 
         ???+ example
             ```nix
@@ -179,8 +179,7 @@ in
               map (m: "'${m}'") (lib.attrNames (lib.filterAttrs (n: _v: !builtins.elem n allowedNames) moduleSet))
             )}
 
-            See: https://docs.clan.lol/guides/clanServices/
-            And: https://docs.clan.lol/guides/authoring/clanServices/
+            See: https://docs.clan.lol/guides/services/community/
           '' moduleSet;
     };
 

lib/test/container-test-driver/nixos-module.nix

@@ -1,4 +1,9 @@
-{ pkgs, lib, ... }:
+{
+  pkgs,
+  lib,
+  options,
+  ...
+}:
 {
   boot.isContainer = true;
 
@@ -7,7 +12,9 @@
 
   # undo qemu stuff
   system.build.initialRamdisk = "";
-  virtualisation.sharedDirectories = lib.mkForce { };
+  virtualisation = lib.optionalAttrs (options ? virtualisation.sharedDirectories) {
+    sharedDirectories = lib.mkForce { };
+  };
   networking.useDHCP = false;
 
   # PAM requires setuid and doesn't work in our containers
@@ -15,11 +22,14 @@
 
   # We use networkd to assign static ip addresses
   networking.useNetworkd = true;
+  networking.useHostResolvConf = false;
   services.resolved.enable = false;
 
-  # Rename the host0 interface to eth0 to match what we expect in VM tests.
+  # Rename the host0 interface to eth1 to match what we expect in VM tests.
   system.activationScripts.renameInterface = ''
-    ${pkgs.iproute2}/bin/ip link set dev host0 name eth1
+    if ${pkgs.iproute2}/bin/ip link show host0 2>/dev/null; then
+      ${pkgs.iproute2}/bin/ip link set dev host0 name eth1
+    fi
   '';
 
   systemd.services.backdoor.enable = false;
@@ -27,6 +37,12 @@
   # we don't have permission to set cpu scheduler in our container
   systemd.services.nix-daemon.serviceConfig.CPUSchedulingPolicy = lib.mkForce "";
 
+  # Disable suid-sgid-wrappers.service as it fails in the nix sandbox
+  systemd.services.suid-sgid-wrappers.enable = false;
+
+  # Disable resolvconf as it can cause issues in containers because it cannot apply posix acl
+  systemd.services.resolvconf.enable = false;
+
   # Adds `Include /nix/store/...` to `/etc/ssh/ssh_config`[1] which will make
   # SSH fail when running inside a container test as SSH checks the permissions
   # of the config files it reads which can't be disabled[2] and all the store

lib/test/container-test-driver/test_driver/__init__.py

@@ -13,13 +13,80 @@ from contextlib import _GeneratorContextManager
 from dataclasses import dataclass
 from functools import cached_property
 from pathlib import Path
-from tempfile import TemporaryDirectory
+from tempfile import NamedTemporaryFile, TemporaryDirectory
 from typing import Any
 
 from colorama import Fore, Style
 
 from .logger import AbstractLogger, CompositeLogger, TerminalLogger
 
+# Global flag to track if test environment has been initialized
+_test_env_initialized = False
+
+
+def init_test_environment() -> None:
+    """Set up the test environment (network bridge, /etc/passwd) once."""
+    global _test_env_initialized
+    if _test_env_initialized:
+        return
+
+    # Set up network bridge
+    subprocess.run(
+        ["ip", "link", "add", "br0", "type", "bridge"], check=True, text=True
+    )
+    subprocess.run(["ip", "link", "set", "br0", "up"], check=True, text=True)
+    subprocess.run(
+        ["ip", "addr", "add", "192.168.1.254/24", "dev", "br0"], check=True, text=True
+    )
+
+    # Set up minimal passwd file for unprivileged operations
+    # Using Nix's convention: UID 1000 for nixbld user, GID 100 for nixbld group
+    passwd_content = """root:x:0:0:Root:/root:/bin/sh
+nixbld:x:1000:100:Nix build user:/tmp:/bin/sh
+nobody:x:65534:65534:Nobody:/:/bin/sh
+"""
+
+    with NamedTemporaryFile(mode="w", delete=False, prefix="test-passwd-") as f:
+        f.write(passwd_content)
+        passwd_path = f.name
+
+    # Set up minimal group file
+    group_content = """root:x:0:
+nixbld:x:100:nixbld
+nogroup:x:65534:
+"""
+
+    with NamedTemporaryFile(mode="w", delete=False, prefix="test-group-") as f:
+        f.write(group_content)
+        group_path = f.name
+
+    # Bind mount our passwd over the system's /etc/passwd
+    result = libc.mount(
+        ctypes.c_char_p(passwd_path.encode()),
+        ctypes.c_char_p(b"/etc/passwd"),
+        ctypes.c_char_p(b"none"),
+        ctypes.c_ulong(MS_BIND),
+        None,
+    )
+    if result != 0:
+        errno = ctypes.get_errno()
+        raise OSError(errno, os.strerror(errno), "Failed to mount passwd")
+
+    # Bind mount our group over the system's /etc/group
+    result = libc.mount(
+        ctypes.c_char_p(group_path.encode()),
+        ctypes.c_char_p(b"/etc/group"),
+        ctypes.c_char_p(b"none"),
+        ctypes.c_ulong(MS_BIND),
+        None,
+    )
+    if result != 0:
+        errno = ctypes.get_errno()
+        raise OSError(errno, os.strerror(errno), "Failed to mount group")
+
+    _test_env_initialized = True
+
+
 # Load the C library
 libc = ctypes.CDLL("libc.so.6", use_errno=True)
 
@@ -123,6 +190,7 @@ class Machine:
 
     def start(self) -> None:
         prepare_machine_root(self.name, self.rootdir)
+        init_test_environment()
         cmd = [
             "systemd-nspawn",
             "--keep-unit",
@@ -146,6 +214,7 @@ class Machine:
     def get_systemd_process(self) -> int:
         assert self.process is not None, "Machine not started"
         assert self.process.stdout is not None, "Machine has no stdout"
+
         for line in self.process.stdout:
             print(line, end="")
             if (
@@ -313,6 +382,18 @@ class Machine:
         command = f"nc -z {shlex.quote(addr)} {port}"
         self.wait_until_succeeds(command, timeout=timeout)
 
+    def wait_for_file(self, filename: str, timeout: int = 30) -> None:
+        """
+        Waits until the file exists in the machine's file system.
+        """
+
+        def check_file(_last_try: bool) -> bool:
+            result = self.execute(f"test -e {filename}")
+            return result.returncode == 0
+
+        with self.nested(f"waiting for file '{filename}'"):
+            retry(check_file, timeout)
+
     def wait_for_unit(self, unit: str, timeout: int = 900) -> None:
         """
         Wait for a systemd unit to get into "active" state.
@@ -407,6 +488,15 @@ def setup_filesystems(container: ContainerInfo) -> None:
     Path("/etc/os-release").touch()
     Path("/etc/machine-id").write_text("a5ea3f98dedc0278b6f3cc8c37eeaeac")
     container.nix_store_dir.mkdir(parents=True)
+    container.nix_store_dir.chmod(0o755)
+
+    # Recreate symlinks
+    for file in Path("/nix/store").iterdir():
+        if file.is_symlink():
+            target = file.readlink()
+            sym = container.nix_store_dir / file.name
+            os.symlink(target, sym)
+
     # Read /proc/mounts and replicate every bind mount
     with Path("/proc/self/mounts").open() as f:
         for line in f:
@@ -471,12 +561,8 @@ class Driver:
             )
 
     def start_all(self) -> None:
-        # child
-        # create bridge
-        subprocess.run(
-            ["ip", "link", "add", "br0", "type", "bridge"], check=True, text=True
-        )
-        subprocess.run(["ip", "link", "set", "br0", "up"], check=True, text=True)
+        # Ensure test environment is set up
+        init_test_environment()
 
         for machine in self.machines:
             print(f"Starting {machine.name}")

nixosModules/clanCore/default.nix

@@ -18,6 +18,7 @@
     ++ lib.optionals (_class == "nixos") [
       ./nixos-facter.nix
       ./vm.nix
+      ./postgresql
       ./machine-id
       ./state-version
       ./wayland-proxy-virtwl.nix

nixosModules/clanCore/outputs.nix

@@ -31,6 +31,7 @@
             The deployment data is now accessed directly from the configuration
             instead of being written to a separate JSON file.
           '';
+          defaultText = "error: deployment.json file generation has been removed in favor of direct selectors.";
         };
         deployment.buildHost = lib.mkOption {
           type = lib.types.nullOr lib.types.str;
@@ -54,10 +55,10 @@
         deployment.nixosMobileWorkaround = lib.mkOption {
           type = lib.types.bool;
           description = ''
-            if true, the deployment will first do a nixos-rebuild switch 
+            if true, the deployment will first do a nixos-rebuild switch
             to register the boot profile the command will fail applying it to the running system
-            which is why afterwards we execute a nixos-rebuild test to apply 
-            the new config without having to reboot. 
+            which is why afterwards we execute a nixos-rebuild test to apply
+            the new config without having to reboot.
             This is a nixos-mobile deployment bug and will be removed in the future
           '';
           default = false;

nixosModules/clanCore/postgresql/default.nix

@@ -0,0 +1,236 @@
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+let
+
+  cfg = config.clan.core.postgresql;
+
+  createDatabaseState =
+    db:
+    let
+      folder = "/var/backup/postgres/${db.name}";
+      current = "${folder}/pg-dump";
+      compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
+    in
+    {
+      folders = [ folder ];
+      preBackupScript = ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+
+        mkdir -p "${folder}"
+        runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
+        mv "${current}.tmp" ${current}
+      '';
+      postRestoreScript = ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+            pkgs.gnugrep
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          (
+            systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
+            trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
+
+            mkdir -p "${folder}"
+            if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
+              runuser -u postgres -- dropdb "${db.name}"
+            fi
+            runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          )
+        else
+          echo No database backup found, skipping restore
+        fi
+      '';
+    };
+
+  createDatabase = db: ''
+    CREATE DATABASE "${db.name}" ${
+      lib.concatStringsSep " " (
+        lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
+      )
+    }
+  '';
+
+  userClauses = lib.mapAttrsToList (
+    _: user:
+    ''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
+  ) cfg.users;
+  databaseClauses = lib.mapAttrsToList (
+    name: db:
+    lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
+  ) cfg.databases;
+in
+{
+  options.clan.core.postgresql = {
+
+    enable = lib.mkEnableOption "Whether to enable PostgreSQL Server";
+
+    # we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
+    databases = lib.mkOption {
+      description = "Databases to create";
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options = {
+              name = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "Database name.";
+              };
+              service = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "Service name that we associate with the database.";
+              };
+              # set to false, in case the upstream module uses ensureDatabase option
+              create.enable = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = "Create the database if it does not exist.";
+              };
+              create.options = lib.mkOption {
+                description = "Options to pass to the CREATE DATABASE command.";
+                type = lib.types.lazyAttrsOf lib.types.str;
+                default = { };
+                example = {
+                  TEMPLATE = "template0";
+                  LC_COLLATE = "C";
+                  LC_CTYPE = "C";
+                  ENCODING = "UTF8";
+                  OWNER = "foo";
+                };
+              };
+              restore.stopOnRestore = lib.mkOption {
+                type = lib.types.listOf lib.types.str;
+                default = [ ];
+                description = "List of systemd services to stop before restoring the database.";
+              };
+            };
+          }
+        )
+      );
+    };
+    users = lib.mkOption {
+      description = "Users to create";
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options.name = lib.mkOption {
+              description = "User name";
+              type = lib.types.str;
+              default = name;
+            };
+          }
+        )
+      );
+    };
+  };
+
+  config = lib.mkIf (config.clan.core.postgresql.enable) {
+
+    clan.core.settings.state-version.enable = true;
+
+    # services.postgresql.package = lib.mkDefault pkgs.postgresql_16;
+
+    services.postgresql.enable = true;
+
+    services.postgresql.settings = {
+      wal_level = "replica";
+      max_wal_senders = 3;
+    };
+
+    # We are duplicating a bit the upstream module but allow to create databases with options
+    systemd.services.postgresql.postStart = ''
+      PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
+
+      while ! $PSQL -d postgres -c "" 2> /dev/null; do
+        if ! kill -0 "$MAINPID"; then exit 1; fi
+        sleep 0.1
+      done
+      ${lib.concatStringsSep "\n" userClauses}
+      ${lib.concatStringsSep "\n" databaseClauses}
+    '';
+
+    clan.core.state = lib.mapAttrs' (
+      _: db: lib.nameValuePair db.service (createDatabaseState db)
+    ) config.clan.core.postgresql.databases;
+
+    environment.systemPackages = builtins.map (
+      db:
+      let
+        folder = "/var/backup/postgres/${db.name}";
+        current = "${folder}/pg-dump";
+      in
+      pkgs.writeShellScriptBin "postgres-db-restore-command-${db.name}" ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+            pkgs.gnugrep
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          (
+            ${lib.optionalString (db.restore.stopOnRestore != [ ]) ''
+              systemctl stop ${builtins.toString db.restore.stopOnRestore}
+              trap "systemctl start ${builtins.toString db.restore.stopOnRestore}" EXIT
+            ''}
+
+            mkdir -p "${folder}"
+            if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
+              runuser -u postgres -- dropdb "${db.name}"
+            fi
+            runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          )
+        else
+          echo No database backup found, skipping restore
+        fi
+      ''
+    ) (builtins.attrValues config.clan.core.postgresql.databases);
+  };
+}

nixosModules/clanCore/postgresql/tests/flake-module.nix

@@ -0,0 +1,106 @@
+{ self, ... }:
+{
+  perSystem =
+    { ... }:
+    {
+      clan.nixosTests.postgresql = {
+
+        name = "service-postgresql";
+
+        clan = {
+          directory = ./.;
+
+          # Workaround until we can use nodes.machine = { };
+          modules."@clan/importer" = ../../../../clanServices/importer;
+
+          inventory = {
+            machines.machine = { };
+            instances.importer = {
+              module.name = "@clan/importer";
+              module.input = "self";
+              roles.default.tags.all = { };
+              roles.default.extraModules = [
+                {
+
+                  imports = [
+                    #   self.nixosModules.clanCore
+                    self.clanModules.localbackup
+                  ];
+
+                  clan.core.postgresql.enable = true;
+                  clan.core.postgresql.users.test = { };
+                  clan.core.postgresql.databases.test.create.options.OWNER = "test";
+                  clan.core.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ];
+                  clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
+                  clan.core.settings.directory = ./.;
+
+                  systemd.services.sample-service = {
+                    wantedBy = [ "multi-user.target" ];
+                    script = ''
+                      while true; do
+                        echo "Hello, world!"
+                        sleep 5
+                      done
+                    '';
+                  };
+
+                }
+              ];
+            };
+          };
+        };
+
+        # TODO: Broken. Use instead of importer after fixing.
+        # nodes.machine = { };
+
+        testScript =
+
+          { nodes, ... }:
+
+          ''
+            start_all()
+            machine.wait_for_unit("postgresql")
+            machine.wait_for_unit("sample-service")
+            # Create a test table
+            machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
+
+            machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
+            timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
+
+            # import time
+            # time.sleep(5400000)
+
+            machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+            machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
+            machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
+            machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+
+            machine.succeed("rm -rf /var/backup/postgres")
+
+            machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
+            machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+
+            machine.succeed("""
+            set -x
+            ${nodes.machine.clan.core.state.test.postRestoreCommand}
+            """)
+            machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
+            machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
+
+            timestamp_after = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
+            assert timestamp_before < timestamp_after, f"{timestamp_before} >= {timestamp_after}: expected sample-service to be restarted after restore"
+
+            # Check that the table is still there
+            machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
+            output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
+            owner = output.split("\n")[1]
+            assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
+
+            # check if restore works if the database does not exist
+            machine.succeed("runuser -u postgres -- dropdb test")
+            machine.succeed("${nodes.machine.clan.core.state.test.postRestoreCommand}")
+            machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
+          '';
+      };
+    };
+}

pkgs/clan-app/clan_app/api/api_bridge.py

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any
 
 from clan_lib.api import ApiResponse
 from clan_lib.api.tasks import WebThread
-from clan_lib.async_run import set_should_cancel
+from clan_lib.async_run import set_current_thread_opkey, set_should_cancel
 
 if TYPE_CHECKING:
     from .middleware import Middleware
@@ -98,7 +98,7 @@ class ApiBridge(ABC):
         *,
         thread_name: str = "ApiBridgeThread",
         wait_for_completion: bool = False,
-        timeout: float = 60.0,
+        timeout: float = 60.0 * 60,  # 1 hour default timeout
     ) -> None:
         """Process an API request in a separate thread with cancellation support.
 
@@ -112,6 +112,7 @@ class ApiBridge(ABC):
 
         def thread_task(stop_event: threading.Event) -> None:
             set_should_cancel(lambda: stop_event.is_set())
+            set_current_thread_opkey(op_key)
             try:
                 log.debug(
                     f"Processing {request.method_name} with args {request.args} "

pkgs/clan-app/clan_app/api/file_gtk.py

@@ -9,6 +9,7 @@ gi.require_version("Gtk", "4.0")
 
 from clan_lib.api import ApiError, ErrorDataClass, SuccessDataClass
 from clan_lib.api.directory import FileRequest
+from clan_lib.async_run import get_current_thread_opkey
 from clan_lib.clan.check import check_clan_valid
 from clan_lib.flake import Flake
 from gi.repository import Gio, GLib, Gtk
@@ -24,7 +25,7 @@ def remove_none(_list: list) -> list:
 RESULT: dict[str, SuccessDataClass[list[str] | None] | ErrorDataClass] = {}
 
 
-def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass:
+def get_clan_folder() -> SuccessDataClass[Flake] | ErrorDataClass:
     """
     Opens the clan folder using the GTK file dialog.
     Returns the path to the clan folder or an error if it fails.
@@ -34,7 +35,10 @@ def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass:
         title="Select Clan Folder",
         initial_folder=str(Path.home()),
     )
-    response = get_system_file(file_request, op_key=op_key)
+
+    response = get_system_file(file_request)
+
+    op_key = response.op_key
 
     if isinstance(response, ErrorDataClass):
         return response
@@ -70,8 +74,13 @@ def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass:
 
 
 def get_system_file(
-    file_request: FileRequest, *, op_key: str
+    file_request: FileRequest,
 ) -> SuccessDataClass[list[str] | None] | ErrorDataClass:
+    op_key = get_current_thread_opkey()
+
+    if not op_key:
+        msg = "No operation key found in the current thread context."
+        raise RuntimeError(msg)
     GLib.idle_add(gtk_open_file, file_request, op_key)
 
     while RESULT.get(op_key) is None:

pkgs/clan-app/clan_app/api/middleware/argument_parsing.py

@@ -21,18 +21,12 @@ class ArgumentParsingMiddleware(Middleware):
             # Convert dictionary arguments to dataclass instances
             reconciled_arguments = {}
             for k, v in context.request.args.items():
-                if k == "op_key":
-                    continue
-
                 # Get the expected argument type from the API
                 arg_class = self.api.get_method_argtype(context.request.method_name, k)
 
                 # Convert dictionary to dataclass instance
                 reconciled_arguments[k] = from_dict(arg_class, v)
 
-            # Add op_key to arguments
-            reconciled_arguments["op_key"] = context.request.op_key
-
             # Create a new request with reconciled arguments
 
             updated_request = BackendRequest(

pkgs/clan-app/clan_app/deps/http/http_bridge.py

@@ -1,13 +1,22 @@
 import json
 import logging
+import threading
 import uuid
 from http.server import BaseHTTPRequestHandler
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 from urllib.parse import urlparse
 
-from clan_lib.api import MethodRegistry, SuccessDataClass, dataclass_to_dict
+from clan_lib.api import (
+    MethodRegistry,
+    SuccessDataClass,
+    dataclass_to_dict,
+)
 from clan_lib.api.tasks import WebThread
+from clan_lib.async_run import (
+    set_current_thread_opkey,
+    set_should_cancel,
+)
 
 from clan_app.api.api_bridge import ApiBridge, BackendRequest, BackendResponse
 
@@ -324,17 +333,34 @@ class HttpBridge(ApiBridge, BaseHTTPRequestHandler):
             msg = f"Operation key '{op_key}' is already in use. Please try again."
             raise ValueError(msg)
 
+    def process_request_in_thread(
+        self,
+        request: BackendRequest,
+        *,
+        thread_name: str = "ApiBridgeThread",
+        wait_for_completion: bool = False,
+        timeout: float = 60.0 * 60,  # 1 hour default timeout
+    ) -> None:
+        pass
+
     def _process_api_request_in_thread(
         self, api_request: BackendRequest, method_name: str
     ) -> None:
         """Process the API request in a separate thread."""
-        # Use the inherited thread processing method
-        self.process_request_in_thread(
-            api_request,
-            thread_name="HttpThread",
-            wait_for_completion=True,
-            timeout=60.0,
+        stop_event = threading.Event()
+        request = api_request
+        op_key = request.op_key or "unknown"
+        set_should_cancel(lambda: stop_event.is_set())
+        set_current_thread_opkey(op_key)
+
+        curr_thread = threading.current_thread()
+        self.threads[op_key] = WebThread(thread=curr_thread, stop_event=stop_event)
+
+        log.debug(
+            f"Processing {request.method_name} with args {request.args} "
+            f"and header {request.header}"
         )
+        self.process_request(request)
 
     def log_message(self, format: str, *args: Any) -> None:  # noqa: A002
         """Override default logging to use our logger."""

pkgs/clan-app/clan_app/deps/webview/_webview_ffi.py

@@ -29,10 +29,7 @@ def _get_lib_names() -> list[str]:
         msg = f"Unsupported architecture: {machine}"
         raise RuntimeError(msg)
     if system == "darwin":
-        if machine == "arm64":
-            return ["libwebview.dylib"]
-        msg = "Not supported"
-        raise RuntimeError(msg)
+        return ["libwebview.dylib"]
     # linux
     return ["libwebview.so"]
 

pkgs/clan-app/process-compose-2d.yaml

@@ -1,39 +0,0 @@
-version: "0.5"
-
-processes:
-  # App Dev
-
-  clan-app-ui:
-    namespace: "app"
-    command: |
-      cd $(git rev-parse --show-toplevel)/pkgs/clan-app/ui-2d
-      npm install
-      vite
-    ready_log_line: "VITE"
-
-  clan-app:
-    namespace: "app"
-    command: |
-      cd $(git rev-parse --show-toplevel)/pkgs/clan-app
-      ./bin/clan-app --debug --content-uri http://localhost:3000
-    depends_on:
-      clan-app-ui:
-        condition: "process_log_ready"
-    is_foreground: true
-    ready_log_line: "Debug mode enabled"
-
-  # Storybook Dev
-
-  storybook:
-    namespace: "storybook"
-    command: |
-      cd $(git rev-parse --show-toplevel)/pkgs/clan-app/ui-2d
-      npm run storybook-dev -- --ci
-    ready_log_line: "started"
-
-  luakit:
-    namespace: "storybook"
-    command: "luakit http://localhost:6006"
-    depends_on:
-      storybook:
-        condition: "process_log_ready"

pkgs/clan-app/ui.nix

@@ -21,6 +21,12 @@ buildNpmPackage (_finalAttrs: {
     mkdir -p api
     cp -r ${clan-ts-api}/* api
     cp -r ${fonts} ".fonts"
+
+    # only needed for the next couple weeks to make sure this file doesn't make it back into the git history
+    if [[ -f "${./ui}/src/routes/Onboarding/background.jpg" ]]; then
+        echo "background.jpg found, exiting"
+        exit 1
+    fi
   '';
 
   # todo figure out why this fails only inside of Nix

pkgs/clan-app/ui/.storybook/main.ts

@@ -3,7 +3,7 @@ import type { StorybookConfig } from "@kachurun/storybook-solid-vite";
 
 const config: StorybookConfig = {
   framework: "@kachurun/storybook-solid-vite",
-  stories: ["../src/components/**/*.mdx", "../src/components/**/*.stories.tsx"],
+  stories: ["../src/**/*.mdx", "../src/**/*.stories.tsx"],
   addons: [
     "@storybook/addon-links",
     "@storybook/addon-docs",

pkgs/clan-app/ui/package-lock.json

@@ -17,6 +17,7 @@
         "@solidjs/router": "^0.15.3",
         "@tanstack/eslint-plugin-query": "^5.51.12",
         "@tanstack/solid-query": "^5.76.0",
+        "@tanstack/solid-query-devtools": "^5.83.0",
         "solid-js": "^1.9.7",
         "solid-toast": "^0.5.0",
         "three": "^0.176.0",
@@ -53,7 +54,6 @@
         "postcss": "^8.4.38",
         "postcss-url": "^10.1.3",
         "prettier": "^3.2.5",
-        "solid-devtools": "^0.34.0",
         "storybook": "^9.0.8",
         "swagger-ui-dist": "^5.26.2",
         "tailwindcss": "^3.4.3",
@@ -360,22 +360,6 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-syntax-typescript": {
-      "version": "7.27.1",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.27.1.tgz",
-      "integrity": "sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@babel/helper-plugin-utils": "^7.27.1"
-      },
-      "engines": {
-        "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
-      }
-    },
     "node_modules/@babel/runtime": {
       "version": "7.27.6",
       "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.27.6.tgz",
@@ -1552,13 +1536,6 @@
         "node": ">= 8"
       }
     },
-    "node_modules/@nothing-but/utils": {
-      "version": "0.17.0",
-      "resolved": "https://registry.npmjs.org/@nothing-but/utils/-/utils-0.17.0.tgz",
-      "integrity": "sha512-TuCHcHLOqDL0SnaAxACfuRHBNRgNJcNn9X0GiH5H3YSDBVquCr3qEIG3FOQAuMyZCbu9w8nk2CHhOsn7IvhIwQ==",
-      "dev": true,
-      "license": "MIT"
-    },
     "node_modules/@oxc-resolver/binding-darwin-arm64": {
       "version": "11.5.0",
       "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-darwin-arm64/-/binding-darwin-arm64-11.5.0.tgz",
@@ -1813,64 +1790,6 @@
         "@sinonjs/commons": "^3.0.1"
       }
     },
-    "node_modules/@solid-devtools/debugger": {
-      "version": "0.28.1",
-      "resolved": "https://registry.npmjs.org/@solid-devtools/debugger/-/debugger-0.28.1.tgz",
-      "integrity": "sha512-6qIUI6VYkXoRnL8oF5bvh2KgH71qlJ18hNw/mwSyY6v48eb80ZR48/5PDXufUa3q+MBSuYa1uqTMwLewpay9eg==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@nothing-but/utils": "~0.17.0",
-        "@solid-devtools/shared": "^0.20.0",
-        "@solid-primitives/bounds": "^0.1.1",
-        "@solid-primitives/event-listener": "^2.4.1",
-        "@solid-primitives/keyboard": "^1.3.1",
-        "@solid-primitives/rootless": "^1.5.1",
-        "@solid-primitives/scheduled": "^1.5.1",
-        "@solid-primitives/static-store": "^0.1.1",
-        "@solid-primitives/utils": "^6.3.1"
-      },
-      "peerDependencies": {
-        "solid-js": "^1.9.0"
-      }
-    },
-    "node_modules/@solid-devtools/shared": {
-      "version": "0.20.0",
-      "resolved": "https://registry.npmjs.org/@solid-devtools/shared/-/shared-0.20.0.tgz",
-      "integrity": "sha512-o5TACmUOQsxpzpOKCjbQqGk8wL8PMi+frXG9WNu4Lh3PQVUB6hs95Kl/S8xc++zwcMguUKZJn8h5URUiMOca6Q==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@nothing-but/utils": "~0.17.0",
-        "@solid-primitives/event-listener": "^2.4.1",
-        "@solid-primitives/media": "^2.3.1",
-        "@solid-primitives/refs": "^1.1.1",
-        "@solid-primitives/rootless": "^1.5.1",
-        "@solid-primitives/scheduled": "^1.5.1",
-        "@solid-primitives/static-store": "^0.1.1",
-        "@solid-primitives/styles": "^0.1.1",
-        "@solid-primitives/utils": "^6.3.1"
-      },
-      "peerDependencies": {
-        "solid-js": "^1.9.0"
-      }
-    },
-    "node_modules/@solid-primitives/bounds": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/@solid-primitives/bounds/-/bounds-0.1.3.tgz",
-      "integrity": "sha512-UbiyKMdSPmtijcEDnYLQL3zzaejpwWDAJJ4Gt5P0hgVs6A72piov0GyNw7V2SroH7NZFwxlYS22YmOr8A5xc1Q==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@solid-primitives/event-listener": "^2.4.3",
-        "@solid-primitives/resize-observer": "^2.1.3",
-        "@solid-primitives/static-store": "^0.1.2",
-        "@solid-primitives/utils": "^6.3.2"
-      },
-      "peerDependencies": {
-        "solid-js": "^1.6.12"
-      }
-    },
     "node_modules/@solid-primitives/event-listener": {
       "version": "2.4.3",
       "resolved": "https://registry.npmjs.org/@solid-primitives/event-listener/-/event-listener-2.4.3.tgz",
@@ -1883,21 +1802,6 @@
         "solid-js": "^1.6.12"
       }
     },
-    "node_modules/@solid-primitives/keyboard": {
-      "version": "1.3.3",
-      "resolved": "https://registry.npmjs.org/@solid-primitives/keyboard/-/keyboard-1.3.3.tgz",
-      "integrity": "sha512-9dQHTTgLBqyAI7aavtO+HnpTVJgWQA1ghBSrmLtMu1SMxLPDuLfuNr+Tk5udb4AL4Ojg7h9JrKOGEEDqsJXWJA==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@solid-primitives/event-listener": "^2.4.3",
-        "@solid-primitives/rootless": "^1.5.2",
-        "@solid-primitives/utils": "^6.3.2"
-      },
-      "peerDependencies": {
-        "solid-js": "^1.6.12"
-      }
-    },
     "node_modules/@solid-primitives/keyed": {
       "version": "1.5.2",
       "resolved": "https://registry.npmjs.org/@solid-primitives/keyed/-/keyed-1.5.2.tgz",
@@ -1985,16 +1889,6 @@
         "solid-js": "^1.6.12"
       }
     },
-    "node_modules/@solid-primitives/scheduled": {
-      "version": "1.5.2",
-      "resolved": "https://registry.npmjs.org/@solid-primitives/scheduled/-/scheduled-1.5.2.tgz",
-      "integrity": "sha512-/j2igE0xyNaHhj6kMfcUQn5rAVSTLbAX+CDEBm25hSNBmNiHLu2lM7Usj2kJJ5j36D67bE8wR1hBNA8hjtvsQA==",
-      "dev": true,
-      "license": "MIT",
-      "peerDependencies": {
-        "solid-js": "^1.6.12"
-      }
-    },
     "node_modules/@solid-primitives/static-store": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/@solid-primitives/static-store/-/static-store-0.1.2.tgz",
@@ -2028,20 +1922,6 @@
         }
       }
     },
-    "node_modules/@solid-primitives/styles": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/@solid-primitives/styles/-/styles-0.1.2.tgz",
-      "integrity": "sha512-7iX5K+J5b1PRrbgw3Ki92uvU2LgQ0Kd/QMsrAZxDg5dpUBwMyTijZkA3bbs1ikZsT1oQhS41bTyKbjrXeU0Awg==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@solid-primitives/rootless": "^1.5.2",
-        "@solid-primitives/utils": "^6.3.2"
-      },
-      "peerDependencies": {
-        "solid-js": "^1.6.12"
-      }
-    },
     "node_modules/@solid-primitives/trigger": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/@solid-primitives/trigger/-/trigger-1.2.2.tgz",
@@ -2281,9 +2161,19 @@
       }
     },
     "node_modules/@tanstack/query-core": {
-      "version": "5.81.5",
-      "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.81.5.tgz",
-      "integrity": "sha512-ZJOgCy/z2qpZXWaj/oxvodDx07XcQa9BF92c0oINjHkoqUPsmm3uG08HpTaviviZ/N9eP1f9CM7mKSEkIo7O1Q==",
+      "version": "5.83.0",
+      "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.83.0.tgz",
+      "integrity": "sha512-0M8dA+amXUkyz5cVUm/B+zSk3xkQAcuXuz5/Q/LveT4ots2rBpPTZOzd7yJa2Utsf8D2Upl5KyjhHRY+9lB/XA==",
+      "license": "MIT",
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/tannerlinsley"
+      }
+    },
+    "node_modules/@tanstack/query-devtools": {
+      "version": "5.81.2",
+      "resolved": "https://registry.npmjs.org/@tanstack/query-devtools/-/query-devtools-5.81.2.tgz",
+      "integrity": "sha512-jCeJcDCwKfoyyBXjXe9+Lo8aTkavygHHsUHAlxQKKaDeyT0qyQNLKl7+UyqYH2dDF6UN/14873IPBHchcsU+Zg==",
       "license": "MIT",
       "funding": {
         "type": "github",
@@ -2291,12 +2181,12 @@
       }
     },
     "node_modules/@tanstack/solid-query": {
-      "version": "5.81.5",
-      "resolved": "https://registry.npmjs.org/@tanstack/solid-query/-/solid-query-5.81.5.tgz",
-      "integrity": "sha512-VqVXaxiJIsKA6B45uApF+RUD3g8Roj/vdAuGpHMjR+RyHqlyQ+hOwgmALkzlbkbIaWCQi8CJOvrbU6WOBuMOxA==",
+      "version": "5.83.0",
+      "resolved": "https://registry.npmjs.org/@tanstack/solid-query/-/solid-query-5.83.0.tgz",
+      "integrity": "sha512-RF8Tv9+6+Kmzj+EafbTzvzzPq+J5SzHtc1Tz3D2MZ/EvlZTH+GL5q4HNnWK3emg7CB6WzyGnTuERmmWJaZs8/w==",
       "license": "MIT",
       "dependencies": {
-        "@tanstack/query-core": "5.81.5"
+        "@tanstack/query-core": "5.83.0"
       },
       "funding": {
         "type": "github",
@@ -2306,6 +2196,23 @@
         "solid-js": "^1.6.0"
       }
     },
+    "node_modules/@tanstack/solid-query-devtools": {
+      "version": "5.83.0",
+      "resolved": "https://registry.npmjs.org/@tanstack/solid-query-devtools/-/solid-query-devtools-5.83.0.tgz",
+      "integrity": "sha512-Z0wQlAWXz/U2bJ/paMRBTDhMoPnB9Te6GmA21sXnI+nDnAAPZRcPxFBiCgYJS3eFsvbkdRGJwoUSQrdIgy0shg==",
+      "license": "MIT",
+      "dependencies": {
+        "@tanstack/query-devtools": "5.81.2"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/tannerlinsley"
+      },
+      "peerDependencies": {
+        "@tanstack/solid-query": "^5.83.0",
+        "solid-js": "^1.6.0"
+      }
+    },
     "node_modules/@testing-library/dom": {
       "version": "10.4.0",
       "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.0.tgz",
@@ -6996,29 +6903,6 @@
         "url": "https://github.com/sponsors/cyyynthia"
       }
     },
-    "node_modules/solid-devtools": {
-      "version": "0.34.3",
-      "resolved": "https://registry.npmjs.org/solid-devtools/-/solid-devtools-0.34.3.tgz",
-      "integrity": "sha512-ZQua959n+Zu3sLbm9g0IRjYUb1YYlYbu83PWLRoKbSsq0a3ItQNhnS2OBU7rQNmOKZiMexNo9Z3izas9BcOKDg==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@babel/core": "^7.27.4",
-        "@babel/plugin-syntax-typescript": "^7.27.1",
-        "@babel/types": "^7.27.6",
-        "@solid-devtools/debugger": "^0.28.1",
-        "@solid-devtools/shared": "^0.20.0"
-      },
-      "peerDependencies": {
-        "solid-js": "^1.9.0",
-        "vite": "^2.2.3 || ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0"
-      },
-      "peerDependenciesMeta": {
-        "vite": {
-          "optional": true
-        }
-      }
-    },
     "node_modules/solid-js": {
       "version": "1.9.7",
       "resolved": "https://registry.npmjs.org/solid-js/-/solid-js-1.9.7.tgz",

pkgs/clan-app/ui/package.json

@@ -52,7 +52,6 @@
     "postcss": "^8.4.38",
     "postcss-url": "^10.1.3",
     "prettier": "^3.2.5",
-    "solid-devtools": "^0.34.0",
     "storybook": "^9.0.8",
     "swagger-ui-dist": "^5.26.2",
     "tailwindcss": "^3.4.3",
@@ -73,6 +72,7 @@
     "@solidjs/router": "^0.15.3",
     "@tanstack/eslint-plugin-query": "^5.51.12",
     "@tanstack/solid-query": "^5.76.0",
+    "@tanstack/solid-query-devtools": "^5.83.0",
     "solid-js": "^1.9.7",
     "solid-toast": "^0.5.0",
     "three": "^0.176.0",

pkgs/clan-app/ui/src/components/Button/Button.css

@@ -123,20 +123,12 @@
     @apply pr-3.5;
   }
 
-  & > div.loader {
-    @apply w-0 opacity-0;
-    @apply top-0 left-0 -mr-2;
-    transition: all 0.5s ease;
-  }
-
   &.loading {
     @apply cursor-wait;
+  }
 
-    & > div.loader {
-      @apply w-4 opacity-100;
-      margin-right: revert;
-      transition: all 0.5s ease;
-    }
+  & > span.typography {
+    @apply max-w-full overflow-hidden whitespace-nowrap text-ellipsis;
   }
 }
 

pkgs/clan-app/ui/src/components/Button/Button.tsx

@@ -67,6 +67,11 @@ export const Button = (props: ButtonProps) => {
 
   const iconSize = iconSizes[local.size || "default"];
 
+  const loadingClass =
+    "w-4 opacity-100 mr-[revert] transition-all duration-500 ease-linear";
+  const idleClass =
+    "hidden w-0 opacity-0 top-0 left-0 -mr-2 transition-all duration-500 ease-linear";
+
   return (
     <KobalteButton
       class={cx(
@@ -83,7 +88,10 @@ export const Button = (props: ButtonProps) => {
       onClick={local.onAction ? onClick : undefined}
       {...other}
     >
-      <Loader hierarchy={hierarchy} />
+      <Loader
+        hierarchy={hierarchy}
+        class={cx({ [idleClass]: !loading(), [loadingClass]: loading() })}
+      />
 
       {local.startIcon && (
         <Icon icon={local.startIcon} class="icon-start" size={iconSize} />

pkgs/clan-app/ui/src/components/Form/Checkbox.tsx

@@ -1,8 +1,10 @@
 import {
-  Checkbox as KCheckbox,
   CheckboxInputProps as KCheckboxInputProps,
   CheckboxRootProps as KCheckboxRootProps,
 } from "@kobalte/core/checkbox";
+
+import { Checkbox as KCheckbox } from "@kobalte/core";
+
 import Icon from "@/src/components/Icon/Icon";
 
 import cx from "classnames";
@@ -11,7 +13,7 @@ import { PolymorphicProps } from "@kobalte/core/polymorphic";
 import "./Checkbox.css";
 import { FieldProps } from "./Field";
 import { Orienter } from "./Orienter";
-import { Show } from "solid-js";
+import { Match, splitProps, Switch } from "solid-js";
 
 export type CheckboxProps = FieldProps &
   KCheckboxRootProps & {
@@ -19,6 +21,9 @@ export type CheckboxProps = FieldProps &
   };
 
 export const Checkbox = (props: CheckboxProps) => {
+  // we need to separate output the input otherwise it interferes with prop binding
+  const [_, rootProps] = splitProps(props, ["input"]);
+
   const alignment = () =>
     (props.orientation || "vertical") == "vertical" ? "start" : "center";
 
@@ -41,34 +46,36 @@ export const Checkbox = (props: CheckboxProps) => {
   );
 
   return (
-    <KCheckbox
+    <KCheckbox.Root
       class={cx("form-field", "checkbox", props.size, props.orientation, {
         inverted: props.inverted,
         ghost: props.ghost,
       })}
-      {...props}
+      {...rootProps}
     >
-      <Orienter orientation={props.orientation} align={alignment()}>
-        <Label
-          labelComponent={KCheckbox.Label}
-          descriptionComponent={KCheckbox.Description}
-          {...props}
-        />
-        <KCheckbox.Input {...props.input} />
-        <KCheckbox.Control class="checkbox-control">
-          {props.readOnly && (
-            <Show
-              when={props.checked || props.defaultChecked}
-              fallback={iconUnchecked}
-            >
-              {iconChecked}
-            </Show>
-          )}
-          {!props.readOnly && (
-            <KCheckbox.Indicator>{iconChecked}</KCheckbox.Indicator>
-          )}
-        </KCheckbox.Control>
-      </Orienter>
-    </KCheckbox>
+      {(state) => (
+        <Orienter orientation={props.orientation} align={alignment()}>
+          <Label
+            labelComponent={KCheckbox.Label}
+            descriptionComponent={KCheckbox.Description}
+            {...props}
+          />
+          <KCheckbox.Input {...props.input} />
+          <KCheckbox.Control class="checkbox-control">
+            <Switch>
+              <Match when={!props.readOnly}>
+                <KCheckbox.Indicator>{iconChecked}</KCheckbox.Indicator>
+              </Match>
+              <Match when={props.readOnly && state.checked()}>
+                {iconChecked}
+              </Match>
+              <Match when={props.readOnly && !state.checked()}>
+                {iconUnchecked}
+              </Match>
+            </Switch>
+          </KCheckbox.Control>
+        </Orienter>
+      )}
+    </KCheckbox.Root>
   );
 };

pkgs/clan-app/ui/src/components/Form/Combobox.tsx

@@ -12,12 +12,20 @@ import cx from "classnames";
 import { FieldProps } from "./Field";
 import { Orienter } from "./Orienter";
 import { Typography } from "@/src/components/Typography/Typography";
-import { Accessor, Component, For, Show, splitProps } from "solid-js";
+import {
+  Accessor,
+  Component,
+  ComponentProps,
+  For,
+  Show,
+  splitProps,
+} from "solid-js";
 import { Tag } from "@/src/components/Tag/Tag";
 
 export type ComboboxProps<Option, OptGroup = never> = FieldProps &
   KComboboxRootOptions<Option, OptGroup> & {
     inverted: boolean;
+    input?: ComponentProps<"select">;
     itemControl?: Component<ComboboxControlState<Option>>;
   };
 
@@ -129,6 +137,7 @@ export const Combobox = <Option, OptGroup = never>(
           {...props}
         />
 
+        <KCombobox.HiddenSelect {...props.input} />
         <KCombobox.Control<Option> class="control">
           {(state) => {
             const [controlProps] = splitProps(props, [

pkgs/clan-app/ui/src/components/Form/Fieldset.stories.tsx

@@ -9,6 +9,7 @@ import { TextInput } from "@/src/components/Form/TextInput";
 import { TextArea } from "@/src/components/Form/TextArea";
 import { Checkbox } from "@/src/components/Form/Checkbox";
 import { FieldProps } from "./Field";
+import { HostFileInput } from "@/src/components/Form/HostFileInput";
 
 const FieldsetExamples = (props: FieldsetProps) => (
   <div class="flex flex-col gap-8">
@@ -26,7 +27,7 @@ const meta = {
         <div
           class={cx({
             "w-[600px]": (context.args.orientation || "vertical") == "vertical",
-            "w-[1024px]": context.args.orientation == "horizontal",
+            "w-[512px]": context.args.orientation == "horizontal",
             "bg-inv-acc-3": context.args.inverted,
           })}
         >
@@ -63,6 +64,11 @@ export const Default: Story = {
           label="Bio"
           input={{ placeholder: "Tell us a bit about yourself", rows: 8 }}
         />
+        <HostFileInput
+          {...props}
+          label="Profile pic"
+          onSelectFile={async () => "/home/foo/bar/baz/fizz/buzz/bla/bizz"}
+        />
         <Checkbox {...props} label="Accept Terms" required={true} />
       </>
     ),

pkgs/clan-app/ui/src/components/Form/HostFileInput.css

@@ -1,5 +0,0 @@
-div.form-field.host-file {
-  button {
-    @apply w-1/2;
-  }
-}

pkgs/clan-app/ui/src/components/Form/HostFileInput.module.css

@@ -0,0 +1,7 @@
+.vertical_button {
+  @apply w-fit;
+}
+
+.horizontal_button {
+  @apply grow max-w-[18rem];
+}

pkgs/clan-app/ui/src/components/Form/HostFileInput.stories.tsx

@@ -58,7 +58,7 @@ export type Story = StoryObj<typeof meta>;
 export const Bare: Story = {
   args: {
     onSelectFile: async () => {
-      return "/home/bob/clans/my-clan";
+      return "/home/github/clans/my-clan/foo/bar/baz/fizz/buzz";
     },
     input: {
       placeholder: "e.g. 11/06/89",

pkgs/clan-app/ui/src/components/Form/HostFileInput.tsx

@@ -7,11 +7,13 @@ import {
 import cx from "classnames";
 import { Label } from "./Label";
 import { Button } from "../Button/Button";
-import "./HostFileInput.css";
+import styles from "./HostFileInput.module.css";
 import { PolymorphicProps } from "@kobalte/core/polymorphic";
 import { FieldProps } from "./Field";
 import { Orienter } from "./Orienter";
 import { createSignal } from "solid-js";
+import { Tooltip } from "@kobalte/core/tooltip";
+import { Typography } from "@/src/components/Typography/Typography";
 
 export type HostFileInputProps = FieldProps &
   TextFieldRootProps & {
@@ -20,39 +22,94 @@ export type HostFileInputProps = FieldProps &
   };
 
 export const HostFileInput = (props: HostFileInputProps) => {
-  const [value, setValue] = createSignal<string | undefined>(undefined);
+  const [value, setValue] = createSignal<string>(props.value || "");
+
+  let actualInputElement: HTMLInputElement | undefined;
 
   const selectFile = async () => {
-    setValue(await props.onSelectFile());
+    try {
+      console.log("selecting file", props.onSelectFile);
+      setValue(await props.onSelectFile());
+      actualInputElement?.dispatchEvent(
+        new Event("input", { bubbles: true, cancelable: true }),
+      );
+    } catch (error) {
+      console.log("Error selecting file", error);
+      // todo work out how to display the error
+    }
   };
 
   return (
     <TextField
-      class={cx("form-field", "host-file", props.size, props.orientation, {
+      class={cx("form-field", props.size, props.orientation, {
         inverted: props.inverted,
         ghost: props.ghost,
       })}
       {...props}
-      value={value()}
-      onChange={setValue}
     >
-      <Orienter orientation={props.orientation} align={"start"}>
+      <Orienter
+        orientation={props.orientation}
+        align={props.orientation == "horizontal" ? "center" : "start"}
+      >
         <Label
           labelComponent={TextField.Label}
           descriptionComponent={TextField.Description}
           {...props}
         />
 
-        <TextField.Input {...props.input} hidden={true} />
+        <TextField.Input
+          {...props.input}
+          hidden={true}
+          value={value()}
+          ref={(el: HTMLInputElement) => {
+            actualInputElement = el; // Capture for local use
+          }}
+        />
 
-        <Button
-          hierarchy="secondary"
-          size={props.size}
-          startIcon="Folder"
-          onClick={selectFile}
-        >
-          {value() ? value() : "No Selection"}
-        </Button>
+        {!value() && (
+          <Button
+            hierarchy="secondary"
+            size={props.size}
+            startIcon="Folder"
+            onClick={selectFile}
+            disabled={props.disabled || props.readOnly}
+            class={cx(
+              props.orientation === "vertical"
+                ? styles.vertical_button
+                : styles.horizontal_button,
+            )}
+          >
+            No Selection
+          </Button>
+        )}
+
+        {value() && (
+          <Tooltip placement="top">
+            <Tooltip.Portal>
+              <Tooltip.Content class="tooltip-content">
+                <Typography
+                  hierarchy="body"
+                  size="xs"
+                  weight="medium"
+                  inverted={!props.inverted}
+                >
+                  {value()}
+                </Typography>
+                <Tooltip.Arrow />
+              </Tooltip.Content>
+            </Tooltip.Portal>
+            <Tooltip.Trigger
+              as={Button}
+              hierarchy="secondary"
+              size={props.size}
+              startIcon="Folder"
+              onClick={selectFile}
+              disabled={props.disabled || props.readOnly}
+            >
+              {value()}
+            </Tooltip.Trigger>
+          </Tooltip>
+        )}
       </Orienter>
     </TextField>
   );

pkgs/clan-app/ui/src/components/Form/Label.css

@@ -22,40 +22,3 @@ div.form-label {
     }
   }
 }
-
-div.tooltip-content {
-  @apply z-50 px-2 py-0.5 bg-inv-4 rounded-[0.125rem] leading-none;
-
-  max-width: min(calc(100vw - 16px), 380px);
-  transform-origin: var(--kb-tooltip-content-transform-origin);
-  animation: tooltipHide 250ms ease-in forwards;
-
-  &[data-expanded] {
-    animation: tooltipShow 250ms ease-out;
-  }
-
-  &.inverted {
-    @apply bg-def-2;
-  }
-}
-
-@keyframes tooltipShow {
-  from {
-    opacity: 0;
-    transform: scale(0.96);
-  }
-  to {
-    opacity: 1;
-    transform: scale(1);
-  }
-}
-@keyframes tooltipHide {
-  from {
-    opacity: 1;
-    transform: scale(1);
-  }
-  to {
-    opacity: 0;
-    transform: scale(0.96);
-  }
-}

pkgs/clan-app/ui/src/components/Form/Label.tsx

@@ -1,12 +1,11 @@
 import { Show } from "solid-js";
 import { Typography } from "@/src/components/Typography/Typography";
-import { Tooltip as KTooltip } from "@kobalte/core/tooltip";
+import { Tooltip } from "@/src/components/Tooltip/Tooltip";
 import Icon from "@/src/components/Icon/Icon";
 import { TextField } from "@kobalte/core/text-field";
 import { Checkbox } from "@kobalte/core/checkbox";
 import { Combobox } from "@kobalte/core/combobox";
 import "./Label.css";
-import cx from "classnames";
 
 export type Size = "default" | "s";
 
@@ -49,31 +48,27 @@ export const Label = (props: LabelProps) => {
             {props.label}
           </Typography>
           {props.tooltip && (
-            <KTooltip placement="top">
-              <KTooltip.Trigger>
+            <Tooltip
+              placement="top"
+              inverted={props.inverted}
+              trigger={
                 <Icon
                   icon="Info"
                   color="tertiary"
                   inverted={props.inverted}
                   size={props.size == "default" ? "0.85em" : "0.75rem"}
                 />
-                <KTooltip.Portal>
-                  <KTooltip.Content
-                    class={cx("tooltip-content", { inverted: props.inverted })}
-                  >
-                    <Typography
-                      hierarchy="body"
-                      size="xs"
-                      weight="medium"
-                      inverted={!props.inverted}
-                    >
-                      {props.tooltip}
-                    </Typography>
-                    <KTooltip.Arrow />
-                  </KTooltip.Content>
-                </KTooltip.Portal>
-              </KTooltip.Trigger>
-            </KTooltip>
+              }
+            >
+              <Typography
+                hierarchy="body"
+                size="xs"
+                weight="medium"
+                inverted={!props.inverted}
+              >
+                {props.tooltip}
+              </Typography>
+            </Tooltip>
           )}
         </props.labelComponent>
         {props.description && (