Merge pull request 'feat(ui): SidebarPane component' (#4248) from ui/sidebar-pane into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4248

.gitea/workflows/create-pr.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Shared script for creating pull requests in Gitea workflows
+set -euo pipefail
+
+# Required environment variables:
+# - CI_BOT_TOKEN: Gitea bot token for authentication
+# - PR_BRANCH: Branch name for the pull request
+# - PR_TITLE: Title of the pull request
+# - PR_BODY: Body/description of the pull request
+
+if [[ -z "${CI_BOT_TOKEN:-}" ]]; then
+  echo "Error: CI_BOT_TOKEN is not set" >&2
+  exit 1
+fi
+
+if [[ -z "${PR_BRANCH:-}" ]]; then
+  echo "Error: PR_BRANCH is not set" >&2
+  exit 1
+fi
+
+if [[ -z "${PR_TITLE:-}" ]]; then
+  echo "Error: PR_TITLE is not set" >&2
+  exit 1
+fi
+
+if [[ -z "${PR_BODY:-}" ]]; then
+  echo "Error: PR_BODY is not set" >&2
+  exit 1
+fi
+
+# Push the branch
+git push origin "+HEAD:${PR_BRANCH}"
+
+# Create pull request
+resp=$(nix run --inputs-from . nixpkgs#curl -- -X POST \
+  -H "Authorization: token $CI_BOT_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d "{
+    \"head\": \"${PR_BRANCH}\",
+    \"base\": \"main\",
+    \"title\": \"${PR_TITLE}\",
+    \"body\": \"${PR_BODY}\"
+  }" \
+  "https://git.clan.lol/api/v1/repos/clan/clan-core/pulls")
+
+pr_number=$(echo "$resp" | jq -r '.number')
+
+if [[ "$pr_number" == "null" ]]; then
+  echo "Error creating pull request:" >&2
+  echo "$resp" | jq . >&2
+  exit 1
+fi
+
+echo "Created pull request #$pr_number"
+
+# Merge when checks succeed
+while true; do
+  resp=$(nix run --inputs-from . nixpkgs#curl -- -X POST \
+    -H "Authorization: token $CI_BOT_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{
+      "Do": "merge",
+      "merge_when_checks_succeed": true,
+      "delete_branch_after_merge": true
+    }' \
+    "https://git.clan.lol/api/v1/repos/clan/clan-core/pulls/$pr_number/merge")
+  msg=$(echo "$resp" | jq -r '.message')
+  if [[ "$msg" != "Please try again later" ]]; then
+    break
+  fi
+  echo "Retrying in 2 seconds..."
+  sleep 2
+done
+
+echo "Pull request #$pr_number merge initiated"

.gitea/workflows/update-clan-core-for-checks.yml

@@ -19,35 +19,10 @@ jobs:
         run: |
           export GIT_AUTHOR_NAME=clan-bot GIT_AUTHOR_EMAIL=clan-bot@clan.lol GIT_COMMITTER_NAME=clan-bot GIT_COMMITTER_EMAIL=clan-bot@clan.lol
           git commit -am "Update pinned clan-core for checks"
-          git push origin +HEAD:update-clan-core-for-checks
-          set -x
-          resp=$(nix run --inputs-from . nixpkgs#curl -- -X POST \
-            -H "Authorization: token $CI_BOT_TOKEN" \
-            -H "Content-Type: application/json" \
-            -d '{
-              "head": "update-clan-core-for-checks",
-              "base": "main",
-              "title": "Update Clan Core for Checks",
-              "body": "This PR updates the pinned clan-core flake input that is used for checks."
-            }' \
-           "https://git.clan.lol/api/v1/repos/clan/clan-core/pulls")
-          pr_number=$(echo "$resp" | jq -r '.number')
 
-          # Merge when succeed
-          while true; do
-            resp=$(nix run --inputs-from . nixpkgs#curl -- -X POST \
-              -H "Authorization: token $CI_BOT_TOKEN" \
-              -H "Content-Type: application/json" \
-              -d '{
-                "Do": "merge",
-                "merge_when_checks_succeed": true,
-                "delete_branch_after_merge": true
-              }' \
-              "https://git.clan.lol/api/v1/repos/clan/clan-core/pulls/$pr_number/merge")
-            msg=$(echo $resp | jq -r '.message')
-            if [[ "$msg" != "Please try again later" ]]; then
-              break
-            fi
-            echo "Retrying in 2 seconds..."
-            sleep 2
-          done
+          # Use shared PR creation script
+          export PR_BRANCH="update-clan-core-for-checks"
+          export PR_TITLE="Update Clan Core for Checks"
+          export PR_BODY="This PR updates the pinned clan-core flake input that is used for checks."
+
+          ./.gitea/workflows/create-pr.sh

.gitea/workflows/update-private-flake-inputs.yml

@@ -0,0 +1,40 @@
+name: "Update private flake inputs"
+on:
+  repository_dispatch:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 3 * * *" # Run daily at 3 AM
+jobs:
+  update-private-flake:
+    runs-on: nix
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Update private flake inputs
+        run: |
+          # Update the private flake lock file
+          cd devFlake/private
+          nix flake update
+          cd ../..
+
+          # Update the narHash
+          bash ./devFlake/update-private-narhash
+      - name: Create pull request
+        env:
+          CI_BOT_TOKEN: ${{ secrets.CI_BOT_TOKEN }}
+        run: |
+          export GIT_AUTHOR_NAME=clan-bot GIT_AUTHOR_EMAIL=clan-bot@clan.lol GIT_COMMITTER_NAME=clan-bot GIT_COMMITTER_EMAIL=clan-bot@clan.lol
+
+          # Check if there are any changes
+          if ! git diff --quiet; then
+            git add devFlake/private/flake.lock devFlake/private.narHash
+            git commit -m "Update dev flake"
+            
+            # Use shared PR creation script
+            export PR_BRANCH="update-dev-flake"
+            export PR_TITLE="Update dev flake"
+            export PR_BODY="This PR updates the dev flake inputs and corresponding narHash."
+          else
+            echo "No changes detected in dev flake inputs"
+          fi

checks/backups/flake-module.nix

@@ -19,11 +19,11 @@
         ...
       }:
       let
-        dependencies = [
-          self
-          pkgs.stdenv.drvPath
-          self.clan.clanInternals.machines.${pkgs.hostPlatform.system}.test-backup.config.system.clan.deployment.file
-        ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
+        dependencies =
+          [
+            pkgs.stdenv.drvPath
+          ]
+          ++ builtins.map (i: i.outPath) (builtins.attrValues (builtins.removeAttrs self.inputs [ "self" ]));
         closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
       in
       {

checks/borgbackup/default.nix

@@ -47,14 +47,6 @@ nixosLib.runTest (
 
       clientone =
         { config, pkgs, ... }:
-        let
-          dependencies = [
-            clan-core
-            pkgs.stdenv.drvPath
-          ] ++ builtins.map (i: i.outPath) (builtins.attrValues clan-core.inputs);
-          closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
-
-        in
         {
 
           services.openssh.enable = true;
@@ -65,15 +57,6 @@ nixosLib.runTest (
 
           environment.systemPackages = [ clan-core.packages.${pkgs.system}.clan-cli ];
 
-          environment.etc.install-closure.source = "${closureInfo}/store-paths";
-          nix.settings = {
-            substituters = pkgs.lib.mkForce [ ];
-            hashed-mirrors = null;
-            connect-timeout = pkgs.lib.mkForce 3;
-            flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
-          };
-          system.extraDependencies = dependencies;
-
           clan.core.state.test-backups.folders = [ "/var/test-backups" ];
         };
 

checks/clan-core-for-checks.nix

@@ -1,6 +1,6 @@
 { fetchgit }:
 fetchgit {
   url = "https://git.clan.lol/clan/clan-core.git";
-  rev = "28131afbbcd379a8ff04c79c66c670ef655ed889";
-  sha256 = "1294cwjlnc341fl6zbggn4rgq8z33gqkcyggjfvk9cf7zdgygrf6";
+  rev = "eea93ea22c9818da67e148ba586277bab9e73cea";
+  sha256 = "sha256-PV0Z+97QuxQbkYSVuNIJwUNXMbHZG/vhsA9M4cDTCOE=";
 }

checks/flake-module.nix

@@ -22,6 +22,7 @@ in
   imports = filter pathExists [
     ./backups/flake-module.nix
     ../nixosModules/clanCore/machine-id/tests/flake-module.nix
+    ../nixosModules/clanCore/state-version/tests/flake-module.nix
     ./devshell/flake-module.nix
     ./flash/flake-module.nix
     ./impure/flake-module.nix

checks/flash/flake-module.nix

@@ -50,8 +50,6 @@
         self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.build.toplevel
         self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.build.diskoScript
         self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.build.diskoScript.drvPath
-        self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.clan.deployment.file
-
       ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
       closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
     in

checks/installation/flake-module.nix

@@ -1,63 +1,9 @@
 {
   self,
   lib,
+
   ...
 }:
-let
-  installer =
-    { modulesPath, pkgs, ... }:
-    let
-      dependencies = [
-        self.clan.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.toplevel
-        self.clan.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.diskoScript
-        self.clan.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.clan.deployment.file
-        pkgs.stdenv.drvPath
-        pkgs.bash.drvPath
-        pkgs.nixos-anywhere
-        pkgs.bubblewrap
-        pkgs.buildPackages.xorg.lndir
-      ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
-      closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
-    in
-    {
-      imports = [
-        (modulesPath + "/../tests/common/auto-format-root-device.nix")
-      ];
-      networking.useNetworkd = true;
-      services.openssh.enable = true;
-      services.openssh.settings.UseDns = false;
-      services.openssh.settings.PasswordAuthentication = false;
-      system.nixos.variant_id = "installer";
-      environment.systemPackages = [
-        self.packages.${pkgs.system}.clan-cli-full
-        pkgs.nixos-facter
-      ];
-      environment.etc."install-closure".source = "${closureInfo}/store-paths";
-      virtualisation.emptyDiskImages = [ 512 ];
-      virtualisation.diskSize = 8 * 1024;
-      virtualisation.rootDevice = "/dev/vdb";
-      # both installer and target need to use the same diskImage
-      virtualisation.diskImage = "./target.qcow2";
-      virtualisation.memorySize = 3048;
-      nix.settings = {
-        substituters = lib.mkForce [ ];
-        hashed-mirrors = null;
-        connect-timeout = lib.mkForce 3;
-        flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
-        experimental-features = [
-          "nix-command"
-          "flakes"
-        ];
-      };
-      users.users.nonrootuser = {
-        isNormalUser = true;
-        openssh.authorizedKeys.keyFiles = [ ../assets/ssh/pubkey ];
-        extraGroups = [ "wheel" ];
-      };
-      security.sudo.wheelNeedsPassword = false;
-      system.extraDependencies = dependencies;
-    };
-in
 {
 
   # The purpose of this test is to ensure `clan machines install` works
@@ -106,6 +52,25 @@ in
 
         environment.etc."install-successful".text = "ok";
 
+        # Enable SSH and add authorized key for testing
+        services.openssh.enable = true;
+        services.openssh.settings.PasswordAuthentication = false;
+        users.users.nonrootuser = {
+          isNormalUser = true;
+          openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ];
+          extraGroups = [ "wheel" ];
+          home = "/home/nonrootuser";
+          createHome = true;
+        };
+        users.users.root.openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ];
+        # Allow users to manage their own SSH keys
+        services.openssh.authorizedKeysFiles = [
+          "/root/.ssh/authorized_keys"
+          "/home/%u/.ssh/authorized_keys"
+          "/etc/ssh/authorized_keys.d/%u"
+        ];
+        security.sudo.wheelNeedsPassword = false;
+
         boot.consoleLogLevel = lib.mkForce 100;
         boot.kernelParams = [ "boot.shell_on_fail" ];
 
@@ -182,55 +147,199 @@ in
       # vm-test-run-test-installation-> target: waiting for the VM to finish booting
       # vm-test-run-test-installation-> target: Guest root shell did not produce any data yet...
       # vm-test-run-test-installation-> target:   To debug, enter the VM and run 'systemctl status backdoor.service'.
-      checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux && !pkgs.stdenv.isAarch64) {
-        nixos-test-installation = self.clanLib.test.baseTest {
-          name = "installation";
-          nodes.target = {
-            services.openssh.enable = true;
-            virtualisation.diskImage = "./target.qcow2";
-            virtualisation.useBootLoader = true;
+      checks =
+        let
+          # Custom Python package for port management utilities
+          closureInfo = pkgs.closureInfo {
+            rootPaths = [
+              self.checks.x86_64-linux.clan-core-for-checks
+              self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.toplevel
+              self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.initialRamdisk
+              self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.diskoScript
+              pkgs.stdenv.drvPath
+              pkgs.bash.drvPath
+              pkgs.buildPackages.xorg.lndir
+            ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
           };
-          nodes.installer = installer;
+        in
+        pkgs.lib.mkIf (pkgs.stdenv.isLinux && !pkgs.stdenv.isAarch64) {
+          nixos-test-installation = self.clanLib.test.baseTest {
+            name = "installation";
+            nodes.target = (import ./test-helpers.nix { inherit lib pkgs self; }).target;
+            extraPythonPackages = _p: [
+              self.legacyPackages.${pkgs.system}.nixosTestLib
+            ];
 
-          testScript = ''
-            installer.start()
+            testScript = ''
+              import tempfile
+              import os
+              import subprocess
+              from nixos_test_lib.ssh import setup_ssh_connection # type: ignore[import-untyped]
+              from nixos_test_lib.nix_setup import prepare_test_flake # type: ignore[import-untyped]
 
-            installer.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../assets/ssh/privkey} /root/.ssh/id_ed25519")
+              def create_test_machine(oldmachine, qemu_test_bin: str, **kwargs):
+                  """Create a new test machine from an installed disk image"""
+                  start_command = [
+                      f"{qemu_test_bin}/bin/qemu-kvm",
+                      "-cpu",
+                      "max",
+                      "-m",
+                      "3048",
+                      "-virtfs",
+                      "local,path=/nix/store,security_model=none,mount_tag=nix-store",
+                      "-drive",
+                      f"file={oldmachine.state_dir}/target.qcow2,id=drive1,if=none,index=1,werror=report",
+                      "-device",
+                      "virtio-blk-pci,drive=drive1",
+                      "-netdev",
+                      "user,id=net0",
+                      "-device",
+                      "virtio-net-pci,netdev=net0",
+                  ]
+                  machine = create_machine(start_command=" ".join(start_command), **kwargs)
+                  driver.machines.append(machine)
+                  return machine
 
-            installer.wait_until_succeeds("timeout 2 ssh -o StrictHostKeyChecking=accept-new -v nonrootuser@localhost hostname")
-            installer.succeed("cp -r ${self.checks.x86_64-linux.clan-core-for-checks} test-flake && chmod -R +w test-flake")
+              target.start()
 
-            installer.succeed("clan machines install --no-reboot --debug --flake test-flake --yes test-install-machine-without-system --target-host nonrootuser@localhost --update-hardware-config nixos-facter >&2")
-            installer.shutdown()
+              # Set up test environment
+              with tempfile.TemporaryDirectory() as temp_dir:
+                  # Prepare test flake and Nix store
+                  flake_dir = prepare_test_flake(
+                      temp_dir,
+                      "${self.checks.x86_64-linux.clan-core-for-checks}",
+                      "${closureInfo}"
+                  )
 
-            # We are missing the test instrumentation somehow. Test this later.
-            target.state_dir = installer.state_dir
-            target.start()
-            target.wait_for_unit("multi-user.target")
-          '';
-        } { inherit pkgs self; };
+                  # Set up SSH connection
+                  ssh_conn = setup_ssh_connection(
+                      target,
+                      temp_dir,
+                      "${../assets/ssh/privkey}"
+                  )
 
-        nixos-test-update-hardware-configuration = self.clanLib.test.baseTest {
-          name = "update-hardware-configuration";
-          nodes.installer = installer;
+                  # Run clan install from host using port forwarding
+                  clan_cmd = [
+                      "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan",
+                      "machines",
+                      "install",
+                      "--phases", "disko,install",
+                      "--debug",
+                      "--flake", flake_dir,
+                      "--yes", "test-install-machine-without-system",
+                      "--target-host", f"nonrootuser@localhost:{ssh_conn.host_port}",
+                      "-i", ssh_conn.ssh_key,
+                      "--option", "store", os.environ['CLAN_TEST_STORE'],
+                      "--update-hardware-config", "nixos-facter",
+                  ]
 
-          testScript = ''
-            installer.start()
-            installer.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../assets/ssh/privkey} /root/.ssh/id_ed25519")
-            installer.wait_until_succeeds("timeout 2 ssh -o StrictHostKeyChecking=accept-new -v nonrootuser@localhost hostname")
-            installer.succeed("cp -r ${self.checks.x86_64-linux.clan-core-for-checks} test-flake && chmod -R +w test-flake")
-            installer.fail("test -f test-flake/machines/test-install-machine/hardware-configuration.nix")
-            installer.fail("test -f test-flake/machines/test-install-machine/facter.json")
+                  subprocess.run(clan_cmd, check=True)
 
-            installer.succeed("clan machines update-hardware-config --debug --flake test-flake test-install-machine-without-system nonrootuser@localhost >&2")
-            installer.succeed("test -f test-flake/machines/test-install-machine-without-system/facter.json")
-            installer.succeed("rm test-flake/machines/test-install-machine-without-system/facter.json")
+              # Shutdown the installer machine gracefully
+              try:
+                  target.shutdown()
+              except BrokenPipeError:
+                  # qemu has already exited
+                  pass
 
-            installer.succeed("clan machines update-hardware-config --debug --backend nixos-generate-config --flake test-flake test-install-machine-without-system nonrootuser@localhost >&2")
-            installer.succeed("test -f test-flake/machines/test-install-machine-without-system/hardware-configuration.nix")
-            installer.succeed("rm test-flake/machines/test-install-machine-without-system/hardware-configuration.nix")
-          '';
-        } { inherit pkgs self; };
-      };
+              # Create a new machine instance that boots from the installed system
+              installed_machine = create_test_machine(target, "${pkgs.qemu_test}", name="after_install")
+              installed_machine.start()
+              installed_machine.wait_for_unit("multi-user.target")
+              installed_machine.succeed("test -f /etc/install-successful")
+            '';
+          } { inherit pkgs self; };
+
+          nixos-test-update-hardware-configuration = self.clanLib.test.baseTest {
+            name = "update-hardware-configuration";
+            nodes.target = (import ./test-helpers.nix { inherit lib pkgs self; }).target;
+            extraPythonPackages = _p: [
+              self.legacyPackages.${pkgs.system}.nixosTestLib
+            ];
+
+            testScript = ''
+              import tempfile
+              import os
+              import subprocess
+              from nixos_test_lib.ssh import setup_ssh_connection # type: ignore[import-untyped]
+              from nixos_test_lib.nix_setup import prepare_test_flake # type: ignore[import-untyped]
+
+              target.start()
+
+              # Set up test environment
+              with tempfile.TemporaryDirectory() as temp_dir:
+                  # Prepare test flake and Nix store
+                  flake_dir = prepare_test_flake(
+                      temp_dir,
+                      "${self.checks.x86_64-linux.clan-core-for-checks}",
+                      "${closureInfo}"
+                  )
+                  
+                  # Set up SSH connection
+                  ssh_conn = setup_ssh_connection(
+                      target,
+                      temp_dir,
+                      "${../assets/ssh/privkey}"
+                  )
+
+                  # Verify files don't exist initially
+                  hw_config_file = os.path.join(flake_dir, "machines/test-install-machine/hardware-configuration.nix")
+                  facter_file = os.path.join(flake_dir, "machines/test-install-machine/facter.json")
+
+                  assert not os.path.exists(hw_config_file), "hardware-configuration.nix should not exist initially"
+                  assert not os.path.exists(facter_file), "facter.json should not exist initially"
+
+                  # Set CLAN_FLAKE for the commands
+                  os.environ["CLAN_FLAKE"] = flake_dir
+
+                  # Test facter backend
+                  clan_cmd = [
+                      "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan",
+                      "machines",
+                      "update-hardware-config",
+                      "--debug",
+                      "--flake", ".",
+                      "--host-key-check", "none",
+                      "test-install-machine-without-system",
+                      "-i", ssh_conn.ssh_key,
+                      "--option", "store", os.environ['CLAN_TEST_STORE'],
+                      f"nonrootuser@localhost:{ssh_conn.host_port}"
+                  ]
+
+                  result = subprocess.run(clan_cmd, capture_output=True, cwd=flake_dir)
+                  if result.returncode != 0:
+                      print(f"Clan update-hardware-config failed: {result.stderr.decode()}")
+                      raise Exception(f"Clan update-hardware-config failed with return code {result.returncode}")
+
+                  facter_without_system_file = os.path.join(flake_dir, "machines/test-install-machine-without-system/facter.json")
+                  assert os.path.exists(facter_without_system_file), "facter.json should exist after update"
+                  os.remove(facter_without_system_file)
+
+                  # Test nixos-generate-config backend
+                  clan_cmd = [
+                      "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan",
+                      "machines",
+                      "update-hardware-config",
+                      "--debug",
+                      "--backend", "nixos-generate-config",
+                      "--host-key-check", "none",
+                      "--flake", ".",
+                      "test-install-machine-without-system",
+                      "-i", ssh_conn.ssh_key,
+                      "--option", "store", os.environ['CLAN_TEST_STORE'],
+                      f"nonrootuser@localhost:{ssh_conn.host_port}"
+                  ]
+
+                  result = subprocess.run(clan_cmd, capture_output=True, cwd=flake_dir)
+                  if result.returncode != 0:
+                      print(f"Clan update-hardware-config (nixos-generate-config) failed: {result.stderr.decode()}")
+                      raise Exception(f"Clan update-hardware-config failed with return code {result.returncode}")
+
+                  hw_config_without_system_file = os.path.join(flake_dir, "machines/test-install-machine-without-system/hardware-configuration.nix")
+                  assert os.path.exists(hw_config_without_system_file), "hardware-configuration.nix should exist after update"
+            '';
+          } { inherit pkgs self; };
+
+        };
     };
 }

checks/installation/pyproject.toml

@@ -0,0 +1,44 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "nixos-test-lib"
+version = "1.0.0"
+description = "NixOS test utilities for clan VM testing"
+authors = [
+    {name = "Clan Core Team"}
+]
+dependencies = []
+
+[project.optional-dependencies]
+dev = [
+    "mypy",
+    "ruff"
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["nixos_test_lib*"]
+
+[tool.setuptools.package-data]
+"nixos_test_lib" = ["py.typed"]
+
+[tool.mypy]
+python_version = "3.12"
+strict = true
+warn_return_any = true
+warn_unused_configs = true
+
+[tool.ruff]
+target-version = "py312"
+line-length = 88
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "D", # docstrings
+    "ANN", # type annotations  
+    "COM812", # trailing comma
+    "ISC001", # string concatenation
+]

checks/installation/test-helpers.nix

@@ -0,0 +1,173 @@
+{
+  lib,
+  pkgs,
+  self,
+  ...
+}:
+let
+  # Common target VM configuration used by both installation and update tests
+  target =
+    { modulesPath, pkgs, ... }:
+    {
+      imports = [
+        (modulesPath + "/../tests/common/auto-format-root-device.nix")
+      ];
+      networking.useNetworkd = true;
+      services.openssh.enable = true;
+      services.openssh.settings.UseDns = false;
+      services.openssh.settings.PasswordAuthentication = false;
+      system.nixos.variant_id = "installer";
+      environment.systemPackages = [
+        pkgs.nixos-facter
+      ];
+      # Disable cache.nixos.org to speed up tests
+      nix.settings.substituters = [ ];
+      nix.settings.trusted-public-keys = [ ];
+      virtualisation.emptyDiskImages = [ 512 ];
+      virtualisation.diskSize = 8 * 1024;
+      virtualisation.rootDevice = "/dev/vdb";
+      # both installer and target need to use the same diskImage
+      virtualisation.diskImage = "./target.qcow2";
+      virtualisation.memorySize = 3048;
+      users.users.nonrootuser = {
+        isNormalUser = true;
+        openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ];
+        extraGroups = [ "wheel" ];
+      };
+      users.users.root.openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ];
+      # Allow users to manage their own SSH keys
+      services.openssh.authorizedKeysFiles = [
+        "/root/.ssh/authorized_keys"
+        "/home/%u/.ssh/authorized_keys"
+        "/etc/ssh/authorized_keys.d/%u"
+      ];
+      security.sudo.wheelNeedsPassword = false;
+    };
+
+  # Common base test machine configuration
+  baseTestMachine =
+    { lib, modulesPath, ... }:
+    {
+      imports = [
+        (modulesPath + "/testing/test-instrumentation.nix")
+        (modulesPath + "/profiles/qemu-guest.nix")
+        self.clanLib.test.minifyModule
+      ];
+
+      # Enable SSH and add authorized key for testing
+      services.openssh.enable = true;
+      services.openssh.settings.PasswordAuthentication = false;
+      users.users.nonrootuser = {
+        isNormalUser = true;
+        openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ];
+        extraGroups = [ "wheel" ];
+        home = "/home/nonrootuser";
+        createHome = true;
+      };
+      users.users.root.openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ];
+      # Allow users to manage their own SSH keys
+      services.openssh.authorizedKeysFiles = [
+        "/root/.ssh/authorized_keys"
+        "/home/%u/.ssh/authorized_keys"
+        "/etc/ssh/authorized_keys.d/%u"
+      ];
+      security.sudo.wheelNeedsPassword = false;
+
+      boot.consoleLogLevel = lib.mkForce 100;
+      boot.kernelParams = [ "boot.shell_on_fail" ];
+
+      # disko config
+      boot.loader.grub.efiSupport = lib.mkDefault true;
+      boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
+      clan.core.vars.settings.secretStore = "vm";
+      clan.core.vars.generators.test = {
+        files.test.neededFor = "partitioning";
+        script = ''
+          echo "notok" > "$out"/test
+        '';
+      };
+      disko.devices = {
+        disk = {
+          main = {
+            type = "disk";
+            device = "/dev/vda";
+
+            preCreateHook = ''
+              test -e /run/partitioning-secrets/test/test
+            '';
+
+            content = {
+              type = "gpt";
+              partitions = {
+                boot = {
+                  size = "1M";
+                  type = "EF02"; # for grub MBR
+                  priority = 1;
+                };
+                ESP = {
+                  size = "512M";
+                  type = "EF00";
+                  content = {
+                    type = "filesystem";
+                    format = "vfat";
+                    mountpoint = "/boot";
+                    mountOptions = [ "umask=0077" ];
+                  };
+                };
+                root = {
+                  size = "100%";
+                  content = {
+                    type = "filesystem";
+                    format = "ext4";
+                    mountpoint = "/";
+                  };
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+
+  # NixOS test library combining port utils and clan VM test utilities
+  nixosTestLib = pkgs.python3Packages.buildPythonPackage {
+    pname = "nixos-test-lib";
+    version = "1.0.0";
+    format = "pyproject";
+    src = lib.fileset.toSource {
+      root = ./.;
+      fileset = lib.fileset.unions [
+        ./pyproject.toml
+        ./nixos_test_lib
+      ];
+    };
+    nativeBuildInputs = with pkgs.python3Packages; [
+      setuptools
+      wheel
+    ];
+    doCheck = false;
+  };
+
+  # Common closure info
+  closureInfo = pkgs.closureInfo {
+    rootPaths = [
+      self.checks.x86_64-linux.clan-core-for-checks
+      self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.toplevel
+      self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.initialRamdisk
+      self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.diskoScript
+      self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.clan.deployment.file
+      pkgs.stdenv.drvPath
+      pkgs.bash.drvPath
+      pkgs.buildPackages.xorg.lndir
+    ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
+  };
+
+in
+{
+  inherit
+    target
+    baseTestMachine
+    nixosTestLib
+    closureInfo
+    ;
+}

checks/morph/flake-module.nix

@@ -35,7 +35,6 @@
                   pkgs.stdenv.drvPath
                   pkgs.stdenvNoCC
                   self.nixosConfigurations.test-morph-machine.config.system.build.toplevel
-                  self.nixosConfigurations.test-morph-machine.config.system.clan.deployment.file
                 ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
                 closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
               in

checks/service-dummy-test-from-flake/default.nix

@@ -23,14 +23,14 @@ nixosLib.runTest (
     clan.test.fromFlake = ./.;
 
     extraPythonPackages = _p: [
-      clan-core.legacyPackages.${hostPkgs.system}.setupNixInNixPythonPackage
+      clan-core.legacyPackages.${hostPkgs.system}.nixosTestLib
     ];
 
     testScript =
       { nodes, ... }:
       ''
-        from setup_nix_in_nix import setup_nix_in_nix # type: ignore[import-untyped]
-        setup_nix_in_nix()
+        from nixos_test_lib.nix_setup import setup_nix_in_nix # type: ignore[import-untyped]
+        setup_nix_in_nix(None)  # No closure info for this test
 
         def run_clan(cmd: list[str], **kwargs) -> str:
             import subprocess

clanModules/borgbackup/roles/client.nix

@@ -196,7 +196,7 @@ in
         pkgs.xkcdpass
       ];
       script = ''
-        ssh-keygen -t ed25519 -N "" -f "$out"/borgbackup.ssh
+        ssh-keygen -t ed25519 -N "" -C "" -f "$out"/borgbackup.ssh
         xkcdpass -n 4 -d - > "$out"/borgbackup.repokey
       '';
     };

clanModules/importer/README.md

@@ -7,7 +7,7 @@ The importer module allows users to configure importing modules in a flexible an
 
 It exposes the `extraModules` functionality of the inventory, without any added configuration.
 
-## Usage:
+## Usage
 
 ```nix
 inventory.services = {

clanModules/sshd/roles/server.nix

@@ -54,7 +54,7 @@ in
         pkgs.openssh
       ];
       script = ''
-        ssh-keygen -t ed25519 -N "" -f "$out"/ssh.id_ed25519
+        ssh-keygen -t ed25519 -N "" -C "" -f "$out"/ssh.id_ed25519
       '';
     };
 
@@ -74,7 +74,7 @@ in
         pkgs.openssh
       ];
       script = ''
-        ssh-keygen -t rsa -b 4096 -N "" -f "$out"/ssh.id_rsa
+        ssh-keygen -t rsa -b 4096 -N "" -C "" -f "$out"/ssh.id_rsa
       '';
     };
 

clanModules/sshd/shared.nix

@@ -36,7 +36,7 @@
             pkgs.openssh
           ];
           script = ''
-            ssh-keygen -t ed25519 -N "" -f "$out"/id_ed25519
+            ssh-keygen -t ed25519 -N "" -C "" -f "$out"/id_ed25519
           '';
         };
 

clanModules/state-version/roles/default.nix

@@ -5,9 +5,12 @@ in
 {
 
   warnings = [
-    "The clan.state-version module is deprecated and will be removed on 2025-07-15.
-      Please migrate to user-maintained configuration or the new equivalent clan services
-      (https://docs.clan.lol/reference/clanServices)."
+    ''
+      The clan.state-version service is deprecated and will be
+      removed on 2025-07-15 in favor of a nix option.
+
+      Please migrate your configuration to use `clan.core.settings.state-version.enable = true` instead.
+    ''
   ];
 
   system.stateVersion = lib.mkDefault (lib.removeSuffix "\n" var.value);

clanServices/borgbackup/default.nix

@@ -256,7 +256,7 @@
                     pkgs.xkcdpass
                   ];
                   script = ''
-                    ssh-keygen -t ed25519 -N "" -f "$out"/borgbackup.ssh
+                    ssh-keygen -t ed25519 -N "" -C "" -f "$out"/borgbackup.ssh
                     xkcdpass -n 4 -d - > "$out"/borgbackup.repokey
                   '';
                 };

clanServices/borgbackup/tests/vm/default.nix

@@ -41,14 +41,6 @@
         clan-core,
         ...
       }:
-      let
-        dependencies = [
-          clan-core
-          pkgs.stdenv.drvPath
-        ] ++ builtins.map (i: i.outPath) (builtins.attrValues clan-core.inputs);
-        closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
-
-      in
       {
 
         services.openssh.enable = true;
@@ -59,15 +51,6 @@
 
         environment.systemPackages = [ clan-core.packages.${pkgs.system}.clan-cli ];
 
-        environment.etc.install-closure.source = "${closureInfo}/store-paths";
-        nix.settings = {
-          substituters = pkgs.lib.mkForce [ ];
-          hashed-mirrors = null;
-          connect-timeout = pkgs.lib.mkForce 3;
-          flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
-        };
-        system.extraDependencies = dependencies;
-
         clan.core.state.test-backups.folders = [ "/var/test-backups" ];
       };
 

clanServices/hello-world/flake-module.nix

@@ -23,7 +23,13 @@ in
       unit-test-module = (
         self.clanLib.test.flakeModules.makeEvalChecks {
           inherit module;
-          inherit self inputs;
+          inherit inputs;
+          fileset = lib.fileset.unions [
+            # The hello-world service being tested
+            ../../clanServices/hello-world
+            # Required modules
+            ../../nixosModules/clanCore
+          ];
           testName = "hello-world";
           tests = ./tests/eval-tests.nix;
           # Optional arguments passed to the test

clanServices/importer/README.md

@@ -1,7 +1,7 @@
 The importer module allows users to configure importing modules in a flexible and structured way.
 It exposes the `extraModules` functionality of the inventory, without any added configuration.
 
-## Usage:
+## Usage
 
 ```nix
 inventory.instances = {

clanServices/sshd/README.md

@@ -0,0 +1,36 @@
+The `sshd` Clan service manages SSH to make it easy to securely access your machines over the internet. The service uses `vars` to store the SSH host keys for each machine to ensure they remain stable across deployments.
+
+`sshd` also generates SSH certificates for both servers and clients allowing for certificate-based authentication for SSH.
+
+The service also disables password-based authentication over SSH, to access your machines you'll need to use public key authentication or certificate-based authentication.
+
+## Usage
+
+```nix
+{
+  inventory.instances = {
+    # By default this service only generates ed25519 host keys
+    sshd-basic = {
+      module = {
+        name = "sshd";
+        input = "clan-core";
+      };
+      roles.server.tags.all = { };
+      roles.client.tags.all = { };
+    };
+
+    # Also generate RSA host keys for all servers
+    sshd-with-rsa = {
+      module = {
+        name = "sshd";
+        input = "clan-core";
+      };
+      roles.server.tags.all = { };
+      roles.server.settings = {
+        hostKeys.rsa.enable = true;
+      };
+      roles.client.tags.all = { };
+    };
+  };
+}
+```

clanServices/sshd/default.nix

@@ -2,7 +2,7 @@
 {
   _class = "clan.service";
   manifest.name = "clan-core/sshd";
-  manifest.description = "Enables secure remote access to the machine over ssh.";
+  manifest.description = "Enables secure remote access to the machine over SSH";
   manifest.categories = [
     "System"
     "Network"
@@ -49,7 +49,7 @@
                 pkgs.openssh
               ];
               script = ''
-                ssh-keygen -t ed25519 -N "" -f "$out"/id_ed25519
+                ssh-keygen -t ed25519 -N "" -C "" -f "$out"/id_ed25519
               '';
             };
 
@@ -109,7 +109,7 @@
                   pkgs.openssh
                 ];
                 script = ''
-                  ssh-keygen -t ed25519 -N "" -f "$out"/id_ed25519
+                  ssh-keygen -t ed25519 -N "" -C "" -f "$out"/id_ed25519
                 '';
               };
 
@@ -151,7 +151,7 @@
                   pkgs.openssh
                 ];
                 script = ''
-                  ssh-keygen -t rsa -b 4096 -N "" -f "$out"/ssh.id_rsa
+                  ssh-keygen -t rsa -b 4096 -N "" -C "" -f "$out"/ssh.id_rsa
                 '';
               };
 
@@ -164,7 +164,7 @@
                   pkgs.openssh
                 ];
                 script = ''
-                  ssh-keygen -t ed25519 -N "" -f "$out"/ssh.id_ed25519
+                  ssh-keygen -t ed25519 -N "" -C "" -f "$out"/ssh.id_ed25519
                 '';
               };
             };

clanServices/state-version/default.nix

@@ -20,6 +20,16 @@
             var = config.clan.core.vars.generators.state-version.files.version or { };
           in
           {
+
+            warnings = [
+              ''
+                The clan.state-version service is deprecated and will be
+                removed on 2025-07-15 in favor of a nix option. 
+
+                Please migrate your configuration to use `clan.core.settings.state-version.enable = true` instead.
+              ''
+            ];
+
             system.stateVersion = lib.mkDefault (lib.removeSuffix "\n" var.value);
 
             clan.core.vars.generators.state-version = {

clanServices/state-version/tests/vm/default.nix

@@ -1,5 +1,6 @@
+{ lib, ... }:
 {
-  name = "state-version";
+  name = "service-state-version";
 
   clan = {
     directory = ./.;
@@ -15,7 +16,7 @@
 
   nodes.server = { };
 
-  testScript = ''
+  testScript = lib.mkDefault ''
     start_all()
   '';
 }

clanServices/users/README.md

@@ -1,30 +1,31 @@
 ## Usage
 
-```
-inventory.instances = {
-
-  # Deploy user alice on all machines. Don't prompt for password (will be
-  # auto-generated).
-
-  user-alice = {
-    module = {
-      name = "users";
-      input = "clan";
+```nix
+{
+  inventory.instances = {
+    # Deploy user alice on all machines. Don't prompt for password (will be
+    # auto-generated).
+    user-alice = {
+      module = {
+        name = "users";
+        input = "clan";
+      };
+      roles.default.tags.all = { };
+      roles.default.settings = {
+        user = "alice";
+        prompt = false;
+      };
     };
-    roles.default.tags.all = { };
-    roles.default.settings = {
-      user = "alice";
-      prompt = false;
+
+    # Deploy user bob only on his laptop. Prompt for a password.
+    user-bob = {
+      module = {
+        name = "users";
+        input = "clan";
+      };
+      roles.default.machines.bobs-laptop = { };
+      roles.default.settings.user = "bob";
     };
   };
-
-  # Deploy user bob only on his laptop. Prompt for a password.
-  user-bob = {
-    module = {
-      name = "users";
-      input = "clan";
-    };
-    roles.default.machines.bobs-laptop = { };
-    roles.default.settings.user = "bob";
-  };
+}
 ```

clanServices/wifi/default.nix

@@ -73,9 +73,10 @@ in
             ];
 
             networking.networkmanager.ensureProfiles.profiles = flip mapAttrs settings.networks (
-              name: _network: {
+              name: networkCfg: {
                 connection.id = "$ssid_${name}";
                 connection.type = "wifi";
+                connection.autoconnect = networkCfg.autoConnect;
                 wifi.mode = "infrastructure";
                 wifi.ssid = "$ssid_${name}";
                 wifi-security.psk = "$pw_${name}";
@@ -102,7 +103,7 @@ in
                   # Generate the secrets file
                   echo "Generating wifi secrets file: $env_file"
                   ${flip (concatMapAttrsStringSep "\n") settings.networks (
-                    name: _network: ''
+                    name: _networkCfg: ''
                       echo "ssid_${name}=\"$(cat "${ssid_path name}")\"" >> /run/secrets/NetworkManager/wifi-secrets
                       echo "pw_${name}=\"$(cat "${password_path name}")\"" >> /run/secrets/NetworkManager/wifi-secrets
                     ''

clanServices/zerotier/flake-module.nix

@@ -15,7 +15,15 @@ in
       unit-test-module = (
         self.clanLib.test.flakeModules.makeEvalChecks {
           inherit module;
-          inherit self inputs;
+          inherit inputs;
+          fileset = lib.fileset.unions [
+            # The zerotier service being tested
+            ../../clanServices/zerotier
+            # Required modules
+            ../../nixosModules/clanCore
+            # Dependencies like clan-cli
+            ../../pkgs/clan-cli
+          ];
           testName = "zerotier";
           tests = ./tests/eval-tests.nix;
           testArgs = { };

devFlake/private.narHash

@@ -0,0 +1 @@
+sha256-pFUj3KhQ4FkzZT19t+FHBru8u8Lspax0rS2cv7nXIgM=

devFlake/private/flake.lock

@@ -0,0 +1,165 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": [
+          "systems"
+        ]
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "ixx": {
+      "inputs": {
+        "flake-utils": [
+          "nuschtos",
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "nuschtos",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1748294338,
+        "narHash": "sha256-FVO01jdmUNArzBS7NmaktLdGA5qA3lUMJ4B7a05Iynw=",
+        "owner": "NuschtOS",
+        "repo": "ixx",
+        "rev": "cc5f390f7caf265461d4aab37e98d2292ebbdb85",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NuschtOS",
+        "ref": "v0.0.8",
+        "repo": "ixx",
+        "type": "github"
+      }
+    },
+    "nixpkgs-dev": {
+      "locked": {
+        "lastModified": 1751867001,
+        "narHash": "sha256-3I49W0s3WVEDBO5S1RxYr74E2LLG7X8Wuvj9AmU0RDk=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "73feb5e20ec7259e280ca6f424ba165059b3bb6b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable-small",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nuschtos": {
+      "inputs": {
+        "flake-utils": "flake-utils_2",
+        "ixx": "ixx",
+        "nixpkgs": [
+          "nixpkgs-dev"
+        ]
+      },
+      "locked": {
+        "lastModified": 1749730855,
+        "narHash": "sha256-L3x2nSlFkXkM6tQPLJP3oCBMIsRifhIDPMQQdHO5xWo=",
+        "owner": "NuschtOS",
+        "repo": "search",
+        "rev": "8dfe5879dd009ff4742b668d9c699bc4b9761742",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NuschtOS",
+        "repo": "search",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs-dev": "nixpkgs-dev",
+        "nuschtos": "nuschtos",
+        "systems": "systems_2",
+        "treefmt-nix": "treefmt-nix"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": []
+      },
+      "locked": {
+        "lastModified": 1750931469,
+        "narHash": "sha256-0IEdQB1nS+uViQw4k3VGUXntjkDp7aAlqcxdewb/hAc=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "ac8e6f32e11e9c7f153823abc3ab007f2a65d3e1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

devFlake/private/flake.nix

@@ -0,0 +1,19 @@
+{
+  description = "private dev inputs";
+
+  # Dev dependencies
+  inputs.nixpkgs-dev.url = "github:NixOS/nixpkgs/nixos-unstable-small";
+
+  inputs.flake-utils.url = "github:numtide/flake-utils";
+  inputs.flake-utils.inputs.systems.follows = "systems";
+
+  inputs.nuschtos.url = "github:NuschtOS/search";
+  inputs.nuschtos.inputs.nixpkgs.follows = "nixpkgs-dev";
+
+  inputs.treefmt-nix.url = "github:numtide/treefmt-nix";
+  inputs.treefmt-nix.inputs.nixpkgs.follows = "";
+
+  inputs.systems.url = "github:nix-systems/default";
+
+  outputs = _: { };
+}

devFlake/update-private-narhash

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Used to update the private dev flake hash reference.
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+echo "Updating $PWD/private.narHash" >&2
+
+nix --extra-experimental-features 'flakes nix-command' flake lock ./private
+nix --extra-experimental-features 'flakes nix-command' hash path ./private >./private.narHash
+
+echo OK

docs/.gitignore

@@ -1,4 +1,5 @@
 /site/reference
 /site/static
 /site/options-page
+/site/openapi.json
 !/site/static/extra.css

docs/mkdocs.yml

@@ -48,13 +48,13 @@ nav:
   - Home: index.md
   - Guides:
       - Getting Started:
-          - Creating Your First Clan: guides/getting-started/index.md
-          - Create USB Installer (optional): guides/getting-started/installer.md
-          - Add Machines: guides/getting-started/add-machines.md
-          - Add Services: guides/getting-started/add-services.md
-          - Secrets & Facts: guides/getting-started/secrets.md
-          - Deploy Machine: guides/getting-started/deploy.md
-          - Continuous Integration: guides/getting-started/check.md
+          - 🚀 Creating Your First Clan: guides/getting-started/index.md
+          - 📀 Create USB Installer (optional): guides/getting-started/installer.md
+          - ⚙️ Add Machines: guides/getting-started/add-machines.md
+          - ⚙️ Add Services: guides/getting-started/add-services.md
+          - 🔐 Secrets & Facts: guides/getting-started/secrets.md
+          - 🚢 Deploy Machine: guides/getting-started/deploy.md
+          - 🧪 Continuous Integration: guides/getting-started/check.md
       - clanServices: guides/clanServices.md
       - Disk Encryption: guides/disk-encryption.md
       - Mesh VPN: guides/mesh-vpn.md
@@ -62,6 +62,7 @@ nav:
       - Vars Backend: guides/vars-backend.md
       - Facts Backend: guides/secrets.md
       - Adding more machines: guides/more-machines.md
+      - Target Host: guides/target-host.md
       - Inventory:
           - Inventory: guides/inventory.md
       - Secure Boot: guides/secure-boot.md
@@ -154,6 +155,7 @@ nav:
           - reference/cli/show.md
           - reference/cli/ssh.md
           - reference/cli/state.md
+          - reference/cli/templates.md
           - reference/cli/vars.md
           - reference/cli/vms.md
       - NixOS Modules:
@@ -181,6 +183,9 @@ nav:
           - 05-deployment-parameters: decisions/05-deployment-parameters.md
           - Template: decisions/_template.md
   - Options: options.md
+  - Developer:
+      - Introduction: intern/index.md
+      - API: intern/api.md
 
 docs_dir: site
 site_dir: out
@@ -238,3 +243,4 @@ extra:
 plugins:
   - search
   - macros
+  - redoc-tag

docs/nix/default.nix

@@ -1,8 +1,8 @@
 {
-  clan-core,
   pkgs,
   module-docs,
   clan-cli-docs,
+  clan-lib-openapi,
   asciinema-player-js,
   asciinema-player-css,
   roboto,
@@ -18,7 +18,17 @@ pkgs.stdenv.mkDerivation {
 
   # Points to repository root.
   # so that we can access directories outside of docs to include code snippets
-  src = clan-core;
+  src = pkgs.lib.fileset.toSource {
+    root = ../..;
+    fileset = pkgs.lib.fileset.unions [
+      # Docs directory
+      ../../docs
+      # Icons needed for the build
+      ../../pkgs/clan-app/ui/icons
+      # Any other directories that might be referenced for code snippets
+      # Add them here as needed based on what mkdocs actually uses
+    ];
+  };
 
   nativeBuildInputs =
     [
@@ -29,6 +39,7 @@ pkgs.stdenv.mkDerivation {
       mkdocs
       mkdocs-material
       mkdocs-macros
+      mkdocs-redoc-tag
     ]);
   configurePhase = ''
     pushd docs
@@ -36,6 +47,10 @@ pkgs.stdenv.mkDerivation {
     mkdir -p ./site/reference/cli
     cp -af ${module-docs}/* ./site/reference/
     cp -af ${clan-cli-docs}/* ./site/reference/cli/
+
+    mkdir -p ./site/reference/internal
+    cp -af ${clan-lib-openapi} ./site/openapi.json
+
     chmod -R +w ./site/reference
     echo "Generated API documentation in './site/reference/' "
 

docs/nix/flake-module.nix

@@ -82,10 +82,9 @@
           }
           ''
             export CLAN_CORE_PATH=${
-              self.filter {
-                include = [
-                  "clanModules"
-                ];
+              inputs.nixpkgs.lib.fileset.toSource {
+                root = ../..;
+                fileset = ../../clanModules;
               }
             }
             export CLAN_CORE_DOCS=${jsonDocs.clanCore}/share/doc/nixos/options.json
@@ -126,8 +125,12 @@
       });
       packages = {
         docs = pkgs.python3.pkgs.callPackage ./default.nix {
-          clan-core = self;
-          inherit (self'.packages) clan-cli-docs docs-options inventory-api-docs;
+          inherit (self'.packages)
+            clan-cli-docs
+            docs-options
+            inventory-api-docs
+            clan-lib-openapi
+            ;
           inherit (inputs) nixpkgs;
           inherit module-docs;
           inherit asciinema-player-js;

docs/nix/options/flake-module.nix

@@ -1,9 +1,15 @@
-{ self, config, ... }:
+{
+  self,
+  config,
+  inputs,
+  privateInputs ? { },
+  ...
+}:
 {
   perSystem =
     {
-      inputs',
       lib,
+      pkgs,
       ...
     }:
     let
@@ -157,11 +163,16 @@
       };
     in
     {
-      packages.docs-options = inputs'.nuschtos.packages.mkMultiSearch {
-        inherit baseHref;
-        title = "Clan Options";
-        # scopes = mapAttrsToList mkScope serviceModules;
-        scopes = [ (mkScope "Clan Inventory" serviceModules) ];
+      packages = lib.optionalAttrs ((privateInputs ? nuschtos) || (inputs ? nuschtos)) {
+        docs-options =
+          (privateInputs.nuschtos or inputs.nuschtos)
+          .packages.${pkgs.stdenv.hostPlatform.system}.mkMultiSearch
+            {
+              inherit baseHref;
+              title = "Clan Options";
+              # scopes = mapAttrsToList mkScope serviceModules;
+              scopes = [ (mkScope "Clan Inventory" serviceModules) ];
+            };
       };
     };
 }

docs/site/decisions/04-fetching-nix-from-python.md

@@ -28,7 +28,7 @@ Benefits:
 * Caching mechanism is very simple.
 
 
-### Method 2: Direct access:
+### Method 2: Direct access
 
 Directly calling the evaluator / build sandbox via `nix build` and `nix eval`within the Python code
 

docs/site/guides/clanServices.md

@@ -17,8 +17,10 @@ For example:
 ```nix
 inventory.instances = {
   borgbackup = {
-    roles.client.machines = [ "laptop" "server1" ];
-    roles.server.machines = [ "backup-box" ];
+    roles.client.machines."laptop" = {};
+    roles.client.machines."server1" = {};
+
+    roles.server.machines."backup-box" = {};
   };
 }
 ```
@@ -40,7 +42,8 @@ Example of instantiating a `borgbackup` service using `clan-core`:
 ```nix
 inventory.instances = {
     # Instance Name: Different name for this 'borgbackup' instance
-    borgbackup-example = {
+    borgbackup = {
+        # Since this is instances."borgbackup" the whole `module = { ... }` below is equivalent and optional.
         module =  {
             name = "borgbackup";  # <-- Name of the module (optional)
             input = "clan-core"; # <-- The flake input where the service is defined (optional)

docs/site/guides/contributing/debugging.md

@@ -105,7 +105,7 @@ git+file:///home/lhebendanz/Projects/clan-core
 │       ├───editor omitted (use '--all-systems' to show)
 └───templates
     ├───default: template: Initialize a new clan flake
-    └───new-clan: template: Initialize a new clan flake
+    └───default: template: Initialize a new clan flake
 ```
 
 You can execute every test separately by following the tree path `nix run .#checks.x86_64-linux.clan-pytest -L` for example.

docs/site/guides/disk-encryption.md

@@ -63,8 +63,7 @@ Replace `kernelModules` with the ethernet module loaded one on your target machi
 }
 ```
 
-
-### Step 1: Copying SSH Public Key
+## Copying SSH Public Key
 
 Before starting the installation process, ensure that the SSH public key is copied to the NixOS installer.
 
@@ -74,7 +73,7 @@ Before starting the installation process, ensure that the SSH public key is copi
 ssh-copy-id -o PreferredAuthentications=password -o PubkeyAuthentication=no root@nixos-installer.local
 ```
 
-### Step 1.5: Prepare Secret Key and Partition Disks
+## Prepare Secret Key and Partition Disks
 
 1. Access the installer using SSH:
 
@@ -100,7 +99,7 @@ blkdiscard /dev/disk/by-id/<installdisk>
 clan machines install gchq-local --target-host root@nixos-installer --phases kexec,disko
 ```
 
-### Step 2: ZFS Pool Import and System Installation
+## ZFS Pool Import and System Installation
 
 1. SSH into the installer once again:
 
@@ -123,8 +122,8 @@ CTRL+D
 4. Locally generate ssh host keys. You only need to generate ones for the algorithms you're using in `authorizedKeys`.
 
 ```bash
-ssh-keygen -q -N "" -t ed25519 -f ./initrd_host_ed25519_key
-ssh-keygen -q -N "" -t rsa -b 4096 -f ./initrd_host_rsa_key
+ssh-keygen -q -N "" -C "" -t ed25519 -f ./initrd_host_ed25519_key
+ssh-keygen -q -N "" -C "" -t rsa -b 4096 -f ./initrd_host_rsa_key
 ```
 
 5. Securely copy your local initrd ssh host keys to the installer's `/mnt` directory:
@@ -151,7 +150,7 @@ zpool export zroot
 
 8. Perform a reboot of the machine and remove the USB installer.
 
-### Step 3: Accessing the Initial Ramdisk (initrd) Environment
+## Accessing the Initial Ramdisk (initrd) Environment
 
 1. SSH into the initrd environment using the `initrd_rsa_key` and provided port:
 

docs/site/guides/flake-parts.md

@@ -3,7 +3,7 @@ Clan supports integration with [flake-parts](https://flake.parts/), a framework
 
 To construct your Clan using flake-parts, follow these steps:
 
-## 1. Update Your Flake Inputs
+## Update Your Flake Inputs
 
 To begin, you'll need to add `flake-parts` as a new dependency in your flake's inputs. This is alongside the already existing dependencies, such as `clan-core` and `nixpkgs`. Here's how you can update your `flake.nix` file:
 
@@ -25,7 +25,7 @@ inputs = {
 }
 ```
 
-## 2. Import the Clan flake-parts Module
+## Import the Clan flake-parts Module
 
 After updating your flake inputs, the next step is to import the Clan flake-parts module. This will make the [Clan options](../reference/nix-api/clan.md) available within `mkFlake`.
 
@@ -43,7 +43,7 @@ After updating your flake inputs, the next step is to import the Clan flake-part
 }
 ```
 
-### 3. Configure Clan Settings and Define Machines
+## Configure Clan Settings and Define Machines
 
 Next you'll need to configure Clan wide settings and define machines, here's an example of how `flake.nix` should look:
 
@@ -91,6 +91,6 @@ Next you'll need to configure Clan wide settings and define machines, here's an
 ```
 
 For detailed information about configuring `flake-parts` and the available options within Clan,
-refer to the Clan module documentation located [here](https://git.clan.lol/clan/clan-core/src/branch/main/flakeModules/clan.nix).
+refer to the [Clan module](https://git.clan.lol/clan/clan-core/src/branch/main/flakeModules/clan.nix) documentation.
 
 ---

docs/site/guides/getting-started/add-machines.md

@@ -119,26 +119,34 @@ clan = {
 1. It is required to define a *targetHost* for each machine before deploying. Best practice has been, to use the zerotier ip/hostname or the ip from the from overlay network you decided to use.
 2. Add your *ssh key* here - That will ensure you can always login to your machine via *ssh* in case something goes wrong.
 
-### (Optional): Renaming Machine
+### (Optional) Renaming a Machine
 
-For renaming jon to your own machine name, you can use the following command:
+Older templates included static machine folders like `jon` and `sara`.
+If your setup still uses such static machines, you can rename a machine folder to match your own machine name:
 
-```
-git mv ./machines/jon ./machines/newname
+```bash
+git mv ./machines/jon ./machines/<your-machine-name>
 ```
 
-Note that our clan lives inside a git repository.
-Only files that have been added with `git add` are recognized by `nix`.
-So for every file that you add or rename you also need to run:
+Since your Clan configuration lives inside a Git repository, remember:
 
-```
-git add ./path/to/my/file
+* Only files tracked by Git (`git add`) are recognized.
+* Whenever you add, rename, or remove files, run:
+
+```bash
+git add ./machines/<your-machine-name>
 ```
 
-### (Optional): Removing a Machine
+to stage the changes.
 
-If you only want to setup a single machine at this point, you can delete `sara` from `flake.nix` as well as from the machines directory:
+---
 
-```
+### (Optional) Removing a Machine
+
+If you want to work with a single machine for now, you can remove other machine entries both from your `flake.nix` and from the `machines` directory. For example, to remove the machine `sara`:
+
+```bash
 git rm -rf ./machines/sara
 ```
+
+Make sure to also remove or update any references to that machine in your `nix files` or `inventory.json` if you have any of that

docs/site/guides/getting-started/deploy.md

@@ -27,7 +27,7 @@ Now that you have created a new machine, we will walk through how to install it.
     !!! Warning "NixOS can cause strange issues when booting in certain cloud environments."
         If on Linode: Make sure that the system uses Direct Disk boot kernel (found in the configuration pannel)
 
-### Step 1. Setting `targetHost`
+## Setting `targetHost`
 
 === "flake.nix (flake-parts)"
 
@@ -98,7 +98,7 @@ Now that you have created a new machine, we will walk through how to install it.
     The use of `root@` in the target address implies SSH access as the `root` user.
     Ensure that the root login is secured and only used when necessary.
 
-### Step 2. Identify the Target Disk
+## Identify the Target Disk
 
 On the setup computer, SSH into the target:
 
@@ -129,7 +129,7 @@ In this example we would copy `nvme-eui.e8238fa6bf530001001b448b4aec2929`
 !!! tip
     For advanced partitioning, see [Disko templates](https://github.com/nix-community/disko-templates) or [Disko examples](https://github.com/nix-community/disko/tree/master/example).
 
-### Step 3. Fill in hardware specific machine configuration
+## Fill in hardware specific machine configuration
 
 Edit the following fields inside the `./machines/<machine_name>/configuration.nix`
 
@@ -164,7 +164,7 @@ Edit the following fields inside the `./machines/<machine_name>/configuration.ni
 !!! Info "Replace `__CHANGE_ME__` with the appropriate `ID-LINK` identifier, such as `nvme-eui.e8238fa6bf530001001b448b4aec2929`"
 !!! Info "Replace `__YOUR_SSH_KEY__` with your personal key, like `ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILoMI0NC5eT9pHlQExrvR5ASV3iW9+BXwhfchq0smXUJ jon@jon-desktop`"
 
-### Step 4. Deploy the machine
+## Deploy the machine
 
 **Finally deployment time!** Use the following command to build and deploy the image via SSH onto your machine.
 
@@ -227,7 +227,7 @@ Edit the following fields inside the `./machines/<machine_name>/configuration.ni
         ```
     2.  The root password for the installer medium.
         This password is autogenerated and meant to be easily typeable.
-    3.  See how to connect the installer medium to wlan [here](./installer.md#optional-connect-to-wifi-manually).
+    3.  See [how to connect to wlan](./installer.md#optional-connect-to-wifi-manually).
 
     !!! tip
         Use [KDE Connect](https://apps.kde.org/de/kdeconnect/) for easyily sharing QR codes from phone to desktop
@@ -236,21 +236,21 @@ Edit the following fields inside the `./machines/<machine_name>/configuration.ni
 
     Just run the command **Option B: Cloud VM** below
 
-#### Deployment Commands
+### Deployment Commands
 
-##### Using password auth
+#### Using password auth
 
 ```bash
 clan machines install [MACHINE] --target-host <IP> --update-hardware-config nixos-facter
 ```
 
-##### Using QR JSON
+#### Using QR JSON
 
 ```bash
 clan machines install [MACHINE] --json "[JSON]" --update-hardware-config nixos-facter
 ```
 
-##### Using QR image file
+#### Using QR image file
 
 ```bash
 clan machines install [MACHINE] --png [PATH] --update-hardware-config nixos-facter

docs/site/guides/getting-started/index.md

@@ -4,8 +4,7 @@ Ready to create your own Clan and manage a fleet of machines? Follow these simpl
 
 By the end of this guide, you'll have a fresh NixOS configuration ready to push to one or more machines. You'll create a new Git repository and a flake, and all you need is at least one machine to push to. This is the easiest way to begin, and we recommend you to copy your existing configuration into this new setup!
 
-
-### Prerequisites
+## Prerequisites
 
 === "**Linux**"
 
@@ -37,22 +36,23 @@ By the end of this guide, you'll have a fresh NixOS configuration ready to push
 
     If you have previously installed Nix, make sure `experimental-features = nix-command flakes` is present in `~/.config/nix/nix.conf` or `/etc/nix/nix.conf`. If this is not the case, please add it to `~/.config/nix/nix.conf`.
 
-### Step 1: Add Clan CLI to Your Shell
+## Add Clan CLI to Your Shell
 
-Add the Clan CLI into your development workflow:
+Add the Clan CLI into your environment:
 
 ```bash
 nix shell git+https://git.clan.lol/clan/clan-core#clan-cli --refresh
 ```
 
-You can find reference documentation for the `clan` CLI program [here](../../reference/cli/index.md).
-
-Alternatively you can check out the help pages directly:
 ```terminalSession
 clan --help
 ```
 
-### Step 2: Initialize Your Project
+Should print the avilable commands.
+
+Also checkout the [cli-reference documentation](../../reference/cli/index.md).
+
+## Initialize Your Project
 
 If you want to migrate an existing project, follow this [guide](../migrations/migration-guide.md).
 
@@ -62,36 +62,29 @@ Set the foundation of your Clan project by initializing it by running:
 clan flakes create my-clan
 ```
 
-This command creates the `flake.nix` and `.clan-flake` files for your project.
-It will also generate files from a default template, to help show general clan usage patterns.
+This command creates a `flake.nix` and some other files for your project.
 
-### Step 3: Verify the Project Structure
+## Explore the Project Structure
 
-Ensure that all project files exist by running:
+Take a lookg at all project files:
 
 ```bash
 cd my-clan
 tree
 ```
 
-This should yield the following:
+For example, you might see something like:
 
 ``` { .console .no-copy }
 .
 ├── flake.nix
-├── machines
-│   ├── jon
-│   │   ├── configuration.nix
-│   │   └── hardware-configuration.nix
-│   └── sara
-│       ├── configuration.nix
-│       └── hardware-configuration.nix
-└── modules
-    └── shared.nix
-
-5 directories, 9 files
+├── machines/
+├── modules/
+└── README.md
 ```
 
+Don’t worry if your output looks different—the template evolves over time.
+
 ??? info "Recommended way of sourcing the `clan` CLI tool"
 
     The default template adds the `clan` CLI tool to the development shell.
@@ -109,17 +102,23 @@ This should yield the following:
     To automatically add the `clan` CLI tool to your environment without having to
     run `nix develop` every time, we recommend setting up [direnv](https://direnv.net/).
 
-
-```bash
+```
 clan machines list
 ```
 
-``` { .console .no-copy }
-jon
-sara
-```
+If you see no output yet, that’s expected — [add machines](./add-machines.md) to populate it.
 
-!!! success
+---
 
-    You just successfully bootstrapped your first Clan.
+## Next Steps
 
+You can continue with **any** of the following steps at your own pace:
+
+- [x] [Install Nix & Clan CLI](./index.md)
+- [x] [Initialize Clan](./index.md#initialize-your-project)
+- [ ] [Create USB Installer (optional)](./installer.md)
+- [ ] [Add Machines](./add-machines.md)
+- [ ] [Add Services](./add-services.md)
+- [ ] [Configure Secrets](./secrets.md)
+- [ ] [Deploy](./deploy.md) - Requires configured secrets
+- [ ] [Setup CI (optional)](./check.md)

docs/site/guides/getting-started/installer.md

@@ -11,13 +11,12 @@ To install Clan on physical machines, you need to use our custom installer image
 ??? info "Reasons for a Custom Install Image"
     Our custom install images are built to include essential tools like [nixos-facter](https://github.com/nix-community/nixos-facter) and support for [ZFS](https://wiki.archlinux.org/title/ZFS). They're also optimized to run on systems with as little as 1 GB of RAM, ensuring efficient performance even on lower-end hardware.
 
-
-### Step 0. Prerequisites
+## Prerequisites
 
 - [x] A free USB Drive with at least 1.5GB (All data on it will be lost)
 - [x] Linux/NixOS Machine with Internet
 
-### Step 1. Identify the USB Flash Drive
+## Identify the USB Flash Drive
 
 1. Insert your USB flash drive into your computer.
 
@@ -45,7 +44,7 @@ To install Clan on physical machines, you need to use our custom installer image
 sudo umount /dev/sdb1
 ```
 
-### Step 2. Installer
+## Installer
 
 === "**Linux OS**"
     **Create a Custom Installer**
@@ -118,7 +117,7 @@ sudo umount /dev/sdb1
     !!! Note
         If you don't have `wget` installed, you can use `curl --progress-bar -OL <url>` instead.
 
-    ### Step 2.5 Flash the Installer to the USB Drive
+    ## Flash the Installer to the USB Drive
 
     !!! Danger "Specifying the wrong device can lead to unrecoverable data loss."
 
@@ -151,11 +150,10 @@ sudo umount /dev/sdb1
       If you need to configure Wi-Fi first, refer to the next section.
       If Multicast-DNS (Avahi) is enabled on your own machine, you can also access the installer using the `nixos-installer.local` address.
 
+## Boot From USB Stick
 
-###  Step 3: Boot From USB Stick
 - To use, boot from the Clan USB drive with **secure boot turned off**. For step by step instructions go to [Disabling Secure Boot](../../guides/secure-boot.md)
 
-
 ## (Optional) Connect to Wifi Manually
 
 If you don't have access via LAN the Installer offers support for connecting via Wifi.
@@ -203,4 +201,3 @@ Press ++ctrl+d++ to exit `IWD`.
     Press ++ctrl+d++ **again** to update the displayed QR code and connection information.
 
 You're all set up
-

docs/site/guides/getting-started/secrets.md

@@ -52,65 +52,6 @@ For more information see the [SOPS] guide on [encrypting with age].
 !!! note
     It's safe to add any secrets created by the clan CLI and placed in your repository to version control systems like `git`.
 
-### Using Age Plugins
-
-If you wish to use a key generated using an [age plugin] as your admin key, extra care is needed.
-
-You must **precede your secret key with a comment that contains its corresponding recipient**.
-
-This is usually output as part of the generation process
-and is only required because there is no unified mechanism for recovering a recipient from a plugin secret key.
-
-Here is an example:
-
-```title="~/.config/sops/age/keys.txt"
-# public key: age1zdy49ek6z60q9r34vf5mmzkx6u43pr9haqdh5lqdg7fh5tpwlfwqea356l
-AGE-PLUGIN-FIDO2-HMAC-1QQPQZRFR7ZZ2WCV...
-```
-
-!!! note
-    The comment that precedes the plugin secret key need only contain the recipient.
-    Any other text is ignored.
-
-    In the example above, you can specify `# recipient: age1zdy...`, `# public: age1zdy....` or even
-    just `# age1zdy....`
-
-You will need to add an entry into your `flake.nix` to ensure that the necessary `age` plugins
-are loaded when using Clan:
-
-```nix title="flake.nix"
-{
-  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-  inputs.nixpkgs.follows = "clan-core/nixpkgs";
-
-  outputs =
-    { self, clan-core, ... }:
-    let
-      clan = clan-core.clanLib.clan {
-        inherit self;
-
-        meta.name = "myclan";
-
-        # Add Yubikey and FIDO2 HMAC plugins
-        # Note: the plugins listed here must be available in nixpkgs.
-        secrets.age.plugins = [
-          "age-plugin-yubikey"
-          "age-plugin-fido2-hmac"
-        ];
-
-        machines = {
-          # elided for brevity
-        };
-      };
-    in
-    {
-      inherit (clan) nixosConfigurations nixosModules clanInternals;
-
-      # elided for brevity
-    };
-}
-```
-
 ### Add Your Public Key(s)
 
 ```console
@@ -176,3 +117,62 @@ clan secrets users remove-key $USER --age-key <your_public_key>
 [age plugin]: https://github.com/FiloSottile/awesome-age?tab=readme-ov-file#plugins
 [sops]: https://github.com/getsops/sops
 [encrypting with age]: https://github.com/getsops/sops?tab=readme-ov-file#encrypting-using-age
+
+## Further: Using Age Plugins
+
+If you wish to use a key generated using an [age plugin] as your admin key, extra care is needed.
+
+You must **precede your secret key with a comment that contains its corresponding recipient**.
+
+This is usually output as part of the generation process
+and is only required because there is no unified mechanism for recovering a recipient from a plugin secret key.
+
+Here is an example:
+
+```title="~/.config/sops/age/keys.txt"
+# public key: age1zdy49ek6z60q9r34vf5mmzkx6u43pr9haqdh5lqdg7fh5tpwlfwqea356l
+AGE-PLUGIN-FIDO2-HMAC-1QQPQZRFR7ZZ2WCV...
+```
+
+!!! note
+    The comment that precedes the plugin secret key need only contain the recipient.
+    Any other text is ignored.
+
+    In the example above, you can specify `# recipient: age1zdy...`, `# public: age1zdy....` or even
+    just `# age1zdy....`
+
+You will need to add an entry into your `flake.nix` to ensure that the necessary `age` plugins
+are loaded when using Clan:
+
+```nix title="flake.nix"
+{
+  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
+  inputs.nixpkgs.follows = "clan-core/nixpkgs";
+
+  outputs =
+    { self, clan-core, ... }:
+    let
+      clan = clan-core.lib.clan {
+        inherit self;
+
+        meta.name = "myclan";
+
+        # Add Yubikey and FIDO2 HMAC plugins
+        # Note: the plugins listed here must be available in nixpkgs.
+        secrets.age.plugins = [
+          "age-plugin-yubikey"
+          "age-plugin-fido2-hmac"
+        ];
+
+        machines = {
+          # elided for brevity
+        };
+      };
+    in
+    {
+      inherit (clan) nixosConfigurations nixosModules clanInternals;
+
+      # elided for brevity
+    };
+}
+```

docs/site/guides/inventory.md

@@ -121,16 +121,3 @@ It is possible to add services to multiple machines via tags as shown
         };
     }
     ```
-
-### API specification
-
-**The complete schema specification is available [here](../reference/nix-api/inventory.md)**
-
-Or it can build anytime via:
-
-```sh
-nix build git+https://git.clan.lol/clan/clan-core#schemas.inventory
-> result
-> ├── schema.cue
-> └── schema.json
-```

docs/site/guides/macos.md

@@ -9,7 +9,7 @@ Currently, Clan supports the following features for macOS:
 - `clan machines update` for existing [nix-darwin](https://github.com/nix-darwin/nix-darwin) installations
 - Support for [vars](../guides/vars-backend.md)
 
-## Step 1: Add Your Machine to Your Clan Flake
+## Add Your Machine to Your Clan Flake
 
 In this example, we'll name the machine `yourmachine`. Replace this with your preferred machine name.
 
@@ -35,7 +35,7 @@ clan-core.lib.clan {
 }
 ```
 
-## Step 2: Add a `configuration.nix` for Your Machine
+## Add a `configuration.nix` for Your Machine
 
 Create the file `./machines/yourmachine/configuration.nix` with the following content (replace `yourmachine` with your chosen machine name):
 
@@ -48,7 +48,7 @@ Create the file `./machines/yourmachine/configuration.nix` with the following co
 
 After creating the file, run `git add` to ensure Nix recognizes it.
 
-## Step 3: Generate Vars (If Needed)
+## Generate Vars (If Needed)
 
 If your machine uses vars, generate them with:
 
@@ -58,12 +58,12 @@ clan vars generate yourmachine
 
 Replace `yourmachine` with your chosen machine name.
 
-## Step 4: Install Nix
+## Install Nix
 
 Install Nix on your macOS machine using one of the methods described in the [nix-darwin prerequisites](https://github.com/nix-darwin/nix-darwin?tab=readme-ov-file#prerequisites).
 
 
-## Step 5: Install nix-darwin
+## Install nix-darwin
 
 Upload your Clan flake to the macOS machine. Then, from within your flake directory, run:
 
@@ -73,7 +73,7 @@ sudo nix run nix-darwin/master#darwin-rebuild -- switch --flake .#yourmachine
 
 Replace `yourmachine` with your chosen machine name.
 
-## Step 6: Manage Your Machine with Clan
+## Manage Your Machine with Clan
 
 Once all the steps above are complete, you can start managing your machine with:
 

docs/site/guides/mesh-vpn.md

@@ -15,140 +15,86 @@ Clan
     Node B
 ```
 
-If you select multiple network technologies at the same time. e.g. (zerotier + yggdrassil)
-You must choose one of them as primary network and the machines are always connected via the primary network.
+This guide shows you how to configure `zerotier` through clan's `Inventory` System.
 
-This guide shows you how to configure `zerotier` either through `NixOS Options` directly, or Clan's `Inventory` System.
+## The Controller
 
+The controller is the initial entrypoint for new machines into the vpn.
+It will sign the id's of new machines.
+Once id's are signed, the controller's continuous operation is not essential.
+A good controller choice is nevertheless a machine that can always be reached for updates - so that new peers can be added to the network.
 
-=== "**Inventory**"
-    ## 1. Choose the Controller
+For the purpose of this guide we have two machines:
 
-    The controller is the initial entrypoint for new machines into the vpn.
-    It will sign the id's of new machines.
-    Once id's are signed, the controller's continuous operation is not essential.
-    A good controller choice is nevertheless a machine that can always be reached for updates - so that new peers can be added to the network.
+- The `controller` machine, which will be the zerotier controller.
+- The `new_machine` machine, which is the machine we want to add to the vpn network.
 
-    For the purpose of this guide we have two machines:
+## Configure the Service
 
-    - The `controller` machine, which will be the zerotier controller.
-    - The `new_machine` machine, which is the machine we want to add to the vpn network.
+```nix {.nix title="flake.nix" hl_lines="19-25"}
+{
+  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
+  inputs.nixpkgs.follows = "clan-core/nixpkgs";
 
-    ## 2. Configure the Inventory
+  outputs =
+    { self, clan-core, ... }:
+    let
+      clan = clan-core.lib.clan {
+        inherit self;
 
-    Note: consider picking a more descriptive name for the VPN than "default".
-    It will be added as an altname for the Zerotier virtual ethernet interface, and
-    will also be visible in the Zerotier app.
+        meta.name = "myclan";
 
-    ```nix
-    clan.inventory = {
-      services.zerotier.default = {
-        roles.controller.machines = [
-          "controller"
-        ];
-        roles.peer.machines = [
-          "new_machine"
-        ];
+        inventory.machines = {
+          controller = {};
+          new_machine = {};
+        };
+
+        inventory.instances = {
+          zerotier = {
+            # Assign the controller machine to the role "controller"
+            roles.controller.machines."controller" = {};
+
+            # All clan machines are zerotier peers
+            roles.peer.tags."all" = {};
+          };
+        };
       };
+    in
+    {
+      inherit (clan) nixosConfigurations nixosModules clanInternals;
+
+      # elided for brevity
     };
-    ```
+}
+```
 
-    ## 3. Apply the Configuration
-    Update the `controller` machine:
+## Apply the Configuration
 
-    ```bash
-    clan machines update controller
-    ```
+Update the `controller` machine first:
 
+```bash
+clan machines update controller
+```
 
-=== "**NixOS Options**"
-    ## 1. Set-Up the VPN Controller
+Then update all other peers:
 
-    The VPN controller is initially essential for providing configuration to new
-    peers. Once addresses are allocated, the controller's continuous operation is not essential.
+```bash
+clan machines update
+```
 
-    1. **Designate a Machine**: Label a machine as the VPN controller in the clan,
-       referred to as `<CONTROLLER>` henceforth in this guide.
-    2. **Add Configuration**: Input the following configuration to the NixOS
-       configuration of the controller machine:
-       ```nix
-       clan.core.networking.zerotier.controller = {
-         enable = true;
-         public = true;
-       };
-       ```
-    3. **Update the Controller Machine**: Execute the following:
-       ```bash
-       clan machines update <CONTROLLER>
-       ```
-       Your machine is now operational as the VPN controller.
+### Verify Connection
 
-    ## 2. Add Machines to the VPN
+On the `new_machine` run:
 
-    To introduce a new machine to the VPN, adhere to the following steps:
+```bash
+$ sudo zerotier-cli info
+```
 
-    1. **Update Configuration**: On the new machine, incorporate the following to its
-       configuration, substituting `<CONTROLLER>` with the controller machine name:
-       ```nix
-       { config, ... }: {
-         clan.core.networking.zerotier.networkId = builtins.readFile ../../vars/per-machine/<CONTROLLER>/zerotier/zerotier-network-id/value;
-       }
-       ```
-    1. **Update the New Machine**: Execute:
-       ```bash
-       $ clan machines update <NEW_MACHINE>
-       ```
-       Replace `<NEW_MACHINE>` with the designated new machine name.
+The status should be "ONLINE":
 
-        !!! Note "For Private Networks"
-            1. **Retrieve Zerotier Metadata**
-
-                === "From the repo"
-                    **Retrieve the ZeroTier IP**: In the clan repo, execute:
-                     ```console
-                     $ clan facts list <NEW_MACHINE> |  jq -r '.["zerotier-ip"]'
-                     ```
-
-                     The returned address is the Zerotier IP address of the machine.
-
-                === "On the new machine"
-                    **Retrieve the ZeroTier ID**: On the `new_machine`, execute:
-                     ```bash
-                     $ sudo zerotier-cli info
-                     ```
-                     Example Output:
-                     ```{.console, .no-copy}
-                     200 info d2c71971db 1.12.1 OFFLINE
-                     ```
-                     , where `d2c71971db` is the ZeroTier ID.
-
-
-            2. **Authorize the New Machine on the Controller**: On the controller machine,
-                 execute:
-
-                === "with ZerotierIP"
-                     ```bash
-                     $ sudo zerotier-members allow --member-ip <IP>
-                     ```
-                     Substitute `<IP>` with the ZeroTier IP obtained previously.
-                === "with ZerotierID"
-                     ```bash
-                     $ sudo zerotier-members allow <ID>
-                     ```
-                     Substitute `<ID>` with the ZeroTier ID obtained previously.
-
-    2. **Verify Connection**: On the `new_machine`, re-execute:
-       ```bash
-       $ sudo zerotier-cli info
-       ```
-       The status should now be "ONLINE":
-       ```{.console, .no-copy}
-       200 info d2c71971db 1.12.1 ONLINE
-       ```
-
-!!! success "Congratulations!"
-    The new machine is now part of the VPN, and the ZeroTier
-    configuration on NixOS within the Clan project is complete.
+```{.console, .no-copy}
+200 info d2c71971db 1.12.1 ONLINE
+```
 
 ## Further
 
@@ -158,3 +104,45 @@ In the future we plan to add additional network technologies like tinc, head/tai
 We chose zerotier because in our tests it was a straight forwards solution to bootstrap.
 It allows you to selfhost a controller and the controller doesn't need to be globally reachable.
 Which made it a good fit for starting the project.
+
+## Debugging
+
+### Retrieve the ZeroTier ID
+
+In the repo:
+
+```console
+$ clan vars list <machineName>
+```
+
+```{.console, .no-copy}
+$ clan vars list controller
+# ... elided
+zerotier/zerotier-identity-secret: ********
+zerotier/zerotier-ip: fd0a:b849:2928:1234:c99:930a:a959:2928
+zerotier/zerotier-network-id: 0aa959282834000c
+```
+
+On the machine:
+
+```bash
+$ sudo zerotier-cli info
+```
+
+#### Manually Authorize a Machine on the Controller
+
+=== "with ZerotierIP"
+
+      ```bash
+      $ sudo zerotier-members allow --member-ip <IP>
+      ```
+
+      Substitute `<IP>` with the ZeroTier IP obtained previously.
+
+=== "with ZerotierID"
+
+      ```bash
+      $ sudo zerotier-members allow <ID>
+      ```
+
+      Substitute `<ID>` with the ZeroTier ID obtained previously.

docs/site/guides/migrations/migrate-inventory-services.md

@@ -74,9 +74,7 @@ instances = {
 
 ## Steps to Migrate
 
-
-
-### 1. Move `services` entries to `instances`
+### Move `services` entries to `instances`
 
 Check if a service that you use has been migrated [In our reference](../../reference/clanServices/index.md)
 
@@ -96,7 +94,7 @@ Each nested service-instance-pair becomes a flat key, like `borgbackup.simple
 
 ---
 
-### 2. Add `module.name` and `module.input`
+### Add `module.name` and `module.input`
 
 Each instance must declare the module name and flake input it comes from:
 
@@ -117,7 +115,7 @@ Then refer to it as `input = "clan-core"`.
 
 ---
 
-### 3. Move role and machine config under `roles`
+### Move role and machine config under `roles`
 
 In the new system:
 

docs/site/guides/secure-boot.md

@@ -1,9 +1,11 @@
 At the moment, NixOS/Clan does not support [Secure Boot](https://wiki.gentoo.org/wiki/Secure_Boot). Therefore, you need to disable it in the BIOS. You can watch this [video guide](https://www.youtube.com/watch?v=BKVShiMUePc) or follow the instructions below:
 
-### Step 1: Insert the USB Stick
+## Insert the USB Stick
+
 - Begin by inserting the USB stick into a USB port on your computer.
 
-### Step 2: Access the UEFI/BIOS Menu
+## Access the UEFI/BIOS Menu
+
 - Restart your computer.
 - As your computer restarts, press the appropriate key to enter the UEFI/BIOS settings.
 ??? tip "The key depends on your laptop or motherboard manufacturer. Click to see a reference list:"
@@ -32,18 +34,22 @@ At the moment, NixOS/Clan does not support [Secure Boot](https://wiki.gentoo.org
     !!! Note
         Pressing the key quickly and repeatedly is sometimes necessary to access the UEFI/BIOS menu, as the window to enter this mode is brief.
 
-### Step 3: Access Advanced Mode (Optional)
+## Access Advanced Mode (Optional)
+
 - If your UEFI/BIOS has a `Simple` or `Easy` mode interface, look for an option labeled `Advanced Mode` (often found in the lower right corner).
 - Click on `Advanced Mode` to access more settings. This step is optional, as your boot settings might be available in the basic view.
 
-### Step 4: Disable Secure Boot
+## Disable Secure Boot
+
 - Locate the `Secure Boot` option in your UEFI/BIOS settings. This is typically found under a `Security` tab, `Boot` tab, or a similarly named section.
 - Set the `Secure Boot` option to `Disabled`.
 
-### Step 5: Change Boot Order
+## Change Boot Order
+
 - Find the option to adjust the boot order—often labeled `Boot Order`, `Boot Sequence`, or `Boot Priority`.
 - Ensure that your USB device is set as the first boot option. This allows your computer to boot from the USB stick.
 
-### Step 6: Save and Exit
+## Save and Exit
+
 - Save your changes before exiting the UEFI/BIOS menu. Look for a `Save & Exit` option or press the corresponding function key (often `F10`).
 - Your computer should now restart and boot from the USB stick.

docs/site/guides/target-host.md

@@ -0,0 +1,82 @@
+# How to Set `targetHost` for a Machine
+
+The `targetHost` defines where the machine can be reached for operations like SSH or deployment. You can set it in two ways, depending on your use case.
+
+---
+
+## ✅ Option 1: Use the Inventory (Recommended for Static Hosts)
+
+If the hostname is **static**, like `server.example.com`, set it in the **inventory**:
+
+```{.nix title="flake.nix" hl_lines="8"}
+{
+  # edlided
+  outputs =
+    { self, clan-core, ... }:
+    let
+      clan = clan-core.lib.clan {
+        inventory.machines.jon = {
+            deploy.targetHost = "root@server.example.com";
+        };
+      };
+    in
+    {
+      inherit (clan.config) nixosConfigurations nixosModules clanInternals;
+      # elided
+    };
+}
+```
+
+This is fast, simple and explicit, and doesn’t require evaluating the NixOS config. We can also displayed it in the clan-cli or clan-app.
+
+---
+
+## ✅ Option 2: Use NixOS (Only for Dynamic Hosts)
+
+If your target host depends on a **dynamic expression** (like using the machine’s evaluated FQDN), set it inside the NixOS module:
+
+```{.nix title="flake.nix" hl_lines="8"}
+{
+  # edlided
+  outputs =
+    { self, clan-core, ... }:
+    let
+      clan = clan-core.lib.clan {
+        machines.jon = {config, ...}: {
+            clan.core.networking.targetHost = "jon@${config.networking.fqdn}";
+        };
+      };
+    in
+    {
+      inherit (clan.config) nixosConfigurations nixosModules clanInternals;
+      # elided
+    };
+}
+```
+
+Use this **only if the value cannot be made static**, because it’s slower and won't be displayed in the clan-cli or clan-app yet.
+
+---
+
+## 📝 TL;DR
+
+| Use Case                  | Use Inventory? | Example                          |
+| ------------------------- | -------------- | -------------------------------- |
+| Static hostname           | ✅ Yes          | `root@server.example.com`        |
+| Dynamic config expression | ❌ No           | `jon@${config.networking.fqdn}` |
+
+---
+
+## 🚀 Coming Soon: Unified Networking Module
+
+We’re working on a new networking module that will automatically do all of this for you.
+
+- Easier to use
+- Sane defaults: You’ll always be able to reach the machine — no need to worry about hostnames.
+- ✨ Migration from **either method** will be supported and simple.
+
+## Summary
+
+- Ask: *Does this hostname dynamically change based on NixOS config?*
+- If **no**, use the inventory.
+- If **yes**, then use NixOS config.

docs/site/intern/api.md

@@ -0,0 +1,7 @@
+---
+template: options.html
+hide:
+  - navigation
+  - toc
+---
+<redoc src="/openapi.json" />

docs/site/intern/index.md

@@ -0,0 +1,25 @@
+# Developer Documentation
+
+!!! Danger
+    This documentation is **not** intended for external users. It may contain low-level details and internal-only interfaces.*
+
+Welcome to the internal developer documentation.
+
+This section is intended for contributors, engineers, and internal stakeholders working directly with our system, tooling, and APIs. It provides a technical overview of core components, internal APIs, conventions, and patterns that support the platform.
+
+Our goal is to make the internal workings of the system **transparent, discoverable, and consistent** — helping you contribute confidently, troubleshoot effectively, and build faster.
+
+## What's Here?
+
+!!! note "docs migration ongoing"
+
+- [ ] **API Reference**: 🚧🚧🚧 Detailed documentation of internal API functions, inputs, and expected outputs. 🚧🚧🚧
+- [ ] **System Concepts**: Architectural overviews and domain-specific guides.
+- [ ] **Development Guides**: How to test, extend, or integrate with key components.
+- [ ] **Design Notes**: Rationales behind major design decisions or patterns.
+
+## Who is This For?
+
+* Developers contributing to the platform
+* Engineers debugging or extending internal systems
+* Anyone needing to understand **how** and **why** things work under the hood

flake.lock

@@ -16,11 +16,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1751413887,
-        "narHash": "sha256-+ut7DrSwamExIvaCFdiTYD88NTSYJFG2CEOvCha59vI=",
-        "rev": "246f0d66547d073af6249e4f7852466197e871ed",
+        "lastModified": 1751846468,
+        "narHash": "sha256-h0mpWZIOIAKj4fmLNyI2HDG+c0YOkbYmyJXSj/bQ9s0=",
+        "rev": "a2166c13b0cb3febdaf36391cd2019aa2ccf4366",
         "type": "tarball",
-        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/246f0d66547d073af6249e4f7852466197e871ed.tar.gz"
+        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/a2166c13b0cb3febdaf36391cd2019aa2ccf4366.tar.gz"
       },
       "original": {
         "type": "tarball",
@@ -34,11 +34,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1750903843,
-        "narHash": "sha256-Ng9+f0H5/dW+mq/XOKvB9uwvGbsuiiO6HrPdAcVglCs=",
+        "lastModified": 1751854533,
+        "narHash": "sha256-U/OQFplExOR1jazZY4KkaQkJqOl59xlh21HP9mI79Vc=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "83c4da299c1d7d300f8c6fd3a72ac46cb0d59aae",
+        "rev": "16b74a1e304197248a1bc663280f2548dbfcae3c",
         "type": "github"
       },
       "original": {
@@ -67,52 +67,6 @@
         "type": "github"
       }
     },
-    "flake-utils": {
-      "inputs": {
-        "systems": [
-          "systems"
-        ]
-      },
-      "locked": {
-        "lastModified": 1731533236,
-        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "ixx": {
-      "inputs": {
-        "flake-utils": [
-          "nuschtos",
-          "flake-utils"
-        ],
-        "nixpkgs": [
-          "nuschtos",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1748294338,
-        "narHash": "sha256-FVO01jdmUNArzBS7NmaktLdGA5qA3lUMJ4B7a05Iynw=",
-        "owner": "NuschtOS",
-        "repo": "ixx",
-        "rev": "cc5f390f7caf265461d4aab37e98d2292ebbdb85",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NuschtOS",
-        "ref": "v0.0.8",
-        "repo": "ixx",
-        "type": "github"
-      }
-    },
     "nix-darwin": {
       "inputs": {
         "nixpkgs": [
@@ -164,51 +118,25 @@
     "nixpkgs": {
       "locked": {
         "lastModified": 315532800,
-        "narHash": "sha256-VgDAFPxHNhCfC7rI5I5wFqdiVJBH43zUefVo8hwo7cI=",
-        "rev": "41da1e3ea8e23e094e5e3eeb1e6b830468a7399e",
+        "narHash": "sha256-mUlYenGbsUFP0A3EhfKJXmUl5+MQGJLhoEop2t3g5p4=",
+        "rev": "ceb24d94c6feaa4e8737a8e2bd3cf71c3a7eaaa0",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre814815.41da1e3ea8e2/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre826033.ceb24d94c6fe/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",
         "url": "https://nixos.org/channels/nixpkgs-unstable/nixexprs.tar.xz"
       }
     },
-    "nuschtos": {
-      "inputs": {
-        "flake-utils": [
-          "flake-utils"
-        ],
-        "ixx": "ixx",
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1749730855,
-        "narHash": "sha256-L3x2nSlFkXkM6tQPLJP3oCBMIsRifhIDPMQQdHO5xWo=",
-        "owner": "NuschtOS",
-        "repo": "search",
-        "rev": "8dfe5879dd009ff4742b668d9c699bc4b9761742",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NuschtOS",
-        "repo": "search",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
         "data-mesher": "data-mesher",
         "disko": "disko",
         "flake-parts": "flake-parts",
-        "flake-utils": "flake-utils",
         "nix-darwin": "nix-darwin",
         "nix-select": "nix-select",
         "nixos-facter-modules": "nixos-facter-modules",
         "nixpkgs": "nixpkgs",
-        "nuschtos": "nuschtos",
         "sops-nix": "sops-nix",
         "systems": "systems",
         "treefmt-nix": "treefmt-nix"
@@ -221,11 +149,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1750119275,
-        "narHash": "sha256-Rr7Pooz9zQbhdVxux16h7URa6mA80Pb/G07T4lHvh0M=",
+        "lastModified": 1751606940,
+        "narHash": "sha256-KrDPXobG7DFKTOteqdSVeL1bMVitDcy7otpVZWDE6MA=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "77c423a03b9b2b79709ea2cb63336312e78b72e2",
+        "rev": "3633fc4acf03f43b260244d94c71e9e14a2f6e0d",
         "type": "github"
       },
       "original": {

flake.nix

@@ -35,19 +35,13 @@
       };
     };
 
-    # dependencies needed for nuschtos
-    flake-utils.url = "github:numtide/flake-utils";
-    flake-utils.inputs.systems.follows = "systems";
-    nuschtos.url = "github:NuschtOS/search";
-    nuschtos.inputs.nixpkgs.follows = "nixpkgs";
-    nuschtos.inputs.flake-utils.follows = "flake-utils";
   };
 
   outputs =
     inputs@{
-      flake-parts,
       nixpkgs,
       systems,
+      flake-parts,
       ...
     }:
     let
@@ -56,10 +50,25 @@
         optional
         pathExists
         ;
+
+      loadDevFlake =
+        path:
+        let
+          flakeHash = nixpkgs.lib.fileContents "${toString path}.narHash";
+          flakePath = "path:${toString path}?narHash=${flakeHash}";
+        in
+        builtins.getFlake (builtins.unsafeDiscardStringContext flakePath);
+
+      devFlake = builtins.tryEval (loadDevFlake ./devFlake/private);
+
+      privateInputs = if devFlake.success then devFlake.value.inputs else { };
     in
     flake-parts.lib.mkFlake { inherit inputs; } (
       { ... }:
       {
+        _module.args = {
+          inherit privateInputs;
+        };
         clan = {
           meta.name = "clan-core";
           inventory = {

formatter.nix

@@ -4,7 +4,7 @@
   perSystem =
     { self', pkgs, ... }:
     {
-      treefmt.projectRootFile = ".git/config";
+      treefmt.projectRootFile = "LICENSE.md";
       treefmt.programs.shellcheck.enable = true;
 
       treefmt.programs.mypy.enable = true;

lib/default.nix

@@ -37,6 +37,7 @@ lib.fix (
     inventory = clanLib.callLib ./modules/inventory { };
     modules = clanLib.callLib ./modules/inventory/frontmatter { };
     test = clanLib.callLib ./test { };
+    flake-inputs = clanLib.callLib ./flake-inputs.nix { };
     # Custom types
     types = clanLib.callLib ./types { };
 

lib/flake-inputs.nix

@@ -0,0 +1,18 @@
+{ ... }:
+{
+  /**
+    Generate nix-unit input overrides for tests
+
+    # Example
+    ```nix
+    inputOverrides = clanLib.flake-inputs.getOverrides inputs;
+    ```
+  */
+  getOverrides =
+    inputs:
+    builtins.concatStringsSep " " (
+      builtins.map (input: " --override-input ${input} ${inputs.${input}}") (
+        builtins.filter (name: name != "self") (builtins.attrNames inputs)
+      )
+    );
+}

lib/introspection/flake-module.nix

@@ -1,8 +1,6 @@
 { self, inputs, ... }:
 let
-  inputOverrides = builtins.concatStringsSep " " (
-    builtins.map (input: " --override-input ${input} ${inputs.${input}}") (builtins.attrNames inputs)
-  );
+  inputOverrides = self.clanLib.flake-inputs.getOverrides inputs;
 in
 {
   perSystem =

lib/modules/clan/module.nix

@@ -229,8 +229,6 @@ in
     clanInternals = {
       inventoryClass =
         let
-          localModuleSet =
-            lib.filterAttrs (n: _: !inventory._legacyModules ? ${n}) inventory.modules // config.modules;
           flakeInputs = config.self.inputs;
         in
         {
@@ -240,7 +238,7 @@ in
           imports = [
             ../inventoryClass/builder/default.nix
             (lib.modules.importApply ../inventoryClass/service-list-from-inputs.nix {
-              inherit flakeInputs clanLib localModuleSet;
+              inherit flakeInputs clanLib;
             })
             {
               inherit inventory directory;

lib/modules/flake-module.nix

@@ -4,9 +4,7 @@
   ...
 }:
 let
-  inputOverrides = builtins.concatStringsSep " " (
-    builtins.map (input: " --override-input ${input} ${inputs.${input}}") (builtins.attrNames inputs)
-  );
+  inputOverrides = self.clanLib.flake-inputs.getOverrides inputs;
 in
 {
   imports = [

lib/modules/inventory/distributed-service/flake-module.nix

@@ -1,8 +1,6 @@
 { self, inputs, ... }:
 let
-  inputOverrides = builtins.concatStringsSep " " (
-    builtins.map (input: " --override-input ${input} ${inputs.${input}}") (builtins.attrNames inputs)
-  );
+  inputOverrides = self.clanLib.flake-inputs.getOverrides inputs;
 in
 {
   perSystem =
@@ -12,6 +10,23 @@ in
       system,
       ...
     }:
+    let
+      # Common filtered source for inventory tests
+      inventoryTestsSrc = lib.fileset.toSource {
+        root = ../../../..;
+        fileset = lib.fileset.unions [
+          ../../../../flake.nix
+          ../../../../flake.lock
+          (lib.fileset.fileFilter (file: file.name == "flake-module.nix") ../../../..)
+          ../../../../flakeModules
+          ../../../../lib
+          ../../../../nixosModules/clanCore
+          ../../../../clanModules/borgbackup
+          ../../../../machines
+          ../../../../inventory.json
+        ];
+      };
+    in
     {
       # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.<attrName>
       legacyPackages.evalTests-distributedServices = import ./tests {
@@ -29,7 +44,7 @@ in
             --extra-experimental-features flakes \
             --show-trace \
             ${inputOverrides} \
-            --flake ${self}#legacyPackages.${system}.evalTests-distributedServices
+            --flake ${inventoryTestsSrc}#legacyPackages.${system}.evalTests-distributedServices
 
           touch $out
         '';
@@ -39,7 +54,7 @@ in
             --extra-experimental-features flakes \
             --show-trace \
             ${inputOverrides} \
-            --flake ${self}#legacyPackages.${system}.eval-tests-resolve-module
+            --flake ${inventoryTestsSrc}#legacyPackages.${system}.eval-tests-resolve-module
 
           touch $out
         '';

lib/modules/inventory/flake-module.nix

@@ -5,9 +5,7 @@
   ...
 }:
 let
-  inputOverrides = builtins.concatStringsSep " " (
-    builtins.map (input: " --override-input ${input} ${inputs.${input}}") (builtins.attrNames inputs)
-  );
+  inputOverrides = self.clanLib.flake-inputs.getOverrides inputs;
 in
 {
   imports = [
@@ -70,12 +68,18 @@ in
             --show-trace \
             ${inputOverrides} \
             --flake ${
-              self.filter {
-                include = [
-                  "flakeModules"
-                  "lib"
-                  "clanModules/flake-module.nix"
-                  "clanModules/borgbackup"
+              lib.fileset.toSource {
+                root = ../../..;
+                fileset = lib.fileset.unions [
+                  ../../../flake.nix
+                  ../../../flake.lock
+                  (lib.fileset.fileFilter (file: file.name == "flake-module.nix") ../../..)
+                  ../../../flakeModules
+                  ../../../lib
+                  ../../../nixosModules/clanCore
+                  ../../../clanModules/borgbackup
+                  ../../../machines
+                  ../../../inventory.json
                 ];
               }
             }#legacyPackages.${system}.evalTests-inventory

lib/modules/inventoryClass/interface.nix

@@ -394,6 +394,7 @@ in
                     options = {
                       # ModuleSpec
                       module = lib.mkOption {
+                        default = { };
                         type = types.submodule {
                           options.input = lib.mkOption {
                             type = types.nullOr types.str;

lib/modules/inventoryClass/service-list-from-inputs.nix

@@ -1,12 +1,9 @@
 {
   flakeInputs,
   clanLib,
-  localModuleSet,
 }:
 { lib, config, ... }:
-
 let
-
   inspectModule =
     inputName: moduleName: module:
     let
@@ -28,16 +25,30 @@ in
 {
   options.modulesPerSource = lib.mkOption {
     # { sourceName :: { moduleName :: {} }}
+    readOnly = true;
+    type = lib.types.raw;
     default =
       let
         inputsWithModules = lib.filterAttrs (_inputName: v: v ? clan.modules) flakeInputs;
-
       in
       lib.mapAttrs (
         inputName: v: lib.mapAttrs (inspectModule inputName) v.clan.modules
       ) inputsWithModules;
   };
   options.localModules = lib.mkOption {
-    default = lib.mapAttrs (inspectModule "self") localModuleSet;
+    readOnly = true;
+    type = lib.types.raw;
+    default = config.modulesPerSource.self;
+  };
+  options.templatesPerSource = lib.mkOption {
+    # { sourceName :: { moduleName :: {} }}
+    readOnly = true;
+    type = lib.types.raw;
+    default =
+      let
+        inputsWithTemplates = lib.filterAttrs (_inputName: v: v ? clan.templates) flakeInputs;
+      in
+      lib.mapAttrs (_inputName: v: lib.mapAttrs (_n: t: t) v.clan.templates) inputsWithTemplates;
+
   };
 }

lib/test/container-test-driver/pyproject.toml

@@ -15,7 +15,7 @@ find = {}
 [tool.setuptools.package-data]
 test_driver = ["py.typed"]
 [tool.mypy]
-python_version = "3.12"
+python_version = "3.13"
 warn_redundant_casts = true
 disallow_untyped_calls = true
 disallow_untyped_defs = true

lib/test/flakeModules.nix

@@ -16,7 +16,7 @@
   */
   makeEvalChecks =
     {
-      self,
+      fileset,
       inputs,
       testName,
       tests,
@@ -24,9 +24,7 @@
       testArgs ? { },
     }:
     let
-      inputOverrides = builtins.concatStringsSep " " (
-        builtins.map (input: " --override-input ${input} ${inputs.${input}}") (builtins.attrNames inputs)
-      );
+      inputOverrides = clanLib.flake-inputs.getOverrides inputs;
       attrName = "eval-tests-${testName}";
     in
     {
@@ -41,16 +39,44 @@
         }
         // testArgs
       );
-      checks.${attrName} = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
-        export HOME="$(realpath .)"
+      checks.${attrName} =
+        let
+          # The root is two directories up from where this file is located
+          root = ../..;
 
-        nix-unit --eval-store "$HOME" \
-          --extra-experimental-features flakes \
-          --show-trace \
-          ${inputOverrides} \
-          --flake ${self}#legacyPackages.${system}.${attrName}
-        touch $out
-      '';
+          # Combine the user-provided fileset with all flake-module.nix files
+          # and other essential files
+          src = lib.fileset.toSource {
+            inherit root;
+            fileset = lib.fileset.unions [
+              # Core flake files
+              (root + "/flake.nix")
+              (root + "/flake.lock")
+
+              # All flake-module.nix files anywhere in the tree
+              (lib.fileset.fileFilter (file: file.name == "flake-module.nix") root)
+
+              # The flakeModules/clan.nix if it exists
+              (lib.fileset.maybeMissing (root + "/flakeModules/clan.nix"))
+
+              # Core libraries
+              (root + "/lib")
+
+              # User-provided fileset
+              fileset
+            ];
+          };
+        in
+        pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
+          export HOME="$(realpath .)"
+
+          nix-unit --eval-store "$HOME" \
+            --extra-experimental-features flakes \
+            --show-trace \
+            ${inputOverrides} \
+            --flake ${src}#legacyPackages.${system}.${attrName}
+          touch $out
+        '';
 
     };
 }

lib/types/flake-module.nix

@@ -1,4 +1,9 @@
-{ self, inputs, ... }:
+{
+  self,
+  inputs,
+  lib,
+  ...
+}:
 {
   perSystem =
     { ... }:
@@ -10,7 +15,11 @@
       test-types-module = (
         self.clanLib.test.flakeModules.makeEvalChecks {
           module = throw "";
-          inherit self inputs;
+          inherit inputs;
+          fileset = lib.fileset.unions [
+            # Only lib is needed for type tests
+            ../../lib
+          ];
           testName = "types";
           tests = ./tests.nix;
           # Optional arguments passed to the test

nixosModules/clanCore/default.nix

@@ -19,6 +19,7 @@
       ./nixos-facter.nix
       ./vm.nix
       ./machine-id
+      ./state-version
       ./wayland-proxy-virtwl.nix
       ./zerotier
       ./zfs.nix

nixosModules/clanCore/outputs.nix

@@ -1,7 +1,6 @@
 {
   config,
   lib,
-  pkgs,
   ...
 }:
 {
@@ -24,6 +23,14 @@
           description = ''
             the location of the deployment.json file
           '';
+          default = throw ''
+            deployment.json file generation has been removed in favor of direct selectors.
+
+            Please upgrade your clan-cli to the latest version.
+
+            The deployment data is now accessed directly from the configuration
+            instead of being written to a separate JSON file.
+          '';
         };
         deployment.buildHost = lib.mkOption {
           type = lib.types.nullOr lib.types.str;
@@ -83,8 +90,5 @@
       inherit (config.system.clan.deployment) nixosMobileWorkaround;
       inherit (config.clan.deployment) requireExplicitUpdate;
     };
-    system.clan.deployment.file = pkgs.writeText "deployment.json" (
-      builtins.toJSON config.system.clan.deployment.data
-    );
   };
 }

nixosModules/clanCore/state-version/tests/sops/users/admin/key.json

@@ -0,0 +1,4 @@
+{
+  "publickey": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+  "type": "age"
+}

nixosModules/clanCore/state-version/tests/vars/per-machine/server/state-version/version/value

@@ -0,0 +1 @@
+25.11

nixosModules/clanCore/vars/default.nix

@@ -40,6 +40,18 @@ in
   };
 
   config = {
+    # Check for removed passBackend option usage
+    assertions = [
+      {
+        assertion = config.clan.core.vars.settings.passBackend == null;
+        message = ''
+          The option `clan.core.vars.settings.passBackend' has been removed.
+          Use clan.core.vars.password-store.passPackage instead.
+          Set it to pkgs.pass for GPG or pkgs.passage for age encryption.
+        '';
+      }
+    ];
+
     # check all that all non-secret files have no owner/group/mode set
     warnings = lib.foldl' (
       warnings: generator:
@@ -73,10 +85,5 @@ in
       ) [ ] (lib.attrValues generator.files)
     ) [ ] (lib.attrValues config.clan.core.vars.generators);
 
-    system.clan.deployment.data = {
-      vars = config.clan.core.vars._serialized;
-      inherit (config.clan.core.networking) targetHost buildHost;
-      inherit (config.clan.core.deployment) requireExplicitUpdate;
-    };
   };
 }

nixosModules/clanCore/vars/eval-tests/default.nix

@@ -1,4 +1,4 @@
-{ lib, pkgs, ... }:
+{ lib, pkgs }:
 let
   eval =
     module:

nixosModules/clanCore/vars/flake-module.nix

@@ -5,18 +5,14 @@
   ...
 }:
 let
-  inputOverrides = builtins.concatStringsSep " " (
-    builtins.map (input: " --override-input ${input} ${inputs.${input}}") (builtins.attrNames inputs)
-  );
+  inputOverrides = self.clanLib.flake-inputs.getOverrides inputs;
 in
 {
   perSystem =
     { system, pkgs, ... }:
     {
       legacyPackages.evalTests-module-clan-vars = import ./eval-tests {
-        inherit lib;
-        clan-core = self;
-        pkgs = inputs.nixpkgs.legacyPackages.${system};
+        inherit lib pkgs;
       };
       checks.eval-module-clan-vars = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
         export HOME="$(realpath .)"
@@ -26,11 +22,15 @@ in
           --show-trace \
           ${inputOverrides} \
           --flake ${
-            self.filter {
-              include = [
-                "flakeModules"
-                "nixosModules"
-                "lib"
+            lib.fileset.toSource {
+              root = ../../..;
+              fileset = lib.fileset.unions [
+                ../../../flake.nix
+                ../../../flake.lock
+                (lib.fileset.fileFilter (file: file.name == "flake-module.nix") ../../..)
+                ../../../flakeModules/clan.nix
+                ../../../lib
+                ../../../nixosModules/clanCore/vars
               ];
             }
           }#legacyPackages.${system}.evalTests-module-clan-vars

nixosModules/clanCore/vars/interface.nix

@@ -34,50 +34,6 @@ let
 in
 {
   options = {
-    _serialized = lib.mkOption {
-      readOnly = true;
-      internal = true;
-      description = ''
-        JSON serialization of the generators.
-        This is read from the python client to generate the specified resources.
-      '';
-      default = {
-        # TODO: We don't support per-machine choice of backends
-        # Configuring different backend doesn't work, this information should be made read only and configured
-        # Via clan.settings instead.
-        inherit (config.settings) secretModule publicModule;
-        # Serialize generators, so that we can use them in the python client
-        # This need to be done because we have some non-serializable values in the generators
-        # Like the finalScript (derivation) or pkgs.
-        generators = lib.flip lib.mapAttrs config.generators (
-          _name: generator: {
-            inherit (generator)
-              name
-              dependencies
-              validationHash
-              migrateFact
-              share
-              prompts
-              ;
-
-            files = lib.flip lib.mapAttrs generator.files (
-              _name: file: {
-                inherit (file)
-                  name
-                  owner
-                  group
-                  mode
-                  deploy
-                  secret
-                  neededFor
-                  ;
-              }
-            );
-          }
-        );
-      };
-    };
-
     settings = import ./settings-opts.nix { inherit lib; };
     generators = lib.mkOption {
       description = ''

nixosModules/clanCore/vars/secret/password-store.nix

@@ -54,7 +54,7 @@ in
 {
   _class = "nixos";
 
-  options.clan.vars.password-store = {
+  options.clan.core.vars.password-store = {
     secretLocation = lib.mkOption {
       type = lib.types.path;
       default = "/etc/secret-vars";
@@ -62,10 +62,15 @@ in
         location where the tarball with the password-store secrets will be uploaded to and the manifest
       '';
     };
+    passPackage = lib.mkOption {
+      type = lib.types.package;
+      default = pkgs.pass;
+      description = ''
+        Password store package to use. Can be pkgs.pass for GPG-based storage or pkgs.passage for age-based storage.
+      '';
+    };
   };
   config = {
-    system.clan.deployment.data.password-store.secretLocation =
-      config.clan.vars.password-store.secretLocation;
     clan.core.vars.settings =
       lib.mkIf (config.clan.core.vars.settings.secretStore == "password-store")
         {
@@ -78,7 +83,7 @@ in
                 else if file.config.neededFor == "services" then
                   "/run/secrets/${file.config.generatorName}/${file.config.name}"
                 else if file.config.neededFor == "activation" then
-                  "${config.clan.password-store.secretLocation}/activation/${file.config.generatorName}/${file.config.name}"
+                  "${config.clan.core.vars.password-store.secretLocation}/activation/${file.config.generatorName}/${file.config.name}"
                 else if file.config.neededFor == "partitioning" then
                   "/run/partitioning-secrets/${file.config.generatorName}/${file.config.name}"
                 else
@@ -97,7 +102,7 @@ in
               ]
               ''
                 [ -e /run/current-system ] || echo setting up secrets...
-                ${installSecretTarball}/bin/install-secret-tarball ${config.clan.vars.password-store.secretLocation}/secrets_for_users.tar.gz /run/user-secrets
+                ${installSecretTarball}/bin/install-secret-tarball ${config.clan.core.vars.password-store.secretLocation}/secrets_for_users.tar.gz /run/user-secrets
               ''
             // lib.optionalAttrs (config.system ? dryActivationScript) {
               supportsDryActivation = true;
@@ -113,7 +118,7 @@ in
               ]
               ''
                 [ -e /run/current-system ] || echo setting up secrets...
-                ${installSecretTarball}/bin/install-secret-tarball ${config.clan.vars.password-store.secretLocation}/secrets.tar.gz /run/secrets
+                ${installSecretTarball}/bin/install-secret-tarball ${config.clan.core.vars.password-store.secretLocation}/secrets.tar.gz /run/secrets
               ''
             // lib.optionalAttrs (config.system ? dryActivationScript) {
               supportsDryActivation = true;
@@ -131,7 +136,7 @@ in
             serviceConfig = {
               Type = "oneshot";
               ExecStart = [
-                "${installSecretTarball}/bin/install-secret-tarball ${config.clan.vars.password-store.secretLocation}/secrets_for_users.tar.gz /run/user-secrets"
+                "${installSecretTarball}/bin/install-secret-tarball ${config.clan.core.vars.password-store.secretLocation}/secrets_for_users.tar.gz /run/user-secrets"
               ];
               RemainAfterExit = true;
             };
@@ -144,7 +149,7 @@ in
             serviceConfig = {
               Type = "oneshot";
               ExecStart = [
-                "${installSecretTarball}/bin/install-secret-tarball ${config.clan.vars.password-store.secretLocation}/secrets.tar.gz /run/secrets"
+                "${installSecretTarball}/bin/install-secret-tarball ${config.clan.core.vars.password-store.secretLocation}/secrets.tar.gz /run/secrets"
               ];
               RemainAfterExit = true;
             };

nixosModules/clanCore/vars/settings-opts.nix

@@ -15,17 +15,6 @@
     '';
   };
 
-  passBackend = lib.mkOption {
-    type = lib.types.enum [
-      "passage"
-      "pass"
-    ];
-    default = "pass";
-    description = ''
-      password-store backend to use. Valid options are `pass` and `passage`
-    '';
-  };
-
   secretModule = lib.mkOption {
     type = lib.types.str;
     internal = true;
@@ -65,4 +54,15 @@
       the python import path to the public module
     '';
   };
+
+  # Legacy option that guides migration
+  passBackend = lib.mkOption {
+    type = lib.types.nullOr lib.types.str;
+    default = null;
+    visible = false;
+    description = ''
+      DEPRECATED: This option has been removed. Use clan.vars.password-store.passPackage instead.
+      Set it to pkgs.pass for GPG or pkgs.passage for age encryption.
+    '';
+  };
 }

pkgs/clan-app/clan_app/api/file_gtk.py

@@ -1,4 +1,3 @@
-# ruff: noqa: N801
 import gi
 
 gi.require_version("Gtk", "4.0")

pkgs/clan-app/clan_app/app.py

@@ -8,14 +8,10 @@ from dataclasses import dataclass
 from pathlib import Path
 
 import clan_lib.machines.actions  # noqa: F401
-from clan_lib.api import API, tasks
-
-# TODO: We have to manually import python files to make the API.register be triggered.
-# We NEED to fix this, as this is super unintuitive and error-prone.
-from clan_lib.api.tasks import list_tasks as dummy_list  # noqa: F401
+from clan_lib.api import API, load_in_all_api_functions, tasks
 from clan_lib.custom_logger import setup_logging
 from clan_lib.dirs import user_data_dir
-from clan_lib.log_manager import LogManager
+from clan_lib.log_manager import LogGroupConfig, LogManager
 from clan_lib.log_manager import api as log_manager_api
 
 from clan_app.api.file_gtk import open_file
@@ -45,16 +41,22 @@ def app_run(app_opts: ClanAppOptions) -> int:
 
     webview = Webview(debug=app_opts.debug)
     webview.title = "Clan App"
-    # This seems to call the gtk api correctly but and gtk also seems to our icon, but somehow the icon is not loaded.
 
-    # Init LogManager global in log_manager_api module
-    log_manager_api.LOG_MANAGER_INSTANCE = LogManager(
-        base_dir=user_data_dir() / "clan-app" / "logs"
+    # Add a log group ["clans", <dynamic_name>, "machines", <dynamic_name>]
+    log_manager = LogManager(base_dir=user_data_dir() / "clan-app" / "logs")
+    clan_log_group = LogGroupConfig("clans", "Clans").add_child(
+        LogGroupConfig("machines", "Machines")
     )
+    log_manager = log_manager.add_root_group_config(clan_log_group)
+    # Init LogManager global in log_manager_api module
+    log_manager_api.LOG_MANAGER_INSTANCE = log_manager
 
-    # Init BAKEND_THREADS in tasks module
+    # Init BAKEND_THREADS global in tasks module
     tasks.BAKEND_THREADS = webview.threads
 
+    # Populate the API global with all functions
+    load_in_all_api_functions()
+
     API.overwrite_fn(open_file)
     webview.bind_jsonschema_api(API, log_manager=log_manager_api.LOG_MANAGER_INSTANCE)
     webview.size = Size(1280, 1024, SizeHint.NONE)

pkgs/clan-app/clan_app/deps/webview/webview.py

@@ -1,3 +1,4 @@
+# ruff: noqa: TRY301
 import functools
 import io
 import json
@@ -66,15 +67,24 @@ class Webview:
     ) -> None:
         op_key = op_key_bytes.decode()
         args = json.loads(request_data.decode())
-        log.debug(f"Calling {method_name}({args})")
+        log.debug(f"Calling {method_name}({json.dumps(args, indent=4)})")
         header: dict[str, Any]
 
         try:
             # Initialize dataclasses from the payload
             reconciled_arguments = {}
-            if len(args) > 1:
-                header = args[1]
-                for k, v in args[0].items():
+            if len(args) == 1:
+                request = args[0]
+                header = request.get("header", {})
+                msg = f"Expected header to be a dict, got {type(header)}"
+                if not isinstance(header, dict):
+                    raise TypeError(msg)
+                body = request.get("body", {})
+                msg = f"Expected body to be a dict, got {type(body)}"
+                if not isinstance(body, dict):
+                    raise TypeError(msg)
+
+                for k, v in body.items():
                     # Some functions expect to be called with dataclass instances
                     # But the js api returns dictionaries.
                     # Introspect the function and create the expected dataclass from dict dynamically
@@ -84,8 +94,11 @@ class Webview:
                     # TODO: rename from_dict into something like construct_checked_value
                     # from_dict really takes Anything and returns an instance of the type/class
                     reconciled_arguments[k] = from_dict(arg_class, v)
-            elif len(args) == 1:
-                header = args[0]
+            elif len(args) > 1:
+                msg = (
+                    "Expected a single argument, got multiple arguments to api_wrapper"
+                )
+                raise ValueError(msg)
 
             reconciled_arguments["op_key"] = op_key
         except Exception as e:
@@ -110,17 +123,39 @@ class Webview:
         def thread_task(stop_event: threading.Event) -> None:
             ctx: AsyncContext = get_async_ctx()
             ctx.should_cancel = lambda: stop_event.is_set()
-            # If the API call has set log_group in metadata,
-            # create the log file under that group.
-            log_group = header.get("logging", {}).get("group", None)
-            if log_group is not None:
-                log.warning(
-                    f"Using log group {log_group} for {method_name} with op_key {op_key}"
-                )
 
-            log_file = log_manager.create_log_file(
-                wrap_method, op_key=op_key, group=log_group
-            ).get_file_path()
+            try:
+                # If the API call has set log_group in metadata,
+                # create the log file under that group.
+                log_group: list[str] = header.get("logging", {}).get("group_path", None)
+                if log_group is not None:
+                    if not isinstance(log_group, list):
+                        msg = f"Expected log_group to be a list, got {type(log_group)}"
+                        raise TypeError(msg)
+                    log.warning(
+                        f"Using log group {log_group} for {method_name} with op_key {op_key}"
+                    )
+
+                log_file = log_manager.create_log_file(
+                    wrap_method, op_key=op_key, group_path=log_group
+                ).get_file_path()
+            except Exception as e:
+                log.exception(f"Error while handling request header of {method_name}")
+                result = ErrorDataClass(
+                    op_key=op_key,
+                    status="error",
+                    errors=[
+                        ApiError(
+                            message="An internal error occured",
+                            description=str(e),
+                            location=["header_middleware", method_name],
+                        )
+                    ],
+                )
+                serialized = json.dumps(
+                    dataclass_to_dict(result), indent=4, ensure_ascii=False
+                )
+                self.return_(op_key, FuncStatus.SUCCESS, serialized)
 
             with log_file.open("ab") as log_f:
                 # Redirect all cmd.run logs to this file.

pkgs/clan-app/flake-module.nix

@@ -1,34 +1,34 @@
 {
   perSystem =
     {
-      lib,
       self',
       pkgs,
       config,
       ...
     }:
     {
-      packages =
-        {
-          webview-lib = pkgs.callPackage ./webview-lib { };
-          clan-app = pkgs.callPackage ./default.nix {
-            inherit (config.packages) clan-cli clan-app-ui webview-lib;
-            pythonRuntime = pkgs.python3;
-          };
+      packages = {
+        webview-lib = pkgs.callPackage ./webview-lib { };
+        clan-app = pkgs.callPackage ./default.nix {
+          inherit (config.packages) clan-cli clan-app-ui webview-lib;
+          pythonRuntime = pkgs.python3;
+        };
 
-          fonts = pkgs.callPackage ./fonts.nix { };
+        fonts = pkgs.callPackage ./fonts.nix { };
 
-          clan-app-ui = pkgs.callPackage ./ui.nix {
-            clan-ts-api = config.packages.clan-ts-api;
-            fonts = config.packages.fonts;
-          };
+        clan-app-ui = pkgs.callPackage ./ui.nix {
+          clan-ts-api = config.packages.clan-ts-api;
+          fonts = config.packages.fonts;
+        };
 
-        }
-        //
-        # todo add darwin support
-        (lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux {
-          clan-app-ui-storybook = self'.packages.clan-app-ui.storybook;
-        });
+      };
+      #        //
+      # todo add darwin support
+      # todo re-enable
+      # see ui.nix for an explanation of why this is disabled for now
+      #        (lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux {
+      #          clan-app-ui-storybook = self'.packages.clan-app-ui.storybook;
+      #        });
 
       devShells.clan-app = pkgs.callPackage ./shell.nix {
         inherit self';

pkgs/clan-app/pyproject.toml

@@ -30,7 +30,7 @@ norecursedirs = "tests/helpers"
 markers = ["impure"]
 
 [tool.mypy]
-python_version = "3.12"
+python_version = "3.13"
 warn_redundant_casts = true
 disallow_untyped_calls = true
 disallow_untyped_defs = true

pkgs/clan-app/tests/wayland.py

@@ -7,7 +7,7 @@ import pytest
 
 
 @pytest.fixture(scope="session")
-def wayland_compositor() -> Generator[Popen, None, None]:
+def wayland_compositor() -> Generator[Popen]:
     # Start the Wayland compositor (e.g., Weston)
     # compositor = Popen(["weston", "--backend=headless-backend.so"])
     compositor = Popen(["weston"])
@@ -20,7 +20,7 @@ GtkProc = NewType("GtkProc", Popen)
 
 
 @pytest.fixture
-def app() -> Generator[GtkProc, None, None]:
+def app() -> Generator[GtkProc]:
     cmd = [sys.executable, "-m", "clan_app"]
     print(f"Running: {cmd}")
     rapp = Popen(

pkgs/clan-app/ui-2d/src/api/index.tsx

@@ -23,42 +23,25 @@ export type SuccessQuery<T extends OperationNames> = Extract<
 >;
 export type SuccessData<T extends OperationNames> = SuccessQuery<T>["data"];
 
-function isMachine(obj: unknown): obj is Machine {
-  return (
-    !!obj &&
-    typeof obj === "object" &&
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    typeof (obj as any).name === "string" &&
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    typeof (obj as any).flake === "object" &&
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    typeof (obj as any).flake.identifier === "string"
-  );
-}
-
-// Machine type with flake for API calls
-interface Machine {
-  name: string;
-  flake: {
-    identifier: string;
-  };
-}
-
-interface BackendOpts {
-  logging?: { group: string | Machine };
+interface SendHeaderType {
+  logging?: { group_path: string[] };
+}
+interface BackendSendType<K extends OperationNames> {
+  body: OperationArgs<K>;
+  header?: SendHeaderType;
 }
 
+// eslint-disable-next-line @typescript-eslint/no-empty-object-type
+interface ReceiveHeaderType {}
 interface BackendReturnType<K extends OperationNames> {
   body: OperationResponse<K>;
-
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  header: Record<string, any>;
+  header: ReceiveHeaderType;
 }
 
 const _callApi = <K extends OperationNames>(
   method: K,
   args: OperationArgs<K>,
-  backendOpts?: BackendOpts,
+  backendOpts?: SendHeaderType,
 ): { promise: Promise<BackendReturnType<K>>; op_key: string } => {
   // if window[method] does not exist, throw an error
   if (!(method in window)) {
@@ -82,26 +65,19 @@ const _callApi = <K extends OperationNames>(
     };
   }
 
-  let header: BackendOpts = {};
-  if (backendOpts != undefined) {
-    header = { ...backendOpts };
-    const group = backendOpts?.logging?.group;
-    if (group != undefined && isMachine(group)) {
-      header = {
-        logging: { group: group.flake.identifier + "#" + group.name },
-      };
-    }
-  }
+  const message: BackendSendType<OperationNames> = {
+    body: args,
+    header: backendOpts,
+  };
 
   const promise = (
     window as unknown as Record<
       OperationNames,
       (
-        args: OperationArgs<OperationNames>,
-        metadata: BackendOpts,
+        args: BackendSendType<OperationNames>,
       ) => Promise<BackendReturnType<OperationNames>>
     >
-  )[method](args, header) as Promise<BackendReturnType<K>>;
+  )[method](message) as Promise<BackendReturnType<K>>;
 
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const op_key = (promise as any)._webviewMessageId as string;
@@ -114,7 +90,7 @@ const handleCancel = async <K extends OperationNames>(
   orig_task: Promise<BackendReturnType<K>>,
 ) => {
   console.log("Canceling operation: ", ops_key);
-  const { promise, op_key } = _callApi("cancel_task", { task_id: ops_key });
+  const { promise, op_key } = _callApi("delete_task", { task_id: ops_key });
   promise.catch((error) => {
     toast.custom(
       (t) => (
@@ -153,7 +129,7 @@ const handleCancel = async <K extends OperationNames>(
 export const callApi = <K extends OperationNames>(
   method: K,
   args: OperationArgs<K>,
-  backendOpts?: BackendOpts,
+  backendOpts?: SendHeaderType,
 ): { promise: Promise<OperationResponse<K>>; op_key: string } => {
   console.log("Calling API", method, args, backendOpts);
 

pkgs/clan-app/ui-2d/src/components/RemoteForm.tsx

@@ -186,6 +186,7 @@ export function RemoteForm(props: RemoteFormProps) {
       props.queryFn,
       props.machine?.name,
       props.machine?.flake,
+      props.machine?.flake.identifier,
       props.field || "targetHost",
     ],
     queryFn: async () => {
@@ -209,7 +210,12 @@ export function RemoteForm(props: RemoteFormProps) {
         },
         {
           logging: {
-            group: { name: props.machine.name, flake: props.machine.flake },
+            group_path: [
+              "clans",
+              props.machine.flake.identifier,
+              "machines",
+              props.machine.name,
+            ],
           },
         },
       ).promise;

pkgs/clan-app/ui-2d/src/components/machine-list-item/index.tsx

@@ -54,7 +54,9 @@ export const MachineListItem = (props: MachineListItemProps) => {
         flake: { identifier: active_clan },
         name: name,
       },
-      { logging: { group: { name, flake: { identifier: active_clan } } } },
+      {
+        logging: { group_path: ["clans", active_clan, "machines", name] },
+      },
     ).promise;
 
     if (target_host.status == "error") {
@@ -73,7 +75,7 @@ export const MachineListItem = (props: MachineListItemProps) => {
     }
 
     setInstalling(true);
-    await callApi("install_machine", {
+    await callApi("run_machine_install", {
       opts: {
         machine: {
           name: name,
@@ -115,7 +117,9 @@ export const MachineListItem = (props: MachineListItemProps) => {
         name: name,
       },
       {
-        logging: { group: { name, flake: { identifier: active_clan } } },
+        logging: {
+          group_path: ["clans", active_clan, "machines", name],
+        },
       },
     ).promise;
 
@@ -141,7 +145,11 @@ export const MachineListItem = (props: MachineListItemProps) => {
         flake: { identifier: active_clan },
         name: name,
       },
-      { logging: { group: { name, flake: { identifier: active_clan } } } },
+      {
+        logging: {
+          group_path: ["clans", active_clan, "machines", name],
+        },
+      },
     ).promise;
 
     if (build_host.status == "error") {
@@ -155,7 +163,7 @@ export const MachineListItem = (props: MachineListItemProps) => {
     }
 
     await callApi(
-      "deploy_machine",
+      "run_machine_deploy",
       {
         machine: {
           name: name,
@@ -166,7 +174,11 @@ export const MachineListItem = (props: MachineListItemProps) => {
         target_host: target_host.data!.data,
         build_host: build_host.data?.data || null,
       },
-      { logging: { group: { name, flake: { identifier: active_clan } } } },
+      {
+        logging: {
+          group_path: ["clans", active_clan, "machines", name],
+        },
+      },
     ).promise;
 
     setUpdating(false);

pkgs/clan-app/ui-2d/src/queries/clan-meta.ts

@@ -13,7 +13,7 @@ export const clanMetaQuery = (uri: string | undefined = undefined) =>
       queryFn: async () => {
         console.log("fetching clan meta", clanURI);
 
-        const result = await callApi("show_clan_meta", {
+        const result = await callApi("get_clan_details", {
           flake: { identifier: clanURI! },
         }).promise;
 

pkgs/clan-app/ui-2d/src/queries/index.ts

@@ -33,27 +33,6 @@ export const createModulesQuery = (
     },
   }));
 
-export const tagsQuery = (uri: string | undefined) =>
-  useQuery<string[]>(() => ({
-    queryKey: [uri, "tags"],
-    placeholderData: [],
-    queryFn: async () => {
-      if (!uri) return [];
-
-      const response = await callApi("get_inventory", {
-        flake: { identifier: uri },
-      }).promise;
-      if (response.status === "error") {
-        console.error("Failed to fetch data");
-      } else {
-        const machines = response.data.machines || {};
-        const tags = Object.values(machines).flatMap((m) => m.tags || []);
-        return tags;
-      }
-      return [];
-    },
-  }));
-
 export const machinesQuery = (uri: string | undefined) =>
   useQuery<string[]>(() => ({
     queryKey: [uri, "machines"],
@@ -61,7 +40,7 @@ export const machinesQuery = (uri: string | undefined) =>
     queryFn: async () => {
       if (!uri) return [];
 
-      const response = await callApi("get_inventory", {
+      const response = await callApi("list_machines", {
         flake: { identifier: uri },
       }).promise;
       if (response.status === "error") {

pkgs/clan-app/ui-2d/src/routes/clans/create.tsx

@@ -66,7 +66,7 @@ export const CreateClan = () => {
     }
 
     // Will generate a key if it doesn't exist, and add a user to the clan
-    const k = await callApi("keygen", {
+    const k = await callApi("create_secrets_user", {
       flake_dir: target_dir[0],
     }).promise;
 
@@ -203,6 +203,6 @@ export const CreateClan = () => {
 };
 
 type Meta = Extract<
-  OperationResponse<"show_clan_meta">,
+  OperationResponse<"get_clan_details">,
   { status: "success" }
 >["data"];

pkgs/clan-app/ui-2d/src/routes/clans/details.tsx

@@ -23,7 +23,7 @@ const EditClanForm = (props: EditClanFormProps) => {
   const handleSubmit: SubmitHandler<GeneralData> = async (values, event) => {
     await toast.promise(
       (async () => {
-        await callApi("update_clan_meta", {
+        await callApi("set_clan_details", {
           options: {
             flake: { identifier: props.directory },
             meta: values,
@@ -128,7 +128,7 @@ const EditClanForm = (props: EditClanFormProps) => {
   );
 };
 
-type GeneralData = SuccessQuery<"show_clan_meta">["data"];
+type GeneralData = SuccessQuery<"get_clan_details">["data"];
 
 export const ClanDetails = () => {
   const params = useParams();

pkgs/clan-app/ui-2d/src/routes/flash/view.tsx

@@ -100,7 +100,7 @@ export const Flash = () => {
   const deviceQuery = createQuery(() => ({
     queryKey: ["block_devices"],
     queryFn: async () => {
-      const result = await callApi("show_block_devices", {}).promise;
+      const result = await callApi("list_block_devices", {}).promise;
       if (result.status === "error") throw new Error("Failed to fetch data");
       return result.data;
     },
@@ -110,7 +110,7 @@ export const Flash = () => {
   const keymapQuery = createQuery(() => ({
     queryKey: ["list_keymaps"],
     queryFn: async () => {
-      const result = await callApi("list_possible_keymaps", {}).promise;
+      const result = await callApi("list_keymaps", {}).promise;
       if (result.status === "error") throw new Error("Failed to fetch data");
       return result.data;
     },
@@ -120,7 +120,7 @@ export const Flash = () => {
   const langQuery = createQuery(() => ({
     queryKey: ["list_languages"],
     queryFn: async () => {
-      const result = await callApi("list_possible_languages", {}).promise;
+      const result = await callApi("list_languages", {}).promise;
       if (result.status === "error") throw new Error("Failed to fetch data");
       return result.data;
     },
@@ -157,7 +157,7 @@ export const Flash = () => {
     console.log("Confirmed flash:", values);
     try {
       await toast.promise(
-        callApi("flash_machine", {
+        callApi("run_machine_flash", {
           machine: {
             name: values.machine.devicePath,
             flake: {

pkgs/clan-app/ui-2d/src/routes/hosts/view.tsx

@@ -4,7 +4,7 @@ import { Button } from "../../components/Button/Button";
 import Icon from "@/src/components/icon";
 
 type ServiceModel = Extract<
-  OperationResponse<"show_mdns">,
+  OperationResponse<"list_mdns_services">,
   { status: "success" }
 >["data"]["services"];
 
@@ -16,7 +16,7 @@ export const HostList: Component = () => {
       <div class="" data-tip="Refresh install targets">
         <Button
           variant="light"
-          onClick={() => callApi("show_mdns", {})}
+          onClick={() => callApi("list_mdns_services", {})}
           startIcon={<Icon icon="Update" />}
         ></Button>
       </div>

pkgs/clan-app/ui-2d/src/routes/machines/components/InstallMachine.tsx

@@ -120,7 +120,7 @@ export function InstallMachine(props: InstallMachineProps) {
         throw new Error("No target host found for the machine");
       }
 
-      const installPromise = callApi("install_machine", {
+      const installPromise = callApi("run_machine_install", {
         opts: {
           machine: {
             name: props.name,

pkgs/clan-app/ui-2d/src/routes/machines/components/MachineForm.tsx

@@ -85,7 +85,7 @@ export function MachineForm(props: MachineFormProps) {
         },
         {
           logging: {
-            group: { name: machine_name, flake: { identifier: base_dir } },
+            group_path: ["clans", base_dir, "machines", machine_name],
           },
         },
       ).promise;
@@ -130,7 +130,9 @@ export function MachineForm(props: MachineFormProps) {
         },
       },
       {
-        logging: { group: { name: machine, flake: { identifier: curr_uri } } },
+        logging: {
+          group_path: ["clans", curr_uri, "machines", machine],
+        },
       },
     ).promise;
 
@@ -147,7 +149,7 @@ export function MachineForm(props: MachineFormProps) {
 
     setIsUpdating(true);
     const r = await callApi(
-      "deploy_machine",
+      "run_machine_deploy",
       {
         machine: {
           name: machine,
@@ -161,7 +163,9 @@ export function MachineForm(props: MachineFormProps) {
         build_host: null,
       },
       {
-        logging: { group: { name: machine, flake: { identifier: curr_uri } } },
+        logging: {
+          group_path: ["clans", curr_uri, "machines", machine],
+        },
       },
     ).promise.finally(() => {
       setIsUpdating(false);

pkgs/clan-app/ui-2d/src/routes/machines/install/hardware-step.tsx

@@ -71,7 +71,7 @@ export const HWStep = (props: StepProps<HardwareValues>) => {
   const hwReportQuery = useQuery(() => ({
     queryKey: [props.dir, props.machine_id, "hw_report"],
     queryFn: async () => {
-      const result = await callApi("show_machine_hardware_config", {
+      const result = await callApi("get_machine_hardware_summary", {
         machine: {
           flake: {
             identifier: props.dir,
@@ -127,7 +127,7 @@ export const HWStep = (props: StepProps<HardwareValues>) => {
       return;
     }
 
-    const r = await callApi("generate_machine_hardware_info", {
+    const r = await callApi("run_machine_hardware_info", {
       opts: {
         machine: {
           name: props.machine_id,

pkgs/clan-app/ui-2d/src/routes/machines/install/vars-step.tsx

@@ -158,7 +158,7 @@ export const VarsStep = (props: VarsStepProps) => {
         },
         {
           logging: {
-            group: { name: props.machine_id, flake: { identifier: props.dir } },
+            group_path: ["clans", props.dir, "machines", props.machine_id],
           },
         },
       ).promise;
@@ -173,7 +173,7 @@ export const VarsStep = (props: VarsStepProps) => {
       toast.error("Error fetching data");
       return;
     }
-    const result = await callApi("generate_vars_for_machine", {
+    const result = await callApi("run_generators", {
       machine_name: props.machine_id,
       base_dir: props.dir,
       generators: generatorsQuery.data.map((generator) => generator.name),