Update nixpkgs-dev in devFlake/private

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4338

.gitea/workflows/update-flake-inputs.yml

@@ -0,0 +1,27 @@
+name: Update Flake Inputs
+
+on:
+  schedule:
+    # Run weekly on Sunday at 4:00 AM UTC
+    - cron: "0 4 * * 0"
+  workflow_dispatch:
+  repository_dispatch:
+
+jobs:
+  update-flake-inputs:
+    runs-on: nix
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Configure git
+        run: |
+          git config --global user.email "clan-bot@clan.lol"
+          git config --global user.name "clan-bot"
+
+      - name: Update flake inputs
+        uses: Mic92/update-flake-inputs-gitea@main
+        env:
+          # Exclude private flakes and update-clan-core checks flake
+          EXCLUDE_PATTERNS: "devFlake/private/flake.nix,checks/impure/flake.nix"

clanModules/disk-id/roles/default.nix

@@ -1,5 +1,4 @@
 {
-  config,
   pkgs,
   ...
 }:
@@ -9,9 +8,14 @@
   config = {
 
     warnings = [
-      "The clan.disk-id module is deprecated and will be removed on 2025-07-15.
-      Please migrate to user-maintained configuration or the new equivalent clan services
-      (https://docs.clan.lol/reference/clanServices)."
+      ''
+        The clan.disk-id module is deprecated and will be removed on 2025-07-15.
+        For migration see: https://docs.clan.lol/guides/migrations/disk-id/
+
+        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+        !!! Please migrate. Otherwise you may not be able to boot your system after that date.  !!!
+        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+      ''
     ];
     clan.core.vars.generators.disk-id = {
       files.diskId.secret = false;

clanServices/admin/default.nix

@@ -9,7 +9,9 @@
     interface =
       { lib, ... }:
       {
-        options.allowedKeys = lib.mkOption {
+
+        options = {
+          allowedKeys = lib.mkOption {
             default = { };
             type = lib.types.attrsOf lib.types.str;
             description = "The allowed public keys for ssh access to the admin user";
@@ -18,6 +20,26 @@
             };
           };
 
+          rsaHostKey.enable = lib.mkEnableOption "Generate RSA host key";
+
+          # TODO: allow per-server domains that we than collect in the inventory
+          #certicficateDomains = lib.mkOption {
+          #  type = lib.types.listOf lib.types.str;
+          #  default = [ ];
+          #  example = [ "git.mydomain.com" ];
+          #  description = "List of domains to include in the certificate. This option will not prepend the machine name in front of each domain.";
+          #};
+
+          certificateSearchDomains = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [ ];
+            example = [ "mydomain.com" ];
+            description = ''
+              List of domains to include in the certificate.
+              This option will prepend the machine name in front of each domain before adding it to the certificate.
+            '';
+          };
+        };
       };
 
     perInstance =
@@ -27,10 +49,15 @@
           { ... }:
           {
             imports = [
-              ../../clanModules/sshd
-              ../../clanModules/root-password
+              # We don't have a good way to specify dependencies between
+              # clanServices for now. When it get's implemtende, we should just
+              # use the ssh and users modules here.
+              ./ssh.nix
+              ./root-password.nix
             ];
 
+            _module.args = { inherit settings; };
+
             users.users.root.openssh.authorizedKeys.keys = builtins.attrValues settings.allowedKeys;
           };
       };

clanServices/admin/root-password.nix

@@ -0,0 +1,39 @@
+# We don't have a way of specifying dependencies between clanServices for now.
+# When it get's added this file should be removed and the users module used instead.
+{
+  config,
+  pkgs,
+  ...
+}:
+{
+
+  users.mutableUsers = false;
+  users.users.root.hashedPasswordFile =
+    config.clan.core.vars.generators.root-password.files.password-hash.path;
+
+  clan.core.vars.generators.root-password = {
+    files.password-hash.neededFor = "users";
+
+    files.password.deploy = false;
+
+    runtimeInputs = [
+      pkgs.coreutils
+      pkgs.mkpasswd
+      pkgs.xkcdpass
+    ];
+
+    prompts.password.type = "hidden";
+    prompts.password.persist = true;
+    prompts.password.description = "You can autogenerate a password, if you leave this prompt blank.";
+
+    script = ''
+      prompt_value="$(cat "$prompts"/password)"
+      if [[ -n "''${prompt_value-}" ]]; then
+        echo "$prompt_value" | tr -d "\n" > "$out"/password
+      else
+        xkcdpass --numwords 5 --delimiter - --count 1 | tr -d "\n" > "$out"/password
+      fi
+      mkpasswd -s -m sha-512 < "$out"/password | tr -d "\n" > "$out"/password-hash
+    '';
+  };
+}

clanServices/admin/ssh.nix

@@ -0,0 +1,115 @@
+{
+  config,
+  pkgs,
+  lib,
+  settings,
+  ...
+}:
+let
+  stringSet = list: builtins.attrNames (builtins.groupBy lib.id list);
+
+  domains = stringSet settings.certificateSearchDomains;
+
+in
+{
+
+  services.openssh = {
+    enable = true;
+    settings.PasswordAuthentication = false;
+
+    settings.HostCertificate = lib.mkIf (
+      settings.certificateSearchDomains != [ ]
+    ) config.clan.core.vars.generators.openssh-cert.files."ssh.id_ed25519-cert.pub".path;
+
+    hostKeys =
+      [
+        {
+          path = config.clan.core.vars.generators.openssh.files."ssh.id_ed25519".path;
+          type = "ed25519";
+        }
+      ]
+      ++ lib.optional settings.rsaHostKey.enable {
+        path = config.clan.core.vars.generators.openssh-rsa.files."ssh.id_rsa".path;
+        type = "rsa";
+      };
+  };
+
+  clan.core.vars.generators.openssh = {
+    files."ssh.id_ed25519" = { };
+    files."ssh.id_ed25519.pub".secret = false;
+    migrateFact = "openssh";
+    runtimeInputs = [
+      pkgs.coreutils
+      pkgs.openssh
+    ];
+    script = ''
+      ssh-keygen -t ed25519 -N "" -C "" -f "$out"/ssh.id_ed25519
+    '';
+  };
+
+  programs.ssh.knownHosts.clan-sshd-self-ed25519 = {
+    hostNames = [
+      "localhost"
+      config.networking.hostName
+    ] ++ (lib.optional (config.networking.domain != null) config.networking.fqdn);
+    publicKey = config.clan.core.vars.generators.openssh.files."ssh.id_ed25519.pub".value;
+  };
+
+  clan.core.vars.generators.openssh-rsa = lib.mkIf settings.rsaHostKey.enable {
+    files."ssh.id_rsa" = { };
+    files."ssh.id_rsa.pub".secret = false;
+    runtimeInputs = [
+      pkgs.coreutils
+      pkgs.openssh
+    ];
+    script = ''
+      ssh-keygen -t rsa -b 4096 -N "" -C "" -f "$out"/ssh.id_rsa
+    '';
+  };
+
+  clan.core.vars.generators.openssh-cert = lib.mkIf (settings.certificateSearchDomains != [ ]) {
+    files."ssh.id_ed25519-cert.pub".secret = false;
+    dependencies = [
+      "openssh"
+      "openssh-ca"
+    ];
+    validation = {
+      name = config.clan.core.settings.machine.name;
+      domains = lib.genAttrs settings.certificateSearchDomains lib.id;
+    };
+    runtimeInputs = [
+      pkgs.openssh
+      pkgs.jq
+    ];
+    script = ''
+      ssh-keygen \
+        -s $in/openssh-ca/id_ed25519 \
+        -I ${config.clan.core.settings.machine.name} \
+        -h \
+        -n ${lib.concatMapStringsSep "," (d: "${config.clan.core.settings.machine.name}.${d}") domains} \
+        $in/openssh/ssh.id_ed25519.pub
+      mv $in/openssh/ssh.id_ed25519-cert.pub "$out"/ssh.id_ed25519-cert.pub
+    '';
+  };
+
+  clan.core.vars.generators.openssh-ca = lib.mkIf (settings.certificateSearchDomains != [ ]) {
+    share = true;
+    files.id_ed25519.deploy = false;
+    files."id_ed25519.pub" = {
+      deploy = false;
+      secret = false;
+    };
+    runtimeInputs = [
+      pkgs.openssh
+    ];
+    script = ''
+      ssh-keygen -t ed25519 -N "" -C "" -f "$out"/id_ed25519
+    '';
+  };
+
+  programs.ssh.knownHosts.ssh-ca = lib.mkIf (settings.certificateSearchDomains != [ ]) {
+    certAuthority = true;
+    extraHostNames = builtins.map (domain: "*.${domain}") settings.certificateSearchDomains;
+    publicKey = config.clan.core.vars.generators.openssh-ca.files."id_ed25519.pub".value;
+  };
+}

clanServices/users/default.nix

@@ -11,7 +11,7 @@
 
   roles.default = {
     interface =
-      { lib, ... }:
+      { config, lib, ... }:
       {
         options = {
           user = lib.mkOption {
@@ -28,8 +28,8 @@
 
               Effects:
 
-              - *enabled* (`true`) - Prompt for a passwort during the machine installation or update workflow.
-              - *disabled* (`false`) - Generate a passwort during the machine installation or update workflow.
+              - *enabled* (`true`) - Prompt for a password during the machine installation or update workflow.
+              - *disabled* (`false`) - Generate a password during the machine installation or update workflow.
 
               The password can be shown in two steps:
 
@@ -37,6 +37,23 @@
               - `clan vars get <machine-name> <name-of-password-variable>`
             '';
           };
+          regularUser = lib.mkOption {
+            type = lib.types.bool;
+            default = config.user != "root";
+            defaultText = lib.literalExpression "config.user != \"root\"";
+            example = false;
+            description = ''
+              Whether the user should be a regular user or a system user.
+
+              Regular users are normal users that can log in and have a home directory.
+
+              System users are used for system services and do not have a home directory.
+
+              !!! Warning
+                  `root` cannot be a regular user.
+                  You must set this to `false` for `root`
+            '';
+          };
           groups = lib.mkOption {
             type = lib.types.listOf lib.types.str;
             default = [ ];
@@ -73,8 +90,8 @@
             ...
           }:
           {
-            users.mutableUsers = false;
             users.users.${settings.user} = {
+              isNormalUser = settings.regularUser;
               extraGroups = settings.groups;
 
               hashedPasswordFile =
@@ -122,4 +139,11 @@
           };
       };
   };
+
+  perMachine = {
+    nixosModule = {
+      # Immutable users to ensure that this module has exclusive control over the users.
+      users.mutableUsers = false;
+    };
+  };
 }

clanServices/users/tests/vm/default.nix

@@ -31,7 +31,6 @@
     server = {
       users.users.testuser.group = "testuser";
       users.groups.testuser = { };
-      users.users.testuser.isNormalUser = true;
     };
   };
 

devFlake/private/flake.lock

@@ -66,11 +66,11 @@
     },
     "nixpkgs-dev": {
       "locked": {
-        "lastModified": 1752039390,
-        "narHash": "sha256-DTHMN6kh1cGoc5hc9O0pYN+VAOnjsyy0wxq4YO5ZRvg=",
+        "lastModified": 1752467518,
+        "narHash": "sha256-7SSvjNlM5ZsFZMP7Nw2uUa7EKYhB6Ny9iNtxtPPhWYY=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6ec4d5f023c3c000cda569255a3486e8710c39bf",
+        "rev": "2f21cef1d1dc734a2dd89f535427cf291aebc8ef",
         "type": "github"
       },
       "original": {

docs/mkdocs.yml

@@ -51,6 +51,7 @@ nav:
           - 🚀 Creating Your First Clan: guides/getting-started/index.md
           - 📀 Create USB Installer (optional): guides/getting-started/installer.md
           - ⚙️ Add Machines: guides/getting-started/add-machines.md
+          - ⚙️ Add User: guides/getting-started/add-user.md
           - ⚙️ Add Services: guides/getting-started/add-services.md
           - 🔐 Secrets & Facts: guides/getting-started/secrets.md
           - 🚢 Deploy Machine: guides/getting-started/deploy.md
@@ -79,6 +80,7 @@ nav:
           - Migrate existing Flakes: guides/migrations/migration-guide.md
           - Migrate inventory Services: guides/migrations/migrate-inventory-services.md
           - Facts Vars Migration: guides/migrations/migration-facts-vars.md
+          - Disk id: guides/migrations/disk-id.md
       - macOS: guides/macos.md
   - Reference:
       - Overview: reference/index.md

docs/nix/render_options/__init__.py

@@ -29,13 +29,13 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
 
-from clan_lib.api.modules import (
+from clan_lib.errors import ClanError
+from clan_lib.services.modules import (
     CategoryInfo,
     Frontmatter,
     extract_frontmatter,
     get_roles,
 )
-from clan_lib.errors import ClanError
 
 # Get environment variables
 CLAN_CORE_PATH = Path(os.environ["CLAN_CORE_PATH"])

docs/site/guides/contributing/debugging.md

@@ -55,9 +55,37 @@ If you're using VSCode, it has a handy feature that makes paths to source code f
 
 ## Finding Print Messages
 
-To identify where a specific print message comes from, you can enable a helpful feature. Simply set the environment variable `export TRACE_PRINT=1`. When you run commands with `--debug` mode, each print message will include information about its source location.
+To trace the origin of print messages in `clan-cli`, you can enable special debugging features using environment variables:
+
+- Set `TRACE_PRINT=1` to include the source location with each print message:
+    ```bash
+    export TRACE_PRINT=1
+    ```
+    When running commands with `--debug`, every print will show where it was triggered in the code.
+
+- To see a deeper stack trace for each print, set `TRACE_DEPTH` to the desired number of stack frames (e.g., 3):
+    ```bash
+    export TRACE_DEPTH=3
+    ```
+
+### Additional Debug Logging
+
+You can enable more detailed logging for specific components by setting these environment variables:
+
+- `CLAN_DEBUG_NIX_SELECTORS=1` — verbose logs for flake.select operations
+- `CLAN_DEBUG_NIX_PREFETCH=1` — verbose logs for flake.prefetch operations
+- `CLAN_DEBUG_COMMANDS=1` — print the diffed environment of executed commands
+
+Example:
+```bash
+export CLAN_DEBUG_NIX_SELECTORS=1
+export CLAN_DEBUG_NIX_PREFETCH=1
+export CLAN_DEBUG_COMMANDS=1
+```
+
+These options help you pinpoint the source and context of print messages and debug logs during development.
+
 
-If you need more details, you can expand the stack trace information that appears with each print by setting the environment variable `export TRACE_DEPTH=3`.
 
 ## Analyzing Performance
 

docs/site/guides/getting-started/add-machines.md

@@ -10,63 +10,22 @@ See the complete [list](../../guides/more-machines.md#automatic-registration) of
 
 ## Create a machine
 
-=== "flake.nix (flake-parts)"
+=== "clan.nix (declarative)"
 
-    ```{.nix hl_lines=12-15}
+    ```{.nix hl_lines="3-4"}
     {
-        inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-        inputs.nixpkgs.follows = "clan-core/nixpkgs";
-        inputs.flake-parts.follows = "clan-core/flake-parts";
-        inputs.flake-parts.inputs.nixpkgs-lib.follows = "clan-core/nixpkgs";
-
-        outputs =
-            inputs@{ flake-parts, ... }:
-            flake-parts.lib.mkFlake { inherit inputs; } {
-                imports = [ inputs.clan-core.flakeModules.default ];
-                clan = {
         inventory.machines = {
             # Define a machine
             jon = { };
         };
-                };
 
-                systems = [
-                    "x86_64-linux"
-                    "aarch64-linux"
-                    "x86_64-darwin"
-                    "aarch64-darwin"
-                ];
-        };
-    }
-    ```
-
-=== "flake.nix (classic)"
-
-    ```{.nix hl_lines=11-14}
-    {
-        inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-        inputs.nixpkgs.follows = "clan-core/nixpkgs";
-
-        outputs =
-            { self, clan-core, ... }:
-            let
-                clan = clan-core.lib.clan {
-                    inherit self;
-
-                    inventory.machines = {
-                        # Define a machine
-                        jon = { };
-                    };
-                };
-            in
-            {
-                inherit (clan.config)
-                    nixosConfigurations
-                    nixosModules
-                    clanInternals
-                    darwinConfigurations
-                    darwinModules
-                    ;
+        # Additional NixOS configuration can be added here.
+        # machines/jon/configuration.nix will be automatically imported.
+        # See: https://docs.clan.lol/guides/more-machines/#automatic-registration
+        machines = {
+            # jon = { config, ... }: {
+            #   environment.systemPackages = [ pkgs.asciinema ];
+            # };
         };
     }
     ```
@@ -89,16 +48,15 @@ See the complete [list](../../guides/more-machines.md#automatic-registration) of
 
     The option: `machines.<name>` is used to add extra *nixosConfiguration* to a machine
 
-```{.nix .annotate title="flake.nix" hl_lines="3-13 18-22"}
-# Sometimes this attribute set is defined in clan.nix
-clan = {
+Add the following to your `clan.nix` file for each machine.
+This example demonstrates what is needed based on a machine called `jon`:
+
+```{.nix .annotate title="clan.nix" hl_lines="3-6 15-19"}
+{
     inventory.machines = {
         jon = {
-            # Define targetHost here
-            # Required before deployment
-            deploy.targetHost = "root@jon"; # (1)
-            # Define tags here
-            tags = [ ];
+            # Define tags here (optional)
+            tags = [ ]; # (1)
         };
         sara = {
             deploy.targetHost = "root@sara";
@@ -117,9 +75,24 @@ clan = {
 }
 ```
 
-1. It is required to define a *targetHost* for each machine before deploying. Best practice has been, to use the zerotier ip/hostname or the ip from the from overlay network you decided to use.
+1. Tags can be used to automatically add this machine to services later on. - You dont need to set this now.
 2. Add your *ssh key* here - That will ensure you can always login to your machine via *ssh* in case something goes wrong.
 
+### (Optional) Create a `configuration.nix`
+
+```nix title="./machines/jon/configuration.nix"
+{
+    imports = [
+        # enables GNOME desktop (optional)
+        ../../modules/gnome.nix
+    ];
+
+    # Set nixosOptions here
+    # Or import your own modules via 'imports'
+    # ...
+}
+```
+
 ### (Optional) Renaming a Machine
 
 Older templates included static machine folders like `jon` and `sara`.

docs/site/guides/getting-started/add-services.md

@@ -17,104 +17,61 @@ To learn more: [Guide about clanService](../clanServices.md)
 
 ## Configure a Zerotier Network (recommended)
 
-```{.nix title="flake.nix" hl_lines="20-28"}
+```{.nix title="clan.nix" hl_lines="8-16"}
 {
-    inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-    inputs.nixpkgs.follows = "clan-core/nixpkgs";
-    inputs.flake-parts.follows = "clan-core/flake-parts";
-    inputs.flake-parts.inputs.nixpkgs-lib.follows = "clan-core/nixpkgs";
-
-    outputs =
-        inputs@{ flake-parts, ... }:
-        flake-parts.lib.mkFlake { inherit inputs; } {
-            imports = [ inputs.clan-core.flakeModules.default ];
-            # Sometimes this attribute set is defined in clan.nix
-            clan = {
     inventory.machines = {
-                    jon = {
-                        targetHost = "root@jon";
-                    };
-                    sara = {
-                        targetHost = "root@jon";
-                    };
+        jon = { };
+        sara = { };
     };
+
     inventory.instances = {
         zerotier = { # (1)
-                        # Defines 'jon' as the controller
-                        roles.controller.machines.jon = {};
-                        # Defines all machines as networking peer.
-                        # The 'all' tag is a clan builtin.
-                        roles.peer.tags.all = {};
+            # Replace with the name (string) of your machine that you will use as zerotier-controller
+            # See: https://docs.zerotier.com/controller/
+            # Deploy this machine first to create the network secrets
+            roles.controller.machines."jon" = { }; # (2)
+            # Peers of the network
+            # this line means 'all' clan machines will be 'peers'
+            roles.peer.tags.all = { }; # (3)
         };
-                }
-            };
-
-            systems = [
-                "x86_64-linux"
-                "aarch64-linux"
-                "x86_64-darwin"
-                "aarch64-darwin"
-            ];
     };
+    # ...
+    # elided
 }
 ```
 
 1. See [reference/clanServices](../../reference/clanServices/index.md) for all available services and how to configure them.
    Or read [authoring/clanServices](../authoring/clanServices/index.md) if you want to bring your own
 
+2. Replace `__YOUR_CONTROLLER_` with the *name* of your machine.
+
+3. This line will add all machines of your clan as `peer` to zerotier
+
 ## Adding more recommended defaults
 
 Adding the following services is recommended for most users:
 
-```{.nix title="flake.nix" hl_lines="25-35"}
+```{.nix title="clan.nix" hl_lines="7-14"}
 {
-    inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-    inputs.nixpkgs.follows = "clan-core/nixpkgs";
-    inputs.flake-parts.follows = "clan-core/flake-parts";
-    inputs.flake-parts.inputs.nixpkgs-lib.follows = "clan-core/nixpkgs";
-
-    outputs =
-        inputs@{ flake-parts, ... }:
-        flake-parts.lib.mkFlake { inherit inputs; } {
-            imports = [ inputs.clan-core.flakeModules.default ];
-            # Sometimes this attribute set is defined in clan.nix
-            clan = {
     inventory.machines = {
-                    jon = {
-                        targetHost = "root@jon";
-                    };
-                    sara = {
-                        targetHost = "root@jon";
-                    };
+        jon = { };
+        sara = { };
     };
     inventory.instances = {
-                    zerotier = {
-                        roles.controller.machines.jon = {};
-                        roles.peer.tags.all = {};
-                    };
         admin = { # (1)
             roles.default.tags.all = { };
             roles.default.settings = {
                 allowedKeys = {
-                                "my-user" = "ssh-ed25519 AAAAC3N..."; # elided
+                    "my-user" = "ssh-ed25519 AAAAC3N..."; # (2)
                 };
             };
         };
-                    state-version = { # (2)
-                        roles.default.tags.all = { };
-                    };
-                };
-            };
-            systems = [
-                "x86_64-linux"
-                "aarch64-linux"
-                "x86_64-darwin"
-                "aarch64-darwin"
-            ];
+        # ...
+        # elided
     };
 }
 ```
 
 1. The `admin` service will generate a **root-password** and **add your ssh-key** that allows for convienient administration.
-
-2. The `state-version` service will generate a [nixos state version](https://wiki.nixos.org/wiki/FAQ/When_do_I_update_stateVersion) for each system once it is deployed.
+2. Equivalent to directly setting `authorizedKeys` like in [configuring a machine](./add-machines.md#configuring-a-machine)
+3. Adds `user = jon` as a user on all machines. Will create a `home` directory, and prompt for a password before deployment.

docs/site/guides/getting-started/add-user.md

@@ -0,0 +1,127 @@
+# How to add users
+
+!!! Note "Under construction"
+
+The users concept of clan is not done yet. This guide outlines some solutions from our community.
+Defining users can be done in many different ways. We want to highlight two approaches:
+
+- Using clan's [users](../../reference/clanServices/users.md) service.
+- Using a custom approach.
+
+## Adding Users using the [users](../../reference/clanServices/users.md) service
+
+To add a first *user* this guide will be leveraging two things:
+
+- [clanServices](../../reference/clanServices/index.md): Allows to bind arbitrary logic to something we call an `ìnstance`.
+- [clanServices/users](../../reference/clanServices/users.md): Implements logic for adding a single user perInstance.
+
+The example shows how to add a user called `jon`:
+
+```{.nix title="clan.nix" hl_lines="7-21"}
+{
+    inventory.machines = {
+        jon = { };
+        sara = { };
+    };
+    inventory.instances = {
+        jon-user = { # (1)
+            module.name = "users";
+
+            roles.default.tags.all = { }; # (2)
+
+            roles.default.settings = {
+                user = "jon"; # (3)
+                groups = [
+                    "wheel" # Allow using 'sudo'
+                    "networkmanager" # Allows to manage network connections.
+                    "video" # Allows to access video devices.
+                    "input" # Allows to access input devices.
+                ];
+            };
+        };
+        # ...
+        # elided
+    };
+}
+```
+
+1. Add `user = jon` as a user on all machines. Will create a `home` directory, and prompt for a password before deployment.
+2. Add this user to `all` machines
+3. Define the `name` of the user to be `jon`
+
+The `users` service creates a `/home/jon` directory, allows `jon` to sign in and will take care of the users password as part of [deployment](./deploy.md).
+
+For more information see [clanService/users](../../reference/clanServices/users.md)
+
+## Using a custom approach
+
+Some people like to define a `users` folder in their repository root.
+That allows to bind all user specific logic to a single place (`default.nix`)
+Which can be imported into individual machines to make the user avilable on that machine.
+
+```bash
+.
+├── machines
+│   ├── jon
+# ......
+├── users
+│   ├── jon
+│   │   └── default.nix # <- a NixOS module; sets some options
+# ... ... ...
+```
+
+## using [home-manager](https://github.com/nix-community/home-manager)
+
+When using clan's `users` service it is possible to define extraModules.
+In fact this is always possible when using clan's services.
+
+We can use this property of clan services to bind a nixosModule to the user, which configures home-manager.
+
+```{.nix title="clan.nix" hl_lines="22"}
+{
+    inventory.machines = {
+        jon = { };
+        sara = { };
+    };
+    inventory.instances = {
+        jon-user = {
+            module.name = "users";
+
+            roles.default.tags.all = { };
+
+            roles.default.settings = {
+                user = "jon",
+                groups = [
+                    "wheel"
+                    "networkmanager"
+                    "video"
+                    "input"
+                ];
+            };
+
+            roles.default.extraModules = [ ./users/jon/home.nix ]; # (1)
+        };
+        # ...
+        # elided
+    };
+}
+```
+
+1. Type `path` or `string`: Must point to a seperate file. Inlining a module is not possible
+
+!!! Note "This is inspiration"
+    Our community might come up with better solutions soon.
+    We are seeking contributions to improve this pattern if you have a nicer solution in mind.
+
+```nix title="users/jon/home.nix"
+# NixOS module to import home-manager and the home-manager configuration of 'jon'
+{ self, ...}:
+{
+  imports = [ self.inputs.home-manager.nixosModules.default ];
+  home-manager.users.jon = {
+    imports = [
+      ./home-configuration.nix
+    ];
+  };
+}
+```

docs/site/guides/getting-started/deploy.md

@@ -1,6 +1,6 @@
 # Deploy a machine
 
-Now that you have created a new machine, we will walk through how to install it.
+Now that you have created a machines, added some services and setup secrets. This guide will walk through how to deploy it.
 
 ## Prerequisites
 
@@ -10,206 +10,58 @@ Now that you have created a new machine, we will walk through how to install it.
     - [x] **Machine configuration**: See our basic [adding and configuring machine guide](./add-machines.md)
     - [x] **Initialized secrets**: See [secrets](secrets.md) for how to initialize your secrets.
 
-=== "**Physical Hardware**"
+## Physical Hardware
 
-    - [x] **USB Flash Drive**: See [Clan Installer](installer.md)
+!!! note "skip this if using a cloud VM"
 
-    !!! Steps
+Steps:
 
-        1. Create a NixOS installer image and transfer it to a bootable USB drive as described in the [installer](./installer.md).
+- Create a NixOS installer image and transfer it to a bootable USB drive as described in the [installer](./installer.md).
+- Boot the target machine and connect it to a network that makes it reachable from your setup computer.
+- Note down a reachable ip adress (*ipv4*, *ipv6* or *tor*)
 
-        2. Boot the target machine and connect it to a network that makes it reachable from your setup computer.
+---
 
-=== "**Cloud VMs**"
+The installer will generate a password and local addresses on boot, then run ssh with these preconfigured.
+The installer shows it's deployment relevant information in two formats, a text form, as well as a QR code.
 
-    - [x] Any cloud machine if it is reachable via SSH and supports `kexec`.
+Sample boot screen shows:
 
-    !!! Warning "NixOS can cause strange issues when booting in certain cloud environments."
-        If on Linode: Make sure that the system uses Direct Disk boot kernel (found in the configuration pannel)
+- Root password
+- IP address
+- Optional Tor and mDNS details
 
-## Setting `targetHost`
-
-=== "flake.nix (flake-parts)"
-
-    ```{.nix hl_lines="22"}
-    {
-        inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-        inputs.nixpkgs.follows = "clan-core/nixpkgs";
-        inputs.flake-parts.follows = "clan-core/flake-parts";
-        inputs.flake-parts.inputs.nixpkgs-lib.follows = "clan-core/nixpkgs";
-
-        outputs =
-            inputs@{ flake-parts, ... }:
-            flake-parts.lib.mkFlake { inherit inputs; } {
-            systems = [
-                "x86_64-linux"
-                "aarch64-linux"
-                "x86_64-darwin"
-                "aarch64-darwin"
-            ];
-            imports = [ inputs.clan-core.flakeModules.default ];
-
-            clan = {
-                inventory.machines = {
-                    jon = {
-                        # targetHost will get picked up by cli commands
-                        deploy.targetHost = "root@jon";
-                    };
-                };
-            };
-        };
-    }
-    ```
-
-=== "flake.nix (classic)"
-
-    ```{.nix hl_lines="14"}
-    {
-        inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
-        inputs.nixpkgs.follows = "clan-core/nixpkgs";
-
-        outputs =
-            { self, clan-core, ... }:
-            let
-                clan = clan-core.lib.clan {
-                    inherit self;
-
-                    inventory.machines = {
-                        jon = {
-                            # targetHost will get picked up by cli commands
-                            deploy.targetHost = "root@jon";
-                        };
-                    };
-                };
-            in
-            {
-                inherit (clan.config)
-                    nixosConfigurations
-                    nixosModules
-                    clanInternals
-                    darwinConfigurations
-                    darwinModules
-                    ;
-            };
-    }
-    ```
-
-!!! warning
-    The use of `root@` in the target address implies SSH access as the `root` user.
-    Ensure that the root login is secured and only used when necessary.
-
-## Identify the Target Disk
-
-On the setup computer, SSH into the target:
-
-```bash title="setup computer"
-ssh root@<IP> lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT
+```{ .bash .annotate .no-copy .nohighlight}
+┌─────────────────────────────────────────────────────────────────────────────────────┐
+│ ┌───────────────────────────┐                                                       │
+│ │███████████████████████████│ # This is the QR Code (1)                             │
+│ │██ ▄▄▄▄▄ █▀▄█▀█▀▄█ ▄▄▄▄▄ ██│                                                       │
+│ │██ █   █ █▀▄▄▄█ ▀█ █   █ ██│                                                       │
+│ │██ █▄▄▄█ █▀▄ ▀▄▄▄█ █▄▄▄█ ██│                                                       │
+│ │██▄▄▄▄▄▄▄█▄▀ ▀▄▀▄█▄▄▄▄▄▄▄██│                                                       │
+│ │███▀▀▀ █▄▄█ ▀▄   ▄▀▄█   ███│                                                       │
+│ │██▄██▄▄█▄▄▀▀██▄▀ ▄▄▄ ▄▀█▀██│                                                       │
+│ │██ ▄▄▄▄▄ █▄▄▄▄ █ █▄█ █▀ ███│                                                       │
+│ │██ █   █ █ █  █ ▄▄▄  ▄▀▀ ██│                                                       │
+│ │██ █▄▄▄█ █ ▄ ▄    ▄ ▀█ ▄███│                                                       │
+│ │██▄▄▄▄▄▄▄█▄▄▄▄▄▄█▄▄▄▄▄█▄███│                                                       │
+│ │███████████████████████████│                                                       │
+│ └───────────────────────────┘                                                       │
+│ ┌─────────────────────────────────────────────────────────────────────────────────┐ │
+│ │Root password: cheesy-capital-unwell  # password (2)                             │ │
+│ │Local network addresses:                                                         │ │
+│ │enp1s0           UP    192.168.178.169/24 metric 1024 fe80::21e:6ff:fe45:3c92/64 │ │
+│ │enp2s0           DOWN                                                            │ │
+│ │wlan0            DOWN # connect to wlan (3)                                      │ │
+│ │Onion address: 6evxy5yhzytwpnhc2vpscrbti3iktxdhpnf6yim6bbs25p4v6beemzyd.onion    │ │
+│ │Multicast DNS: nixos-installer.local                                             │ │
+│ └─────────────────────────────────────────────────────────────────────────────────┘ │
+│ Press 'Ctrl-C' for console access                                                   │
+│                                                                                     │
+└─────────────────────────────────────────────────────────────────────────────────────┘
 ```
 
-Replace `<IP>` with the machine's IP or hostname if mDNS (i.e. Avahi) is available.
-
-Which should show something like:
-
-```{.shellSession hl_lines="6" .no-copy}
-NAME        ID-LINK                                         FSTYPE   SIZE MOUNTPOINT
-sda         usb-ST_16GB_AA6271026J1000000509-0:0                    14.9G
-├─sda1      usb-ST_16GB_AA6271026J1000000509-0:0-part1                 1M
-├─sda2      usb-ST_16GB_AA6271026J1000000509-0:0-part2      vfat     100M /boot
-└─sda3      usb-ST_16GB_AA6271026J1000000509-0:0-part3      ext4     2.9G /
-nvme0n1     nvme-eui.e8238fa6bf530001001b448b4aec2929              476.9G
-├─nvme0n1p1 nvme-eui.e8238fa6bf530001001b448b4aec2929-part1 vfat     512M
-├─nvme0n1p2 nvme-eui.e8238fa6bf530001001b448b4aec2929-part2 ext4   459.6G
-└─nvme0n1p3 nvme-eui.e8238fa6bf530001001b448b4aec2929-part3 swap    16.8G
-```
-
-Look for the top-level disk device (e.g., nvme0n1 or sda) and copy its `ID-LINK`. Avoid using partition IDs like `nvme0n1p1`.
-
-In this example we would copy `nvme-eui.e8238fa6bf530001001b448b4aec2929`
-
-!!! tip
-    For advanced partitioning, see [Disko templates](https://github.com/nix-community/disko-templates) or [Disko examples](https://github.com/nix-community/disko/tree/master/example).
-
-## Fill in hardware specific machine configuration
-
-Edit the following fields inside the `./machines/<machine_name>/configuration.nix`
-
-<!-- Note: Use "jon" instead of "<machine>" as "<" is not supported in title tag -->
-
-```nix title="./machines/jon/configuration.nix" hl_lines="12 15 19"
-{
-    imports = [
-    # contains your disk format and partitioning configuration.
-    ../../modules/disko.nix
-    # this file is shared among all machines
-    ../../modules/shared.nix
-    # enables GNOME desktop (optional)
-    ../../modules/gnome.nix
-    ];
-
-    # Put your username here for login
-    users.users.user.name = "__YOUR_USERNAME__";
-
-    # Replace this __CHANGE_ME__ with the copied result of the lsblk command
-    disko.devices.disk.main.device = "/dev/disk/by-id/__CHANGE_ME__";
-
-    # IMPORTANT! Add your SSH key here
-    # e.g. > cat ~/.ssh/id_ed25519.pub
-    users.users.root.openssh.authorizedKeys.keys = [ "__YOUR_SSH_KEY__" ];
-
-    # ...
-}
-```
-
-!!! Info "Replace `__YOUR_USERNAME__` with the ip of your machine, if you use avahi you can also use your hostname"
-!!! Info "Replace `__CHANGE_ME__` with the appropriate `ID-LINK` identifier, such as `nvme-eui.e8238fa6bf530001001b448b4aec2929`"
-!!! Info "Replace `__YOUR_SSH_KEY__` with your personal key, like `ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILoMI0NC5eT9pHlQExrvR5ASV3iW9+BXwhfchq0smXUJ jon@jon-desktop`"
-
-## Deploy the machine
-
-**Finally deployment time!** Use the following command to build and deploy the image via SSH onto your machine.
-
-=== "**Image Installer**"
-
-    The installer will generate a password and local addresses on boot, then run ssh with these preconfigured.
-    The installer shows it's deployment relevant information in two formats, a text form, as well as a QR code.
-
-    Sample boot screen shows:
-
-    - Root password
-    - IP address
-    - Optional Tor and mDNS details
-
-    ```{ .bash .annotate .no-copy .nohighlight}
-    ┌─────────────────────────────────────────────────────────────────────────────────────┐
-    │ ┌───────────────────────────┐                                                       │
-    │ │███████████████████████████│ # This is the QR Code (1)                             │
-    │ │██ ▄▄▄▄▄ █▀▄█▀█▀▄█ ▄▄▄▄▄ ██│                                                       │
-    │ │██ █   █ █▀▄▄▄█ ▀█ █   █ ██│                                                       │
-    │ │██ █▄▄▄█ █▀▄ ▀▄▄▄█ █▄▄▄█ ██│                                                       │
-    │ │██▄▄▄▄▄▄▄█▄▀ ▀▄▀▄█▄▄▄▄▄▄▄██│                                                       │
-    │ │███▀▀▀ █▄▄█ ▀▄   ▄▀▄█   ███│                                                       │
-    │ │██▄██▄▄█▄▄▀▀██▄▀ ▄▄▄ ▄▀█▀██│                                                       │
-    │ │██ ▄▄▄▄▄ █▄▄▄▄ █ █▄█ █▀ ███│                                                       │
-    │ │██ █   █ █ █  █ ▄▄▄  ▄▀▀ ██│                                                       │
-    │ │██ █▄▄▄█ █ ▄ ▄    ▄ ▀█ ▄███│                                                       │
-    │ │██▄▄▄▄▄▄▄█▄▄▄▄▄▄█▄▄▄▄▄█▄███│                                                       │
-    │ │███████████████████████████│                                                       │
-    │ └───────────────────────────┘                                                       │
-    │ ┌─────────────────────────────────────────────────────────────────────────────────┐ │
-    │ │Root password: cheesy-capital-unwell  # password (2)                             │ │
-    │ │Local network addresses:                                                         │ │
-    │ │enp1s0           UP    192.168.178.169/24 metric 1024 fe80::21e:6ff:fe45:3c92/64 │ │
-    │ │enp2s0           DOWN                                                            │ │
-    │ │wlan0            DOWN # connect to wlan (3)                                      │ │
-    │ │Onion address: 6evxy5yhzytwpnhc2vpscrbti3iktxdhpnf6yim6bbs25p4v6beemzyd.onion    │ │
-    │ │Multicast DNS: nixos-installer.local                                             │ │
-    │ └─────────────────────────────────────────────────────────────────────────────────┘ │
-    │ Press 'Ctrl-C' for console access                                                   │
-    │                                                                                     │
-    └─────────────────────────────────────────────────────────────────────────────────────┘
-    ```
-
-    1.  This is not an actual QR code, because it is displayed rather poorly on text sites.
+1.  This is not an actual QR code, because it is displayed rather poorly on text sites.
     This would be the actual content of this specific QR code prettified:
     ```json
     {
@@ -225,41 +77,145 @@ Edit the following fields inside the `./machines/<machine_name>/configuration.ni
     ```shellSession
     echo '{"pass":"cheesy-capital-unwell","tor":"6evxy5yhzytwpnhc2vpscrbti3iktxdhpnf6yim6bbs25p4v6beemzyd.onion","addrs":["2001:9e8:347:ca00:21e:6ff:fe45:3c92"]}' | nix run nixpkgs#qrencode -- -s 2 -m 2 -t utf8
     ```
-    2.  The root password for the installer medium.
+2.  The root password for the installer medium.
     This password is autogenerated and meant to be easily typeable.
-    3.  See [how to connect to wlan](./installer.md#optional-connect-to-wifi-manually).
+3.  See [how to connect to wlan](./installer.md#optional-connect-to-wifi-manually).
 
-    !!! tip
+!!! tip
     Use [KDE Connect](https://apps.kde.org/de/kdeconnect/) for easyily sharing QR codes from phone to desktop
 
-=== "**Cloud VM**"
+## Cloud VMs
 
-    Just run the command **Option B: Cloud VM** below
+!!! note "skip this if using a physical machine"
+
+Clan supports any cloud machine if it is reachable via SSH and supports `kexec`.
+
+Steps:
+
+- Go to the configuration panel and note down how to connect to the machine via ssh.
+
+!!! tip "NixOS can cause strange issues when booting in certain cloud environments."
+    If on Linode: Make sure that the system uses "Direct Disk boot kernel" (found in the configuration panel)
+
+## Setting `targetHost`
+
+In your nix files set the targetHost (reachable ip) that you retrieved in the previous step.
+
+```{.nix title="clan.nix" hl_lines="9"}
+{
+  # Ensure this is unique among all clans you want to use.
+  meta.name = "my-clan";
+
+  inventory.machines = {
+    # Define machines here.
+    # The machine name will be used as the hostname.
+    jon = {
+        deploy.targetHost = "root@192.168.192.4"; # (1)
+    };
+  };
+  # ...
+  # elided
+}
+```
+
+1. Use the ip address of your targetMachine that you want to deploy. If using the [flash-installer](./installer.md) it should display its local ip-address when booted.
+
+!!! warning
+    The use of `root@` in the target address implies SSH access as the `root` user.
+    Ensure that the root login is secured and only used when necessary.
+
+See also [how to set TargetHost](../target-host.md) for other methods.
+
+## Retrieve the hardware report
+
+By default clan uses [nixos-facter](https://github.com/nix-community/nixos-facter) which captures detailed information about the machine or virtual environment.
+
+To generate the hardware-report (`facter.json`) run:
+
+```bash
+clan machines update-hardware-config <machineName>
+```
+
+Example output:
+
+```shell-session
+$ clan machines update-hardware-config jon
+[jon] $ nixos-facter
+Successfully generated: ./machines/jon/facter.json
+```
+
+See [update-hardware-config cli reference](../../reference/cli/machines.md#machines-update-hardware-config) for further configuration possibilities if needed.
+
+## Configure your disk schema
+
+By default clan uses [disko](https://github.com/nix-community/disko) which allows for declarative disk partitioning.
+
+To setup a disk schema for a machine run
+
+```bash
+clan templates apply disk single-disk jon --set mainDisk ""
+```
+
+Which should fail and give the valid options for the specific hardware:
+
+```shellSession
+Invalid value  for placeholder mainDisk - Valid options:
+/dev/disk/by-id/nvme-WD_PC_SN740_SDDQNQD-512G-1201_232557804368
+```
+
+Re-run the command with the correct disk:
+
+```bash
+clan templates apply disk single-disk jon --set mainDisk "/dev/disk/by-id/nvme-WD_PC_SN740_SDDQNQD-512G-1201_232557804368"
+```
+
+Should now be succesfull
+
+```shellSession
+Applied disk template 'single-disk' to machine 'jon'
+```
+
+A disko.nix file should be created in `machines/jon`
+You can have a look and customize it if needed.
+
+!!! tip
+    For advanced partitioning, see [Disko templates](https://github.com/nix-community/disko-templates) or [Disko examples](https://github.com/nix-community/disko/tree/master/example).
+
+!!! Danger
+    Don't change the `disko.nix` after the machine is installed for the first time.
+
+    Changing disko configuration requires wiping and reinstalling the machine.
+
+    Unless you really know what you are doing.
+
+## Deploy the machine
+
+**Finally deployment time!** Use one of the following commands to build and deploy the image via SSH onto your machine.
 
 ### Deployment Commands
 
 #### Using password auth
 
 ```bash
-clan machines install [MACHINE] --target-host <IP> --update-hardware-config nixos-facter
+clan machines install [MACHINE] --target-host <IP>
 ```
 
 #### Using QR JSON
 
 ```bash
-clan machines install [MACHINE] --json "[JSON]" --update-hardware-config nixos-facter
+clan machines install [MACHINE] --json "[JSON]"
 ```
 
 #### Using QR image file
 
 ```bash
-clan machines install [MACHINE] --png [PATH] --update-hardware-config nixos-facter
+clan machines install [MACHINE] --png [PATH]
 ```
 
 #### Option B: Cloud VM
 
 ```bash
-clan machines install [MACHINE] --target-host <IP> --update-hardware-config nixos-facter
+clan machines install [MACHINE] --target-host <IP>
 ```
 
 !!! success
@@ -318,4 +274,3 @@ clan {
 ```
 
 This is useful for machines that are not always online or are not part of the regular update cycle.
-

docs/site/guides/getting-started/index.md

@@ -38,31 +38,24 @@ By the end of this guide, you'll have a fresh NixOS configuration ready to push
 
 ## Add Clan CLI to Your Shell
 
-Add the Clan CLI into your environment:
+Create a new clan
 
 ```bash
-nix shell git+https://git.clan.lol/clan/clan-core#clan-cli --refresh
+nix run git+https://git.clan.lol/clan/clan-core#clan-cli --refresh -- flakes create
 ```
 
+This should prompt for a *name*:
+
 ```terminalSession
-clan --help
+Enter a name for the new clan: my-clan
 ```
 
-Should print the avilable commands.
+Enter a *name*, confirm with *enter*. A directory with that name will be created and initialized.
 
-Also checkout the [cli-reference documentation](../../reference/cli/index.md).
+!!! Note
+    This command uses the `default` template
 
-## Initialize Your Project
-
-If you want to migrate an existing project, follow this [guide](../migrations/migration-guide.md).
-
-Set the foundation of your Clan project by initializing it by running:
-
-```bash
-clan flakes create my-clan
-```
-
-This command creates a `flake.nix` and some other files for your project.
+    See `clan templates list` and the `--help` reference for how to use other templates.
 
 ## Explore the Project Structure
 
@@ -83,36 +76,48 @@ For example, you might see something like:
 └── README.md
 ```
 
-Don’t worry if your output looks different—the template evolves over time.
+Don’t worry if your output looks different — Clan templates evolve over time.
 
-??? info "Recommended way of sourcing the `clan` CLI tool"
+To interact with your newly created clan the you need to load the `clan` cli-package it into your environment by running:
 
-    The default template adds the `clan` CLI tool to the development shell.
-    This means that you can access the `clan` CLI tool directly from the folder
-    you are in right now.
+=== "Automatic (direnv, recommended)"
+    - prerequisite: [install nix-direnv](https://github.com/nix-community/nix-direnv)
 
-    In the `my-clan` directory, run the following command:
+    ```
+    direnv allow
+    ```
+
+=== "Manual (nix develop)"
 
     ```
     nix develop
     ```
 
-    This will ensure the `clan` CLI tool is available in your shell environment.
+verify that you can run `clan` commands:
 
-    To automatically add the `clan` CLI tool to your environment without having to
-    run `nix develop` every time, we recommend setting up [direnv](https://direnv.net/).
-
-```
+```bash
 clan show
 ```
 
 You should see something like this:
 
-```terminal-session
-Name: my-clan
+```shellSession
+Name: __CHANGE_ME__
 Description: None
 ```
 
+To change the name of your clan edit `meta.name` in the `clan.nix` or `flake.nix` file
+
+```{.nix title="clan.nix" hl_lines="3"}
+{
+  # Ensure this is unique among all clans you want to use.
+  meta.name = "__CHANGE_ME__";
+
+  # ...
+  # elided
+}
+```
+
 ---
 
 ## Next Steps
@@ -123,6 +128,7 @@ You can continue with **any** of the following steps at your own pace:
 - [x] [Initialize Clan](./index.md#initialize-your-project)
 - [ ] [Create USB Installer (optional)](./installer.md)
 - [ ] [Add Machines](./add-machines.md)
+- [ ] [Add a User](./add-user.md)
 - [ ] [Add Services](./add-services.md)
 - [ ] [Configure Secrets](./secrets.md)
 - [ ] [Deploy](./deploy.md) - Requires configured secrets

docs/site/guides/migrations/disk-id.md

@@ -0,0 +1,98 @@
+# Migrate disko config from `clanModules.disk-id`
+
+If you previously bootstrapped a machine's disk using `clanModules.disk-id`, you should now migrate to a standalone, self-contained disko configuration. This ensures long-term stability and avoids reliance on dynamic values from Clan.
+
+If your `disko.nix` currently looks something like this:
+
+```nix title="disko.nix"
+{
+  lib,
+  clan-core,
+  config,
+  ...
+}:
+
+let
+  suffix = config.clan.core.vars.generators.disk-id.files.diskId.value;
+in
+{
+  imports = [
+    clan-core.clanModules.disk-id
+  ];
+
+  # DO NOT EDIT THIS FILE AFTER INSTALLATION of a machine
+  # Otherwise your system might not boot because of missing partitions / filesystems
+  boot.loader.grub.efiSupport = lib.mkDefault true;
+  boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
+  disko.devices = {
+    disk = {
+      "main" = {
+        # suffix is to prevent disk name collisions
+        name = "main-" + suffix;
+        type = "disk";
+        # Set the following in flake.nix for each maschine:
+        # device = <uuid>;
+        content = {
+          # edlied
+        };
+      };
+    };
+  };
+}
+```
+
+## Step 1: Retrieve your `disk-id`
+
+Run the following command to retrieve the generated disk ID for your machine:
+
+```bash
+clan vars list <machineName>
+```
+
+Which should print the generated `disk-id/diskId` value in clear text
+You should see output like:
+
+```shellSession
+disk-id/diskId: fcef30a749f8451d8f60c46e1ead726f
+# ...
+# elided
+```
+
+Copy this value — you'll need it in the next step.
+
+## ✍️ Step 2: Replace Dynamic Configuration with Static Values
+
+✅ Goal: Make your disko.nix file standalone.
+
+We are going to make three changes:
+
+- Remove `let in, imports, {lib,clan-core,config, ...}:` to isolate the file.
+- Replace `suffix` with the actual disk-id
+- Move `disko.devices.disk.main.device` from `flake.nix` or `configuration.nix` into this file.
+
+```{.nix title="disko.nix" hl_lines="7-9 11-14"}
+{
+  boot.loader.grub.efiSupport = lib.mkDefault true;
+  boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
+  disko.devices = {
+    disk = {
+      "main" = {
+        #       ↓ Copy the disk-id into place
+        name = "main-fcef30a749f8451d8f60c46e1ead726f";
+        type = "disk";
+
+        # Some earlier guides had this line in a flake.nix
+        # disko.devices.disk.main.device = "/dev/disk/by-id/__CHANGE_ME__";
+        #        ↓ Copy the '/dev/disk/by-id' into here instead
+        device = "/dev/disk/by-id/nvme-eui.e8238fa6bf530001001b448b4aec2929";
+
+        # edlied;
+      };
+    };
+  };
+}
+```
+
+These steps are only needed for existing configurations that depend on the `diskId` module.
+
+For newer machines clan offers simple *disk templates* via its [templates cli](../../reference/cli/templates.md)

flake.lock

@@ -16,11 +16,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1751846468,
-        "narHash": "sha256-h0mpWZIOIAKj4fmLNyI2HDG+c0YOkbYmyJXSj/bQ9s0=",
-        "rev": "a2166c13b0cb3febdaf36391cd2019aa2ccf4366",
+        "lastModified": 1752451292,
+        "narHash": "sha256-jvLbfYFvcS5f0AEpUlFS2xZRnK770r9TRM2smpUFFaU=",
+        "rev": "309e06fbc9a6d133ab6dd1c7d8e4876526e058bb",
         "type": "tarball",
-        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/a2166c13b0cb3febdaf36391cd2019aa2ccf4366.tar.gz"
+        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/309e06fbc9a6d133ab6dd1c7d8e4876526e058bb.tar.gz"
       },
       "original": {
         "type": "tarball",

lib/modules/inventoryClass/service-list-from-inputs.nix

@@ -35,10 +35,20 @@ in
         inputName: v: lib.mapAttrs (inspectModule inputName) v.clan.modules
       ) inputsWithModules;
   };
-  options.localModules = lib.mkOption {
+  options.moduleSchemas = lib.mkOption {
+    # { sourceName :: { moduleName :: { roleName :: Schema }}}
     readOnly = true;
     type = lib.types.raw;
-    default = config.modulesPerSource.self;
+    default = lib.mapAttrs (
+      _inputName: moduleSet:
+      lib.mapAttrs (
+        _moduleName: module:
+        (clanLib.evalService {
+          modules = [ module ];
+          prefix = [ ];
+        }).config.result.api.schema
+      ) moduleSet
+    ) config.modulesPerSource;
   };
   options.templatesPerSource = lib.mkOption {
     # { sourceName :: { moduleName :: {} }}

nixosModules/clanCore/vars/interface.nix

@@ -22,6 +22,7 @@ let
     package
     path
     str
+    strMatching
     submoduleWith
     ;
   # the original types.submodule has strange behavior
@@ -47,7 +48,7 @@ in
           imports = [ ./generator.nix ];
           options = {
             name = lib.mkOption {
-              type = lib.types.str;
+              type = str;
               description = ''
                 The name of the generator.
                 This name will be used to refer to the generator in other generators.
@@ -153,7 +154,7 @@ in
                   options =
                     {
                       name = lib.mkOption {
-                        type = lib.types.str;
+                        type = str;
                         description = ''
                           name of the public fact
                         '';
@@ -162,7 +163,7 @@ in
                         defaultText = "Name of the file";
                       };
                       generatorName = lib.mkOption {
-                        type = lib.types.str;
+                        type = str;
                         description = ''
                           name of the generator
                         '';
@@ -171,7 +172,7 @@ in
                         defaultText = "Name of the generator that generates this file";
                       };
                       share = lib.mkOption {
-                        type = lib.types.bool;
+                        type = bool;
                         description = ''
                           Whether the generated vars should be shared between machines.
                           Shared vars are only generated once, when the first machine using it is deployed.
@@ -233,7 +234,7 @@ in
                           By setting this to `user`, the secret will be deployed prior to users and groups are created, allowing
                           users' passwords to be managed by vars. The secret will be stored in `/run/secrets-for-users` and `owner` and `group` must be `root`.
                         '';
-                        type = lib.types.enum [
+                        type = enum [
                           "partitioning"
                           "activation"
                           "users"
@@ -251,7 +252,7 @@ in
                         defaultText = lib.literalExpression ''if _class == "darwin" then "wheel" else "root"'';
                       };
                       mode = lib.mkOption {
-                        type = lib.types.strMatching "^[0-7]{4}$";
+                        type = strMatching "^[0-7]{4}$";
                         description = "The unix file mode of the file. Must be a 4-digit octal number.";
                         default = "0400";
                       };
@@ -375,7 +376,7 @@ in
                   - all required programs are in PATH
                   - sandbox is set up correctly
               '';
-              type = lib.types.path;
+              type = path;
               readOnly = true;
               internal = true;
             };

nixosModules/clanCore/vars/secret/sops/collectFiles.nix

@@ -0,0 +1,37 @@
+# collectFiles helper function
+{
+  lib ? import <nixpkgs/lib>,
+}:
+let
+  inherit (lib)
+    filterAttrs
+    flatten
+    mapAttrsToList
+    ;
+in
+generators:
+let
+  relevantFiles =
+    generator:
+    filterAttrs (
+      _name: f: f.secret && f.deploy && (f.neededFor == "users" || f.neededFor == "services")
+    ) generator.files;
+  allFiles = flatten (
+    mapAttrsToList (
+      gen_name: generator:
+      mapAttrsToList (fname: file: {
+        name = fname;
+        generator = gen_name;
+        neededForUsers = file.neededFor == "users";
+        inherit (generator) share;
+        inherit (file)
+          owner
+          group
+          mode
+          restartUnits
+          ;
+      }) (relevantFiles generator)
+    ) generators
+  );
+in
+allFiles

nixosModules/clanCore/vars/secret/sops/default.nix

@@ -7,7 +7,7 @@
 }:
 let
 
-  inherit (import ./funcs.nix { inherit lib; }) collectFiles;
+  collectFiles = import ./collectFiles.nix { inherit lib; };
 
   machineName = config.clan.core.settings.machine.name;
 

nixosModules/clanCore/vars/secret/sops/funcs.nix

@@ -1,42 +0,0 @@
-{
-  lib ? import <nixpkgs/lib>,
-  ...
-}:
-let
-
-  inherit (lib)
-    filterAttrs
-    flatten
-    mapAttrsToList
-    ;
-in
-{
-
-  collectFiles =
-    generators:
-    let
-      relevantFiles =
-        generator:
-        filterAttrs (
-          _name: f: f.secret && f.deploy && (f.neededFor == "users" || f.neededFor == "services")
-        ) generator.files;
-      allFiles = flatten (
-        mapAttrsToList (
-          gen_name: generator:
-          mapAttrsToList (fname: file: {
-            name = fname;
-            generator = gen_name;
-            neededForUsers = file.neededFor == "users";
-            inherit (generator) share;
-            inherit (file)
-              owner
-              group
-              mode
-              restartUnits
-              ;
-          }) (relevantFiles generator)
-        ) generators
-      );
-    in
-    allFiles;
-}

pkgs/clan-app/ui/src/index.tsx

@@ -3,7 +3,7 @@ import { render } from "solid-js/web";
 
 import "./index.css";
 import { QueryClient } from "@tanstack/solid-query";
-import { CubeScene } from "./scene/qubes";
+import { CubeScene } from "./scene/cubes";
 
 export const client = new QueryClient();
 

pkgs/clan-app/ui/src/scene/cubes.tsx

@@ -402,24 +402,16 @@ export function CubeScene() {
       if (selected) {
         // When selected, make all faces red-ish but maintain the lighting difference
         materials.forEach((material, index) => {
-          (material as THREE.MeshBasicMaterial).color.set(
-            index === 2
-              ? 0xff6666 // Top face - lighter red
-              : index === 0 || index === 4
-                ? 0xdce4e5 // Front/right faces - keep
-                : 0xa4b3b5, // Shadow faces - keep
-          );
+          if (index === 2) {
+            (material as THREE.MeshBasicMaterial).color.set(0xff6666);
+          }
         });
       } else {
         // Normal colors - restore original face colors
         materials.forEach((material, index) => {
-          (material as THREE.MeshBasicMaterial).color.set(
-            index === 2
-              ? 0xffffff // Top face - light
-              : index === 0 || index === 4
-                ? 0xdce4e5 // Front/right faces - medium
-                : 0xa4b3b5, // Shadow faces - dark
-          );
+          if (index === 2) {
+            (material as THREE.MeshBasicMaterial).color.set(0xffffff);
+          }
         });
       }
     }

pkgs/clan-cli/bin/clan

@@ -6,7 +6,7 @@ sys.path.insert(
     0, os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
 )
 
-from clan_cli import main  # NOQA
+from clan_cli.cli import main  # NOQA
 
 if __name__ == "__main__":
     main()

pkgs/clan-cli/bin/clan-config

@@ -6,7 +6,7 @@ sys.path.insert(
     0, os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
 )
 
-from clan_cli import config  # NOQA
+from clan_cli.cli import config  # NOQA
 
 if __name__ == "__main__":
     config.main()

pkgs/clan-cli/clan_cli/__init__.py

@@ -1,486 +0,0 @@
-import argparse
-import contextlib
-import logging
-import sys
-from pathlib import Path
-from types import ModuleType
-
-from clan_lib.custom_logger import setup_logging
-from clan_lib.dirs import get_clan_flake_toplevel_or_env
-from clan_lib.errors import ClanError
-from clan_lib.flake import Flake
-
-from . import (
-    backups,
-    clan,
-    secrets,
-    select,
-    state,
-    templates,
-    vms,
-)
-from .arg_actions import AppendOptionAction
-from .clan import show
-from .facts import cli as facts
-from .flash import cli as flash_cli
-from .hyperlink import help_hyperlink
-from .machines import cli as machines
-from .profiler import profile
-from .ssh import deploy_info as ssh_cli
-from .vars import cli as vars_cli
-
-log = logging.getLogger(__name__)
-
-argcomplete: ModuleType | None = None
-with contextlib.suppress(ImportError):
-    import argcomplete  # type: ignore[no-redef]
-
-
-def flake_path(arg: str) -> str:
-    flake_dir = Path(arg).resolve()
-    if flake_dir.exists() and flake_dir.is_dir():
-        return str(flake_dir)
-    return arg
-
-
-def default_flake() -> str | None:
-    val = get_clan_flake_toplevel_or_env()
-    if val:
-        return str(val)
-    return None
-
-
-def create_flake_from_args(args: argparse.Namespace) -> Flake:
-    """Create a Flake object from parsed arguments, including nix_options."""
-    flake_path_str = args.flake
-    nix_options = getattr(args, "option", [])
-    return Flake(flake_path_str, nix_options=nix_options)
-
-
-def add_common_flags(parser: argparse.ArgumentParser) -> None:
-    def argument_exists(parser: argparse.ArgumentParser, arg: str) -> bool:
-        """
-        Check if an argparse argument already exists.
-        This is needed because the aliases subcommand doesn't *really*
-        create an alias - it duplicates the actual parser in the tree
-        making duplication inevitable while naively traversing.
-
-        The error that would be thrown by argparse:
-        - argparse.ArgumentError
-        """
-        return any(
-            arg in action.option_strings
-            for action in parser._actions  # noqa: SLF001
-        )
-
-    if not argument_exists(parser, "--debug"):
-        parser.add_argument(
-            "--debug",
-            help="Enable debug logging",
-            action="store_true",
-            default=False,
-        )
-
-    if not argument_exists(parser, "--option"):
-        parser.add_argument(
-            "--option",
-            help="Nix option to set",
-            nargs=2,
-            metavar=("name", "value"),
-            action=AppendOptionAction,
-            default=[],
-        )
-
-    if not argument_exists(parser, "--flake"):
-        parser.add_argument(
-            "--flake",
-            help="path to the flake where the clan resides in, can be a remote flake or local, can be set through the [CLAN_DIR] environment variable",
-            default=default_flake(),
-            metavar="PATH",
-            type=flake_path,
-        )
-
-
-def register_common_flags(parser: argparse.ArgumentParser) -> None:
-    has_subparsers = False
-    for action in parser._actions:  # noqa: SLF001
-        if isinstance(action, argparse._SubParsersAction):  # noqa: SLF001
-            for _choice, child_parser in action.choices.items():
-                has_subparsers = True
-                register_common_flags(child_parser)
-
-    if not has_subparsers:
-        add_common_flags(parser)
-
-
-def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(
-        prog=prog,
-        usage="%(prog)s [-h] [SUBCOMMAND]",
-        description="The clan cli tool",
-        epilog=(
-            f"""
-Online reference for the clan cli tool: {help_hyperlink("cli reference", "https://docs.clan.lol/reference/cli")}
-For more detailed information, visit: {help_hyperlink("docs", "https://docs.clan.lol")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-
-    subparsers = parser.add_subparsers()
-
-    show_parser = subparsers.add_parser(
-        "show",
-        help="Show meta information about the clan",
-        description="Show meta information about the clan",
-        epilog=(
-            """
-This command prints the metadata of a clan.
-
-Examples:
-
-  $ clan show --flake [PATH]
-  Name: My Empty Clan
-  Description: some nice description
-  Icon: A path to the png
-"""
-        ),
-    )
-    show_parser.set_defaults(func=show.show_command)
-
-    parser_backups = subparsers.add_parser(
-        "backups",
-        aliases=["b"],
-        help="Manage backups of clan machines",
-        description="Manage backups of clan machines",
-        epilog=(
-            f"""
-This subcommand provides an interface to backups that clan machines expose.
-
-Examples:
-
-  $ clan backups list [MACHINE]
-  List backups for the machine [MACHINE]
-
-  $ clan backups create [MACHINE]
-  Create a backup for the machine [MACHINE].
-
-  $ clan backups restore [MACHINE] [PROVIDER] [NAME]
-  The backup to restore for the machine [MACHINE] with the configured [PROVIDER]
-  with the name [NAME].
-
-For more detailed information visit: {help_hyperlink("backups", "https://docs.clan.lol/guides/backups")}.
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    backups.register_parser(parser_backups)
-
-    parser_flake = subparsers.add_parser(
-        "flakes",
-        aliases=["f"],
-        help="Create a clan flake inside the current directory",
-        description="Create a clan flake inside the current directory",
-        epilog=(
-            f"""
-Examples:
-  $ clan flakes create [DIR]
-  Will create a new clan flake in the specified directory and create it if it
-  doesn't exist yet. The flake will be created from a default template.
-
-For more detailed information, visit: {help_hyperlink("getting-started", "https://docs.clan.lol/guides/getting-started/index.html")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-
-    clan.register_parser(parser_flake)
-
-    parser_templates = subparsers.add_parser(
-        "templates",
-        help="Subcommands to interact with templates",
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    templates.register_parser(parser_templates)
-
-    parser_flash = subparsers.add_parser(
-        "flash",
-        help="Flashes your machine to an USB drive",
-        description="Flashes your machine to an USB drive",
-        epilog=(
-            f"""
-Examples:
-  $ clan flash write mymachine --disk main /dev/sd<X> --ssh-pubkey ~/.ssh/id_rsa.pub
-  Will flash the machine 'mymachine' to the disk '/dev/sd<X>' with the ssh public key '~/.ssh/id_rsa.pub'.
-
-For more detailed information, visit: {help_hyperlink("getting-started", "https://docs.clan.lol/guides/getting-started/installer")}
-            """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    flash_cli.register_parser(parser_flash)
-
-    parser_ssh = subparsers.add_parser(
-        "ssh",
-        help="Ssh to a remote machine",
-        description="Ssh to a remote machine",
-        epilog=(
-            f"""
-This subcommand allows seamless ssh access to the nixos-image builders or a machine of your clan.
-
-Examples:
-
-  $ clan ssh [ssh_args ...] berlin`
-
-  Will ssh in to the machine called `berlin`, using the
-  `clan.core.networking.targetHost` specified in its configuration
-
-  $ clan ssh [ssh_args ...] --json [JSON]
-  Will ssh in to the machine based on the deployment information contained in
-  the json string. [JSON] can either be a json formatted string itself, or point
-  towards a file containing the deployment information
-
-For more detailed information, visit: {help_hyperlink("deploy", "https://docs.clan.lol/guides/getting-started/deploy")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    ssh_cli.register_parser(parser_ssh)
-
-    parser_secrets = subparsers.add_parser(
-        "secrets",
-        help="Manage secrets",
-        description="Manage secrets",
-        epilog=(
-            f"""
-This subcommand provides an interface to secrets.
-
-Examples:
-
-  $ clan secrets list [regex]
-  Will list secrets for all managed machines.
-  It accepts an optional regex, allowing easy filtering of returned secrets.
-
-  $ clan secrets get [SECRET]
-  Will display the content of the specified secret.
-
-For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    secrets.register_parser(parser_secrets)
-
-    parser_facts = subparsers.add_parser(
-        "facts",
-        help="Manage facts",
-        description="Manage facts",
-        epilog=(
-            f"""
-
-This subcommand provides an interface to facts of clan machines.
-Facts are artifacts that a service can generate.
-There are public and secret facts.
-Public facts can be referenced by other machines directly.
-Public facts can include: ip addresses, public keys.
-Secret facts can include: passwords, private keys.
-
-A service is an included clan-module that implements facts generation functionality.
-For example the zerotier module will generate private and public facts.
-In this case the public fact will be the resulting zerotier-ip of the machine.
-The secret fact will be the zerotier-identity-secret, which is used by zerotier
-to prove the machine has control of the zerotier-ip.
-
-Examples:
-
-  $ clan facts generate
-  Will generate facts for all machines.
-
-  $ clan facts generate --service [SERVICE] --regenerate
-  Will regenerate facts, if they are already generated for a specific service.
-  This is especially useful for resetting certain passwords while leaving the rest
-  of the facts for a machine in place.
-
-For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    facts.register_parser(parser_facts)
-
-    # like facts but with vars instead of facts
-    parser_vars = subparsers.add_parser(
-        "vars",
-        aliases=["va"],
-        help="Manage vars",
-        description="Manage vars",
-        epilog=(
-            f"""
-This subcommand provides an interface to `vars` of clan machines.
-Vars are variables that a service can generate.
-There are public and secret vars.
-Public vars can be referenced by other machines directly.
-Public vars can include: ip addresses, public keys.
-Secret vars can include: passwords, private keys.
-
-A service is an included clan-module that implements vars generation functionality.
-For example the zerotier module will generate private and public vars.
-In this case the public var will be the resulting zerotier-ip of the machine.
-The secret var will be the zerotier-identity-secret, which is used by zerotier
-to prove the machine has control of the zerotier-ip.
-
-Examples:
-
-    $ clan vars generate
-    Will generate vars for all machines.
-
-    $ clan vars generate --service [SERVICE] --regenerate
-    Will regenerate vars, if they are already generated for a specific service.
-    This is especially useful for resetting certain passwords while leaving the rest
-    of the vars for a machine in place.
-
-For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    vars_cli.register_parser(parser_vars)
-
-    parser_machine = subparsers.add_parser(
-        "machines",
-        aliases=["m"],
-        help="Manage machines and their configuration",
-        description="Manage machines and their configuration",
-        epilog=(
-            f"""
-This subcommand provides an interface to machines managed by Clan.
-
-Examples:
-
-  $ clan machines list
-  List all the machines managed by Clan.
-
-  $ clan machines update [MACHINES]
-  Will update the specified machines [MACHINES], if [MACHINES] is omitted, the command
-  will attempt to update every configured machine.
-
-  $ clan machines install [MACHINE] --target-host [TARGET_HOST]
-  Will install the specified machine [MACHINE] to the specified [TARGET_HOST].
-  If the `--target-host` flag is omitted will try to find host information by
-  checking the deployment configuration inside the specified machine.
-
-For more detailed information, visit: {help_hyperlink("deploy", "https://docs.clan.lol/guides/getting-started/deploy")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    machines.register_parser(parser_machine)
-
-    parser_vms = subparsers.add_parser(
-        "vms", help="Manage virtual machines", description="Manage virtual machines"
-    )
-    vms.register_parser(parser_vms)
-
-    parser_select = subparsers.add_parser(
-        "select",
-        aliases=["se"],
-        help="Select nixos values from the flake",
-        description="Select nixos values from the flake",
-        epilog=(
-            """
-This subcommand provides an interface nix values defined in the flake.
-
-Examples:
-
-  $ clan select nixosConfigurations.*.config.networking.hostName
-    List hostnames of all nixos configurations as JSON.
-
-  $ clan select nixosConfigurations.{jon,alice}.config.clan.core.vars.generators.*.name
-    List all vars generators for jon and alice.
-
-  $ clan select nixosConfigurations.jon.config.envirnonment.systemPackages.1
-    List the first system package for jon.
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    select.register_parser(parser_select)
-
-    parser_state = subparsers.add_parser(
-        "state",
-        aliases=["st"],
-        help="Query state information about machines",
-        description="Query state information about machines",
-        epilog=(
-            f"""
-This subcommand provides an interface to the state managed by Clan.
-
-State can be folders and databases that modules depend on managed by Clan.
-
-State directories can be added to on a per machine basis:
-```
-  config.clan.core.state.[SERVICE_NAME].folders = [
-    "/home"
-    "/root"
-  ];
-```
-Here [SERVICE_NAME] can be set freely, if the user sets them extra `userdata`
-can be a good choice.
-
-Examples:
-
-  $ clan state list [MACHINE]
-  List state of the machines managed by Clan.
-
-For more detailed information, visit: {help_hyperlink("getting-started", "https://docs.clan.lol/guides/backups")}
-        """
-        ),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    state.register_parser(parser_state)
-
-    if argcomplete:
-        argcomplete.autocomplete(parser, exclude=["morph"])
-
-    register_common_flags(parser)
-
-    return parser
-
-
-# this will be the entrypoint under /bin/clan (see pyproject.toml config)
-@profile
-def main() -> None:
-    parser = create_parser()
-    args = parser.parse_args()
-
-    if len(sys.argv) == 1:
-        parser.print_help()
-
-    if debug := getattr(args, "debug", False):
-        setup_logging(logging.DEBUG)
-        log.debug("Debug log activated")
-    else:
-        setup_logging(logging.INFO)
-
-    if not hasattr(args, "func"):
-        return
-
-    # Convert flake path to Flake object with nix_options if flake argument exists
-    if hasattr(args, "flake") and args.flake is not None:
-        args.flake = create_flake_from_args(args)
-
-    try:
-        args.func(args)
-    except ClanError as e:
-        if debug:
-            log.exception("Exited with error")
-        else:
-            log.error("%s", e)
-        sys.exit(1)
-    except KeyboardInterrupt as ex:
-        log.warning("Interrupted by user", exc_info=ex)
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

pkgs/clan-cli/clan_cli/__main__.py

@@ -1,4 +1,4 @@
-from . import main
+from .cli import main
 
 if __name__ == "__main__":
     main()

pkgs/clan-cli/clan_cli/clan/create.py

@@ -4,6 +4,7 @@ import logging
 from pathlib import Path
 
 from clan_lib.clan.create import CreateOptions, create_clan
+from clan_lib.errors import ClanError
 
 log = logging.getLogger(__name__)
 
@@ -26,10 +27,10 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
     )
 
     parser.add_argument(
-        "path",
-        type=Path,
-        help="Path where to write the clan template to",
-        default=Path(),
+        "name",
+        type=str,
+        nargs="?",
+        help="Name of the clan to create. If not provided, will prompt for a name.",
     )
 
     parser.add_argument(
@@ -40,9 +41,18 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
     )
 
     def create_flake_command(args: argparse.Namespace) -> None:
+        # Ask for a path interactively if none provided
+        if args.name is None:
+            user_input = input("Enter a name for the new clan: ").strip()
+            if not user_input:
+                msg = "Error: name is required."
+                raise ClanError(msg)
+
+            args.name = Path(user_input)
+
         create_clan(
             CreateOptions(
-                dest=args.path,
+                dest=Path(args.name),
                 template=args.template,
                 setup_git=not args.no_git,
                 src_flake=args.flake,

pkgs/clan-cli/clan_cli/cli.py

@@ -0,0 +1,506 @@
+import argparse
+import contextlib
+import logging
+import sys
+from pathlib import Path
+from types import ModuleType
+
+from clan_lib.custom_logger import setup_logging
+from clan_lib.dirs import get_clan_flake_toplevel_or_env
+from clan_lib.errors import ClanError
+from clan_lib.flake import Flake
+
+from . import (
+    backups,
+    clan,
+    secrets,
+    select,
+    state,
+    templates,
+    vms,
+)
+from .arg_actions import AppendOptionAction
+from .clan import show
+from .facts import cli as facts
+from .flash import cli as flash_cli
+from .hyperlink import help_hyperlink
+from .machines import cli as machines
+from .profiler import profile
+from .ssh import deploy_info as ssh_cli
+from .vars import cli as vars_cli
+
+log = logging.getLogger(__name__)
+
+argcomplete: ModuleType | None = None
+with contextlib.suppress(ImportError):
+    import argcomplete  # type: ignore[no-redef]
+
+
+def flake_path(arg: str) -> str:
+    flake_dir = Path(arg).resolve()
+    if flake_dir.exists() and flake_dir.is_dir():
+        return str(flake_dir)
+    return arg
+
+
+def default_flake() -> str | None:
+    val = get_clan_flake_toplevel_or_env()
+    if val:
+        return str(val)
+    return None
+
+
+def create_flake_from_args(args: argparse.Namespace) -> Flake:
+    """Create a Flake object from parsed arguments, including nix_options."""
+    flake_path_str = args.flake
+    nix_options = getattr(args, "option", [])
+    return Flake(flake_path_str, nix_options=nix_options)
+
+
+def add_common_flags(parser: argparse.ArgumentParser) -> None:
+    def argument_exists(parser: argparse.ArgumentParser, arg: str) -> bool:
+        """
+        Check if an argparse argument already exists.
+        This is needed because the aliases subcommand doesn't *really*
+        create an alias - it duplicates the actual parser in the tree
+        making duplication inevitable while naively traversing.
+
+        The error that would be thrown by argparse:
+        - argparse.ArgumentError
+        """
+        return any(
+            arg in action.option_strings
+            for action in parser._actions  # noqa: SLF001
+        )
+
+    if not argument_exists(parser, "--debug"):
+        parser.add_argument(
+            "--debug",
+            help="Enable debug logging",
+            action="store_true",
+            default=False,
+        )
+
+    if not argument_exists(parser, "--option"):
+        parser.add_argument(
+            "--option",
+            help="Nix option to set",
+            nargs=2,
+            metavar=("name", "value"),
+            action=AppendOptionAction,
+            default=[],
+        )
+
+    if not argument_exists(parser, "--flake"):
+        parser.add_argument(
+            "--flake",
+            help="path to the flake where the clan resides in, can be a remote flake or local, can be set through the [CLAN_DIR] environment variable",
+            default=default_flake(),
+            metavar="PATH",
+            type=flake_path,
+        )
+
+
+def register_common_flags(parser: argparse.ArgumentParser) -> None:
+    has_subparsers = False
+    for action in parser._actions:  # noqa: SLF001
+        if isinstance(action, argparse._SubParsersAction):  # noqa: SLF001
+            for _choice, child_parser in action.choices.items():
+                has_subparsers = True
+                register_common_flags(child_parser)
+
+    if not has_subparsers:
+        add_common_flags(parser)
+
+
+def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog=prog,
+        usage="%(prog)s [-h] [SUBCOMMAND]",
+        description="The clan cli tool",
+        epilog=(
+            f"""
+Online reference for the clan cli tool: {help_hyperlink("cli reference", "https://docs.clan.lol/reference/cli")}
+For more detailed information, visit: {help_hyperlink("docs", "https://docs.clan.lol")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+
+    subparsers = parser.add_subparsers()
+
+    show_parser = subparsers.add_parser(
+        "show",
+        help="Show meta information about the clan",
+        description="Show meta information about the clan",
+        epilog=(
+            """
+This command prints the metadata of a clan.
+
+Examples:
+
+  $ clan show --flake [PATH]
+  Name: My Empty Clan
+  Description: some nice description
+  Icon: A path to the png
+"""
+        ),
+    )
+    show_parser.set_defaults(func=show.show_command)
+
+    parser_backups = subparsers.add_parser(
+        "backups",
+        aliases=["b"],
+        help="Manage backups of clan machines",
+        description="Manage backups of clan machines",
+        epilog=(
+            f"""
+This subcommand provides an interface to backups that clan machines expose.
+
+Examples:
+
+  $ clan backups list [MACHINE]
+  List backups for the machine [MACHINE]
+
+  $ clan backups create [MACHINE]
+  Create a backup for the machine [MACHINE].
+
+  $ clan backups restore [MACHINE] [PROVIDER] [NAME]
+  The backup to restore for the machine [MACHINE] with the configured [PROVIDER]
+  with the name [NAME].
+
+For more detailed information visit: {help_hyperlink("backups", "https://docs.clan.lol/guides/backups")}.
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    backups.register_parser(parser_backups)
+
+    parser_flake = subparsers.add_parser(
+        "flakes",
+        aliases=["f"],
+        help="Create a clan flake inside the current directory",
+        description="Create a clan flake inside the current directory",
+        epilog=(
+            f"""
+Examples:
+  $ clan flakes create [DIR]
+  Will create a new clan flake in the specified directory and create it if it
+  doesn't exist yet. The flake will be created from a default template.
+
+For more detailed information, visit: {help_hyperlink("getting-started", "https://docs.clan.lol/guides/getting-started/index.html")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+
+    clan.register_parser(parser_flake)
+
+    parser_templates = subparsers.add_parser(
+        "templates",
+        help="Interact with templates",
+        description="Interact with templates",
+        epilog=(
+            """
+This subcommand provides an interface to templates provided by clan.
+
+Examples:
+
+  $ clan templates list
+  List all the machines managed by Clan.
+
+  $ clan templates apply disk [TEMPLATE] [MACHINE]
+  Will apply the specified [TEMPLATE] to the [MACHINE]
+
+  Many templates require to *set* variables via the `--set` flag.
+  $ clan templates apply disk [TEMPLATE] [MACHINE] --set key1 value1 --set key2 value2
+
+  Real world example
+  $ clan templates apply disk single-disk jon --set mainDisk "/dev/disk/by-id/nvme-WD_PC_SN740_SDDQNQD-512G-1201_232557804368"
+"""
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    templates.register_parser(parser_templates)
+
+    parser_flash = subparsers.add_parser(
+        "flash",
+        help="Flashes your machine to an USB drive",
+        description="Flashes your machine to an USB drive",
+        epilog=(
+            f"""
+Examples:
+  $ clan flash write mymachine --disk main /dev/sd<X> --ssh-pubkey ~/.ssh/id_rsa.pub
+  Will flash the machine 'mymachine' to the disk '/dev/sd<X>' with the ssh public key '~/.ssh/id_rsa.pub'.
+
+For more detailed information, visit: {help_hyperlink("getting-started", "https://docs.clan.lol/guides/getting-started/installer")}
+            """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    flash_cli.register_parser(parser_flash)
+
+    parser_ssh = subparsers.add_parser(
+        "ssh",
+        help="Ssh to a remote machine",
+        description="Ssh to a remote machine",
+        epilog=(
+            f"""
+This subcommand allows seamless ssh access to the nixos-image builders or a machine of your clan.
+
+Examples:
+
+  $ clan ssh berlin
+
+  Will ssh in to the machine called `berlin`, using the
+  `clan.core.networking.targetHost` specified in its configuration
+
+  $ clan ssh --json [JSON] --host-key-check none
+  Will ssh in to the machine based on the deployment information contained in
+  the json string. [JSON] can either be a json formatted string itself, or point
+  towards a file containing the deployment information
+
+For more detailed information, visit: {help_hyperlink("deploy", "https://docs.clan.lol/guides/getting-started/deploy")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    ssh_cli.register_parser(parser_ssh)
+
+    parser_secrets = subparsers.add_parser(
+        "secrets",
+        help="Manage secrets",
+        description="Manage secrets",
+        epilog=(
+            f"""
+This subcommand provides an interface to secrets.
+
+Examples:
+
+  $ clan secrets list [regex]
+  Will list secrets for all managed machines.
+  It accepts an optional regex, allowing easy filtering of returned secrets.
+
+  $ clan secrets get [SECRET]
+  Will display the content of the specified secret.
+
+For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    secrets.register_parser(parser_secrets)
+
+    parser_facts = subparsers.add_parser(
+        "facts",
+        help="Manage facts",
+        description="Manage facts",
+        epilog=(
+            f"""
+
+This subcommand provides an interface to facts of clan machines.
+Facts are artifacts that a service can generate.
+There are public and secret facts.
+Public facts can be referenced by other machines directly.
+Public facts can include: ip addresses, public keys.
+Secret facts can include: passwords, private keys.
+
+A service is an included clan-module that implements facts generation functionality.
+For example the zerotier module will generate private and public facts.
+In this case the public fact will be the resulting zerotier-ip of the machine.
+The secret fact will be the zerotier-identity-secret, which is used by zerotier
+to prove the machine has control of the zerotier-ip.
+
+Examples:
+
+  $ clan facts generate
+  Will generate facts for all machines.
+
+  $ clan facts generate --service [SERVICE] --regenerate
+  Will regenerate facts, if they are already generated for a specific service.
+  This is especially useful for resetting certain passwords while leaving the rest
+  of the facts for a machine in place.
+
+For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    facts.register_parser(parser_facts)
+
+    # like facts but with vars instead of facts
+    parser_vars = subparsers.add_parser(
+        "vars",
+        aliases=["va"],
+        help="Manage vars",
+        description="Manage vars",
+        epilog=(
+            f"""
+This subcommand provides an interface to `vars` of clan machines.
+Vars are variables that a service can generate.
+There are public and secret vars.
+Public vars can be referenced by other machines directly.
+Public vars can include: ip addresses, public keys.
+Secret vars can include: passwords, private keys.
+
+A service is an included clan-module that implements vars generation functionality.
+For example the zerotier module will generate private and public vars.
+In this case the public var will be the resulting zerotier-ip of the machine.
+The secret var will be the zerotier-identity-secret, which is used by zerotier
+to prove the machine has control of the zerotier-ip.
+
+Examples:
+
+    $ clan vars generate
+    Will generate vars for all machines.
+
+    $ clan vars generate --service [SERVICE] --regenerate
+    Will regenerate vars, if they are already generated for a specific service.
+    This is especially useful for resetting certain passwords while leaving the rest
+    of the vars for a machine in place.
+
+For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/guides/getting-started/secrets")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    vars_cli.register_parser(parser_vars)
+
+    parser_machine = subparsers.add_parser(
+        "machines",
+        aliases=["m"],
+        help="Manage machines and their configuration",
+        description="Manage machines and their configuration",
+        epilog=(
+            f"""
+This subcommand provides an interface to machines managed by Clan.
+
+Examples:
+
+  $ clan machines list
+  List all the machines managed by Clan.
+
+  $ clan machines update [MACHINES]
+  Will update the specified machines [MACHINES], if [MACHINES] is omitted, the command
+  will attempt to update every configured machine.
+
+  $ clan machines install [MACHINE] --target-host [TARGET_HOST]
+  Will install the specified machine [MACHINE] to the specified [TARGET_HOST].
+  If the `--target-host` flag is omitted will try to find host information by
+  checking the deployment configuration inside the specified machine.
+
+For more detailed information, visit: {help_hyperlink("deploy", "https://docs.clan.lol/guides/getting-started/deploy")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    machines.register_parser(parser_machine)
+
+    parser_vms = subparsers.add_parser(
+        "vms", help="Manage virtual machines", description="Manage virtual machines"
+    )
+    vms.register_parser(parser_vms)
+
+    parser_select = subparsers.add_parser(
+        "select",
+        aliases=["se"],
+        help="Select nixos values from the flake",
+        description="Select nixos values from the flake",
+        epilog=(
+            """
+This subcommand provides an interface nix values defined in the flake.
+
+Examples:
+
+  $ clan select nixosConfigurations.*.config.networking.hostName
+    List hostnames of all nixos configurations as JSON.
+
+  $ clan select nixosConfigurations.{jon,alice}.config.clan.core.vars.generators.*.name
+    List all vars generators for jon and alice.
+
+  $ clan select nixosConfigurations.jon.config.envirnonment.systemPackages.1
+    List the first system package for jon.
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    select.register_parser(parser_select)
+
+    parser_state = subparsers.add_parser(
+        "state",
+        aliases=["st"],
+        help="Query state information about machines",
+        description="Query state information about machines",
+        epilog=(
+            f"""
+This subcommand provides an interface to the state managed by Clan.
+
+State can be folders and databases that modules depend on managed by Clan.
+
+State directories can be added to on a per machine basis:
+```
+  config.clan.core.state.[SERVICE_NAME].folders = [
+    "/home"
+    "/root"
+  ];
+```
+Here [SERVICE_NAME] can be set freely, if the user sets them extra `userdata`
+can be a good choice.
+
+Examples:
+
+  $ clan state list [MACHINE]
+  List state of the machines managed by Clan.
+
+For more detailed information, visit: {help_hyperlink("getting-started", "https://docs.clan.lol/guides/backups")}
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    state.register_parser(parser_state)
+
+    if argcomplete:
+        argcomplete.autocomplete(parser, exclude=["morph"])
+
+    register_common_flags(parser)
+
+    return parser
+
+
+# this will be the entrypoint under /bin/clan (see pyproject.toml config)
+@profile
+def main() -> None:
+    parser = create_parser()
+    args = parser.parse_args()
+
+    if len(sys.argv) == 1:
+        parser.print_help()
+
+    if debug := getattr(args, "debug", False):
+        setup_logging(logging.DEBUG)
+        log.debug("Debug log activated")
+    else:
+        setup_logging(logging.INFO)
+
+    if not hasattr(args, "func"):
+        return
+
+    # Convert flake path to Flake object with nix_options if flake argument exists
+    if hasattr(args, "flake") and args.flake is not None:
+        args.flake = create_flake_from_args(args)
+
+    try:
+        args.func(args)
+    except ClanError as e:
+        if debug:
+            log.exception("Exited with error")
+        else:
+            log.error("%s", e)
+        sys.exit(1)
+    except KeyboardInterrupt as ex:
+        log.warning("Interrupted by user", exc_info=ex)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

pkgs/clan-cli/clan_cli/machines/install.py

@@ -24,12 +24,14 @@ def install_command(args: argparse.Namespace) -> None:
         # Only if the caller did not specify a target_host via args.target_host
         # Find a suitable target_host that is reachable
         target_host_str = args.target_host
-        deploy_info: DeployInfo | None = ssh_command_parse(args)
+        deploy_info: DeployInfo | None = (
+            ssh_command_parse(args) if target_host_str is None else None
+        )
 
         use_tor = False
-        if deploy_info and not args.target_host:
+        if deploy_info:
             host = find_reachable_host(deploy_info)
-            if host is None:
+            if host is None or host.tor_socks:
                 use_tor = True
                 target_host_str = deploy_info.tor.target
             else:

pkgs/clan-cli/clan_cli/ssh/deploy_info.py

@@ -1,12 +1,14 @@
 import argparse
 import json
 import logging
+import textwrap
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
 from clan_lib.cmd import run
 from clan_lib.errors import ClanError
+from clan_lib.machines.machines import Machine
 from clan_lib.nix import nix_shell
 from clan_lib.ssh.remote import HostKeyCheck, Remote
 
@@ -37,20 +39,23 @@ class DeployInfo:
             raise ClanError(msg)
         return addrs[0]
 
-    @staticmethod
-    def from_hostnames(
-        hostname: list[str], host_key_check: HostKeyCheck
+    def overwrite_remotes(
+        self,
+        host_key_check: HostKeyCheck | None = None,
+        private_key: Path | None = None,
+        ssh_options: dict[str, str] | None = None,
     ) -> "DeployInfo":
-        remotes = []
-        for host in hostname:
-            if not host:
-                msg = "Hostname cannot be empty."
-                raise ClanError(msg)
-            remote = Remote.from_ssh_uri(
-                machine_name="clan-installer", address=host
-            ).override(host_key_check=host_key_check)
-            remotes.append(remote)
-        return DeployInfo(addrs=remotes)
+        """Return a new DeployInfo with all Remotes overridden with the given host_key_check."""
+        return DeployInfo(
+            addrs=[
+                addr.override(
+                    host_key_check=host_key_check,
+                    private_key=private_key,
+                    ssh_options=ssh_options,
+                )
+                for addr in self.addrs
+            ]
+        )
 
     @staticmethod
     def from_json(data: dict[str, Any], host_key_check: HostKeyCheck) -> "DeployInfo":
@@ -103,9 +108,22 @@ def find_reachable_host(deploy_info: DeployInfo) -> Remote | None:
     return None
 
 
-def ssh_shell_from_deploy(deploy_info: DeployInfo) -> None:
+def ssh_shell_from_deploy(
+    deploy_info: DeployInfo, command: list[str] | None = None
+) -> None:
+    if command and len(command) == 1 and command[0].count(" ") > 0:
+        msg = (
+            textwrap.dedent("""
+            It looks like you quoted the remote command.
+            The first argument should be the command to run, not a quoted string.
+        """)
+            .lstrip("\n")
+            .rstrip("\n")
+        )
+        raise ClanError(msg)
+
     if host := find_reachable_host(deploy_info):
-        host.interactive_ssh()
+        host.interactive_ssh(command)
         return
 
     log.info("Could not reach host via clearnet 'addrs'")
@@ -127,7 +145,7 @@ def ssh_shell_from_deploy(deploy_info: DeployInfo) -> None:
                 log.info(
                     "Host reachable via tor address, starting interactive ssh session."
                 )
-                tor_addr.interactive_ssh()
+                tor_addr.interactive_ssh(command)
                 return
 
         log.error("Could not reach host via tor address.")
@@ -135,56 +153,99 @@ def ssh_shell_from_deploy(deploy_info: DeployInfo) -> None:
 
 def ssh_command_parse(args: argparse.Namespace) -> DeployInfo | None:
     host_key_check = args.host_key_check
+    deploy = None
+
     if args.json:
         json_file = Path(args.json)
         if json_file.is_file():
             data = json.loads(json_file.read_text())
             return DeployInfo.from_json(data, host_key_check)
         data = json.loads(args.json)
-        return DeployInfo.from_json(data, host_key_check)
-    if args.png:
-        return DeployInfo.from_qr_code(Path(args.png), host_key_check)
-
-    if hasattr(args, "machines"):
-        return DeployInfo.from_hostnames(args.machines, host_key_check)
+        deploy = DeployInfo.from_json(data, host_key_check)
+    elif args.png:
+        deploy = DeployInfo.from_qr_code(Path(args.png), host_key_check)
+    elif hasattr(args, "machine") and args.machine:
+        machine = Machine(args.machine, args.flake)
+        target = machine.target_host().override(
+            command_prefix=machine.name, host_key_check=host_key_check
+        )
+        deploy = DeployInfo(addrs=[target])
+    else:
         return None
 
+    ssh_options = None
+    if hasattr(args, "ssh_option") and args.ssh_option:
+        for name, value in args.ssh_option:
+            ssh_options = {}
+            ssh_options[name] = value
+
+    deploy = deploy.overwrite_remotes(ssh_options=ssh_options)
+
+    return deploy
+
 
 def ssh_command(args: argparse.Namespace) -> None:
     deploy_info = ssh_command_parse(args)
     if not deploy_info:
         msg = "No MACHINE, --json or --png data provided"
         raise ClanError(msg)
-
-    ssh_shell_from_deploy(deploy_info)
+    ssh_shell_from_deploy(deploy_info, args.remote_command)
 
 
 def register_parser(parser: argparse.ArgumentParser) -> None:
-    group = parser.add_mutually_exclusive_group(required=True)
-    machines_parser = group.add_argument(
-        "machines",
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        "machine",
         type=str,
-        nargs="*",
-        default=[],
+        nargs="?",
         metavar="MACHINE",
-        help="Machine to ssh into.",
+        help="Machine to ssh into (uses clan.core.networking.targetHost from configuration).",
     )
-    add_dynamic_completer(machines_parser, complete_machines)
 
     group.add_argument(
         "-j",
         "--json",
-        help="specify the json file for ssh data (generated by starting the clan installer)",
+        type=str,
+        help=(
+            "Deployment information as a JSON string or path to a JSON file "
+            "(generated by starting the clan installer)."
+        ),
     )
     group.add_argument(
         "-P",
         "--png",
-        help="specify the json file for ssh data as the qrcode image (generated by starting the clan installer)",
+        type=str,
+        help="Deployment information as a QR code image file (generated by starting the clan installer).",
     )
+
     parser.add_argument(
         "--host-key-check",
         choices=["strict", "ask", "tofu", "none"],
         default="tofu",
         help="Host key (.ssh/known_hosts) check mode.",
     )
+
+    parser.add_argument(
+        "--ssh-option",
+        help="SSH option to set (can be specified multiple times)",
+        nargs=2,
+        metavar=("name", "value"),
+        action="append",
+        default=[],
+    )
+
+    parser.add_argument(
+        "-c",
+        "--remote-command",
+        type=str,
+        metavar="COMMAND",
+        nargs=argparse.REMAINDER,
+        help="Command to execute on the remote host, needs to be the LAST argument as it takes all remaining arguments.",
+    )
+
+    add_dynamic_completer(
+        parser._actions[1],  # noqa: SLF001
+        complete_machines,
+    )  # assumes 'machine' is the first positional
+
     parser.set_defaults(func=ssh_command)

pkgs/clan-cli/clan_cli/ssh/test_deploy_info.py

@@ -7,6 +7,8 @@ from clan_lib.nix import nix_shell
 from clan_lib.ssh.remote import Remote
 
 from clan_cli.ssh.deploy_info import DeployInfo, find_reachable_host
+from clan_cli.tests.fixtures_flakes import ClanFlake
+from clan_cli.tests.helpers import cli
 
 
 def test_qrcode_scan(temp_dir: Path) -> None:
@@ -69,7 +71,10 @@ def test_from_json() -> None:
 @pytest.mark.with_core
 def test_find_reachable_host(hosts: list[Remote]) -> None:
     host = hosts[0]
-    deploy_info = DeployInfo.from_hostnames(["172.19.1.2", host.ssh_url()], "none")
+
+    uris = ["172.19.1.2", host.ssh_url()]
+    remotes = [Remote.from_ssh_uri(machine_name="some", address=uri) for uri in uris]
+    deploy_info = DeployInfo(addrs=remotes)
 
     assert deploy_info.addrs[0].address == "172.19.1.2"
 
@@ -77,3 +82,40 @@ def test_find_reachable_host(hosts: list[Remote]) -> None:
 
     assert remote is not None
     assert remote.ssh_url() == host.ssh_url()
+
+
+@pytest.mark.with_core
+def test_ssh_shell_from_deploy(
+    hosts: list[Remote],
+    flake: ClanFlake,
+) -> None:
+    host = hosts[0]
+
+    machine1_config = flake.machines["m1_machine"]
+    machine1_config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
+    machine1_config["clan"]["networking"]["targetHost"] = host.ssh_url()
+    flake.refresh()
+
+    assert host.private_key
+
+    success_txt = flake.path / "success.txt"
+    assert not success_txt.exists()
+    cli.run(
+        [
+            "ssh",
+            "--flake",
+            str(flake.path),
+            "m1_machine",
+            "--host-key-check=none",
+            "--ssh-option",
+            "IdentityFile",
+            str(host.private_key),
+            "--remote-command",
+            "touch",
+            str(success_txt),
+            "&&",
+            "exit 0",
+        ]
+    )
+
+    assert success_txt.exists()

pkgs/clan-cli/clan_cli/templates/__init__.py

@@ -1,6 +1,7 @@
 # !/usr/bin/env python3
 import argparse
 
+from .apply import register_apply_parser
 from .list import register_list_parser
 
 
@@ -12,5 +13,9 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
         help="the command to run",
         required=True,
     )
-    list_parser = subparser.add_parser("list", help="List avilable templates")
+    list_parser = subparser.add_parser("list", help="List available templates")
+    apply_parser = subparser.add_parser(
+        "apply", help="Apply a template of the specified type"
+    )
     register_list_parser(list_parser)
+    register_apply_parser(apply_parser)

pkgs/clan-cli/clan_cli/templates/apply.py

@@ -0,0 +1,15 @@
+import argparse
+
+from .apply_disk import register_apply_disk_template_parser
+
+
+def register_apply_parser(parser: argparse.ArgumentParser) -> None:
+    subparser = parser.add_subparsers(
+        title="template_type",
+        description="the template type to apply",
+        help="the template type to apply",
+        required=True,
+    )
+    disk_parser = subparser.add_parser("disk", help="Apply a disk template")
+
+    register_apply_disk_template_parser(disk_parser)

pkgs/clan-cli/clan_cli/templates/apply_disk.py

@@ -0,0 +1,82 @@
+import argparse
+import logging
+from collections.abc import Sequence
+from typing import Any
+
+from clan_lib.machines.machines import Machine
+from clan_lib.templates.disk import set_machine_disk_schema
+
+from clan_cli.completions import (
+    add_dynamic_completer,
+    complete_machines,
+)
+
+log = logging.getLogger(__name__)
+
+
+class AppendSetAction(argparse.Action):
+    def __init__(self, option_strings: str, dest: str, **kwargs: Any) -> None:
+        super().__init__(option_strings, dest, **kwargs)
+
+    def __call__(
+        self,
+        parser: argparse.ArgumentParser,
+        namespace: argparse.Namespace,
+        values: str | Sequence[str] | None,
+        option_string: str | None = None,
+    ) -> None:
+        lst = getattr(namespace, self.dest)
+        assert isinstance(values, list), "values must be a list"
+        lst.append((values[0], values[1]))
+
+
+def apply_command(args: argparse.Namespace) -> None:
+    """Apply a disk template to a machine."""
+    set_tuples: list[tuple[str, str]] = args.set
+
+    placeholders = dict(set_tuples)
+
+    set_machine_disk_schema(
+        Machine(args.machine, args.flake),
+        args.template,
+        placeholders,
+        force=args.force,
+        check_hw=not args.skip_hardware_check,
+    )
+    log.info(f"Applied disk template '{args.template}' to machine '{args.machine}' ")
+
+
+def register_apply_disk_template_parser(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "template",
+        type=str,
+        help="The name of the disk template to apply",
+    )
+    machine_action = parser.add_argument(
+        "machine",
+        type=str,
+        help="The machine to apply the template to",
+    )
+    add_dynamic_completer(machine_action, complete_machines)
+    parser.add_argument(
+        "--set",
+        help="Set a placeholder in the template to a value",
+        nargs=2,
+        metavar=("placeholder", "value"),
+        action=AppendSetAction,
+        default=[],
+    )
+    parser.add_argument(
+        "--force",
+        help="Force apply the template even if the machine already has a disk schema",
+        action="store_true",
+        default=False,
+    )
+    parser.add_argument(
+        "--skip-hardware-check",
+        help="Disables hardware checking. By default this command checks that the facter.json report exists and validates provided options",
+        action="store_true",
+        default=False,
+    )
+
+    parser.set_defaults(func=apply_command)

pkgs/clan-cli/clan_cli/tests/helpers/cli.py

@@ -2,7 +2,7 @@ import argparse
 import logging
 import shlex
 
-from clan_cli import create_flake_from_args, create_parser
+from clan_cli.cli import create_flake_from_args, create_parser
 from clan_lib.custom_logger import print_trace
 
 log = logging.getLogger(__name__)

pkgs/clan-cli/clan_cli/tests/test_modules.py

@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING
 import pytest
 from clan_cli.machines.create import CreateOptions, create_machine
 from clan_cli.tests.fixtures_flakes import FlakeForTest
-from clan_lib.api.modules import list_modules
 from clan_lib.flake import Flake
 from clan_lib.nix import nix_eval, run
 from clan_lib.nix_models.clan import (
@@ -16,6 +15,7 @@ from clan_lib.nix_models.clan import (
 )
 from clan_lib.persist.inventory_store import InventoryStore
 from clan_lib.persist.util import set_value_by_path
+from clan_lib.services.modules import list_service_modules
 
 if TYPE_CHECKING:
     from .age_keys import KeyPair
@@ -27,10 +27,9 @@ from clan_lib.machines.machines import Machine as MachineMachine
 @pytest.mark.with_core
 def test_list_modules(test_flake_with_core: FlakeForTest) -> None:
     base_path = test_flake_with_core.path
-    modules_info = list_modules(str(base_path))
+    modules_info = list_service_modules(Flake(str(base_path)))
 
-    assert "localModules" in modules_info
-    assert "modulesPerSource" in modules_info
+    assert "modules" in modules_info
 
 
 @pytest.mark.impure

pkgs/clan-cli/clan_lib/flake/flake.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import textwrap
 from dataclasses import asdict, dataclass, field
 from enum import Enum
 from hashlib import sha1
@@ -588,7 +589,7 @@ class FlakeCache:
 
     def load_from_file(self, path: Path) -> None:
         with path.open("r") as f:
-            log.debug(f"Loading cache from {path}")
+            log.debug("Loading flake cache from file")
             data = json.load(f)
             self.cache = FlakeCacheEntry.from_json(data["cache"])
 
@@ -662,7 +663,7 @@ class Flake:
         """
         Loads the flake into the store and populates self.store_path and self.hash such that the flake can evaluate locally and offline
         """
-        from clan_lib.cmd import run
+        from clan_lib.cmd import RunOpts, run
         from clan_lib.nix import (
             nix_command,
         )
@@ -681,7 +682,10 @@ class Flake:
             self.identifier,
         ]
 
-        flake_prefetch = run(nix_command(cmd))
+        trace_prefetch = os.environ.get("CLAN_DEBUG_NIX_PREFETCH", "0") == "1"
+        if not trace_prefetch:
+            log.debug(f"Prefetching flake {self.identifier}")
+        flake_prefetch = run(nix_command(cmd), RunOpts(trace=trace_prefetch))
         flake_metadata = json.loads(flake_prefetch.stdout)
         self.store_path = flake_metadata["storePath"]
         self.hash = flake_metadata["hash"]
@@ -698,8 +702,6 @@ class Flake:
             nix_metadata,
         )
 
-        log.debug(f"Invalidating cache for {self.identifier}")
-
         self.prefetch()
 
         self._cache = FlakeCache()
@@ -813,12 +815,15 @@ class Flake:
                 ];
               }}
         """
-        if len(selectors) > 1:
-            log.debug(f"""
-selecting: {selectors}
-to debug run:
-nix repl --expr 'rec {{
-  flake = builtins.getFlake "self.identifier";
+        if len(selectors) > 1 :
+            msg = textwrap.dedent(f"""
+                clan select "{selectors}"
+            """).lstrip("\n").rstrip("\n")
+            if os.environ.get("CLAN_DEBUG_NIX_SELECTORS"):
+                msg += textwrap.dedent(f"""
+                    to debug run:
+                    nix repl --expr 'rec {{
+                    flake = builtins.getFlake "{self.identifier}";
                     selectLib = (builtins.getFlake "path:{select_source()}?narHash={select_hash}").lib;
                     query = [
                         {" ".join(
@@ -828,21 +833,24 @@ nix repl --expr 'rec {{
                             ]
                         )}
                     ];
-}}'
-            """)
+                    }}'
+                """).lstrip("\n")
+            log.debug(msg)
         # fmt: on
         elif len(selectors) == 1:
-            log.debug(
-                f"""
-selecting: {selectors[0]}
-to debug run:
-nix repl --expr 'rec {{
+            msg = textwrap.dedent(f"""
+               $ clan select "{selectors[0]}"
+            """).lstrip("\n").rstrip("\n")
+            if os.environ.get("CLAN_DEBUG_NIX_SELECTORS"):
+                msg += textwrap.dedent(f"""
+                to debug run:
+                    nix repl --expr 'rec {{
                     flake = builtins.getFlake "{self.identifier}";
                     selectLib = (builtins.getFlake "path:{select_source()}?narHash={select_hash}").lib;
                     query = selectLib.select '"''{selectors[0]}''"' flake;
-}}'
-            """
-            )
+                    }}'
+                    """).lstrip("\n")
+            log.debug(msg)
 
         build_output = Path(
             run(

pkgs/clan-cli/clan_lib/machines/actions.py

@@ -65,14 +65,14 @@ def get_machine(flake: Flake, name: str) -> InventoryMachine:
         InventoryMachine: An instance representing the machine's inventory details.
 
     Raises:
-        ClanError: If the machine with the specified name is not found in the inventory.
+        ClanError: If the machine with the specified name is not found in the clan
     """
     inventory_store = InventoryStore(flake=flake)
     inventory = inventory_store.read()
 
     machine_inv = inventory.get("machines", {}).get(name)
     if machine_inv is None:
-        msg = f"Machine {name} not found in inventory"
+        msg = f"Machine {name} does not exist"
         raise ClanError(msg)
 
     return InventoryMachine(**machine_inv)

pkgs/clan-cli/clan_lib/machines/hardware.py

@@ -118,7 +118,7 @@ def run_machine_hardware_info(
     commit_file(
         hw_file,
         opts.machine.flake.path,
-        f"machines/{opts.machine}/{hw_file.name}: update hardware configuration",
+        f"machines/{opts.machine.name}/{hw_file.name}: update hardware configuration",
     )
     try:
         get_machine_target_platform(opts.machine)

pkgs/clan-cli/clan_lib/machines/list.py

@@ -5,13 +5,13 @@ from dataclasses import dataclass
 from clan_cli.machines.hardware import HardwareConfig
 
 from clan_lib.api import API
-from clan_lib.api.disk import MachineDiskMatter
-from clan_lib.api.modules import parse_frontmatter
 from clan_lib.dirs import specific_machine_dir
 from clan_lib.flake import Flake
 from clan_lib.machines.actions import get_machine, list_machines
 from clan_lib.machines.machines import Machine
 from clan_lib.nix_models.clan import InventoryMachine
+from clan_lib.services.modules import parse_frontmatter
+from clan_lib.templates.disk import MachineDiskMatter
 
 log = logging.getLogger(__name__)
 

pkgs/clan-cli/clan_lib/services/instances.py

@@ -0,0 +1,115 @@
+from typing import TypedDict
+
+from clan_lib.api import API
+from clan_lib.errors import ClanError
+from clan_lib.flake.flake import Flake
+from clan_lib.nix_models.clan import (
+    InventoryInstanceModule,
+    InventoryInstanceRolesType,
+    InventoryInstancesType,
+    InventoryMachinesType,
+)
+from clan_lib.persist.inventory_store import InventoryStore
+from clan_lib.persist.util import set_value_by_path
+from clan_lib.services.modules import (
+    get_service_module,
+)
+
+# TODO: move imports out of cli/__init__.py causing import cycles
+# from clan_lib.machines.actions import list_machines
+
+
+@API.register
+def list_service_instances(flake: Flake) -> InventoryInstancesType:
+    """
+    Returns all currently present service instances including their full configuration
+    """
+
+    inventory_store = InventoryStore(flake)
+    inventory = inventory_store.read()
+    instances = inventory.get("instances", {})
+    return instances
+
+
+def collect_tags(machines: InventoryMachinesType) -> set[str]:
+    res = set()
+    for _, machine in machines.items():
+        res |= set(machine.get("tags", []))
+
+    return res
+
+
+# Removed 'module' ref - Needs to be passed seperately
+class InstanceConfig(TypedDict):
+    roles: InventoryInstanceRolesType
+
+
+@API.register
+def create_service_instance(
+    flake: Flake,
+    module_ref: InventoryInstanceModule,
+    instance_name: str,
+    instance_config: InstanceConfig,
+) -> None:
+    module = get_service_module(flake, module_ref)
+
+    inventory_store = InventoryStore(flake)
+    inventory = inventory_store.read()
+
+    instances = inventory.get("instances", {})
+    if instance_name in instances:
+        msg = f"service instance '{instance_name}' already exists."
+        raise ClanError(msg)
+
+    target_roles = instance_config.get("roles")
+    if not target_roles:
+        msg = "Creating a service instance requires adding roles"
+        raise ClanError(msg)
+
+    available_roles = set(module.get("roles", {}).keys())
+
+    unavailable_roles = list(filter(lambda r: r not in available_roles, target_roles))
+    if unavailable_roles:
+        msg = f"Unknown roles: {unavailable_roles}. Use one of {available_roles}"
+        raise ClanError(msg)
+
+    role_configs = instance_config.get("roles")
+    if not role_configs:
+        return
+
+    ## Validate machine references
+    all_machines = inventory.get("machines", {})
+    available_machine_refs = set(all_machines.keys())
+    available_tag_refs = collect_tags(all_machines)
+
+    for role_name, role_members in role_configs.items():
+        machine_refs = role_members.get("machines")
+        msg = f"Role: '{role_name}' - "
+        if machine_refs:
+            unavailable_machines = list(
+                filter(lambda m: m not in available_machine_refs, machine_refs)
+            )
+            if unavailable_machines:
+                msg += f"Unknown machine reference: {unavailable_machines}. Use one of {available_machine_refs}"
+                raise ClanError(msg)
+
+        tag_refs = role_members.get("tags")
+        if tag_refs:
+            unavailable_tags = list(
+                filter(lambda m: m not in available_tag_refs, tag_refs)
+            )
+
+            if unavailable_tags:
+                msg += (
+                    f"Unknown tags: {unavailable_tags}. Use one of {available_tag_refs}"
+                )
+                raise ClanError(msg)
+
+    # TODO:
+    # Validate instance_config roles settings against role schema
+
+    set_value_by_path(inventory, f"instances.{instance_name}", instance_config)
+    set_value_by_path(inventory, f"instances.{instance_name}.module", module_ref)
+    inventory_store.write(
+        inventory, message=f"services: instance '{instance_name}' init"
+    )

pkgs/clan-cli/clan_lib/services/modules.py

@@ -4,10 +4,10 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, TypedDict
 
+from clan_lib.api import API
 from clan_lib.errors import ClanError
 from clan_lib.flake import Flake
-
-from . import API
+from clan_lib.nix_models.clan import InventoryInstanceModuleType
 
 
 class CategoryInfo(TypedDict):
@@ -154,22 +154,96 @@ class ModuleInfo(TypedDict):
     roles: dict[str, None]
 
 
-class ModuleLists(TypedDict):
-    modulesPerSource: dict[str, dict[str, ModuleInfo]]
-    localModules: dict[str, ModuleInfo]
+class ModuleList(TypedDict):
+    modules: dict[str, dict[str, ModuleInfo]]
 
 
 @API.register
-def list_modules(base_path: str) -> ModuleLists:
+def list_service_modules(flake: Flake) -> ModuleList:
     """
     Show information about a module
     """
-    flake = Flake(base_path)
-    modules = flake.select(
-        "clanInternals.inventoryClass.{?modulesPerSource,?localModules}"
-    )
+    modules = flake.select("clanInternals.inventoryClass.modulesPerSource")
 
-    return modules
+    return ModuleList({"modules": modules})
+
+
+@API.register
+def get_service_module(
+    flake: Flake, module_ref: InventoryInstanceModuleType
+) -> ModuleInfo:
+    """
+    Returns the module information for a given module reference
+
+    :param module_ref: The module reference to get the information for
+    :return: Dict of module information
+    :raises ClanError: If the module_ref is invalid or missing required fields
+    """
+
+    input_name, module_name = check_service_module_ref(flake, module_ref)
+
+    avilable_modules = list_service_modules(flake)
+    module_set = avilable_modules.get("modules", {}).get(input_name)
+
+    assert module_set is not None  # Since check_service_module_ref already checks this
+
+    module = module_set.get(module_name)
+
+    assert module is not None  # Since check_service_module_ref already checks this
+
+    return module
+
+
+def check_service_module_ref(
+    flake: Flake,
+    module_ref: InventoryInstanceModuleType,
+) -> tuple[str, str]:
+    """
+    Checks if the module reference is valid
+
+    :param module_ref: The module reference to check
+    :raises ClanError: If the module_ref is invalid or missing required fields
+    """
+    avilable_modules = list_service_modules(flake)
+
+    input_ref = module_ref.get("input", None)
+    if input_ref is None:
+        msg = "Setting module_ref.input is currently required"
+        raise ClanError(msg)
+
+    module_set = avilable_modules.get("modules", {}).get(input_ref)
+
+    if module_set is None:
+        msg = f"module set for input '{input_ref}' not found"
+        msg += f"\nAvilable input_refs: {avilable_modules.get('modules', {}).keys()}"
+        raise ClanError(msg)
+
+    module_name = module_ref.get("name")
+    assert module_name
+    module = module_set.get(module_name)
+    if module is None:
+        msg = f"module with name '{module_name}' not found"
+        raise ClanError(msg)
+
+    return (input_ref, module_name)
+
+
+@API.register
+def get_service_module_schema(
+    flake: Flake, module_ref: InventoryInstanceModuleType
+) -> dict[str, Any]:
+    """
+    Returns the schema for a service module
+
+    :param module_ref: The module reference to get the schema for
+    :return: Dict of schemas for the service module roles
+    :raises ClanError: If the module_ref is invalid or missing required fields
+    """
+    input_name, module_name = check_service_module_ref(flake, module_ref)
+
+    return flake.select(
+        f"clanInternals.inventoryClass.moduleSchemas.{input_name}.{module_name}"
+    )
 
 
 @dataclass

pkgs/clan-cli/clan_lib/ssh/remote.py

@@ -16,7 +16,7 @@ from tempfile import TemporaryDirectory
 from clan_lib.api import API
 from clan_lib.cmd import ClanCmdError, ClanCmdTimeoutError, CmdOut, RunOpts, run
 from clan_lib.colors import AnsiColor
-from clan_lib.errors import ClanError  # Assuming these are available
+from clan_lib.errors import ClanError, indent_command  # Assuming these are available
 from clan_lib.nix import nix_shell
 from clan_lib.ssh.host_key import HostKeyCheck, hostkey_to_ssh_opts
 from clan_lib.ssh.parse import parse_ssh_uri
@@ -61,6 +61,9 @@ class Remote:
         private_key: Path | None = None,
         password: str | None = None,
         tor_socks: bool | None = None,
+        command_prefix: str | None = None,
+        port: int | None = None,
+        ssh_options: dict[str, str] | None = None,
     ) -> "Remote":
         """
         Returns a new Remote instance with the same data but with a different host_key_check.
@@ -68,8 +71,8 @@ class Remote:
         return Remote(
             address=self.address,
             user=self.user,
-            command_prefix=self.command_prefix,
-            port=self.port,
+            command_prefix=command_prefix or self.command_prefix,
+            port=port or self.port,
             private_key=private_key if private_key is not None else self.private_key,
             password=password if password is not None else self.password,
             forward_agent=self.forward_agent,
@@ -77,7 +80,7 @@ class Remote:
                 host_key_check if host_key_check is not None else self.host_key_check
             ),
             verbose_ssh=self.verbose_ssh,
-            ssh_options=self.ssh_options,
+            ssh_options=ssh_options or self.ssh_options,
             tor_socks=tor_socks if tor_socks is not None else self.tor_socks,
             _control_path_dir=self._control_path_dir,
             _askpass_path=self._askpass_path,
@@ -418,10 +421,30 @@ class Remote:
                 msg = f"SSH command failed with return code {res.returncode}"
                 raise ClanError(msg)
 
-    def interactive_ssh(self) -> None:
-        cmd_list = self.ssh_cmd(tty=True, control_master=False)
-        res = subprocess.run(cmd_list, check=False)
+    def interactive_ssh(self, command: list[str] | None = None) -> None:
+        ssh_cmd = self.ssh_cmd(tty=True, control_master=False)
+        if command:
+            ssh_cmd = [
+                *self.ssh_cmd(tty=True, control_master=False),
+                "--",
+                "bash",
+                "-c",
+                quote('exec "$@"'),
+                "--",
+                " ".join(map(quote, command)),
+            ]
+        cmdlog.info(
+            f"{indent_command(ssh_cmd)}",
+            extra={
+                "command_prefix": self.command_prefix,
+                "color": AnsiColor.GREEN.value,
+            },
+        )
+        res = subprocess.run(ssh_cmd, check=False)
 
+        # We only check the error code if a password is set, as sshpass is used.
+        # AS sshpass swallows all output.
+        if self.password:
             self.check_sshpass_errorcode(res)
 
     def check_machine_ssh_reachable(self) -> bool:
@@ -431,7 +454,7 @@ class Remote:
 @dataclass(frozen=True)
 class ConnectionOptions:
     timeout: int = 2
-    retries: int = 10
+    retries: int = 5
 
 
 @dataclass
@@ -504,6 +527,10 @@ def check_machine_ssh_reachable(
     if opts is None:
         opts = ConnectionOptions()
 
+    cmdlog.debug(
+        f"Checking SSH reachability for {remote.target} on port {remote.port or 22}",
+    )
+
     address_family = socket.AF_INET6 if ":" in remote.address else socket.AF_INET
     for _ in range(opts.retries):
         with socket.socket(address_family, socket.SOCK_STREAM) as sock:

pkgs/clan-cli/clan_lib/templates/disk.py

@@ -6,12 +6,12 @@ from typing import Any, TypedDict
 from uuid import uuid4
 
 from clan_lib.api import API
-from clan_lib.api.modules import Frontmatter, extract_frontmatter
 from clan_lib.dirs import TemplateType, clan_templates
 from clan_lib.errors import ClanError
 from clan_lib.git import commit_file
 from clan_lib.machines.hardware import HardwareConfig, get_machine_hardware_config
 from clan_lib.machines.machines import Machine
+from clan_lib.services.modules import Frontmatter, extract_frontmatter
 
 log = logging.getLogger(__name__)
 
@@ -72,8 +72,18 @@ templates: dict[str, dict[str, Callable[[dict[str, Any]], Placeholder]]] = {
 }
 
 
+def get_empty_placeholder(label: str) -> Placeholder:
+    return Placeholder(
+        label,
+        options=None,
+        required=not label.endswith("*"),
+    )
+
+
 @API.register
-def get_machine_disk_schemas(machine: Machine) -> dict[str, DiskSchema]:
+def get_machine_disk_schemas(
+    machine: Machine, check_hw: bool = True
+) -> dict[str, DiskSchema]:
     """
     Get the available disk schemas.
     This function reads the disk schemas from the templates directory and returns them as a dictionary.
@@ -89,9 +99,11 @@ def get_machine_disk_schemas(machine: Machine) -> dict[str, DiskSchema]:
     hw_report = {}
 
     hw_report_path = HardwareConfig.NIXOS_FACTER.config_path(machine)
-    if not hw_report_path.exists():
+    if check_hw and not hw_report_path.exists():
         msg = "Hardware configuration missing"
         raise ClanError(msg)
+
+    if hw_report_path.exists():
         with hw_report_path.open("r") as hw_report_file:
             hw_report = json.load(hw_report_file)
 
@@ -109,7 +121,10 @@ def get_machine_disk_schemas(machine: Machine) -> dict[str, DiskSchema]:
             placeholders = {}
 
             if placeholder_getters:
-                placeholders = {k: v(hw_report) for k, v in placeholder_getters.items()}
+                placeholders = {
+                    k: v(hw_report) if hw_report else get_empty_placeholder(k)
+                    for k, v in placeholder_getters.items()
+                }
 
             raw_readme = (disk_template / "README.md").read_text()
             frontmatter, readme = extract_frontmatter(
@@ -139,16 +154,22 @@ def set_machine_disk_schema(
     # Use get disk schemas to get the placeholders and their options
     placeholders: dict[str, str],
     force: bool = False,
+    check_hw: bool = True,
 ) -> None:
     """
     Set the disk placeholders of the template
     """
+    # Ensure the machine exists
+    machine.get_inv_machine()
+
     # Assert the hw-config must exist before setting the disk
     hw_config = get_machine_hardware_config(machine)
     hw_config_path = hw_config.config_path(machine)
 
+    if check_hw:
         if not hw_config_path.exists():
-        msg = "Hardware configuration must exist before applying disk schema"
+            msg = "Hardware configuration must exist for checking."
+            msg += f"\nrun 'clan machines update-hardware-config {machine.name}' to generate a hardware report. Alternatively disable hardware checking to skip this check"
             raise ClanError(msg)
 
         if hw_config != HardwareConfig.NIXOS_FACTER:
@@ -158,15 +179,16 @@ def set_machine_disk_schema(
     disk_schema_path = clan_templates(TemplateType.DISK) / f"{schema_name}/default.nix"
 
     if not disk_schema_path.exists():
-        msg = f"Disk schema not found at {disk_schema_path}"
+        msg = f"Disk schema '{schema_name}' not found at {disk_schema_path}"
+        msg += f"\nAvailable schemas: {', '.join([p.name for p in clan_templates(TemplateType.DISK).iterdir()])}"
         raise ClanError(msg)
 
     # Check that the placeholders are valid
-    disk_schema = get_machine_disk_schemas(machine)[schema_name]
+    disk_schema = get_machine_disk_schemas(machine, check_hw)[schema_name]
     # check that all required placeholders are present
     for placeholder_name, schema_placeholder in disk_schema.placeholders.items():
         if schema_placeholder.required and placeholder_name not in placeholders:
-            msg = f"Required placeholder {placeholder_name} - {schema_placeholder} missing"
+            msg = f"Required to set template variable {placeholder_name}"
             raise ClanError(msg)
 
     # For every placeholder check that the value is valid
@@ -183,12 +205,15 @@ def set_machine_disk_schema(
                 description=f"Available placeholders: {disk_schema.placeholders.keys()}",
             )
 
-        # Invalid value. Check if the value is one of the provided options
-        if ph.options and placeholder_value not in ph.options:
+        # Checking invalid value: if the value is one of the provided options
+        if check_hw and ph.options and placeholder_value not in ph.options:
             msg = (
                 f"Invalid value {placeholder_value} for placeholder {placeholder_name}"
             )
-            raise ClanError(msg, description=f"Valid options: {ph.options}")
+            raise ClanError(
+                msg,
+                description=f"Valid options: \n{'\n'.join(ph.options)}",
+            )
 
     placeholders_toml = "\n".join(
         [f"""# {k} = "{v}" """ for k, v in placeholders.items() if v is not None]
@@ -221,6 +246,9 @@ def set_machine_disk_schema(
             disk_config.write(header)
             disk_config.write(config_str)
 
+        # TODO: return files to commit
+        # Don't commit here
+        # The top level command will usually collect files and commit them in batches
         commit_file(
             disko_file_path,
             machine.flake.path,

pkgs/clan-cli/clan_lib/templates/filesystem.py

@@ -1,3 +1,4 @@
+import shutil
 from pathlib import Path
 
 from clan_lib.flake import Flake
@@ -35,5 +36,7 @@ def copy_from_nixstore(src: Path, dest: Path) -> None:
     Uses `cp -r` to recursively copy the directory.
     Ensures the destination directory is writable by the user.
     """
-    run(["cp", "-r", str(src), str(dest)])
-    run(["chmod", "-R", "u+w", str(dest)])
+    shutil.copytree(src, dest, dirs_exist_ok=True, symlinks=True)
+    run(
+        ["chmod", "-R", "u+w", str(dest)]
+    )  # Ensure the destination is writable by the user

pkgs/clan-cli/clan_lib/tests/test_create.py

@@ -16,8 +16,6 @@ from clan_cli.secrets.sops import maybe_get_admin_public_keys
 from clan_cli.secrets.users import add_user
 from clan_cli.vars.generate import get_generators, run_generators
 
-from clan_lib.api.disk import hw_main_disk_options, set_machine_disk_schema
-from clan_lib.api.modules import list_modules
 from clan_lib.cmd import RunOpts, run
 from clan_lib.dirs import specific_machine_dir
 from clan_lib.errors import ClanError
@@ -33,7 +31,9 @@ from clan_lib.nix_models.clan import (
 from clan_lib.nix_models.clan import InventoryMachineDeploy as MachineDeploy
 from clan_lib.persist.inventory_store import InventoryStore
 from clan_lib.persist.util import set_value_by_path
+from clan_lib.services.modules import list_service_modules
 from clan_lib.ssh.remote import Remote, check_machine_ssh_login
+from clan_lib.templates.disk import hw_main_disk_options, set_machine_disk_schema
 
 log = logging.getLogger(__name__)
 
@@ -206,9 +206,9 @@ def test_clan_create_api(
     store = InventoryStore(clan_dir_flake)
     inventory = store.read()
 
-    modules = list_modules(str(clan_dir_flake.path))
+    modules = list_service_modules(clan_dir_flake)
     assert (
-        modules["modulesPerSource"]["clan-core"]["admin"]["manifest"]["name"]
+        modules["modules"]["clan-core"]["admin"]["manifest"]["name"]
         == "clan-core/admin"
     )
 

pkgs/clan-cli/docs.py

@@ -4,7 +4,7 @@ import sys
 from dataclasses import dataclass
 from pathlib import Path
 
-from clan_cli import create_parser
+from clan_cli.cli import create_parser
 
 hidden_subcommands = ["machine", "b", "f", "m", "se", "st", "va"]
 

pkgs/clan-cli/openapi.py

@@ -41,7 +41,7 @@ TOP_LEVEL_RESOURCES = {
     "secret",  # sops & key operations
     "log",  # log operations
     "generator",  # vars generators operations
-    "module",  # module (clan.service) management
+    "service",  # clan.service management
     "system",  # system operations
 }
 

pkgs/clan-cli/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "clan"
 description = "clan cli tool"
 dynamic = ["version"]
-scripts = { clan = "clan_cli:main" }
+scripts = { clan = "clan_cli.cli:main" }
 license = { text = "MIT" }
 
 [project.urls]

renovate.json

@@ -1,15 +0,0 @@
-{
-  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": ["config:recommended"],
-  "lockFileMaintenance": { "enabled": true },
-  "nix": {
-    "enabled": true
-  },
-  "packageRules": [
-    {
-      "matchManagers": ["npm"],
-      "matchPaths": ["pkgs/clan-app/ui/**"],
-      "enabled": false
-    }
-  ]
-}

templates/clan/default/clan.nix

@@ -2,6 +2,11 @@
   # Ensure this is unique among all clans you want to use.
   meta.name = "__CHANGE_ME__";
 
+  inventory.machines = {
+    # Define machines here.
+    # jon = { };
+  };
+
   # Docs: See https://docs.clan.lol/reference/clanServices
   inventory.instances = {
 

templates/clan/default/modules/disko.nix

@@ -1,61 +0,0 @@
-{
-  lib,
-  clan-core,
-  config,
-  ...
-}:
-
-let
-  suffix = config.clan.core.vars.generators.disk-id.files.diskId.value;
-in
-{
-  imports = [
-    clan-core.clanModules.disk-id
-  ];
-
-  # DO NOT EDIT THIS FILE AFTER INSTALLATION of a machine
-  # Otherwise your system might not boot because of missing partitions / filesystems
-  boot.loader.grub.efiSupport = lib.mkDefault true;
-  boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
-  disko.devices = {
-    disk = {
-      "main" = {
-        # suffix is to prevent disk name collisions
-        name = "main-" + suffix;
-        type = "disk";
-        # Set the following in flake.nix for each maschine:
-        # device = <uuid>;
-        content = {
-          type = "gpt";
-          partitions = {
-            "boot" = {
-              size = "1M";
-              type = "EF02"; # for grub MBR
-              priority = 1;
-            };
-            "ESP" = {
-              size = "512M";
-              type = "EF00";
-              content = {
-                type = "filesystem";
-                format = "vfat";
-                mountpoint = "/boot";
-                mountOptions = [ "nofail" ];
-              };
-            };
-            "root" = {
-              size = "100%";
-              content = {
-                type = "filesystem";
-                format = "ext4";
-                # format = "btrfs";
-                # format = "bcachefs";
-                mountpoint = "/";
-              };
-            };
-          };
-        };
-      };
-    };
-  };
-}

templates/clan/default/modules/shared.nix

@@ -1,28 +0,0 @@
-{
-  clan-core,
-  # Optional, if you want to access other flakes:
-  # self,
-  ...
-}:
-{
-  imports = [
-    clan-core.clanModules.user-password
-  ];
-
-  # Locale service discovery and mDNS
-  services.avahi.enable = true;
-
-  # generate a random password for our user below
-  # can be read using `clan secrets get <machine-name>-user-password` command
-  clan.user-password.user = "user";
-  users.users.user = {
-    isNormalUser = true;
-    extraGroups = [
-      "wheel"
-      "networkmanager"
-      "video"
-      "input"
-    ];
-    uid = 1000;
-  };
-}

templates/clan/flake-parts/clan.nix

@@ -1,81 +1,50 @@
-{ self }:
 {
-  meta.name = "__CHANGE_ME__"; # Ensure this is unique among all clans you want to use.
+  # Ensure this is unique among all clans you want to use.
+  meta.name = "__CHANGE_ME__";
 
-  inherit self;
+  inventory.machines = {
+    # Define machines here.
+    # jon = { };
+  };
+
+  # Docs: See https://docs.clan.lol/reference/clanServices
+  inventory.instances = {
+
+    # Docs: https://docs.clan.lol/reference/clanServices/admin/
+    # Admin service for managing machines
+    # This service adds a root password and SSH access.
+    admin = {
+      roles.default.tags.all = { };
+      roles.default.settings.allowedKeys = {
+        # Insert the public key that you want to use for SSH access.
+        # All keys will have ssh access to all machines ("tags.all" means 'all machines').
+        # Alternatively set 'users.users.root.openssh.authorizedKeys.keys' in each machine
+        "admin-machine-1" = "__YOUR_PUBLIC_KEY__";
+      };
+    };
+
+    # Docs: https://docs.clan.lol/reference/clanServices/zerotier/
+    # The lines below will define a zerotier network and add all machines as 'peer' to it.
+    # !!! Manual steps required:
+    #   - Define a controller machine for the zerotier network.
+    #   - Deploy the controller machine first to initilize the network.
+    zerotier = {
+      # Replace with the name (string) of your machine that you will use as zerotier-controller
+      # See: https://docs.zerotier.com/controller/
+      # Deploy this machine first to create the network secrets
+      roles.controller.machines."__YOUR_CONTROLLER__" = { };
+      # Peers of the network
+      # tags.all means 'all machines' will joined
+      roles.peer.tags.all = { };
+    };
+  };
+
+  # Additional NixOS configuration can be added here.
+  # machines/jon/configuration.nix will be automatically imported.
+  # See: https://docs.clan.lol/guides/more-machines/#automatic-registration
   machines = {
-    # "jon" will be the hostname of the machine
-    jon =
-      { pkgs, ... }:
-      {
-        imports = [
-          ./modules/shared.nix
-          ./modules/disko.nix
-          ./machines/jon/configuration.nix
-        ];
-
-        nixpkgs.hostPlatform = "x86_64-linux";
-
-        # Set this for clan commands use ssh i.e. `clan machines update`
-        # If you change the hostname, you need to update this line to root@<new-hostname>
-        # This only works however if you have avahi running on your admin machine else use IP
-        clan.core.networking.targetHost = pkgs.lib.mkDefault "root@jon";
-
-        # You can get your disk id by running the following command on the installer:
-        # Replace <IP> with the IP of the installer printed on the screen or by running the `ip addr` command.
-        # ssh root@<IP> lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT
-        disko.devices.disk.main = {
-          device = "/dev/disk/by-id/__CHANGE_ME__";
-        };
-
-        # IMPORTANT! Add your SSH key here
-        # e.g. > cat ~/.ssh/id_ed25519.pub
-        users.users.root.openssh.authorizedKeys.keys = throw ''
-          Don't forget to add your SSH key here!
-          users.users.root.openssh.authorizedKeys.keys = [ "<YOUR SSH_KEY>" ]
-        '';
-
-        # Zerotier needs one controller to accept new nodes. Once accepted
-        # the controller can be offline and routing still works.
-        clan.core.networking.zerotier.controller.enable = true;
-      };
-    # "sara" will be the hostname of the machine
-    sara =
-      { pkgs, ... }:
-      {
-        imports = [
-          ./modules/shared.nix
-          ./modules/disko.nix
-          ./machines/sara/configuration.nix
-        ];
-
-        nixpkgs.hostPlatform = "x86_64-linux";
-
-        # Set this for clan commands use ssh i.e. `clan machines update`
-        # If you change the hostname, you need to update this line to root@<new-hostname>
-        # This only works however if you have avahi running on your admin machine else use IP
-        clan.core.networking.targetHost = pkgs.lib.mkDefault "root@sara";
-
-        # You can get your disk id by running the following command on the installer:
-        # Replace <IP> with the IP of the installer printed on the screen or by running the `ip addr` command.
-        # ssh root@<IP> lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT
-        disko.devices.disk.main = {
-          device = "/dev/disk/by-id/__CHANGE_ME__";
-        };
-
-        # IMPORTANT! Add your SSH key here
-        # e.g. > cat ~/.ssh/id_ed25519.pub
-        users.users.root.openssh.authorizedKeys.keys = throw ''
-          Don't forget to add your SSH key here!
-          users.users.root.openssh.authorizedKeys.keys = [ "<YOUR SSH_KEY>" ]
-        '';
-
-        /*
-          After jon is deployed, uncomment the following line
-          This will allow sara to share the VPN overlay network with jon
-          The networkId is generated by the first deployment of jon
-        */
-        # clan.core.networking.zerotier.networkId = builtins.readFile ../../vars/per-machine/jon/zerotier/zerotier-network-id/value;
-      };
+    # jon = { config, ... }: {
+    #   environment.systemPackages = [ pkgs.asciinema ];
+    # };
   };
 }

templates/clan/flake-parts/flake.nix

@@ -6,7 +6,6 @@
 
   outputs =
     inputs@{
-      self,
       flake-parts,
       ...
     }:
@@ -22,7 +21,9 @@
       ];
 
       # https://docs.clan.lol/guides/getting-started/flake-parts/
-      clan = import ./clan.nix { inherit self; };
+      clan = {
+        imports = [ ./clan.nix ];
+      };
 
       perSystem =
         { pkgs, inputs', ... }:

templates/clan/flake-parts/machines/jon/configuration.nix

@@ -1,38 +0,0 @@
-{ config, ... }:
-let
-  username = config.networking.hostName;
-in
-{
-  imports = [ ./hardware-configuration.nix ];
-
-  # Locale service discovery and mDNS
-  services.avahi.enable = true;
-
-  services.xserver.enable = true;
-  services.xserver.desktopManager.gnome.enable = true;
-  services.xserver.displayManager.gdm.enable = true;
-  # Disable the default gnome apps to speed up deployment
-  services.gnome.core-utilities.enable = false;
-
-  # Enable automatic login for the user.
-  services.displayManager.autoLogin = {
-    enable = true;
-    user = username;
-  };
-
-  users.users.${username} = {
-    initialPassword = username;
-    isNormalUser = true;
-    extraGroups = [
-      "wheel"
-      "networkmanager"
-      "video"
-      "audio"
-      "input"
-      "dialout"
-      "disk"
-    ];
-    uid = 1000;
-    openssh.authorizedKeys.keys = config.users.users.root.openssh.authorizedKeys.keys;
-  };
-}

templates/clan/flake-parts/machines/sara/configuration.nix

@@ -1,39 +0,0 @@
-{ config, ... }:
-
-let
-  username = config.networking.hostName;
-in
-{
-  imports = [ ./hardware-configuration.nix ];
-
-  # Locale service discovery and mDNS
-  services.avahi.enable = true;
-
-  services.xserver.enable = true;
-  services.xserver.desktopManager.gnome.enable = true;
-  services.xserver.displayManager.gdm.enable = true;
-  # Disable the default gnome apps to speed up deployment
-  services.gnome.core-utilities.enable = false;
-
-  # Enable automatic login for the user.
-  services.displayManager.autoLogin = {
-    enable = true;
-    user = username;
-  };
-
-  users.users.${username} = {
-    initialPassword = username;
-    isNormalUser = true;
-    extraGroups = [
-      "wheel"
-      "networkmanager"
-      "video"
-      "audio"
-      "input"
-      "dialout"
-      "disk"
-    ];
-    uid = 1000;
-    openssh.authorizedKeys.keys = config.users.users.root.openssh.authorizedKeys.keys;
-  };
-}

templates/clan/flake-parts/modules/disko.nix

@@ -1,51 +0,0 @@
-{ lib, clan-core, ... }:
-
-let
-  suffix = config.clan.core.vars.generators.disk-id.files.diskId.value;
-in
-{
-  imports = [
-    clan-core.clanModules.disk-id
-  ];
-
-  boot.loader.grub.efiSupport = lib.mkDefault true;
-  boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
-  disko.devices = {
-    disk = {
-      "main" = {
-        # suffix is to prevent disk name collisions
-        name = "main-" + suffix;
-        type = "disk";
-        # Set the following in flake.nix for each maschine:
-        # device = <uuid>;
-        content = {
-          type = "gpt";
-          partitions = {
-            "boot" = {
-              size = "1M";
-              type = "EF02"; # for grub MBR
-              priority = 1;
-            };
-            "ESP" = {
-              size = "512M";
-              type = "EF00";
-              content = {
-                type = "filesystem";
-                format = "vfat";
-                mountpoint = "/boot";
-              };
-            };
-            "root" = {
-              size = "100%";
-              content = {
-                type = "filesystem";
-                format = "ext4";
-                mountpoint = "/";
-              };
-            };
-          };
-        };
-      };
-    };
-  };
-}

templates/flake-module.nix

@@ -5,6 +5,27 @@
   };
 
   flake = {
+    checks.x86_64-linux.equal-templates =
+      inputs.nixpkgs.legacyPackages.x86_64-linux.runCommand "minimal-clan-flake" { }
+        ''
+          file1=${./clan/default/clan.nix}
+          file2=${./clan/flake-parts/clan.nix}
+
+          echo "Comparing $file1 and $file2"
+          if cmp -s "$file1" "$file2"; then
+            echo "clan.nix files are identical"
+          else
+            echo "clan.nix files are out of sync"
+            echo "Please make sure to keep templates clan.nix files in sync."
+            echo "files: templates/clan/default/clan.nix templates/clan/flake-parts/clan.nix"
+            echo "--------------------------------\n"
+            diff "$file1" "$file2"
+            echo "--------------------------------\n\n"
+            exit 1
+          fi
+
+          touch $out
+        '';
     checks.x86_64-linux.template-minimal =
       let
         path = self.clan.templates.clan.minimal.path;