clan: add run_no_stdout function suppressing stdout

Add the `run_no_stdout` function suppressing stdout by default.
This keeps the noise down on most commands, while still staying
debuggable.
Stdout will be active when the `--debug` flag is passed to the cli.

Fixes #1443

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-# Contributing to cLAN
+# Contributing to Clan
 
 ## Live-reloading documentation
 

README.md

@@ -1,6 +1,6 @@
 # Clan Core Repository
 
-Welcome to the Clan Core Repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo is the foundation of Clan, a revolutionary open-source project aimed at restoring fun, freedom, and functionality to computing. Here, you'll find all the essential packages, NixOS modules, CLI tools, and tests needed to contribute to and work with the cLAN project. Clan leverages the Nix system to ensure reliability, security, and seamless management of digital environments, putting the power back into the hands of users.
+Welcome to the Clan Core Repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo is the foundation of Clan, a revolutionary open-source project aimed at restoring fun, freedom, and functionality to computing. Here, you'll find all the essential packages, NixOS modules, CLI tools, and tests needed to contribute to and work with the Clan project. Clan leverages the Nix system to ensure reliability, security, and seamless management of digital environments, putting the power back into the hands of users.
 
 ## Why Clan?
 

checks/impure/flake-module.nix

@@ -7,6 +7,8 @@
         #!${pkgs.bash}/bin/bash
         set -euo pipefail
 
+        unset CLAN_DIR
+
         export PATH="${
           lib.makeBinPath [
             pkgs.gitMinimal

clanModules/flake-module.nix

@@ -17,6 +17,7 @@
     static-hosts = ./static-hosts;
     syncthing = ./syncthing;
     thelounge = ./thelounge;
+    trusted-nix-caches = ./trusted-nix-caches;
     user-password = ./user-password;
     xfce = ./xfce;
     zerotier-static-peers = ./zerotier-static-peers;

clanModules/trusted-nix-caches/README.md

@@ -0,0 +1,2 @@
+This module sets the `clan.lol` and `nix-community` cache up as a trusted cache.
+----

clanModules/trusted-nix-caches/default.nix

@@ -0,0 +1,10 @@
+{
+  nix.settings.trusted-substituters = [
+    "https://cache.clan.lol"
+    "https://nix-community.cachix.org"
+  ];
+  nix.settings.trusted-public-keys = [
+    "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
+    "cache.clan.lol-1:3KztgSAB5R1M+Dz7vzkBGzXdodizbgLXGXKXlcQLA28="
+  ];
+}

devShell.nix

@@ -1,3 +1,4 @@
+{ ... }:
 {
   perSystem =
     {
@@ -34,9 +35,6 @@
           config.treefmt.build.wrapper
         ];
         shellHook = ''
-          # no longer used
-          rm -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit"
-
           echo -e "${ansiEscapes.green}switch to another dev-shell using: select-shell${ansiEscapes.reset}"
         '';
       };

docs/_internal/use-cases/01-self-host.md

@@ -20,11 +20,11 @@ There are several reasons for choosing to self-host. These can include:
 
 Alice wants to self-host a mumble server for her family.
 
-- She visits to the cLAN website, and follows the instructions on how to install cLAN-OS on her server.
+- She visits to the Clan website, and follows the instructions on how to install Clan-OS on her server.
-- Alice logs into a terminal on her server via SSH (alternatively uses cLAN GUI app)
+- Alice logs into a terminal on her server via SSH (alternatively uses Clan GUI app)
-- Using the cLAN CLI or GUI tool, alice creates a new private network for her family (VPN)
+- Using the Clan CLI or GUI tool, alice creates a new private network for her family (VPN)
-- Alice now browses a list of curated cLAN modules and finds a module for mumble.
+- Alice now browses a list of curated Clan modules and finds a module for mumble.
-- She adds this module to her network using the cLAN tool.
+- She adds this module to her network using the Clan tool.
 - After that, she uses the clan tool to invite her family members to her network
 - Other family members join the private network via the invitation.
 - By accepting the invitation, other members automatically install all required software to interact with the network on their machine.
@@ -33,7 +33,7 @@ Alice wants to self-host a mumble server for her family.
 
 Alice wants to add a photos app to her private network
 
-- She uses the clan CLI or GUI tool to manage her existing private cLAN family network
+- She uses the clan CLI or GUI tool to manage her existing private Clan family network
 - She discovers a module for photoprism, and adds it to her server using the tool
 - Other members who are already part of her network, will receive a notification that an update is required to their environment
 - After accepting, all new software and services to interact with the new photoprism service will be installed automatically.

docs/_internal/use-cases/02-join-network.md

@@ -1,4 +1,4 @@
-# Joining a cLAN network
+# Joining a Clan network
 
 ## General Description
 
@@ -8,13 +8,13 @@ Joining a self-hosted infrastructure involves connecting to a network, server, o
 
 ### Story 1: Joining a private network
 
-Alice' son Bob has never heard of cLAN, but receives an invitation URL from Alice who already set up private cLAN network for her family.
+Alice' son Bob has never heard of Clan, but receives an invitation URL from Alice who already set up private Clan network for her family.
 
-Bob opens the invitation link and lands on the cLAN website. He quickly learns about what cLAN is and can see that the invitation is for a private network of his family that hosts a number of services, like a private voice chat and a photo sharing platform.
+Bob opens the invitation link and lands on the Clan website. He quickly learns about what Clan is and can see that the invitation is for a private network of his family that hosts a number of services, like a private voice chat and a photo sharing platform.
 
-Bob decides to join the network and follows the instructions to install the cLAN tool on his computer.
+Bob decides to join the network and follows the instructions to install the Clan tool on his computer.
 
-Feeding the invitation link to the cLAN tool, bob registers his machine with the network.
+Feeding the invitation link to the Clan tool, bob registers his machine with the network.
 
 All programs required to interact with the network will be installed and configured automatically and securely.
 
@@ -22,7 +22,7 @@ Optionally, bob can customize the configuration of these programs through a simp
 
 ### Story 2: Receiving breaking changes
 
-The cLAN family network which Bob is part of received an update.
+The Clan family network which Bob is part of received an update.
 
 The existing photo sharing service has been removed and replaced with another alternative service. The new photo sharing service requires a different client app to view and upload photos.
 
@@ -30,7 +30,7 @@ Bob accepts the update. Now his environment will be updated. The old client soft
 
 Because Bob has customized the previous photo viewing app, he is notified that this customization is no longer valid, as the software has been removed (deprecation message).l
 
-Optionally, Bob can now customize the new photo viewing software through his cLAN configuration app or via a config file.
+Optionally, Bob can now customize the new photo viewing software through his Clan configuration app or via a config file.
 
 ## Challenges
 

docs/_internal/use-cases/03-maintaining-clan-modules.md

@@ -1,10 +1,10 @@
-# cLAN module maintaining
+# Clan module maintaining
 
 ## General Description
 
-cLAN modules are pieces of software that can be used by admins to build a private or public infrastructure.
+Clan modules are pieces of software that can be used by admins to build a private or public infrastructure.
 
-cLAN modules should have the following properties:
+Clan modules should have the following properties:
 
 1. Documented: It should be clear what the module does and how to use it.
 1. Self contained: A module should be usable as is. If it requires any other software or settings, those should be delivered with the module itself.

docs/mkdocs.yml

@@ -28,6 +28,7 @@ markdown_extensions:
   - pymdownx.highlight:
       use_pygments: true
       anchor_linenums: true
+  - pymdownx.keys
   - toc:
       title: On this page
 
@@ -64,6 +65,7 @@ nav:
           - reference/clanModules/syncthing.md
           - reference/clanModules/static-hosts.md
           - reference/clanModules/thelounge.md
+          - reference/clanModules/trusted-nix-caches.md
           - reference/clanModules/user-password.md
           - reference/clanModules/xfce.md
           - reference/clanModules/zerotier-static-peers.md
@@ -92,8 +94,8 @@ docs_dir: site
 site_dir: out
 
 theme:
-  logo: static/clan-white.png
+  logo: https://clan.lol/static/logo/clan-white.png
-  favicon: static/clan-dark.png
+  favicon: https://clan.lol/static/logo/clan-dark.png
   name: material
   features:
     - navigation.instant
@@ -103,6 +105,8 @@ theme:
     - content.tabs.link
   icon:
     repo: fontawesome/brands/git
+  font:
+    code: Roboto Mono
 
   palette:
     # Palette toggle for light mode

docs/nix/default.nix

@@ -2,6 +2,8 @@
   pkgs,
   module-docs,
   clan-cli-docs,
+  asciinema-player-js,
+  asciinema-player-css,
   ...
 }:
 let
@@ -27,6 +29,10 @@ pkgs.stdenv.mkDerivation {
     mkdir -p ./site/reference/cli
     cp -af ${module-docs}/* ./site/reference/
     cp -af ${clan-cli-docs}/* ./site/reference/cli/
+
+    mkdir -p ./site/static/asciinema-player
+    ln -snf ${asciinema-player-js} ./site/static/asciinema-player/asciinema-player.min.js
+    ln -snf ${asciinema-player-css} ./site/static/asciinema-player/asciinema-player.css
   '';
 
   buildPhase = ''

docs/nix/flake-module.nix

@@ -40,6 +40,15 @@
             mypy --strict $out
           '';
 
+      asciinema-player-js = pkgs.fetchurl {
+        url = "https://github.com/asciinema/asciinema-player/releases/download/v3.7.0/asciinema-player.min.js";
+        sha256 = "sha256-Ymco/+FinDr5YOrV72ehclpp4amrczjo5EU3jfr/zxs=";
+      };
+      asciinema-player-css = pkgs.fetchurl {
+        url = "https://github.com/asciinema/asciinema-player/releases/download/v3.7.0/asciinema-player.css";
+        sha256 = "sha256-GZMeZFFGvP5GMqqh516mjJKfQaiJ6bL38bSYOXkaohc=";
+      };
+
       module-docs = pkgs.runCommand "rendered" { nativeBuildInputs = [ pkgs.python3 ]; } ''
         export CLAN_CORE=${jsonDocs.clanCore}/share/doc/nixos/options.json 
         # A file that contains the links to all clanModule docs
@@ -56,12 +65,16 @@
       devShells.docs = pkgs.callPackage ./shell.nix {
         inherit (self'.packages) docs clan-cli-docs;
         inherit module-docs;
+        inherit asciinema-player-js;
+        inherit asciinema-player-css;
       };
       packages = {
         docs = pkgs.python3.pkgs.callPackage ./default.nix {
           inherit (self'.packages) clan-cli-docs;
           inherit (inputs) nixpkgs;
           inherit module-docs;
+          inherit asciinema-player-js;
+          inherit asciinema-player-css;
         };
         deploy-docs = pkgs.callPackage ./deploy-docs.nix { inherit (config.packages) docs; };
         inherit module-docs;

docs/nix/shell.nix

@@ -3,6 +3,8 @@
   pkgs,
   module-docs,
   clan-cli-docs,
+  asciinema-player-js,
+  asciinema-player-css,
   ...
 }:
 pkgs.mkShell {
@@ -14,5 +16,9 @@ pkgs.mkShell {
     chmod +w ./site/reference/*
 
     echo "Generated API documentation in './site/reference/' "
+
+    mkdir -p ./site/static/asciinema-player
+    ln -snf ${asciinema-player-js} ./site/static/asciinema-player/asciinema-player.min.js
+    ln -snf ${asciinema-player-css} ./site/static/asciinema-player/asciinema-player.css
   '';
 }

docs/site/blog/.authors.yml

@@ -1,21 +1,26 @@
 authors:
+  DavHau:
+    name: "DavHau"
+    description: "Core Developer"
+    avatar: "https://clan.lol/static/profiles/davhau.jpg"
+    url: "https://DavHau.com"
   Lassulus:
     name: "Lassulus"
-    description: "Contributor to Clan"
+    description: "Core Developer"
-    avatar: "https://avatars.githubusercontent.com/u/621759?v=4"
+    avatar: "https://clan.lol/static/profiles/lassulus.jpg"
     url: "https://http://lassul.us/"
   Mic92:
     name: "Mic92"
-    description: "Contributor to Clan"
+    description: "Core Developer"
-    avatar: "https://avatars.githubusercontent.com/u/96200?v=4"
+    avatar: "https://clan.lol/static/profiles/mic92.jpg"
     url: "https://thalheim.io"
   W:
     name: "W"
     description: "Founder of Clan"
-    avatar: "/static/w_profile.webp"
+    avatar: "https://clan.lol/static/profiles/w_profile.webp"
     url: ""
   Qubasa:
     name: "Qubasa"
-    description: "Contributor to Clan"
+    description: "Core Developer"
-    avatar: "https://avatars.githubusercontent.com/u/22085373?v=4"
+    avatar: "https://clan.lol/static/profiles/qubasa.png"
     url: "https://github.com/Qubasa"

docs/site/blog/posts/jsonschema.md

@@ -0,0 +1,194 @@
+---
+title: "Dev Report: Introducing the NixOS to JSON Schema Converter"
+description: "Discover our new library designed to extract JSON schema interfaces from NixOS modules, streamlining frontend development"
+authors:
+  - DavHau
+date: 2024-05-25
+slug: jsonschema-converter
+---
+
+## Overview
+
+We’ve developed a new library designed to extract interfaces from NixOS modules and convert them into JSON schemas, paving the way for effortless GUI generation. This blog post outlines the motivations behind this development, demonstrates the capabilities of the library, and guides you through leveraging it to create GUIs seamlessly.
+
+## Motivation
+
+In recent months, our team has been exploring various graphical user interfaces (GUIs) to streamline NixOS machine configuration. While our opinionated Clan modules simplify NixOS configurations, there's a need to configure these modules from diverse frontends, such as:
+
+- Command-line interfaces (CLIs)
+- Web-based UIs
+- Desktop applications
+- Mobile applications
+- Large Language Models (LLMs)
+
+Given this need, a universal format like JSON is a natural choice. It is already possible as of now, to import json based NixOS configurations, as illustrated below:
+
+`configuration.json`:
+```json
+{ "networking": { "hostName": "my-machine" } }
+```
+
+This configuration can be then imported inside a classic NixOS config:
+```nix
+{config, lib, pkgs, ...}: {
+  imports = [
+    (lib.importJSON ./configuration.json)
+  ];
+}
+```
+
+This straightforward approach allows us to build a frontend that generates JSON, enabling the configuration of NixOS machines. But, two critical questions arise:
+
+1. How does the frontend learn about existing configuration options?
+2. How can it verify user input without running Nix?
+
+Introducing [JSON schema](https://json-schema.org/), a widely supported standard that defines interfaces in JSON and validates input against them.
+
+Example schema for `networking.hostName`:
+```json
+{
+  "type": "object",
+  "properties": {
+    "networking": {
+      "type": "object",
+      "properties": {
+        "hostName": {
+          "type": "string",
+          "pattern": "^$|^[a-z0-9]([a-z0-9_-]{0,61}[a-z0-9])?$"
+        }
+      }
+    }
+  }
+}
+```
+
+## Client-Side Input Validation
+
+Validating input against JSON schemas is both efficient and well-supported across numerous programming languages. Using JSON schema validators, you can accurately check configurations like our `configuration.json`.
+
+Validation example:
+
+```shell
+$ nix-shell -p check-jsonschema
+$ jsonschema -o pretty ./schema.json -i ./configuration.json
+===[SUCCESS]===(./configuration.json)===
+```
+
+In case of invalid input, schema validators provide explicit error messages:
+
+```shell
+$ echo '{ "networking": { "hostName": "my/machine" } }' > configuration.json
+$ jsonschema -o pretty ./schema.json -i ./configuration.json
+===[ValidationError]===(./configuration.json)===
+
+'my/machine' does not match '^$|^[a-z0-9]([a-z0-9_-]{0,61}[a-z0-9])?$'
+
+Failed validating 'pattern' in schema['properties']['networking']['properties']['hostName']:
+    {'pattern': '^$|^[a-z0-9]([a-z0-9_-]{0,61}[a-z0-9])?$',
+     'type': 'string'}
+
+On instance['networking']['hostName']:
+    'my/machine'
+```
+
+## Automatic GUI Generation
+
+Certain libraries facilitate straightforward GUI generation from JSON schemas. For instance, the [react-jsonschema-form playground](https://rjsf-team.github.io/react-jsonschema-form/) auto-generates a form for any given schema.
+
+## NixOS Module to JSON Schema Converter
+
+To enable the development of responsive frontends, our library allows the extraction of interfaces from NixOS modules to JSON schemas. Open-sourced for community collaboration, this library supports building sophisticated user interfaces for NixOS.
+
+Here’s a preview of our library's functions exposed through the [clan-core](https://git.clan.lol/clan/clan-core) flake:
+
+- `lib.jsonschema.parseModule` - Generates a schema for a NixOS module.
+- `lib.jsonschema.parseOption` - Generates a schema for a single NixOS option.
+- `lib.jsonschema.parseOptions` - Generates a schema from an attrset of NixOS options.
+
+Example:
+`module.nix`:
+```nix
+{lib, config, pkgs, ...}: {
+  # a simple service with two options
+  options.services.example-web-service = {
+    enable = lib.mkEnableOption "Example web service";
+    port = lib.mkOption {
+      type = lib.types.int;
+      description = "Port used to serve the content";
+    };
+  };
+}
+```
+
+Converted, using the `parseModule` function:
+```shell
+$ cd clan-core
+$ nix eval --json --impure --expr \
+  '(import ./lib/jsonschema {}).parseModule ./module.nix' | jq | head
+{
+  "properties": {
+    "services": {
+      "properties": {
+        "example-web-service": {
+          "properties": {
+            "enable": {
+              "default": false,
+              "description": "Whether to enable Example web service.",
+              "examples": [
+...
+```
+
+This utility can also generate interfaces for existing NixOS modules or options.
+
+## GUI for NGINX in Under a Minute
+
+Creating a prototype GUI for the NGINX module using our library and [react-jsonschema-form playground](https://rjsf-team.github.io/react-jsonschema-form/) can be done quickly:
+
+1. Export all NGINX options into a JSON schema using a Nix expression:
+
+```nix
+# export.nix
+let
+  pkgs = import <nixpkgs> {};
+  clan-core = builtins.getFlake "git+https://git.clan.lol/clan/clan-core";
+  options = (pkgs.nixos {}).options.services.nginx;
+in
+  clan-core.lib.jsonschema.parseOption options
+```
+
+2. Write the schema into a file:
+```shell
+$ nix eval --json -f ./export.nix | jq > nginx.json
+```
+
+3. Open the [react-jsonschema-form playground](https://rjsf-team.github.io/react-jsonschema-form/), select `Blank` and paste the `nginx.json` contents.
+
+This provides a quick look at a potential GUI (screenshot is cropped).
+
+![Image title](https://clan.lol/static/blog-post-jsonschema/nginx-gui.jpg)
+
+## Limitations
+
+### Laziness
+
+JSON schema mandates the declaration of all required fields upfront, which might be configured implicitly or remain unused. For instance, `services.nginx.virtualHosts.<name>.sslCertificate` must be specified even if SSL isn’t enabled.
+
+### Limited Types
+
+Certain NixOS module types, like `types.functionTo` and `types.package`, do not map straightforwardly to JSON. For full compatibility, adjustments to NixOS modules might be necessary, such as substituting `listOf package` with `listOf str`.
+
+### Parsing NixOS Modules
+
+Currently, our converter relies on the `options` attribute of evaluated NixOS modules, extracting information from the `type.name` attribute, which is suboptimal. Enhanced introspection capabilities within the NixOS module system would be beneficial.
+
+## Future Prospects
+
+We hope these experiments inspire the community, encourage contributions and further development in this space. Share your ideas and contributions through our issue tracker or matrix channel!
+
+## Links
+
+- [Comments on NixOS Discourse](https://discourse.nixos.org/t/introducing-the-nixos-to-json-schema-converter/45948)
+- [Source Code of the JSON Schema Library](https://git.clan.lol/clan/clan-core/src/branch/main/lib/jsonschema)
+- [Our Issue Tracker](https://git.clan.lol/clan/clan-core/issues)
+- [Our Matrix Channel](https://matrix.to/#/#clan:lassul.us)
+- [react-jsonschema-form Playground](https://rjsf-team.github.io/react-jsonschema-form/)

docs/site/drafts/deployment-framework.md

@@ -0,0 +1,63 @@
+---
+title: "Git Based Machine Deployment with Clan-Core"
+description: ""
+authors:
+  - Qubasa
+date: 2024-05-25
+---
+## Revolutionizing Server Management
+
+In the world of server management, countless tools claim to offer seamless deployment of multiple machines. Yet, many fall short, leaving server admins and self-hosting enthusiasts grappling with complexity. Enter the Clan-Core Framework—a groundbreaking all in one solution designed to transform decentralized self-hosting into an effortless and scalable endeavor.
+
+### The Power of Clan-Core
+
+Imagine having the power to manage your servers with unparalleled ease, scaling your IT infrastructure like never before. Clan-Core empowers you to do just that. At its core, Clan-Core leverages a single Git repository to define everything about your machines. This central repository utilizes Nix or JSON files to specify configurations, including disk formatting, ensuring a streamlined and unified approach.
+
+### Simplified Deployment Process
+
+With Clan-Core, the cumbersome task of bootstrapping a specific ISO is a thing of the past. All you need is SSH access to your Linux server. Clan-Core allows you to overwrite any existing Linux distribution live over SSH, eliminating time-consuming setup processes. This capability means you can deploy updates or new configurations swiftly and efficiently, maximizing uptime and minimizing hassle.
+
+### Secure and Efficient Secret Management
+
+Security is paramount in server management, and Clan-Core takes it seriously. Passwords and other sensitive information are encrypted within the Git repository, automatically decrypted during deployment. This not only ensures the safety of your secrets but also simplifies their management. Clan-Core supports sharing secrets with other admins, fostering collaboration and maintaining reproducibillity and security without sacrificing convenience.
+
+### Services as Apps
+
+Setting up a service can be quite difficult. Many server adjustments need to be made, from setting up a database to adjusting webserver configurations and generating the correct private keys. However, Clan-Core aims to make setting up a service as easy as installing an application. Through Clan-Core's Module system, everything down to secrets can be automatically set up. This transforms the often daunting task of service setup into a smooth, automated process, making it accessible to all.
+
+### Decentralized Mesh VPN
+
+Building on these features is a self-configuring decentralized mesh VPN that interconnects all your machines into a private darknet. This ensures that sensitive services, which might have too much attack surface to be hosted on the public internet, can still be made available privately without the need to worry about potential system compromise. By creating a secure, private network, Clan-Core offers an additional layer of protection for your most critical services.
+
+### Decentralized Domain Name System
+
+Current DNS implementations are distributed but not truly decentralized. For Clan-Core, we implemented our own truly decentralized DNS module. This module uses simple flooding and caching algorithms to discover available domains inside the darknet. This approach ensures that your internal domain name system is robust, reliable, and independent of external control, enhancing the resilience and security of your infrastructure.
+
+
+### A New Era of Decentralized Self-Hosting
+
+Clan-Core is more than just a tool; it's a paradigm shift in server management. By consolidating machine definitions, secrets and network configuration, into a single, secure repository, it transforms how you manage and scale your infrastructure. Whether you're a seasoned server admin or a self-hosting enthusiast, Clan-Core offers a powerful, user-friendly solution to take your capabilities to the next level.
+
+
+### Key Features of Clan-Core:
+
+- **Unified Git Repository**: All machine configurations and secrets stored in a single repository.
+- **Live Overwrites**: Deploy configurations over existing Linux distributions via SSH.
+- **Automated Service Setup**: Easily set up services with Clan-Core's Module system.
+- **Decentralized Mesh VPN**: Securely interconnect all machines into a private darknet.
+- **Decentralized DNS**: Robust, independent DNS using flooding and caching algorithms.
+- **Automated Secret Management**: Encrypted secrets that are automatically decrypted during deployment.
+- **Collaboration Support**: Share secrets securely with other admins.
+
+
+## Clan-Cores Future
+
+Our vision for Clan-Core extends far beyond being just another deployment tool. Clan-Core is a framework we've developed to achieve something much greater. We want to put the "personal" back into "personal computing." Our goal is for everyday users to fully customize their phones or laptops and create truly private spaces for friends and family.
+
+Our first major step is to develop a Graphical User Interface (GUI) that makes configuring all this possible. Initial tests have shown that AI can be leveraged as an alternative to traditional GUIs. This paves the way for a future where people can simply talk to their computers, and they will configure themselves according to the users' wishes. 
+
+By adopting Clan, you're not just embracing a tool—you're joining a movement towards a more efficient, secure, and scalable approach to server management. Join us and revolutionize your IT infrastructure today.
+
+
+
+

docs/site/getting-started/configure.md

@@ -59,7 +59,7 @@ Adding or configuring a new machine requires two simple steps:
 
     Which should show something like:
 
-    ```bash hl_lines="6"
+    ```{.shellSession hl_lines="6" .no-copy}
     NAME        ID-LINK                                         FSTYPE   SIZE MOUNTPOINT
     sda         usb-ST_16GB_AA6271026J1000000509-0:0                    14.9G 
     ├─sda1      usb-ST_16GB_AA6271026J1000000509-0:0-part1                 1M 

docs/site/getting-started/deploy.md

@@ -52,7 +52,7 @@ This process involves preparing a suitable hardware and disk partitioning config
 
     This is an example of the booted installer.
 
-    ```{ .bash .annotate .no-copy }
+    ```{ .bash .annotate .no-copy .nohighlight}
     ┌─────────────────────────────────────────────────────────────────────────────────────┐
     │ ┌───────────────────────────┐                                                       │
     │ │███████████████████████████│ # This is the QR Code (1)                             │
@@ -151,7 +151,7 @@ Clan CLI enables you to remotely update your machines over SSH. This requires se
 ### Setting the Target Host
 
 Replace `root@jon` with the actual hostname or IP address of your target machine:
-```nix hl_lines="9"
+```{.nix hl_lines="9" .no-copy}
 buildClan {
     # ...
     machines = {
@@ -192,7 +192,7 @@ it is also possible to specify a build host instead.
 During an update, the cli will ssh into the build host and run `nixos-rebuild` from there.
 
 
-```nix hl_lines="5"
+```{.nix hl_lines="5" .no-copy}
 buildClan {
     # ...
     machines = {
@@ -208,7 +208,7 @@ buildClan {
 To exclude machines from being updated when running `clan machines update` without any machines specified,
 one can set the `clan.deployment.requireExplicitUpdate` option to true:
 
-```nix hl_lines="5"
+```{.nix hl_lines="5" .no-copy}
 buildClan {
     # ...
     machines = {

docs/site/getting-started/flake-parts.md

@@ -94,9 +94,4 @@ Below is a guide on how to structure this in your flake.nix:
 For detailed information about configuring `flake-parts` and the available options within Clan,
 refer to the Clan module documentation located [here](https://git.clan.lol/clan/clan-core/src/branch/main/flakeModules/clan.nix).
 
-## Whats next?
-
-- [Configure Machines](configure.md): Customize machine configuration
-- [Deploying](deploy.md): Deploying a Machine configuration
-
 ---

docs/site/getting-started/installer.md

@@ -22,7 +22,7 @@ Follow our step-by-step guide to create and transfer this image onto a bootable
     lsblk
     ```
 
-    ```shellSession hl_lines="2"
+    ```{.shellSession hl_lines="2" .no-copy}
     NAME                                          MAJ:MIN RM   SIZE RO TYPE  MOUNTPOINTS
     sdb                                             8:0    1 117,2G  0 disk
     └─sdb1                                          8:1    1 117,2G  0 part  /run/media/qubasa/INTENSO
@@ -123,7 +123,7 @@ This will enter `iwd`
 
 Now run the following command to connect to your Wifi:
 
-```shellSession
+```{.shellSession .no-copy}
 # Identify your network device.
 device list
 
@@ -148,10 +148,10 @@ Connected network     FRITZ!Box (Your router device)
 IPv4 address          192.168.188.50 (Your new local ip)
 ```
 
-Press `ctrl-d` to exit `IWD`.
+Press ++ctrl+d++ to exit `IWD`.
 
 !!! Important
-    Press `ctrl-d` **again** to update the displayed QR code and connection information.
+    Press ++ctrl+d++ **again** to update the displayed QR code and connection information.
 
 You're all set up
 

docs/site/getting-started/mesh-vpn.md

@@ -9,7 +9,7 @@ include a new machine into the VPN.
 
 By default all machines within one clan are connected via a chosen network technology.
 
-```
+```{.no-copy}
 Clan 
     Node A
     <-> (zerotier / mycelium / ...)
@@ -36,7 +36,7 @@ peers. Once addresses are allocated, the controller's continuous operation is no
    ```
 3. **Update the Controller Machine**: Execute the following:
    ```bash
-   $ clan machines update <CONTROLLER>
+   clan machines update <CONTROLLER>
    ```
    Your machine is now operational as the VPN controller.
 

docs/site/getting-started/secrets.md

@@ -4,10 +4,15 @@ Clan enables encryption of secrets (such as passwords & keys) ensuring security
 
 Clan utilizes the [sops](https://github.com/getsops/sops) format and integrates with [sops-nix](https://github.com/Mic92/sops-nix) on NixOS machines.
 
+This guide will walk you through:
 
-### Create Your Master Keypair
+- **Creating a Keypair for Your User**: Learn how to generate a keypair for $USER to securely control all secrets.
+- **Creating Your First Secret**: Step-by-step instructions on creating your initial secret.
+- **Assigning Machine Access to the Secret**: Understand how to grant a machine access to the newly created secret.
 
-To get started, you'll need to create **Your master keypair**.
+## Create Your Admin Keypair
+
+To get started, you'll need to create **Your admin keypair**.
 
 !!! info
     Don't worry — if you've already made one before, this step won't change or overwrite it.
@@ -27,7 +32,7 @@ Also add your age public key to the repository with 'clan secrets users add YOUR
 
 !!! warning
     Make sure to keep a safe backup of the private key you've just created.
-    If it's lost, you won't be able to get to your secrets anymore because they all need the master key to be unlocked.
+    If it's lost, you won't be able to get to your secrets anymore because they all need the admin key to be unlocked.
 
 !!! note
     It's safe to add any secrets created by the clan CLI and placed in your repository to version control systems like `git`.
@@ -35,7 +40,7 @@ Also add your age public key to the repository with 'clan secrets users add YOUR
 ### Add Your Public Key
 
 ```bash
-clan secrets users add <your_username> <your_public_key>
+clan secrets users add $USER <your_public_key>
 ```
 
 It's best to choose the same username as on your Setup/Admin Machine that you use to control the deployment with.

docs/site/static/asciinema-player/asciinema-player.css

@@ -0,0 +1 @@
+/nix/store/8y5h98wk5p94mv1wyb2c4gkrr7bswd19-asciinema-player.css

docs/site/static/asciinema-player/asciinema-player.min.js

@@ -0,0 +1 @@
+/nix/store/w0i3f9qzn9n6jmfnfgiw5wnab2f9ssdw-asciinema-player.min.js

docs/site/static/asciinema-player/custom-theme.css

@@ -1,45 +0,0 @@
-.asciinema-player-theme-alabaster-auto {
-        --term-color-foreground: #000000; /* Black for foreground text */
-        --term-color-background: #f7f7f7; /* Very light gray for background */
-
-        --term-color-0: #000000; /* Black */
-        --term-color-1: #aa3731; /* Red */
-        --term-color-2: #448c37; /* Green */
-        --term-color-3: #cb9000; /* Yellow */
-        --term-color-4: #325cc0; /* Blue */
-        --term-color-5: #7a3e9d; /* Magenta */
-        --term-color-6: #0083b2; /* Cyan */
-        --term-color-7: #bbbbbb; /* White */
-        --term-color-8: #777777; /* Bright black (gray) */
-        --term-color-9: #f05050; /* Bright red */
-        --term-color-10: #60cb00; /* Bright green */
-        --term-color-11: #ffbc5d; /* Bright yellow */
-        --term-color-12: #007acc; /* Bright blue */
-        --term-color-13: #e64ce6; /* Bright magenta */
-        --term-color-14: #00aacb; /* Bright cyan */
-        --term-color-15: #ffffff; /* Bright white */
-    }
-
-  @media (prefers-color-scheme: dark) {
-    .asciinema-player-theme-solarized-auto {
-      --term-color-foreground: #839496;
-      --term-color-background: #002b36;
-
-      --term-color-0: #073642;
-      --term-color-1: #dc322f;
-      --term-color-2: #859900;
-      --term-color-3: #b58900;
-      --term-color-4: #268bd2;
-      --term-color-5: #d33682;
-      --term-color-6: #2aa198;
-      --term-color-7: #eee8d5;
-      --term-color-8: #002b36;
-      --term-color-9: #cb4b16;
-      --term-color-10: #586e75;
-      --term-color-11: #657b83;
-      --term-color-12: #839496;
-      --term-color-13: #6c71c4;
-      --term-color-14: #93a1a1;
-      --term-color-15: #fdf6e3;
-    }
-  }

flake.lock

@@ -7,15 +7,15 @@
         ]
       },
       "locked": {
-        "lastModified": 1714400597,
+        "lastModified": 1716394172,
-        "narHash": "sha256-AA1TCyEl4O6+6F5man/V5VH9Zl9HPBpK91tSkZ16i2E=",
+        "narHash": "sha256-B+pNhV8GFeCj9/MoH+qtGqKbgv6fU4hGaw2+NoYYtB0=",
-        "owner": "Qubasa",
+        "owner": "nix-community",
         "repo": "disko",
-        "rev": "58785136b8c37aeb2f67081387b48f663b166331",
+        "rev": "23c63fb09334c3e8958b57e2ddc3870b75b9111d",
         "type": "github"
       },
       "original": {
-        "owner": "Qubasa",
+        "owner": "nix-community",
         "repo": "disko",
         "type": "github"
       }

flake.nix

@@ -8,7 +8,7 @@
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
-    disko.url = "github:Qubasa/disko";
+    disko.url = "github:nix-community/disko";
     disko.inputs.nixpkgs.follows = "nixpkgs";
     sops-nix.url = "github:Mic92/sops-nix";
     sops-nix.inputs.nixpkgs.follows = "nixpkgs";

lib/jsonschema/default.nix

@@ -83,20 +83,20 @@ rec {
     in
 
     # either type
-    # TODO: if all nested optiosn are excluded, the parent sould be excluded too
+    # TODO: if all nested options are excluded, the parent should be excluded too
     if
-      option.type.name or null == "either"
+      option.type.name or null == "either" || option.type.name or null == "coercedTo"
     # return jsonschema property definition for either
     then
       let
         optionsList' = [
           {
-            type = option.type.nestedTypes.left;
+            type = option.type.nestedTypes.left or option.type.nestedTypes.coercedType;
             _type = "option";
             loc = option.loc;
           }
           {
-            type = option.type.nestedTypes.right;
+            type = option.type.nestedTypes.right or option.type.nestedTypes.finalType;
             _type = "option";
             loc = option.loc;
           }
@@ -157,12 +157,21 @@ rec {
 
     # TODO: Add support for intMatching in jsonschema
     # parse port type aka. "unsignedInt16"
-    else if option.type.name == "unsignedInt16" then
+    else if
+      option.type.name == "unsignedInt16"
+      || option.type.name == "unsignedInt"
+      || option.type.name == "pkcs11"
+      || option.type.name == "intBetween"
+    then
       default // example // description // { type = "integer"; }
 
     # parse string
+    # TODO: parse more precise string types
     else if
       option.type.name == "str"
+      || option.type.name == "singleLineStr"
+      || option.type.name == "passwdEntry str"
+      || option.type.name == "passwdEntry path"
     # return jsonschema property definition for string
     then
       default // example // description // { type = "string"; }

pkgs/clan-cli/api.py

@@ -1,10 +1,12 @@
+import json
+
 from clan_cli.api import API
 
 
 def main() -> None:
     schema = API.to_json_schema()
     print(
-        f"""export const schema = {schema} as const;
+        f"""export const schema = {json.dumps(schema, indent=2)} as const;
 """
     )
 

pkgs/clan-cli/clan_cli/__init__.py

@@ -52,7 +52,17 @@ class AppendOptionAction(argparse.Action):
 
 
 def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(prog=prog, description="cLAN tool")
+    parser = argparse.ArgumentParser(
+        prog=prog,
+        description="The clan cli tool.",
+        epilog=(
+            """
+Online reference for the clan cli tool: https://docs.clan.lol/reference/cli/
+For more detailed information, visit: https://docs.clan.lol
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
 
     parser.add_argument(
         "--debug",
@@ -90,28 +100,160 @@ def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
         "backups",
         help="manage backups of clan machines",
         description="manage backups of clan machines",
+        epilog=(
+            """
+This subcommand provides an interface to backups that clan machines expose.
+
+Examples:
+
+  $ clan backups list [MACHINE]
+  List backups for the machine [MACHINE]
+
+  $ clan backups create [MACHINE]
+  Create a backup for the machine [MACHINE].
+
+  $ clan backups restore [MACHINE] [PROVIDER] [NAME]
+  The backup to restore for the machine [MACHINE] with the configured [PROVIDER]
+  with the name [NAME].
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/backups/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     backups.register_parser(parser_backups)
 
     parser_flake = subparsers.add_parser(
-        "flakes", help="create a clan flake inside the current directory"
+        "flakes",
+        help="create a clan flake inside the current directory",
+        epilog=(
+            """
+Examples:
+  $ clan flakes create [DIR]
+  Will create a new clan flake in the specified directory and create it if it
+  doesn't exist yet. The flake will be created from a default template.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     flakes.register_parser(parser_flake)
 
-    parser_config = subparsers.add_parser("config", help="set nixos configuration")
+    parser_config = subparsers.add_parser(
+        "config",
+        help="set nixos configuration",
+        epilog=(
+            """
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     config.register_parser(parser_config)
 
-    parser_ssh = subparsers.add_parser("ssh", help="ssh to a remote machine")
+    parser_ssh = subparsers.add_parser(
+        "ssh",
+        help="ssh to a remote machine",
+        epilog=(
+            """
+This subcommand allows seamless ssh access to the nixos-image builders.
+
+Examples:
+
+  $ clan ssh [ssh_args ...] --json [JSON]
+  Will ssh in to the machine based on the deployment information contained in
+  the json string. [JSON] can either be a json formatted string itself, or point
+  towards a file containing the deployment information
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/deploy
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     ssh_cli.register_parser(parser_ssh)
 
-    parser_secrets = subparsers.add_parser("secrets", help="manage secrets")
+    parser_secrets = subparsers.add_parser(
+        "secrets",
+        help="manage secrets",
+        epilog=(
+            """
+This subcommand provides an interface to secret facts.
+
+Examples:
+
+  $ clan secrets list [regex]
+  Will list secrets for all managed machines.
+  It accepts an optional regex, allowing easy filtering of returned secrets.
+
+  $ clan secrets get [SECRET]
+  Will display the content of the specified secret.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/secrets/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     secrets.register_parser(parser_secrets)
 
-    parser_facts = subparsers.add_parser("facts", help="manage facts")
+    parser_facts = subparsers.add_parser(
+        "facts",
+        help="manage facts",
+        epilog=(
+            """
+
+This subcommand provides an interface to facts of clan machines.
+Facts are artifacts that a service can generate.
+There are public and secret facts. 
+Public facts can be referenced by other machines directly.
+Public facts can include: ip addresses, public keys.
+Secret facts can include: passwords, private keys.
+
+A service is an included clan-module that implements facts generation functionality.
+For example the zerotier module will generate private and public facts.
+In this case the public fact will be the resulting zerotier-ip of the machine.
+The secret fact will be the zerotier-identity-secret, which is used by zerotier
+to prove the machine has control of the zerotier-ip.
+
+Examples:
+
+  $ clan facts generate
+  Will generate facts for all machines.
+   
+  $ clan facts generate --service [SERVICE] --regenerate
+  Will regenerate facts, if they are already generated for a specific service.
+  This is especially useful for resetting certain passwords while leaving the rest
+  of the facts for a machine in place.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/secrets/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     facts.register_parser(parser_facts)
 
     parser_machine = subparsers.add_parser(
-        "machines", help="manage machines and their configuration"
+        "machines",
+        help="manage machines and their configuration",
+        epilog=(
+            """
+This subcommand provides an interface to machines managed by clan.
+
+Examples:
+
+  $ clan machines list
+  List all the machines managed by clan.
+
+  $ clan machines update [MACHINES]
+  Will update the specified machine [MACHINE], if [MACHINE] is omitted, the command 
+  will attempt to update every configured machine.
+
+  $ clan machines install [MACHINES] [TARGET_HOST]
+  Will install the specified machine [MACHINE], to the specified [TARGET_HOST].
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/deploy
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     machines.register_parser(parser_machine)
 

pkgs/clan-cli/clan_cli/api/__init__.py

@@ -1,18 +1,37 @@
 from collections.abc import Callable
-from typing import Any
+from dataclasses import dataclass
+from typing import Any, Generic, Literal, TypeVar
+
+T = TypeVar("T")
+
+
+ResponseDataType = TypeVar("ResponseDataType")
+
+
+@dataclass
+class ApiError:
+    message: str
+    description: str | None
+    location: list[str] | None
+
+
+@dataclass
+class ApiResponse(Generic[ResponseDataType]):
+    status: Literal["success", "error"]
+    errors: list[ApiError] | None
+    data: ResponseDataType | None
 
 
 class _MethodRegistry:
     def __init__(self) -> None:
-        self._registry: dict[str, Callable] = {}
+        self._registry: dict[str, Callable[[Any], Any]] = {}
 
-    def register(self, fn: Callable) -> Callable:
+    def register(self, fn: Callable[..., T]) -> Callable[..., T]:
         self._registry[fn.__name__] = fn
         return fn
 
-    def to_json_schema(self) -> str:
+    def to_json_schema(self) -> dict[str, Any]:
         # Import only when needed
-        import json
         from typing import get_type_hints
 
         from clan_cli.api.util import type_to_dict
@@ -21,25 +40,51 @@ class _MethodRegistry:
             "$comment": "An object containing API methods. ",
             "type": "object",
             "additionalProperties": False,
-            "required": ["list_machines"],
+            "required": [func_name for func_name in self._registry.keys()],
             "properties": {},
         }
+
         for name, func in self._registry.items():
             hints = get_type_hints(func)
+
             serialized_hints = {
-                "argument" if key != "return" else "return": type_to_dict(
+                key: type_to_dict(
                     value, scope=name + " argument" if key != "return" else "return"
                 )
                 for key, value in hints.items()
             }
+
+            return_type = serialized_hints.pop("return")
+
             api_schema["properties"][name] = {
+                "type": "object",
+                "required": ["arguments", "return"],
+                "additionalProperties": False,
+                "properties": {
+                    "return": return_type,
+                    "arguments": {
                         "type": "object",
                         "required": [k for k in serialized_hints.keys()],
                         "additionalProperties": False,
-                "properties": {**serialized_hints},
+                        "properties": serialized_hints,
+                    },
+                },
             }
 
-        return json.dumps(api_schema, indent=2)
+        return api_schema
+
+    def get_method_argtype(self, method_name: str, arg_name: str) -> Any:
+        from inspect import signature
+
+        func = self._registry.get(method_name, None)
+        if func:
+            sig = signature(func)
+            param = sig.parameters.get(arg_name)
+            if param:
+                param_class = param.annotation
+                return param_class
+
+        return None
 
 
 API = _MethodRegistry()

pkgs/clan-cli/clan_cli/api/util.py

@@ -42,8 +42,13 @@ def type_to_dict(t: Any, scope: str = "") -> dict:
             return {"type": "array", "items": type_to_dict(t.__args__[0], scope)}
 
         elif issubclass(origin, dict):
+            value_type = t.__args__[1]
+            if value_type is Any:
+                return {"type": "object", "additionalProperties": True}
+            else:
                 return {
                     "type": "object",
+                    "additionalProperties": type_to_dict(value_type, scope),
                 }
 
         raise BaseException(f"Error api type not yet supported {t!s}")

pkgs/clan-cli/clan_cli/cmd.py

@@ -140,3 +140,23 @@ def run(
         raise ClanCmdError(cmd_out)
 
     return cmd_out
+
+
+def run_no_stdout(
+    cmd: list[str],
+    *,
+    env: dict[str, str] | None = None,
+    cwd: Path = Path.cwd(),
+    log: Log = Log.STDERR,
+    check: bool = True,
+    error_msg: str | None = None,
+) -> CmdOut:
+    """
+    Like run, but automatically suppresses stdout, if not in DEBUG log level.
+    If in DEBUG log level the stdout of commands will be shown.
+    """
+    if logging.getLogger(__name__.split(".")[0]).isEnabledFor(logging.DEBUG):
+        return run(cmd, env=env, log=log, check=check, error_msg=error_msg)
+    else:
+        log = Log.NONE
+        return run(cmd, env=env, log=log, check=check, error_msg=error_msg)

pkgs/clan-cli/clan_cli/config/__init__.py

@@ -8,7 +8,7 @@ import sys
 from pathlib import Path
 from typing import Any, get_origin
 
-from clan_cli.cmd import run
+from clan_cli.cmd import run_no_stdout
 from clan_cli.dirs import machine_settings_file
 from clan_cli.errors import ClanError
 from clan_cli.git import commit_file
@@ -116,7 +116,7 @@ def options_for_machine(
         f"{clan_dir}#nixosConfigurations.{machine_name}.config.clanCore.optionsNix"
     )
     cmd = nix_eval(flags=flags)
-    proc = run(
+    proc = run_no_stdout(
         cmd,
         error_msg=f"Failed to read options for machine {machine_name}",
     )
@@ -136,7 +136,7 @@ def read_machine_option_value(
             f"{clan_dir}#nixosConfigurations.{machine_name}.config.{option}",
         ],
     )
-    proc = run(cmd, error_msg=f"Failed to read option {option}")
+    proc = run_no_stdout(cmd, error_msg=f"Failed to read option {option}")
 
     value = json.loads(proc.stdout)
     # print the value so that the output can be copied and fed as an input.

pkgs/clan-cli/clan_cli/config/parsing.py

@@ -2,7 +2,7 @@ import json
 from pathlib import Path
 from typing import Any
 
-from ..cmd import run
+from ..cmd import run_no_stdout
 from ..errors import ClanError
 from ..nix import nix_eval
 
@@ -32,7 +32,7 @@ def schema_from_module_file(
     """
     # run the nix expression and parse the output as json
     cmd = nix_eval(["--expr", nix_expr])
-    proc = run(cmd)
+    proc = run_no_stdout(cmd)
     return json.loads(proc.stdout)
 
 

pkgs/clan-cli/clan_cli/facts/__init__.py

@@ -23,7 +23,42 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
     register_list_parser(list_parser)
 
     parser_generate = subparser.add_parser(
-        "generate", help="generate secrets for machines if they don't exist yet"
+        "generate",
+        help="generate public and secret facts for machines",
+        epilog=(
+            """
+This subcommand allows control of the generation of facts.
+Often this function will be invoked automatically on deploying machines,
+but there are situations the user may want to have more granular control,
+especially for the regeneration of certain services.
+
+A service is an included clan-module that implements facts generation functionality.
+For example the zerotier module will generate private and public facts.
+In this case the public fact will be the resulting zerotier-ip of the machine.
+The secret fact will be the zerotier-identity-secret, which is used by zerotier
+to prove the machine has control of the zerotier-ip.
+
+
+Examples:
+
+  $ clan facts generate
+  Will generate facts for all machines.
+   
+  $ clan facts generate [MACHINE]
+  Will generate facts for the specified machine.
+
+  $ clan facts generate [MACHINE] --service [SERVICE]
+  Will generate facts for the specified machine for the specified service.
+
+  $ clan facts generate --service [SERVICE] --regenerate
+  Will regenerate facts, if they are already generated for a specific service.
+  This is especially useful for resetting certain passwords while leaving the rest
+  of the facts for a machine in place.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/secrets/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     register_generate_parser(parser_generate)
 

pkgs/clan-cli/clan_cli/facts/generate.py

@@ -33,6 +33,7 @@ def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str:
 def generate_service_facts(
     machine: Machine,
     service: str,
+    regenerate: bool,
     secret_facts_store: SecretStoreBase,
     public_facts_store: FactStoreBase,
     tmpdir: Path,
@@ -42,7 +43,7 @@ def generate_service_facts(
     # check if all secrets exist and generate them if at least one is missing
     needs_regeneration = not check_secrets(machine, service=service)
     log.debug(f"{service} needs_regeneration: {needs_regeneration}")
-    if not needs_regeneration:
+    if not (needs_regeneration or regenerate):
         return False
     if not isinstance(machine.flake, Path):
         msg = f"flake is not a Path: {machine.flake}"
@@ -134,7 +135,11 @@ def prompt_func(text: str) -> str:
 
 
 def _generate_facts_for_machine(
-    machine: Machine, tmpdir: Path, prompt: Callable[[str], str] = prompt_func
+    machine: Machine,
+    service: str | None,
+    regenerate: bool,
+    tmpdir: Path,
+    prompt: Callable[[str], str] = prompt_func,
 ) -> bool:
     local_temp = tmpdir / machine.name
     local_temp.mkdir()
@@ -145,10 +150,23 @@ def _generate_facts_for_machine(
     public_facts_store = public_facts_module.FactStore(machine=machine)
 
     machine_updated = False
-    for service in machine.facts_data:
+
+    if service and service not in machine.facts_data:
+        services = list(machine.facts_data.keys())
+        raise ClanError(
+            f"Could not find service with name: {service}. The following services are available: {services}"
+        )
+
+    if service:
+        machine_service_facts = {service: machine.facts_data[service]}
+    else:
+        machine_service_facts = machine.facts_data
+
+    for service in machine_service_facts:
         machine_updated |= generate_service_facts(
             machine=machine,
             service=service,
+            regenerate=regenerate,
             secret_facts_store=secret_facts_store,
             public_facts_store=public_facts_store,
             tmpdir=local_temp,
@@ -161,7 +179,10 @@ def _generate_facts_for_machine(
 
 
 def generate_facts(
-    machines: list[Machine], prompt: Callable[[str], str] = prompt_func
+    machines: list[Machine],
+    service: str | None,
+    regenerate: bool,
+    prompt: Callable[[str], str] = prompt_func,
 ) -> bool:
     was_regenerated = False
     with TemporaryDirectory() as tmp:
@@ -170,7 +191,9 @@ def generate_facts(
         for machine in machines:
             errors = 0
             try:
-                was_regenerated |= _generate_facts_for_machine(machine, tmpdir, prompt)
+                was_regenerated |= _generate_facts_for_machine(
+                    machine, service, regenerate, tmpdir, prompt
+                )
             except Exception as exc:
                 log.error(f"Failed to generate facts for {machine.name}: {exc}")
                 errors += 1
@@ -189,7 +212,7 @@ def generate_command(args: argparse.Namespace) -> None:
         machines = get_all_machines(args.flake)
     else:
         machines = get_selected_machines(args.flake, args.machines)
-    generate_facts(machines)
+    generate_facts(machines, args.service, args.regenerate)
 
 
 def register_generate_parser(parser: argparse.ArgumentParser) -> None:
@@ -200,4 +223,17 @@ def register_generate_parser(parser: argparse.ArgumentParser) -> None:
         nargs="*",
         default=[],
     )
+    parser.add_argument(
+        "--service",
+        type=str,
+        help="service to generate facts for, if empty, generate facts for every service",
+        default=None,
+    )
+    parser.add_argument(
+        "--regenerate",
+        type=bool,
+        action=argparse.BooleanOptionalAction,
+        help="whether to regenerate facts for the specified machine",
+        default=None,
+    )
     parser.set_defaults(func=generate_command)

pkgs/clan-cli/clan_cli/facts/list.py

@@ -26,7 +26,14 @@ def get_all_facts(machine: Machine) -> dict:
 
 def get_command(args: argparse.Namespace) -> None:
     machine = Machine(name=args.machine, flake=args.flake)
-    print(json.dumps(get_all_facts(machine), indent=4))
+
+    # the raw_facts are bytestrings making them not json serializable
+    raw_facts = get_all_facts(machine)
+    facts = dict()
+    for key in raw_facts["TODO"]:
+        facts[key] = raw_facts["TODO"][key].decode("utf8")
+
+    print(json.dumps(facts, indent=4))
 
 
 def register_list_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/flakes/create.py

@@ -2,7 +2,7 @@
 import argparse
 from pathlib import Path
 
-from ..cmd import CmdOut, run
+from ..cmd import CmdOut, run_no_stdout
 from ..errors import ClanError
 from ..nix import nix_command, nix_shell
 
@@ -23,28 +23,28 @@ def create_flake(directory: Path, url: str) -> dict[str, CmdOut]:
             url,
         ]
     )
-    out = run(command, cwd=directory)
+    out = run_no_stdout(command, cwd=directory)
 
     command = nix_shell(["nixpkgs#git"], ["git", "init"])
-    out = run(command, cwd=directory)
+    out = run_no_stdout(command, cwd=directory)
     response["git init"] = out
 
     command = nix_shell(["nixpkgs#git"], ["git", "add", "."])
-    out = run(command, cwd=directory)
+    out = run_no_stdout(command, cwd=directory)
     response["git add"] = out
 
     command = nix_shell(["nixpkgs#git"], ["git", "config", "user.name", "clan-tool"])
-    out = run(command, cwd=directory)
+    out = run_no_stdout(command, cwd=directory)
     response["git config"] = out
 
     command = nix_shell(
         ["nixpkgs#git"], ["git", "config", "user.email", "clan@example.com"]
     )
-    out = run(command, cwd=directory)
+    out = run_no_stdout(command, cwd=directory)
     response["git config"] = out
 
     command = ["nix", "flake", "update"]
-    out = run(command, cwd=directory)
+    out = run_no_stdout(command, cwd=directory)
     response["flake update"] = out
 
     return response

pkgs/clan-cli/clan_cli/flakes/inspect.py

@@ -2,7 +2,7 @@ import argparse
 from dataclasses import dataclass
 from pathlib import Path
 
-from ..cmd import run
+from ..cmd import run_no_stdout
 from ..dirs import machine_gcroot
 from ..errors import ClanError
 from ..machines.list import list_machines
@@ -30,7 +30,7 @@ class FlakeConfig:
 
 
 def run_cmd(cmd: list[str]) -> str:
-    proc = run(cmd)
+    proc = run_no_stdout(cmd)
     return proc.stdout.strip()
 
 
@@ -39,7 +39,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig:
     system = config["system"]
 
     # Check if the machine exists
-    machines = list_machines(flake_url)
+    machines = list_machines(flake_url, False)
     if machine_name not in machines:
         raise ClanError(
             f"Machine {machine_name} not found in {flake_url}. Available machines: {', '.join(machines)}"
@@ -53,7 +53,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig:
         gcroot_icon: Path = machine_gcroot(flake_url=str(flake_url)) / vm.machine_name
         nix_add_to_gcroots(vm.machine_icon, gcroot_icon)
 
-    # Get the cLAN name
+    # Get the Clan name
     cmd = nix_eval(
         [
             f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clanCore.clanName'
@@ -70,7 +70,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig:
     )
     res = run_cmd(cmd)
 
-    # If the icon is null, no icon is set for this cLAN
+    # If the icon is null, no icon is set for this Clan
     if res == "null":
         icon_path = None
     else:
@@ -113,7 +113,7 @@ def inspect_command(args: argparse.Namespace) -> None:
     res = inspect_flake(
         flake_url=inspect_options.flake, machine_name=inspect_options.machine
     )
-    print("cLAN name:", res.clan_name)
+    print("Clan name:", res.clan_name)
     print("Icon:", res.icon)
     print("Description:", res.description)
     print("Last updated:", res.last_updated)

pkgs/clan-cli/clan_cli/flash.py

@@ -12,7 +12,7 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any
 
-from .cmd import Log, run
+from .cmd import Log, run, run_no_stdout
 from .errors import ClanError
 from .facts.secret_modules import SecretStoreBase
 from .machines.machines import Machine
@@ -60,7 +60,7 @@ def get_keymap_and_locale() -> dict[str, str]:
     keymap = "en"
 
     # Execute the `localectl status` command
-    result = run(["localectl", "status"])
+    result = run_no_stdout(["localectl", "status"])
 
     if result.returncode == 0:
         output = result.stdout

pkgs/clan-cli/clan_cli/git.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from clan_cli.errors import ClanError
 from clan_cli.nix import nix_shell
 
-from .cmd import Log, run
+from .cmd import Log, run, run_no_stdout
 from .locked_open import locked_open
 
 
@@ -78,7 +78,7 @@ def _commit_file_to_git(
             ["git", "-C", str(repo_dir), "diff", "--cached", "--exit-code"]
             + [str(file_path) for file_path in file_paths],
         )
-        result = run(cmd, check=False, cwd=repo_dir)
+        result = run_no_stdout(cmd, check=False, cwd=repo_dir)
         # if there is no diff, return
         if result.returncode == 0:
             return
@@ -97,6 +97,6 @@ def _commit_file_to_git(
             + [str(file_path) for file_path in file_paths],
         )
 
-        run(
+        run_no_stdout(
             cmd, error_msg=f"Failed to commit {file_paths} to git repository {repo_dir}"
         )

pkgs/clan-cli/clan_cli/machines/__init__.py

@@ -17,7 +17,26 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
         required=True,
     )
 
-    update_parser = subparser.add_parser("update", help="Update a machine")
+    update_parser = subparser.add_parser(
+        "update",
+        help="Update a machine",
+        epilog=(
+            """
+This subcommand provides an interface to update machines managed by clan.
+
+Examples:
+
+  $ clan machines update [MACHINES]
+  Will update the specified machine [MACHINE], if [MACHINE] is omitted, the command
+  will attempt to update every configured machine.
+  To exclude machines being updated `clan.deployment.requireExplicitUpdate = true;`
+  can be set in the machine config.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/deploy
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     register_update_parser(update_parser)
 
     create_parser = subparser.add_parser("create", help="Create a machine")
@@ -26,7 +45,21 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
     delete_parser = subparser.add_parser("delete", help="Delete a machine")
     register_delete_parser(delete_parser)
 
-    list_parser = subparser.add_parser("list", help="List machines")
+    list_parser = subparser.add_parser(
+        "list",
+        help="List machines",
+        epilog=(
+            """
+This subcommand lists all machines managed by this clan.
+
+Examples:
+
+  $ clan machines list
+  Lists all the machines and their descriptions.
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     register_list_parser(list_parser)
 
     install_parser = subparser.add_parser(
@@ -37,5 +70,23 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
         The target must be a Linux based system reachable via SSH.
         Installing a machine means overwriting the target's disk.
         """,
+        epilog=(
+            """
+This subcommand provides an interface to install machines managed by clan.
+
+Examples:
+
+  $ clan machines install [MACHINE] [TARGET_HOST]
+  Will install the specified machine [MACHINE], to the specified [TARGET_HOST].
+
+  $ clan machines install [MACHINE] --json [JSON]
+  Will install the specified machine [MACHINE] to the host exposed by 
+  the deployment information of the [JSON] deployment string.
+
+For information on how to set up the installer see: https://docs.clan.lol/getting-started/installer/
+For more detailed information, visit: https://docs.clan.lol/getting-started/deploy
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     register_install_parser(install_parser)

pkgs/clan-cli/clan_cli/machines/create.py

@@ -1,13 +1,27 @@
 import argparse
 import logging
+from dataclasses import dataclass
+from pathlib import Path
 
+from clan_cli.api import API
 from clan_cli.config.machine import set_config_for_machine
 
 log = logging.getLogger(__name__)
 
 
+@dataclass
+class MachineCreateRequest:
+    name: str
+    config: dict[str, int]
+
+
+@API.register
+def create_machine(flake_dir: str | Path, machine: MachineCreateRequest) -> None:
+    set_config_for_machine(Path(flake_dir), machine.name, machine.config)
+
+
 def create_command(args: argparse.Namespace) -> None:
-    set_config_for_machine(args.flake, args.machine, dict())
+    create_machine(args.flake, MachineCreateRequest(args.machine, dict()))
 
 
 def register_create_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/machines/install.py

@@ -35,7 +35,7 @@ def install_nixos(
     target_host = f"{h.user or 'root'}@{h.host}"
     log.info(f"target host: {target_host}")
 
-    generate_facts([machine])
+    generate_facts([machine], None, False)
 
     with TemporaryDirectory() as tmpdir_:
         tmpdir = Path(tmpdir_)

pkgs/clan-cli/clan_cli/machines/inventory.py

@@ -1,7 +1,7 @@
 import json
 from pathlib import Path
 
-from ..cmd import run
+from ..cmd import run_no_stdout
 from ..nix import nix_build, nix_config
 from .machines import Machine
 
@@ -10,7 +10,7 @@ from .machines import Machine
 def get_all_machines(flake_dir: Path) -> list[Machine]:
     config = nix_config()
     system = config["system"]
-    json_path = run(
+    json_path = run_no_stdout(
         nix_build([f'{flake_dir}#clanInternals.all-machines-json."{system}"'])
     ).stdout
 

pkgs/clan-cli/clan_cli/machines/list.py

@@ -1,37 +1,63 @@
 import argparse
+import dataclasses
 import json
 import logging
 from pathlib import Path
 
 from clan_cli.api import API
 
-from ..cmd import run
+from ..cmd import run_no_stdout
 from ..nix import nix_config, nix_eval
 
 log = logging.getLogger(__name__)
 
 
+@dataclasses.dataclass
+class MachineInfo:
+    machine_name: str
+    machine_description: str | None
+    machine_icon: str | None
+
+
 @API.register
-def list_machines(flake_url: Path | str) -> list[str]:
+def list_machines(flake_url: str | Path, debug: bool) -> dict[str, MachineInfo]:
     config = nix_config()
     system = config["system"]
     cmd = nix_eval(
         [
             f"{flake_url}#clanInternals.machines.{system}",
             "--apply",
-            "builtins.attrNames",
+            """builtins.mapAttrs (name: attrs: {
+  inherit (attrs.config.clanCore) machineDescription machineIcon machineName;
+})""",
             "--json",
         ]
     )
-    proc = run(cmd)
+
+    proc = run_no_stdout(cmd)
 
     res = proc.stdout.strip()
-    return json.loads(res)
+    machines_dict = json.loads(res)
+
+    return {
+        k: MachineInfo(
+            machine_name=v.get("machineName"),
+            machine_description=v.get("machineDescription", None),
+            machine_icon=v.get("machineIcon", None),
+        )
+        for k, v in machines_dict.items()
+    }
 
 
 def list_command(args: argparse.Namespace) -> None:
-    for machine in list_machines(Path(args.flake)):
+    flake_path = Path(args.flake).resolve()
-        print(machine)
+    print("Listing all machines:\n")
+    print("Source: ", flake_path)
+    print("-" * 40)
+    for name, machine in list_machines(flake_path, args.debug).items():
+        description = machine.machine_description or "[no description]"
+        print(f"{name}\n: {description}\n")
+    print("-" * 40)
 
 
 def register_list_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/machines/machines.py

@@ -10,7 +10,7 @@ from clan_cli.clan_uri import ClanURI, MachineData
 from clan_cli.dirs import vm_state_dir
 from clan_cli.qemu.qmp import QEMUMonitorProtocol
 
-from ..cmd import run
+from ..cmd import run_no_stdout
 from ..errors import ClanError
 from ..nix import nix_build, nix_config, nix_eval, nix_metadata
 from ..ssh import Host, parse_deployment_address
@@ -197,15 +197,15 @@ class Machine:
             config_json.flush()
 
             file_info = json.loads(
-                run(
+                run_no_stdout(
                     nix_eval(
                         [
                             "--impure",
                             "--expr",
                             f'let x = (builtins.fetchTree {{ type = "file"; url = "file://{config_json.name}"; }}); in {{ narHash = x.narHash; path = x.outPath; }}',
                         ]
-                    )
+                    ),
-                ).stdout.strip()
+                ).stdout.strip(),
             )
 
         args = []
@@ -247,10 +247,10 @@ class Machine:
             ]
 
         if method == "eval":
-            output = run(nix_eval(args)).stdout.strip()
+            output = run_no_stdout(nix_eval(args)).stdout.strip()
             return output
         elif method == "build":
-            outpath = run(nix_build(args)).stdout.strip()
+            outpath = run_no_stdout(nix_build(args)).stdout.strip()
             return Path(outpath)
         else:
             raise ValueError(f"Unknown method {method}")

pkgs/clan-cli/clan_cli/machines/update.py

@@ -98,7 +98,7 @@ def deploy_nixos(machines: MachineGroup) -> None:
         env = os.environ.copy()
         env["NIX_SSHOPTS"] = ssh_arg
 
-        generate_facts([machine])
+        generate_facts([machine], None, False)
         upload_secrets(machine)
 
         path = upload_sources(".", target)

pkgs/clan-cli/clan_cli/nix.py

@@ -4,7 +4,7 @@ import tempfile
 from pathlib import Path
 from typing import Any
 
-from .cmd import run
+from .cmd import run_no_stdout
 from .dirs import nixpkgs_flake, nixpkgs_source
 
 
@@ -55,12 +55,12 @@ def nix_build(flags: list[str], gcroot: Path | None = None) -> list[str]:
 
 def nix_add_to_gcroots(nix_path: Path, dest: Path) -> None:
     cmd = ["nix-store", "--realise", f"{nix_path}", "--add-root", f"{dest}"]
-    run(cmd)
+    run_no_stdout(cmd)
 
 
 def nix_config() -> dict[str, Any]:
     cmd = nix_command(["show-config", "--json"])
-    proc = run(cmd)
+    proc = run_no_stdout(cmd)
     data = json.loads(proc.stdout)
     config = {}
     for key, value in data.items():
@@ -95,7 +95,7 @@ def nix_eval(flags: list[str]) -> list[str]:
 
 def nix_metadata(flake_url: str | Path) -> dict[str, Any]:
     cmd = nix_command(["flake", "metadata", "--json", f"{flake_url}"])
-    proc = run(cmd)
+    proc = run_no_stdout(cmd)
     data = json.loads(proc.stdout)
     return data
 
@@ -106,7 +106,7 @@ def nix_shell(packages: list[str], cmd: list[str]) -> list[str]:
     if os.environ.get("IN_NIX_SANDBOX"):
         return cmd
     return [
-        *nix_command(["shell", "--offline", "--inputs-from", f"{nixpkgs_flake()!s}"]),
+        *nix_command(["shell", "--inputs-from", f"{nixpkgs_flake()!s}"]),
         *packages,
         "-c",
         *cmd,

pkgs/clan-cli/clan_cli/vms/run.py

@@ -69,7 +69,7 @@ def get_secrets(
     secret_facts_module = importlib.import_module(machine.secret_facts_module)
     secret_facts_store = secret_facts_module.SecretStore(machine=machine)
 
-    generate_facts([machine])
+    generate_facts([machine], None, False)
 
     secret_facts_store.upload(secrets_dir)
     return secrets_dir

pkgs/clan-cli/default.nix

@@ -83,7 +83,7 @@ let
     cp -r ${./.} $out
     chmod -R +w $out
     rm $out/clan_cli/config/jsonschema
-    ln -s ${nixpkgs'} $out/clan_cli/nixpkgs
+    ln -sf ${nixpkgs'} $out/clan_cli/nixpkgs
     cp -r ${../../lib/jsonschema} $out/clan_cli/config/jsonschema
   '';
 
@@ -101,7 +101,7 @@ let
       outputs = _inputs: { };
     }
     EOF
-    ln -s ${nixpkgs} $out/path
+    ln -sf ${nixpkgs} $out/path
     nix flake update $out \
       --store ./. \
       --extra-experimental-features 'nix-command flakes'

pkgs/clan-cli/tests/test_create_flake.py

@@ -26,6 +26,14 @@ def test_create_flake(
     cli.run(["machines", "create", "machine1"])
     capsys.readouterr()  # flush cache
 
+    # create a hardware-configuration.nix that doesn't throw an eval error
+
+    for patch_machine in ["jon", "sara"]:
+        with open(
+            flake_dir / "machines" / f"{patch_machine}/hardware-configuration.nix", "w"
+        ) as hw_config_nix:
+            hw_config_nix.write("{}")
+
     cli.run(["machines", "list"])
     assert "machine1" in capsys.readouterr().out
     flake_show = subprocess.run(

pkgs/clan-cli/tests/test_machines_cli.py

@@ -16,7 +16,10 @@ def test_machine_subcommands(
     cli.run(["--flake", str(test_flake_with_core.path), "machines", "list"])
 
     out = capsys.readouterr()
-    assert "machine1\nvm1\nvm2\n" == out.out
+
+    assert "machine1" in out.out
+    assert "vm1" in out.out
+    assert "vm2" in out.out
 
     cli.run(
         ["--flake", str(test_flake_with_core.path), "machines", "delete", "machine1"]
@@ -25,4 +28,7 @@ def test_machine_subcommands(
     capsys.readouterr()
     cli.run(["--flake", str(test_flake_with_core.path), "machines", "list"])
     out = capsys.readouterr()
-    assert "vm1\nvm2\n" == out.out
+
+    assert "machine1" not in out.out
+    assert "vm1" in out.out
+    assert "vm2" in out.out

pkgs/clan-vm-manager/clan_vm_manager/app.py

@@ -106,7 +106,7 @@ class MainApplication(Adw.Application):
     def on_activate(self, source: "MainApplication") -> None:
         if not self.window:
             self.init_style()
-            self.window = MainWindow(config=ClanConfig(initial_view="webview"))
+            self.window = MainWindow(config=ClanConfig(initial_view="list"))
             self.window.set_application(self)
 
         self.window.show()

pkgs/clan-vm-manager/clan_vm_manager/views/webview.py

@@ -1,3 +1,4 @@
+import dataclasses
 import json
 import logging
 import sys
@@ -8,6 +9,7 @@ from threading import Lock
 from typing import Any
 
 import gi
+from clan_cli.api import API
 
 gi.require_version("WebKit", "6.0")
 
@@ -22,6 +24,23 @@ site_index: Path = (
 log = logging.getLogger(__name__)
 
 
+def dataclass_to_dict(obj: Any) -> Any:
+    """
+    Utility function to convert dataclasses to dictionaries
+    It converts all nested dataclasses, lists, tuples, and dictionaries to dictionaries
+
+    It does NOT convert member functions.
+    """
+    if dataclasses.is_dataclass(obj):
+        return {k: dataclass_to_dict(v) for k, v in dataclasses.asdict(obj).items()}
+    elif isinstance(obj, list | tuple):
+        return [dataclass_to_dict(item) for item in obj]
+    elif isinstance(obj, dict):
+        return {k: dataclass_to_dict(v) for k, v in obj.items()}
+    else:
+        return obj
+
+
 class WebView:
     def __init__(self, methods: dict[str, Callable]) -> None:
         self.method_registry: dict[str, Callable] = methods
@@ -77,12 +96,35 @@ class WebView:
         self.queue_size += 1
 
     def threaded_handler(
-        self, handler_fn: Callable[[Any], Any], data: Any, method_name: str
+        self,
+        handler_fn: Callable[
+            ...,
+            Any,
+        ],
+        data: dict[str, Any] | None,
+        method_name: str,
     ) -> None:
         with self.mutex_lock:
             log.debug("Executing... ", method_name)
-            result = handler_fn(data)
+            log.debug(f"{data}")
-            serialized = json.dumps(result)
+            if data is None:
+                result = handler_fn()
+            else:
+                reconciled_arguments = {}
+                for k, v in data.items():
+                    # Some functions expect to be called with dataclass instances
+                    # But the js api returns dictionaries.
+                    # Introspect the function and create the expected dataclass from dict dynamically
+                    # Depending on the introspected argument_type
+                    arg_type = API.get_method_argtype(method_name, k)
+                    if dataclasses.is_dataclass(arg_type):
+                        reconciled_arguments[k] = arg_type(**v)
+                    else:
+                        reconciled_arguments[k] = v
+
+                result = handler_fn(**reconciled_arguments)
+
+            serialized = json.dumps(dataclass_to_dict(result))
 
             # Use idle_add to queue the response call to js on the main GTK thread
             GLib.idle_add(self.return_data_to_js, method_name, serialized)

pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py

@@ -26,7 +26,7 @@ log = logging.getLogger(__name__)
 class MainWindow(Adw.ApplicationWindow):
     def __init__(self, config: ClanConfig) -> None:
         super().__init__()
-        self.set_title("cLAN Manager")
+        self.set_title("Clan Manager")
         self.set_default_size(980, 850)
 
         overlay = ToastOverlay.use().overlay
@@ -62,8 +62,7 @@ class MainWindow(Adw.ApplicationWindow):
         stack_view.add_named(Logs(), "logs")
 
         webview = WebView(methods=API._registry)
-
+        stack_view.add_named(webview.get_webview(), "webview")
-        stack_view.add_named(webview.get_webview(), "list")
 
         stack_view.set_visible_child_name(config.initial_view)
 

pkgs/clan-vm-manager/default.nix

@@ -27,7 +27,7 @@ let
     name = "org.clan.vm-manager";
     exec = "clan-vm-manager %u";
     icon = ./clan_vm_manager/assets/clan_white.png;
-    desktopName = "cLAN Manager";
+    desktopName = "Clan Manager";
     startupWMClass = "clan";
     mimeTypes = [ "x-scheme-handler/clan" ];
   };

pkgs/editor/clan-edit-codium.nix

@@ -0,0 +1,40 @@
+{
+  lib,
+  coreutils,
+  nil,
+  nixd,
+  nixpkgs-fmt,
+  direnv,
+  vscode-extensions,
+  vscode-with-extensions,
+  vscodium,
+  writeShellApplication,
+}:
+let
+  codium = vscode-with-extensions.override {
+    vscode = vscodium;
+    vscodeExtensions = [
+      vscode-extensions.jnoortheen.nix-ide
+      vscode-extensions.mkhl.direnv
+    ];
+  };
+in
+writeShellApplication {
+  name = "clan-edit-codium";
+  runtimeInputs = [
+    coreutils
+    nil
+    nixd
+    nixpkgs-fmt
+    direnv
+  ];
+  text = ''
+    set -eux
+    DATA_DIR="''${XDG_CACHE_HOME:-$HOME/.cache}/clan-edit-codium"
+    SETTINGS="$DATA_DIR"/User/settings.json
+    ${coreutils}/bin/mkdir -p "$DATA_DIR/User"
+    cat ${./settings.json} > "$SETTINGS"
+
+    exec ${lib.getExe codium} --user-data-dir "$DATA_DIR" "$@"
+  '';
+}

pkgs/editor/default.nix

@@ -0,0 +1,4 @@
+{ pkgs }:
+{
+  clan-edit-codium = pkgs.callPackage ./clan-edit-codium.nix;
+}

pkgs/editor/settings.json

@@ -0,0 +1,21 @@
+{
+  "security.workspace.trust.enabled": false,
+  "nix.enableLanguageServer": true,
+  "nix.serverPath": "nixd",
+  "nix.formatterPath": "nixpkgs-fmt",
+  "nix.serverSettings": {
+    "nixd": {
+      "formatting": {
+        "command": "nixpkgs-fmt"
+      },
+      "options": {
+        "nixos": {
+          "expr": "(builtins.getFlake \"github:nixos/nixpkgs\").nixosConfigurations.<name>.options"
+        },
+        "home-manager": {
+          "expr": "(builtins.getFlake \"github:nix-community/home-manager\").homeConfigurations.<name>.options"
+        }
+      }
+    }
+  }
+}

pkgs/flake-module.nix

@@ -25,6 +25,7 @@
           moonlight-sunshine-accept = pkgs.callPackage ./moonlight-sunshine-accept { };
           merge-after-ci = pkgs.callPackage ./merge-after-ci { inherit (config.packages) tea-create-pr; };
           pending-reviews = pkgs.callPackage ./pending-reviews { };
+          editor = pkgs.callPackage ./editor/clan-edit-codium.nix { };
         }
         // lib.optionalAttrs pkgs.stdenv.isLinux {
           wayland-proxy-virtwl = pkgs.callPackage ./wayland-proxy-virtwl { };

pkgs/webview-ui/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "typescript.tsdk": "node_modules/typescript/lib",
+    "tailwindCSS.experimental.classRegex": [["cx\\(([^)]*)\\)", "[\"'`]([^\"'`]*)[\"'`]"]],
+    "editor.wordWrap": "on"
+}

pkgs/webview-ui/app/eslint.config.mjs

@@ -0,0 +1,26 @@
+import eslint from "@eslint/js";
+import tseslint from "typescript-eslint";
+import tailwind from "eslint-plugin-tailwindcss";
+
+export default tseslint.config(
+  eslint.configs.recommended,
+  ...tseslint.configs.strict,
+  ...tseslint.configs.stylistic,
+  ...tailwind.configs["flat/recommended"],
+  {
+    rules: {
+      "tailwindcss/no-contradicting-classname": [
+        "error",
+        {
+          callees: ["cx"],
+        },
+      ],
+      "tailwindcss/no-custom-classname": [
+        "error",
+        {
+          callees: ["cx"],
+        },
+      ],
+    },
+  }
+);

pkgs/webview-ui/app/package.json

@@ -8,22 +8,29 @@
     "build": "npm run check && vite build && npm run convert-html",
     "convert-html": "node gtk.webview.js",
     "serve": "vite preview",
-    "check": "tsc --noEmit --skipLibCheck"
+    "check": "tsc --noEmit --skipLibCheck && eslint ./src"
   },
   "license": "MIT",
   "devDependencies": {
+    "@eslint/js": "^9.3.0",
+    "@types/node": "^20.12.12",
+    "@typescript-eslint/parser": "^7.10.0",
     "autoprefixer": "^10.4.19",
+    "classnames": "^2.5.1",
     "daisyui": "^4.11.1",
+    "eslint": "^8.57.0",
+    "json-schema-to-ts": "^3.1.0",
     "postcss": "^8.4.38",
+    "prettier": "^3.2.5",
     "solid-devtools": "^0.29.2",
     "tailwindcss": "^3.4.3",
-    "typescript": "^5.3.3",
+    "typescript": "^5.4.5",
+    "typescript-eslint": "^7.10.0",
     "vite": "^5.0.11",
-    "vite-plugin-solid": "^2.8.2"
+    "vite-plugin-solid": "^2.8.2",
+    "eslint-plugin-tailwindcss": "^3.17.0"
   },
   "dependencies": {
-    "@types/node": "^20.12.12",
-    "json-schema-to-ts": "^3.1.0",
     "solid-js": "^1.8.11"
   }
 }

pkgs/webview-ui/app/src/App.tsx

@@ -1,34 +1,60 @@
 import { Match, Switch, createSignal, type Component } from "solid-js";
 import { CountProvider } from "./Config";
+// import { Nested } from "./nested";
+import { Layout } from "./layout/layout";
+import cx from "classnames";
 import { Nested } from "./nested";
 
-type Route = "home" | "graph";
+type Route = "home" | "machines";
 
 const App: Component = () => {
   const [route, setRoute] = createSignal<Route>("home");
   return (
     <CountProvider>
-      <div class="w-full flex items-center flex-col gap-2 my-2">
+      <Layout>
-        <div>Clan</div>
+        <div class="col-span-1">
-        <p>Current route: {route()}</p>
+          <div class={cx("text-zinc-500")}>Navigation</div>
-
+          <ul>
-        <div class="flex items-center">
+            <li>
               <button
-            onClick={() => setRoute((o) => (o === "graph" ? "home" : "graph"))}
+                onClick={() => setRoute("home")}
-            class="btn btn-link"
+                classList={{ "bg-blue-500": route() === "home" }}
               >
-            Navigate to {route() === "home" ? "graph" : "home"}
+                Home
               </button>
+            </li>
+            <li>
+              {" "}
+              <button
+                onClick={() => setRoute("machines")}
+                classList={{ "bg-blue-500": route() === "machines" }}
+              >
+                Machines
+              </button>
+            </li>
+          </ul>
         </div>
+
+        <div class="col-span-7">
+          <div>{route()}</div>
           <Switch fallback={<p>{route()} not found</p>}>
             <Match when={route() == "home"}>
               <Nested />
             </Match>
-          <Match when={route() == "graph"}>
+            <Match when={route() == "machines"}>
-            <p></p>
+              <div class="grid grid-cols-3 gap-2">
+                <div class="h-10 w-20 bg-red-500">red</div>
+                <div class="h-10 w-20 bg-green-500">green</div>
+                <div class="h-10 w-20 bg-blue-500">blue</div>
+                <div class="h-10 w-20 bg-yellow-500">yellow</div>
+                <div class="h-10 w-20 bg-purple-500">purple</div>
+                <div class="h-10 w-20 bg-cyan-500">cyan</div>
+                <div class="h-10 w-20 bg-pink-500">pink</div>
+              </div>
             </Match>
           </Switch>
         </div>
+      </Layout>
     </CountProvider>
   );
 };

pkgs/webview-ui/app/src/Config.tsx

@@ -1,8 +1,16 @@
-import { createSignal, createContext, useContext, JSXElement } from "solid-js";
+import {
-import { pyApi } from "./message";
+  createSignal,
+  createContext,
+  useContext,
+  JSXElement,
+  createEffect,
+} from "solid-js";
+import { OperationResponse, pyApi } from "./message";
 
 export const makeCountContext = () => {
-  const [machines, setMachines] = createSignal<string[]>([]);
+  const [machines, setMachines] = createSignal<
+    OperationResponse<"list_machines">
+  >({});
   const [loading, setLoading] = createSignal(false);
 
   pyApi.list_machines.receive((machines) => {
@@ -10,13 +18,17 @@ export const makeCountContext = () => {
     setMachines(machines);
   });
 
+  createEffect(() => {
+    console.log("The count is now", machines());
+  });
+
   return [
     { loading, machines },
     {
       getMachines: () => {
         // When the gtk function sends its data the loading state will be set to false
         setLoading(true);
-        pyApi.list_machines.dispatch(".");
+        pyApi.list_machines.dispatch({ debug: true, flake_url: "." });
       },
     },
   ] as const;
@@ -25,8 +37,14 @@ export const makeCountContext = () => {
 type CountContextType = ReturnType<typeof makeCountContext>;
 
 export const CountContext = createContext<CountContextType>([
-  { loading: () => false, machines: () => [] },
   {
+    loading: () => false,
+
+    // eslint-disable-next-line
+    machines: () => ({}),
+  },
+  {
+    // eslint-disable-next-line
     getMachines: () => {},
   },
 ]);

pkgs/webview-ui/app/src/index.tsx

@@ -6,7 +6,6 @@ import App from "./App";
 
 const root = document.getElementById("app");
 
-// @ts-ignore: add the clan scope to the window object so we can register callbacks for gtk
 window.clan = window.clan || {};
 
 if (import.meta.env.DEV && !(root instanceof HTMLElement)) {
@@ -15,4 +14,5 @@ if (import.meta.env.DEV && !(root instanceof HTMLElement)) {
   );
 }
 
+// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 render(() => <App />, root!);

pkgs/webview-ui/app/src/layout/layout.tsx

@@ -0,0 +1,9 @@
+import { Component, JSXElement } from "solid-js";
+
+interface LayoutProps {
+  children: JSXElement;
+}
+
+export const Layout: Component<LayoutProps> = (props) => {
+  return <div class="grid grid-cols-8">{props.children}</div>;
+};

pkgs/webview-ui/app/src/message.ts

@@ -1,11 +1,11 @@
 import { FromSchema } from "json-schema-to-ts";
 import { schema } from "@/api";
 
-type API = FromSchema<typeof schema>;
+export type API = FromSchema<typeof schema>;
 
-type OperationNames = keyof API;
+export type OperationNames = keyof API;
-type OperationArgs<T extends OperationNames> = API[T]["argument"];
+export type OperationArgs<T extends OperationNames> = API[T]["arguments"];
-type OperationResponse<T extends OperationNames> = API[T]["return"];
+export type OperationResponse<T extends OperationNames> = API[T]["return"];
 
 declare global {
   interface Window {
@@ -15,7 +15,10 @@ declare global {
     webkit: {
       messageHandlers: {
         gtk: {
-          postMessage: (message: { method: OperationNames; data: any }) => void;
+          postMessage: (message: {
+            method: OperationNames;
+            data: OperationArgs<OperationNames>;
+          }) => void;
         };
       };
     };
@@ -31,7 +34,7 @@ function createFunctions<K extends OperationNames>(
   return {
     dispatch: (args: OperationArgs<K>) => {
       console.log(
-        `Operation: ${operationName}, Arguments: ${JSON.stringify(args)}`
+        `Operation: ${String(operationName)}, Arguments: ${JSON.stringify(args)}`
       );
       // Send the data to the gtk app
       window.webkit.messageHandlers.gtk.postMessage({
@@ -40,7 +43,7 @@ function createFunctions<K extends OperationNames>(
       });
     },
     receive: (fn: (response: OperationResponse<K>) => void) => {
-      window.clan.list_machines = deserialize(fn);
+      window.clan[operationName] = deserialize(fn);
     },
   };
 }
@@ -59,6 +62,7 @@ const deserialize =
   <T>(fn: (response: T) => void) =>
   (str: string) => {
     try {
+      console.debug("Received data: ", str);
       fn(JSON.parse(str) as T);
     } catch (e) {
       alert(`Error parsing JSON: ${e}`);
@@ -68,7 +72,10 @@ const deserialize =
 // Create the API object
 
 const pyApi: PyApi = {} as PyApi;
-operationNames.forEach((name) => {
+operationNames.forEach((opName) => {
+  const name = opName as OperationNames;
+  // @ts-expect-error - TODO: Fix this. Typescript is not recognizing the receive function correctly
   pyApi[name] = createFunctions(name);
 });
+
 export { pyApi };

pkgs/webview-ui/app/src/nested.tsx

@@ -1,24 +1,34 @@
-import { For, Match, Switch, type Component } from "solid-js";
+import { For, Match, Switch, createEffect, type Component } from "solid-js";
 import { useCountContext } from "./Config";
 
 export const Nested: Component = () => {
   const [{ machines, loading }, { getMachines }] = useCountContext();
+
+  const list = () => Object.values(machines());
+
+  createEffect(() => {
+    console.log("1", list());
+  });
+  createEffect(() => {
+    console.log("2", machines());
+  });
   return (
     <div>
       <button onClick={() => getMachines()} class="btn btn-primary">
         Get machines
       </button>
-      <hr />
+      <div></div>
       <Switch>
         <Match when={loading()}>Loading...</Match>
-        <Match when={!loading() && machines().length === 0}>
+        <Match when={!loading() && Object.entries(machines()).length === 0}>
           No machines found
         </Match>
-        <Match when={!loading() && machines().length}>
+        <Match when={!loading()}>
-          <For each={machines()}>
+          <For each={list()}>
-            {(machine, i) => (
+            {(entry, i) => (
               <li>
-                {i() + 1}: {machine}
+                {i() + 1}: {entry.machine_name}{" "}
+                {entry.machine_description || "No description"}
               </li>
             )}
           </For>

pkgs/webview-ui/default.nix

@@ -1,32 +0,0 @@
-{
-  dream2nix,
-  config,
-  src,
-  ...
-}:
-{
-  imports = [ dream2nix.modules.dream2nix.WIP-nodejs-builder-v3 ];
-
-  mkDerivation = {
-    inherit src;
-  };
-
-  deps =
-    { nixpkgs, ... }:
-    {
-      inherit (nixpkgs) stdenv;
-    };
-
-  WIP-nodejs-builder-v3 = {
-    packageLockFile = "${config.mkDerivation.src}/package-lock.json";
-    # config.groups.all.packages.${config.name}.${config.version}.
-  };
-  public.out = {
-    checkPhase = ''
-      echo "Running tests"
-      echo "Tests passed"
-    '';
-  };
-  name = "@clan/webview-ui";
-  version = "0.0.1";
-}

pkgs/webview-ui/flake-module.nix

@@ -9,10 +9,9 @@
 
         src = ./app;
 
-        # npmDepsHash = "sha256-bRD2vzijhdOOvcEi6XaG/neSqhkVQMqIX/8bxvRQkTc=";
         npmDeps = pkgs.fetchNpmDeps {
           src = ./app;
-          hash = "sha256-bRD2vzijhdOOvcEi6XaG/neSqhkVQMqIX/8bxvRQkTc=";
+          hash = "sha256-E0++hupVKnDqmLk7ljoMcqcI4w+DIMlfRYRPbKUsT2c=";
         };
         # The prepack script runs the build script, which we'd rather do in the build phase.
         npmPackFlags = [ "--ignore-scripts" ];

scripts/pre-commit

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# To install:
+# ln -sf ../../scripts/pre-commit .git/hooks/pre-commit
+set -o errexit
+set -o nounset
+set -o pipefail
+
+readarray staged < <(git diff --name-only --cached)
+[[ ${#staged[@]} = 0 ]] && exit
+unstash() {
+  local ret=$?
+  set +e
+  git stash pop -q
+  exit "$ret"
+}
+git stash push --quiet --keep-index --message "pre-commit"
+trap unstash EXIT
+nix fmt
+{
+  changed=$(git diff --name-only --exit-code);
+  status=$?;
+} || true
+if [[ $status -ne 0 ]]; then
+  exec 1>&2
+  echo Files changed by pre-commit hook:
+  echo "$changed"
+  exit $status
+fi

templates/new-clan/flake.nix

@@ -85,9 +85,9 @@
       };
     in
     {
-      # all machines managed by cLAN
+      # all machines managed by Clan
       inherit (clan) nixosConfigurations clanInternals;
-      # add the cLAN cli tool to the dev shell
+      # add the Clan cli tool to the dev shell
       devShells.${system}.default = pkgs.mkShell {
         packages = [ clan-core.packages.${system}.clan-cli ];
       };