WIP: try to improve nixd completion

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/1484

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-# Contributing to cLAN
+# Contributing to Clan
 
 ## Live-reloading documentation
 

README.md

@@ -1,6 +1,6 @@
 # Clan Core Repository
 
-Welcome to the Clan Core Repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo is the foundation of Clan, a revolutionary open-source project aimed at restoring fun, freedom, and functionality to computing. Here, you'll find all the essential packages, NixOS modules, CLI tools, and tests needed to contribute to and work with the cLAN project. Clan leverages the Nix system to ensure reliability, security, and seamless management of digital environments, putting the power back into the hands of users.
+Welcome to the Clan Core Repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo is the foundation of Clan, a revolutionary open-source project aimed at restoring fun, freedom, and functionality to computing. Here, you'll find all the essential packages, NixOS modules, CLI tools, and tests needed to contribute to and work with the Clan project. Clan leverages the Nix system to ensure reliability, security, and seamless management of digital environments, putting the power back into the hands of users.
 
 ## Why Clan?
 

checks/impure/flake-module.nix

@@ -7,6 +7,8 @@
         #!${pkgs.bash}/bin/bash
         set -euo pipefail
 
+        unset CLAN_DIR
+
         export PATH="${
           lib.makeBinPath [
             pkgs.gitMinimal

clanModules/flake-module.nix

@@ -17,6 +17,7 @@
     static-hosts = ./static-hosts;
     syncthing = ./syncthing;
     thelounge = ./thelounge;
+    trusted-nix-caches = ./trusted-nix-caches;
     user-password = ./user-password;
     xfce = ./xfce;
     zerotier-static-peers = ./zerotier-static-peers;

clanModules/localsend/default.nix

@@ -9,7 +9,7 @@
   # - cli frontend: https://github.com/localsend/localsend/issues/11
   # - ipv6 support: https://github.com/localsend/localsend/issues/549
   options.clan.localsend = {
-    enable = lib.mkEnableOption (lib.mdDoc "enable the localsend module");
+    enable = lib.mkEnableOption "enable the localsend module";
     defaultLocation = lib.mkOption {
       type = lib.types.str;
       description = "The default download location";

clanModules/trusted-nix-caches/README.md

@@ -0,0 +1,2 @@
+This module sets the `clan.lol` and `nix-community` cache up as a trusted cache.
+----

clanModules/trusted-nix-caches/default.nix

@@ -0,0 +1,10 @@
+{
+  nix.settings.trusted-substituters = [
+    "https://cache.clan.lol"
+    "https://nix-community.cachix.org"
+  ];
+  nix.settings.trusted-public-keys = [
+    "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
+    "cache.clan.lol-1:3KztgSAB5R1M+Dz7vzkBGzXdodizbgLXGXKXlcQLA28="
+  ];
+}

devShell.nix

@@ -1,3 +1,4 @@
+{ ... }:
 {
   perSystem =
     {
@@ -34,9 +35,6 @@
           config.treefmt.build.wrapper
         ];
         shellHook = ''
-          # no longer used
-          rm -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit"
-
           echo -e "${ansiEscapes.green}switch to another dev-shell using: select-shell${ansiEscapes.reset}"
         '';
       };

docs/.gitignore

@@ -1 +1,3 @@
 /site/reference
+/site/static/Roboto-Regular.ttf
+/site/static/FiraCode-VF.ttf

docs/CONTRIBUTING.md

@@ -22,9 +22,9 @@ Let's get your development environment up and running:
 
 2. **Install direnv**:
 
-   - Download the direnv package from [here](https://direnv.net/docs/installation.html) or run the following command:
+   - To automatically setup a devshell on entering the directory
      ```bash
-     curl -sfL https://direnv.net/install.sh | bash
+     nix profile install nixpkgs#nix-direnv-flakes
      ```
 
 3. **Add direnv to your shell**:
@@ -36,9 +36,14 @@ Let's get your development environment up and running:
    echo 'eval "$(direnv hook zsh)"' >> ~/.zshrc && echo 'eval "$(direnv hook bash)"' >> ~/.bashrc && eval "$SHELL"
    ```
 
-4. **Clone the Repository and Navigate**:
-
-   - Clone this repository and navigate to it.
+4. **Create a Gitea Account**:
+   - Register an account on https://git.clan.lol
+   - Fork the [clan-core](https://git.clan.lol/clan/clan-core) repository
+   - Clone the repository and navigate to it
+   - Add a new remote called upstream:
+      ```bash
+      git remote add upstream gitea@git.clan.lol:clan/clan-core.git
+      ```
 
 5. **Allow .envrc**:
 
@@ -48,59 +53,68 @@ Let's get your development environment up and running:
      ```
    - Execute `direnv allow` to automatically execute the shell script `.envrc` when entering the directory.
 
-# Setting Up Your Git Workflow
-
-Let's set up your Git workflow to collaborate effectively:
-
-1. **Register Your Gitea Account Locally**:
-
-   - Execute the following command to add your Gitea account locally:
+6. **(Optional) Install Git Hooks**:
+   - To syntax check your code you can run:
       ```bash
-     tea login add
+      nix fmt
       ```
-   - Fill out the prompt as follows:
-     - URL of Gitea instance: `https://git.clan.lol`
-     - Name of new Login [gitea.gchq.icu]: `gitea.gchq.icu:7171`
-     - Do you have an access token? No
-     - Username: YourUsername
-     - Password: YourPassword
-     - Set Optional settings: No
-
-2. **Git Workflow**:
-
-   1. Add your changes to Git using `git add <file1> <file2>`.
-   2. Run `nix fmt` to lint your files.
-   3. Commit your changes with a descriptive message: `git commit -a -m "My descriptive commit message"`.
-   4. Make sure your branch has the latest changes from upstream by executing:
+   - To make this automatic install the git hooks
       ```bash
-      git fetch && git rebase origin/main --autostash
-      ```
-   5. Use `git status` to check for merge conflicts.
-   6. If conflicts exist, resolve them. Here's a tutorial for resolving conflicts in [VSCode](https://code.visualstudio.com/docs/sourcecontrol/overview#_merge-conflicts).
-   7. After resolving conflicts, execute `git merge --continue` and repeat step 5 until there are no conflicts.
-
-3. **Create a Pull Request**:
-
-   - To automatically open a pull request that gets merged if all tests pass, execute:
-     ```bash
-     merge-after-ci
+      ./scripts/pre-commit
       ```
 
-4. **Review Your Pull Request**:
-
-   - Visit https://git.clan.lol and go to the project page. Check under "Pull Requests" for any issues with your pull request.
-
-5. **Push Your Changes**:
-   - If there are issues, fix them and redo step 2. Afterward, execute:
-     ```bash
-     git push origin HEAD:YourUsername-main
-     ```
-   - This will directly push to your open pull request.
+7. **Open a Pull Request**:
+   - Go to the webinterface and open up a pull request
 
 # Debugging
 
 Here are some methods for debugging and testing the clan-cli:
 
+## See all possible packages and tests
+
+To quickly show all possible packages and tests execute:
+
+```bash
+nix flake show --system no-eval
+```
+
+Under `checks` you will find all tests that are executed in our CI. Under `packages` you find all our projects.
+
+```
+git+file:///home/lhebendanz/Projects/clan-core
+├───apps
+│   └───x86_64-linux
+│       ├───install-vm: app
+│       └───install-vm-nogui: app
+├───checks
+│   └───x86_64-linux
+│       ├───borgbackup omitted (use '--all-systems' to show)
+│       ├───check-for-breakpoints omitted (use '--all-systems' to show)
+│       ├───clan-dep-age omitted (use '--all-systems' to show)
+│       ├───clan-dep-bash omitted (use '--all-systems' to show)
+│       ├───clan-dep-e2fsprogs omitted (use '--all-systems' to show)
+│       ├───clan-dep-fakeroot omitted (use '--all-systems' to show)
+│       ├───clan-dep-git omitted (use '--all-systems' to show)
+│       ├───clan-dep-nix omitted (use '--all-systems' to show)
+│       ├───clan-dep-openssh omitted (use '--all-systems' to show)
+│       ├───"clan-dep-python3.11-mypy" omitted (use '--all-systems' to show)
+├───packages
+│   └───x86_64-linux
+│       ├───clan-cli omitted (use '--all-systems' to show)
+│       ├───clan-cli-docs omitted (use '--all-systems' to show)
+│       ├───clan-ts-api omitted (use '--all-systems' to show)
+│       ├───clan-vm-manager omitted (use '--all-systems' to show)
+│       ├───default omitted (use '--all-systems' to show)
+│       ├───deploy-docs omitted (use '--all-systems' to show)
+│       ├───docs omitted (use '--all-systems' to show)
+│       ├───editor omitted (use '--all-systems' to show)
+└───templates
+    ├───default: template: Initialize a new clan flake
+    └───new-clan: template: Initialize a new clan flake
+```
+
+You can execute every test separately by following the tree path `nix build .#checks.x86_64-linux.clan-pytest` for example.
+
 ## Test Locally in Devshell with Breakpoints
 
 To test the cli locally in a development environment and set breakpoints for debugging, follow these steps:
@@ -150,12 +164,14 @@ If you need to inspect the Nix sandbox while running tests, follow these steps:
 2. Use `cntr` and `psgrep` to attach to the Nix sandbox. This allows you to interactively debug your code while it's paused. For example:
 
    ```bash
-   cntr exec -w your_sandbox_name
    psgrep -a -x your_python_process_name
+   cntr attach <container id, container name or process id>
    ```
 
+Or you can also use the [nix breakpoint hook](https://nixos.org/manual/nixpkgs/stable/#breakpointhook)
+
 
 # Standards
 
-Every new module name should be in kebab-case.
-Every fact definition, where possible should be in kebab-case.
+- Every new module name should be in kebab-case.
+- Every fact definition, where possible should be in kebab-case.

docs/_internal/use-cases/01-self-host.md

@@ -20,11 +20,11 @@ There are several reasons for choosing to self-host. These can include:
 
 Alice wants to self-host a mumble server for her family.
 
-- She visits to the cLAN website, and follows the instructions on how to install cLAN-OS on her server.
-- Alice logs into a terminal on her server via SSH (alternatively uses cLAN GUI app)
-- Using the cLAN CLI or GUI tool, alice creates a new private network for her family (VPN)
-- Alice now browses a list of curated cLAN modules and finds a module for mumble.
-- She adds this module to her network using the cLAN tool.
+- She visits to the Clan website, and follows the instructions on how to install Clan-OS on her server.
+- Alice logs into a terminal on her server via SSH (alternatively uses Clan GUI app)
+- Using the Clan CLI or GUI tool, alice creates a new private network for her family (VPN)
+- Alice now browses a list of curated Clan modules and finds a module for mumble.
+- She adds this module to her network using the Clan tool.
 - After that, she uses the clan tool to invite her family members to her network
 - Other family members join the private network via the invitation.
 - By accepting the invitation, other members automatically install all required software to interact with the network on their machine.
@@ -33,7 +33,7 @@ Alice wants to self-host a mumble server for her family.
 
 Alice wants to add a photos app to her private network
 
-- She uses the clan CLI or GUI tool to manage her existing private cLAN family network
+- She uses the clan CLI or GUI tool to manage her existing private Clan family network
 - She discovers a module for photoprism, and adds it to her server using the tool
 - Other members who are already part of her network, will receive a notification that an update is required to their environment
 - After accepting, all new software and services to interact with the new photoprism service will be installed automatically.

docs/_internal/use-cases/02-join-network.md

@@ -1,4 +1,4 @@
-# Joining a cLAN network
+# Joining a Clan network
 
 ## General Description
 
@@ -8,13 +8,13 @@ Joining a self-hosted infrastructure involves connecting to a network, server, o
 
 ### Story 1: Joining a private network
 
-Alice' son Bob has never heard of cLAN, but receives an invitation URL from Alice who already set up private cLAN network for her family.
+Alice' son Bob has never heard of Clan, but receives an invitation URL from Alice who already set up private Clan network for her family.
 
-Bob opens the invitation link and lands on the cLAN website. He quickly learns about what cLAN is and can see that the invitation is for a private network of his family that hosts a number of services, like a private voice chat and a photo sharing platform.
+Bob opens the invitation link and lands on the Clan website. He quickly learns about what Clan is and can see that the invitation is for a private network of his family that hosts a number of services, like a private voice chat and a photo sharing platform.
 
-Bob decides to join the network and follows the instructions to install the cLAN tool on his computer.
+Bob decides to join the network and follows the instructions to install the Clan tool on his computer.
 
-Feeding the invitation link to the cLAN tool, bob registers his machine with the network.
+Feeding the invitation link to the Clan tool, bob registers his machine with the network.
 
 All programs required to interact with the network will be installed and configured automatically and securely.
 
@@ -22,7 +22,7 @@ Optionally, bob can customize the configuration of these programs through a simp
 
 ### Story 2: Receiving breaking changes
 
-The cLAN family network which Bob is part of received an update.
+The Clan family network which Bob is part of received an update.
 
 The existing photo sharing service has been removed and replaced with another alternative service. The new photo sharing service requires a different client app to view and upload photos.
 
@@ -30,7 +30,7 @@ Bob accepts the update. Now his environment will be updated. The old client soft
 
 Because Bob has customized the previous photo viewing app, he is notified that this customization is no longer valid, as the software has been removed (deprecation message).l
 
-Optionally, Bob can now customize the new photo viewing software through his cLAN configuration app or via a config file.
+Optionally, Bob can now customize the new photo viewing software through his Clan configuration app or via a config file.
 
 ## Challenges
 

docs/_internal/use-cases/03-maintaining-clan-modules.md

@@ -1,10 +1,10 @@
-# cLAN module maintaining
+# Clan module maintaining
 
 ## General Description
 
-cLAN modules are pieces of software that can be used by admins to build a private or public infrastructure.
+Clan modules are pieces of software that can be used by admins to build a private or public infrastructure.
 
-cLAN modules should have the following properties:
+Clan modules should have the following properties:
 
 1. Documented: It should be clear what the module does and how to use it.
 1. Self contained: A module should be usable as is. If it requires any other software or settings, those should be delivered with the module itself.

docs/mkdocs.yml

@@ -28,6 +28,7 @@ markdown_extensions:
   - pymdownx.highlight:
       use_pygments: true
       anchor_linenums: true
+  - pymdownx.keys
   - toc:
       title: On this page
 
@@ -64,6 +65,7 @@ nav:
           - reference/clanModules/syncthing.md
           - reference/clanModules/static-hosts.md
           - reference/clanModules/thelounge.md
+          - reference/clanModules/trusted-nix-caches.md
           - reference/clanModules/user-password.md
           - reference/clanModules/xfce.md
           - reference/clanModules/zerotier-static-peers.md
@@ -92,8 +94,9 @@ docs_dir: site
 site_dir: out
 
 theme:
-  logo: static/clan-white.png
-  favicon: static/clan-dark.png
+  font: false
+  logo: https://clan.lol/static/logo/clan-white.png
+  favicon: https://clan.lol/static/logo/clan-dark.png
   name: material
   features:
     - navigation.instant
@@ -124,6 +127,7 @@ theme:
         name: Switch to light mode
 
 extra_css:
+  - static/extra.css
   - static/asciinema-player/custom-theme.css
   - static/asciinema-player/asciinema-player.css
 
@@ -138,7 +142,6 @@ extra:
     - icon: fontawesome/solid/rss
       link: /feed_rss_created.xml
 
-
 plugins:
   - search
   - blog

docs/nix/default.nix

@@ -2,6 +2,10 @@
   pkgs,
   module-docs,
   clan-cli-docs,
+  asciinema-player-js,
+  asciinema-player-css,
+  roboto,
+  fira-code,
   ...
 }:
 let
@@ -27,6 +31,14 @@ pkgs.stdenv.mkDerivation {
     mkdir -p ./site/reference/cli
     cp -af ${module-docs}/* ./site/reference/
     cp -af ${clan-cli-docs}/* ./site/reference/cli/
+
+    mkdir -p ./site/static/asciinema-player
+    ln -snf ${asciinema-player-js} ./site/static/asciinema-player/asciinema-player.min.js
+    ln -snf ${asciinema-player-css} ./site/static/asciinema-player/asciinema-player.css
+
+    # Link to fonts
+    ln -snf ${roboto}/share/fonts/truetype/Roboto-Regular.ttf ./site/static/
+    ln -snf ${fira-code}/share/fonts/truetype/FiraCode-VF.ttf ./site/static/
   '';
 
   buildPhase = ''

docs/nix/flake-module.nix

@@ -40,6 +40,15 @@
             mypy --strict $out
           '';
 
+      asciinema-player-js = pkgs.fetchurl {
+        url = "https://github.com/asciinema/asciinema-player/releases/download/v3.7.0/asciinema-player.min.js";
+        sha256 = "sha256-Ymco/+FinDr5YOrV72ehclpp4amrczjo5EU3jfr/zxs=";
+      };
+      asciinema-player-css = pkgs.fetchurl {
+        url = "https://github.com/asciinema/asciinema-player/releases/download/v3.7.0/asciinema-player.css";
+        sha256 = "sha256-GZMeZFFGvP5GMqqh516mjJKfQaiJ6bL38bSYOXkaohc=";
+      };
+
       module-docs = pkgs.runCommand "rendered" { nativeBuildInputs = [ pkgs.python3 ]; } ''
         export CLAN_CORE=${jsonDocs.clanCore}/share/doc/nixos/options.json 
         # A file that contains the links to all clanModule docs
@@ -56,12 +65,16 @@
       devShells.docs = pkgs.callPackage ./shell.nix {
         inherit (self'.packages) docs clan-cli-docs;
         inherit module-docs;
+        inherit asciinema-player-js;
+        inherit asciinema-player-css;
       };
       packages = {
         docs = pkgs.python3.pkgs.callPackage ./default.nix {
           inherit (self'.packages) clan-cli-docs;
           inherit (inputs) nixpkgs;
           inherit module-docs;
+          inherit asciinema-player-js;
+          inherit asciinema-player-css;
         };
         deploy-docs = pkgs.callPackage ./deploy-docs.nix { inherit (config.packages) docs; };
         inherit module-docs;

docs/nix/scripts/renderOptions.py

@@ -40,13 +40,14 @@ def sanitize(text: str) -> str:
     return text.replace(">", "\\>")
 
 
-def replace_store_path(text: str) -> Path:
+def replace_store_path(text: str) -> tuple[str, str]:
     res = text
     if text.startswith("/nix/store/"):
         res = "https://git.clan.lol/clan/clan-core/src/branch/main/" + str(
             Path(*Path(text).parts[4:])
         )
-    return Path(res)
+    name = Path(res).name
+    return (res, name)
 
 
 def render_option_header(name: str) -> str:
@@ -108,9 +109,10 @@ def render_option(name: str, option: dict[str, Any], level: int = 3) -> str:
 """
 
     decls = option.get("declarations", [])
-    source_path = replace_store_path(decls[0])
+    source_path, name = replace_store_path(decls[0])
+    print(source_path, name)
     res += f"""
-:simple-git: [{source_path.name}]({source_path})
+:simple-git: [{name}]({source_path})
 """
     res += "\n"
 
@@ -160,7 +162,7 @@ def produce_clan_core_docs() -> None:
         for option_name, info in options.items():
             outfile = f"{module_name}/index.md"
 
-            # Create seperate files for nested options
+            # Create separate files for nested options
             if len(option_name.split(".")) <= 2:
                 # i.e. clan-core.clanDir
                 output = core_outputs.get(

docs/nix/shell.nix

@@ -3,6 +3,10 @@
   pkgs,
   module-docs,
   clan-cli-docs,
+  asciinema-player-js,
+  asciinema-player-css,
+  roboto,
+  fira-code,
   ...
 }:
 pkgs.mkShell {
@@ -14,5 +18,14 @@ pkgs.mkShell {
     chmod +w ./site/reference/*
 
     echo "Generated API documentation in './site/reference/' "
+
+    mkdir -p ./site/static/asciinema-player
+
+    ln -snf ${asciinema-player-js} ./site/static/asciinema-player/asciinema-player.min.js
+    ln -snf ${asciinema-player-css} ./site/static/asciinema-player/asciinema-player.css
+
+    # Link to fonts
+    ln -snf ${roboto}/share/fonts/truetype/Roboto-Regular.ttf ./site/static/
+    ln -snf ${fira-code}/share/fonts/truetype/FiraCode-VF.ttf ./site/static/
   '';
 }

docs/site/blog/.authors.yml

@@ -1,21 +1,26 @@
 authors:
+  DavHau:
+    name: "DavHau"
+    description: "Core Developer"
+    avatar: "https://clan.lol/static/profiles/davhau.jpg"
+    url: "https://DavHau.com"
   Lassulus:
     name: "Lassulus"
-    description: "Contributor to Clan"
-    avatar: "https://avatars.githubusercontent.com/u/621759?v=4"
+    description: "Core Developer"
+    avatar: "https://clan.lol/static/profiles/lassulus.jpg"
     url: "https://http://lassul.us/"
   Mic92:
     name: "Mic92"
-    description: "Contributor to Clan"
-    avatar: "https://avatars.githubusercontent.com/u/96200?v=4"
+    description: "Core Developer"
+    avatar: "https://clan.lol/static/profiles/mic92.jpg"
     url: "https://thalheim.io"
   W:
     name: "W"
     description: "Founder of Clan"
-    avatar: "/static/w_profile.webp"
+    avatar: "https://clan.lol/static/profiles/w_profile.webp"
     url: ""
   Qubasa:
     name: "Qubasa"
-    description: "Contributor to Clan"
-    avatar: "https://avatars.githubusercontent.com/u/22085373?v=4"
+    description: "Core Developer"
+    avatar: "https://clan.lol/static/profiles/qubasa.png"
     url: "https://github.com/Qubasa"

docs/site/blog/posts/jsonschema.md

@@ -0,0 +1,194 @@
+---
+title: "Dev Report: Introducing the NixOS to JSON Schema Converter"
+description: "Discover our new library designed to extract JSON schema interfaces from NixOS modules, streamlining frontend development"
+authors:
+  - DavHau
+date: 2024-05-25
+slug: jsonschema-converter
+---
+
+## Overview
+
+We’ve developed a new library designed to extract interfaces from NixOS modules and convert them into JSON schemas, paving the way for effortless GUI generation. This blog post outlines the motivations behind this development, demonstrates the capabilities of the library, and guides you through leveraging it to create GUIs seamlessly.
+
+## Motivation
+
+In recent months, our team has been exploring various graphical user interfaces (GUIs) to streamline NixOS machine configuration. While our opinionated Clan modules simplify NixOS configurations, there's a need to configure these modules from diverse frontends, such as:
+
+- Command-line interfaces (CLIs)
+- Web-based UIs
+- Desktop applications
+- Mobile applications
+- Large Language Models (LLMs)
+
+Given this need, a universal format like JSON is a natural choice. It is already possible as of now, to import json based NixOS configurations, as illustrated below:
+
+`configuration.json`:
+```json
+{ "networking": { "hostName": "my-machine" } }
+```
+
+This configuration can be then imported inside a classic NixOS config:
+```nix
+{config, lib, pkgs, ...}: {
+  imports = [
+    (lib.importJSON ./configuration.json)
+  ];
+}
+```
+
+This straightforward approach allows us to build a frontend that generates JSON, enabling the configuration of NixOS machines. But, two critical questions arise:
+
+1. How does the frontend learn about existing configuration options?
+2. How can it verify user input without running Nix?
+
+Introducing [JSON schema](https://json-schema.org/), a widely supported standard that defines interfaces in JSON and validates input against them.
+
+Example schema for `networking.hostName`:
+```json
+{
+  "type": "object",
+  "properties": {
+    "networking": {
+      "type": "object",
+      "properties": {
+        "hostName": {
+          "type": "string",
+          "pattern": "^$|^[a-z0-9]([a-z0-9_-]{0,61}[a-z0-9])?$"
+        }
+      }
+    }
+  }
+}
+```
+
+## Client-Side Input Validation
+
+Validating input against JSON schemas is both efficient and well-supported across numerous programming languages. Using JSON schema validators, you can accurately check configurations like our `configuration.json`.
+
+Validation example:
+
+```shell
+$ nix-shell -p check-jsonschema
+$ jsonschema -o pretty ./schema.json -i ./configuration.json
+===[SUCCESS]===(./configuration.json)===
+```
+
+In case of invalid input, schema validators provide explicit error messages:
+
+```shell
+$ echo '{ "networking": { "hostName": "my/machine" } }' > configuration.json
+$ jsonschema -o pretty ./schema.json -i ./configuration.json
+===[ValidationError]===(./configuration.json)===
+
+'my/machine' does not match '^$|^[a-z0-9]([a-z0-9_-]{0,61}[a-z0-9])?$'
+
+Failed validating 'pattern' in schema['properties']['networking']['properties']['hostName']:
+    {'pattern': '^$|^[a-z0-9]([a-z0-9_-]{0,61}[a-z0-9])?$',
+     'type': 'string'}
+
+On instance['networking']['hostName']:
+    'my/machine'
+```
+
+## Automatic GUI Generation
+
+Certain libraries facilitate straightforward GUI generation from JSON schemas. For instance, the [react-jsonschema-form playground](https://rjsf-team.github.io/react-jsonschema-form/) auto-generates a form for any given schema.
+
+## NixOS Module to JSON Schema Converter
+
+To enable the development of responsive frontends, our library allows the extraction of interfaces from NixOS modules to JSON schemas. Open-sourced for community collaboration, this library supports building sophisticated user interfaces for NixOS.
+
+Here’s a preview of our library's functions exposed through the [clan-core](https://git.clan.lol/clan/clan-core) flake:
+
+- `lib.jsonschema.parseModule` - Generates a schema for a NixOS module.
+- `lib.jsonschema.parseOption` - Generates a schema for a single NixOS option.
+- `lib.jsonschema.parseOptions` - Generates a schema from an attrset of NixOS options.
+
+Example:
+`module.nix`:
+```nix
+{lib, config, pkgs, ...}: {
+  # a simple service with two options
+  options.services.example-web-service = {
+    enable = lib.mkEnableOption "Example web service";
+    port = lib.mkOption {
+      type = lib.types.int;
+      description = "Port used to serve the content";
+    };
+  };
+}
+```
+
+Converted, using the `parseModule` function:
+```shell
+$ cd clan-core
+$ nix eval --json --impure --expr \
+  '(import ./lib/jsonschema {}).parseModule ./module.nix' | jq | head
+{
+  "properties": {
+    "services": {
+      "properties": {
+        "example-web-service": {
+          "properties": {
+            "enable": {
+              "default": false,
+              "description": "Whether to enable Example web service.",
+              "examples": [
+...
+```
+
+This utility can also generate interfaces for existing NixOS modules or options.
+
+## GUI for NGINX in Under a Minute
+
+Creating a prototype GUI for the NGINX module using our library and [react-jsonschema-form playground](https://rjsf-team.github.io/react-jsonschema-form/) can be done quickly:
+
+1. Export all NGINX options into a JSON schema using a Nix expression:
+
+```nix
+# export.nix
+let
+  pkgs = import <nixpkgs> {};
+  clan-core = builtins.getFlake "git+https://git.clan.lol/clan/clan-core";
+  options = (pkgs.nixos {}).options.services.nginx;
+in
+  clan-core.lib.jsonschema.parseOption options
+```
+
+2. Write the schema into a file:
+```shell
+$ nix eval --json -f ./export.nix | jq > nginx.json
+```
+
+3. Open the [react-jsonschema-form playground](https://rjsf-team.github.io/react-jsonschema-form/), select `Blank` and paste the `nginx.json` contents.
+
+This provides a quick look at a potential GUI (screenshot is cropped).
+
+![Image title](https://clan.lol/static/blog-post-jsonschema/nginx-gui.jpg)
+
+## Limitations
+
+### Laziness
+
+JSON schema mandates the declaration of all required fields upfront, which might be configured implicitly or remain unused. For instance, `services.nginx.virtualHosts.<name>.sslCertificate` must be specified even if SSL isn’t enabled.
+
+### Limited Types
+
+Certain NixOS module types, like `types.functionTo` and `types.package`, do not map straightforwardly to JSON. For full compatibility, adjustments to NixOS modules might be necessary, such as substituting `listOf package` with `listOf str`.
+
+### Parsing NixOS Modules
+
+Currently, our converter relies on the `options` attribute of evaluated NixOS modules, extracting information from the `type.name` attribute, which is suboptimal. Enhanced introspection capabilities within the NixOS module system would be beneficial.
+
+## Future Prospects
+
+We hope these experiments inspire the community, encourage contributions and further development in this space. Share your ideas and contributions through our issue tracker or matrix channel!
+
+## Links
+
+- [Comments on NixOS Discourse](https://discourse.nixos.org/t/introducing-the-nixos-to-json-schema-converter/45948)
+- [Source Code of the JSON Schema Library](https://git.clan.lol/clan/clan-core/src/branch/main/lib/jsonschema)
+- [Our Issue Tracker](https://git.clan.lol/clan/clan-core/issues)
+- [Our Matrix Channel](https://matrix.to/#/#clan:lassul.us)
+- [react-jsonschema-form Playground](https://rjsf-team.github.io/react-jsonschema-form/)

docs/site/drafts/deployment-framework.md

@@ -0,0 +1,63 @@
+---
+title: "Git Based Machine Deployment with Clan-Core"
+description: ""
+authors:
+  - Qubasa
+date: 2024-05-25
+---
+## Revolutionizing Server Management
+
+In the world of server management, countless tools claim to offer seamless deployment of multiple machines. Yet, many fall short, leaving server admins and self-hosting enthusiasts grappling with complexity. Enter the Clan-Core Framework—a groundbreaking all in one solution designed to transform decentralized self-hosting into an effortless and scalable endeavor.
+
+### The Power of Clan-Core
+
+Imagine having the power to manage your servers with unparalleled ease, scaling your IT infrastructure like never before. Clan-Core empowers you to do just that. At its core, Clan-Core leverages a single Git repository to define everything about your machines. This central repository utilizes Nix or JSON files to specify configurations, including disk formatting, ensuring a streamlined and unified approach.
+
+### Simplified Deployment Process
+
+With Clan-Core, the cumbersome task of bootstrapping a specific ISO is a thing of the past. All you need is SSH access to your Linux server. Clan-Core allows you to overwrite any existing Linux distribution live over SSH, eliminating time-consuming setup processes. This capability means you can deploy updates or new configurations swiftly and efficiently, maximizing uptime and minimizing hassle.
+
+### Secure and Efficient Secret Management
+
+Security is paramount in server management, and Clan-Core takes it seriously. Passwords and other sensitive information are encrypted within the Git repository, automatically decrypted during deployment. This not only ensures the safety of your secrets but also simplifies their management. Clan-Core supports sharing secrets with other admins, fostering collaboration and maintaining reproducibillity and security without sacrificing convenience.
+
+### Services as Apps
+
+Setting up a service can be quite difficult. Many server adjustments need to be made, from setting up a database to adjusting webserver configurations and generating the correct private keys. However, Clan-Core aims to make setting up a service as easy as installing an application. Through Clan-Core's Module system, everything down to secrets can be automatically set up. This transforms the often daunting task of service setup into a smooth, automated process, making it accessible to all.
+
+### Decentralized Mesh VPN
+
+Building on these features is a self-configuring decentralized mesh VPN that interconnects all your machines into a private darknet. This ensures that sensitive services, which might have too much attack surface to be hosted on the public internet, can still be made available privately without the need to worry about potential system compromise. By creating a secure, private network, Clan-Core offers an additional layer of protection for your most critical services.
+
+### Decentralized Domain Name System
+
+Current DNS implementations are distributed but not truly decentralized. For Clan-Core, we implemented our own truly decentralized DNS module. This module uses simple flooding and caching algorithms to discover available domains inside the darknet. This approach ensures that your internal domain name system is robust, reliable, and independent of external control, enhancing the resilience and security of your infrastructure.
+
+
+### A New Era of Decentralized Self-Hosting
+
+Clan-Core is more than just a tool; it's a paradigm shift in server management. By consolidating machine definitions, secrets and network configuration, into a single, secure repository, it transforms how you manage and scale your infrastructure. Whether you're a seasoned server admin or a self-hosting enthusiast, Clan-Core offers a powerful, user-friendly solution to take your capabilities to the next level.
+
+
+### Key Features of Clan-Core:
+
+- **Unified Git Repository**: All machine configurations and secrets stored in a single repository.
+- **Live Overwrites**: Deploy configurations over existing Linux distributions via SSH.
+- **Automated Service Setup**: Easily set up services with Clan-Core's Module system.
+- **Decentralized Mesh VPN**: Securely interconnect all machines into a private darknet.
+- **Decentralized DNS**: Robust, independent DNS using flooding and caching algorithms.
+- **Automated Secret Management**: Encrypted secrets that are automatically decrypted during deployment.
+- **Collaboration Support**: Share secrets securely with other admins.
+
+
+## Clan-Cores Future
+
+Our vision for Clan-Core extends far beyond being just another deployment tool. Clan-Core is a framework we've developed to achieve something much greater. We want to put the "personal" back into "personal computing." Our goal is for everyday users to fully customize their phones or laptops and create truly private spaces for friends and family.
+
+Our first major step is to develop a Graphical User Interface (GUI) that makes configuring all this possible. Initial tests have shown that AI can be leveraged as an alternative to traditional GUIs. This paves the way for a future where people can simply talk to their computers, and they will configure themselves according to the users' wishes. 
+
+By adopting Clan, you're not just embracing a tool—you're joining a movement towards a more efficient, secure, and scalable approach to server management. Join us and revolutionize your IT infrastructure today.
+
+
+
+

docs/site/getting-started/configure.md

@@ -59,7 +59,7 @@ Adding or configuring a new machine requires two simple steps:
 
     Which should show something like:
 
-    ```bash hl_lines="6"
+    ```{.shellSession hl_lines="6" .no-copy}
     NAME        ID-LINK                                         FSTYPE   SIZE MOUNTPOINT
     sda         usb-ST_16GB_AA6271026J1000000509-0:0                    14.9G 
     ├─sda1      usb-ST_16GB_AA6271026J1000000509-0:0-part1                 1M 

docs/site/getting-started/deploy.md

@@ -52,7 +52,7 @@ This process involves preparing a suitable hardware and disk partitioning config
 
     This is an example of the booted installer.
 
-    ```{ .bash .annotate .no-copy }
+    ```{ .bash .annotate .no-copy .nohighlight}
     ┌─────────────────────────────────────────────────────────────────────────────────────┐
     │ ┌───────────────────────────┐                                                       │
     │ │███████████████████████████│ # This is the QR Code (1)                             │
@@ -151,7 +151,7 @@ Clan CLI enables you to remotely update your machines over SSH. This requires se
 ### Setting the Target Host
 
 Replace `root@jon` with the actual hostname or IP address of your target machine:
-```nix hl_lines="9"
+```{.nix hl_lines="9" .no-copy}
 buildClan {
     # ...
     machines = {
@@ -192,7 +192,7 @@ it is also possible to specify a build host instead.
 During an update, the cli will ssh into the build host and run `nixos-rebuild` from there.
 
 
-```nix hl_lines="5"
+```{.nix hl_lines="5" .no-copy}
 buildClan {
     # ...
     machines = {
@@ -208,7 +208,7 @@ buildClan {
 To exclude machines from being updated when running `clan machines update` without any machines specified,
 one can set the `clan.deployment.requireExplicitUpdate` option to true:
 
-```nix hl_lines="5"
+```{.nix hl_lines="5" .no-copy}
 buildClan {
     # ...
     machines = {

docs/site/getting-started/flake-parts.md

@@ -94,9 +94,4 @@ Below is a guide on how to structure this in your flake.nix:
 For detailed information about configuring `flake-parts` and the available options within Clan,
 refer to the Clan module documentation located [here](https://git.clan.lol/clan/clan-core/src/branch/main/flakeModules/clan.nix).
 
-## Whats next?
-
-- [Configure Machines](configure.md): Customize machine configuration
-- [Deploying](deploy.md): Deploying a Machine configuration
-
 ---

docs/site/getting-started/installer.md

@@ -22,7 +22,7 @@ Follow our step-by-step guide to create and transfer this image onto a bootable
     lsblk
     ```
 
-    ```shellSession hl_lines="2"
+    ```{.shellSession hl_lines="2" .no-copy}
     NAME                                          MAJ:MIN RM   SIZE RO TYPE  MOUNTPOINTS
     sdb                                             8:0    1 117,2G  0 disk
     └─sdb1                                          8:1    1 117,2G  0 part  /run/media/qubasa/INTENSO
@@ -123,7 +123,7 @@ This will enter `iwd`
 
 Now run the following command to connect to your Wifi:
 
-```shellSession
+```{.shellSession .no-copy}
 # Identify your network device.
 device list
 
@@ -148,10 +148,10 @@ Connected network     FRITZ!Box (Your router device)
 IPv4 address          192.168.188.50 (Your new local ip)
 ```
 
-Press `ctrl-d` to exit `IWD`.
+Press ++ctrl+d++ to exit `IWD`.
 
 !!! Important
-    Press `ctrl-d` **again** to update the displayed QR code and connection information.
+    Press ++ctrl+d++ **again** to update the displayed QR code and connection information.
 
 You're all set up
 

docs/site/getting-started/mesh-vpn.md

@@ -9,7 +9,7 @@ include a new machine into the VPN.
 
 By default all machines within one clan are connected via a chosen network technology.
 
-```
+```{.no-copy}
 Clan 
     Node A
     <-> (zerotier / mycelium / ...)
@@ -36,7 +36,7 @@ peers. Once addresses are allocated, the controller's continuous operation is no
    ```
 3. **Update the Controller Machine**: Execute the following:
    ```bash
-   $ clan machines update <CONTROLLER>
+   clan machines update <CONTROLLER>
    ```
    Your machine is now operational as the VPN controller.
 

docs/site/getting-started/secrets.md

@@ -4,10 +4,15 @@ Clan enables encryption of secrets (such as passwords & keys) ensuring security
 
 Clan utilizes the [sops](https://github.com/getsops/sops) format and integrates with [sops-nix](https://github.com/Mic92/sops-nix) on NixOS machines.
 
+This guide will walk you through:
 
-### Create Your Master Keypair
+- **Creating a Keypair for Your User**: Learn how to generate a keypair for $USER to securely control all secrets.
+- **Creating Your First Secret**: Step-by-step instructions on creating your initial secret.
+- **Assigning Machine Access to the Secret**: Understand how to grant a machine access to the newly created secret.
 
-To get started, you'll need to create **Your master keypair**.
+## Create Your Admin Keypair
+
+To get started, you'll need to create **Your admin keypair**.
 
 !!! info
     Don't worry — if you've already made one before, this step won't change or overwrite it.
@@ -27,7 +32,7 @@ Also add your age public key to the repository with 'clan secrets users add YOUR
 
 !!! warning
     Make sure to keep a safe backup of the private key you've just created.
-    If it's lost, you won't be able to get to your secrets anymore because they all need the master key to be unlocked.
+    If it's lost, you won't be able to get to your secrets anymore because they all need the admin key to be unlocked.
 
 !!! note
     It's safe to add any secrets created by the clan CLI and placed in your repository to version control systems like `git`.
@@ -35,7 +40,7 @@ Also add your age public key to the repository with 'clan secrets users add YOUR
 ### Add Your Public Key
 
 ```bash
-clan secrets users add <your_username> <your_public_key>
+clan secrets users add $USER <your_public_key>
 ```
 
 It's best to choose the same username as on your Setup/Admin Machine that you use to control the deployment with.
@@ -101,17 +106,20 @@ In your nixos configuration you can get a path to secrets like this `config.sops
 
 ### Assigning Access
 
-By default, secrets are encrypted for your key. To specify which users and machines can access a secret:
+When using `clan secrets set <secret>` without arguments, secrets are encrypted for the key of the user named like your current $USER.
 
-```bash
-clan secrets set --machine <machine1> --machine <machine2> --user <user1> --user <user2> <secret_name>
-```
-You can also just add machines/users to existing secrets:
+To add machines/users to an existing secret use:
 
 ```bash
  clan secrets machines add-secret <machine_name> <secret_name>
 ```
 
+Alternatively specify users and machines while creating a secret:
+
+```bash
+clan secrets set --machine <machine1> --machine <machine2> --user <user1> --user <user2> <secret_name>
+```
+
 ## Advanced
 
 In this section we go into more advanced secret management topics.
@@ -183,11 +191,9 @@ Since our clan secret module will auto-import secrets that are encrypted for a p
 you can now remove `sops.secrets.<secrets> = { };` unless you need to specify more options for the secret like owner/group of the secret file.
 
 
-
 ## Indepth Explanation
 
 
-
 The secrets system conceptually knows two different entities:
 
 - **Machine**: consumes secrets

docs/site/static/asciinema-player/asciinema-player.css

@@ -0,0 +1 @@
+/nix/store/8y5h98wk5p94mv1wyb2c4gkrr7bswd19-asciinema-player.css

docs/site/static/asciinema-player/asciinema-player.min.js

@@ -0,0 +1 @@
+/nix/store/w0i3f9qzn9n6jmfnfgiw5wnab2f9ssdw-asciinema-player.min.js

docs/site/static/asciinema-player/custom-theme.css

@@ -1,45 +0,0 @@
-.asciinema-player-theme-alabaster-auto {
-        --term-color-foreground: #000000; /* Black for foreground text */
-        --term-color-background: #f7f7f7; /* Very light gray for background */
-
-        --term-color-0: #000000; /* Black */
-        --term-color-1: #aa3731; /* Red */
-        --term-color-2: #448c37; /* Green */
-        --term-color-3: #cb9000; /* Yellow */
-        --term-color-4: #325cc0; /* Blue */
-        --term-color-5: #7a3e9d; /* Magenta */
-        --term-color-6: #0083b2; /* Cyan */
-        --term-color-7: #bbbbbb; /* White */
-        --term-color-8: #777777; /* Bright black (gray) */
-        --term-color-9: #f05050; /* Bright red */
-        --term-color-10: #60cb00; /* Bright green */
-        --term-color-11: #ffbc5d; /* Bright yellow */
-        --term-color-12: #007acc; /* Bright blue */
-        --term-color-13: #e64ce6; /* Bright magenta */
-        --term-color-14: #00aacb; /* Bright cyan */
-        --term-color-15: #ffffff; /* Bright white */
-    }
-
-  @media (prefers-color-scheme: dark) {
-    .asciinema-player-theme-solarized-auto {
-      --term-color-foreground: #839496;
-      --term-color-background: #002b36;
-
-      --term-color-0: #073642;
-      --term-color-1: #dc322f;
-      --term-color-2: #859900;
-      --term-color-3: #b58900;
-      --term-color-4: #268bd2;
-      --term-color-5: #d33682;
-      --term-color-6: #2aa198;
-      --term-color-7: #eee8d5;
-      --term-color-8: #002b36;
-      --term-color-9: #cb4b16;
-      --term-color-10: #586e75;
-      --term-color-11: #657b83;
-      --term-color-12: #839496;
-      --term-color-13: #6c71c4;
-      --term-color-14: #93a1a1;
-      --term-color-15: #fdf6e3;
-    }
-  }

docs/site/static/extra.css

@@ -0,0 +1,13 @@
+@font-face {
+    font-family: "Roboto";
+    src: url(./Roboto-Regular.ttf) format('truetype');
+}
+@font-face {
+    font-family: "Fira Code";
+    src: url(./FiraCode-VF.ttf) format('truetype');
+}
+
+:root {
+    --md-text-font: "Roboto"; 
+    --md-code-font: "Fira Code";
+}

flake.lock

@@ -7,15 +7,15 @@
         ]
       },
       "locked": {
-        "lastModified": 1714400597,
-        "narHash": "sha256-AA1TCyEl4O6+6F5man/V5VH9Zl9HPBpK91tSkZ16i2E=",
-        "owner": "Qubasa",
+        "lastModified": 1716394172,
+        "narHash": "sha256-B+pNhV8GFeCj9/MoH+qtGqKbgv6fU4hGaw2+NoYYtB0=",
+        "owner": "nix-community",
         "repo": "disko",
-        "rev": "58785136b8c37aeb2f67081387b48f663b166331",
+        "rev": "23c63fb09334c3e8958b57e2ddc3870b75b9111d",
         "type": "github"
       },
       "original": {
-        "owner": "Qubasa",
+        "owner": "nix-community",
         "repo": "disko",
         "type": "github"
       }

flake.nix

@@ -8,7 +8,7 @@
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
-    disko.url = "github:Qubasa/disko";
+    disko.url = "github:nix-community/disko";
     disko.inputs.nixpkgs.follows = "nixpkgs";
     sops-nix.url = "github:Mic92/sops-nix";
     sops-nix.inputs.nixpkgs.follows = "nixpkgs";

lib/jsonschema/default.nix

@@ -83,20 +83,20 @@ rec {
     in
 
     # either type
-    # TODO: if all nested optiosn are excluded, the parent sould be excluded too
+    # TODO: if all nested options are excluded, the parent should be excluded too
     if
-      option.type.name or null == "either"
+      option.type.name or null == "either" || option.type.name or null == "coercedTo"
     # return jsonschema property definition for either
     then
       let
         optionsList' = [
           {
-            type = option.type.nestedTypes.left;
+            type = option.type.nestedTypes.left or option.type.nestedTypes.coercedType;
             _type = "option";
             loc = option.loc;
           }
           {
-            type = option.type.nestedTypes.right;
+            type = option.type.nestedTypes.right or option.type.nestedTypes.finalType;
             _type = "option";
             loc = option.loc;
           }
@@ -157,12 +157,21 @@ rec {
 
     # TODO: Add support for intMatching in jsonschema
     # parse port type aka. "unsignedInt16"
-    else if option.type.name == "unsignedInt16" then
+    else if
+      option.type.name == "unsignedInt16"
+      || option.type.name == "unsignedInt"
+      || option.type.name == "pkcs11"
+      || option.type.name == "intBetween"
+    then
       default // example // description // { type = "integer"; }
 
     # parse string
+    # TODO: parse more precise string types
     else if
       option.type.name == "str"
+      || option.type.name == "singleLineStr"
+      || option.type.name == "passwdEntry str"
+      || option.type.name == "passwdEntry path"
     # return jsonschema property definition for string
     then
       default // example // description // { type = "string"; }

nixosModules/clanCore/vm.nix

@@ -122,7 +122,7 @@ in
       cores = lib.mkOption {
         type = lib.types.ints.positive;
         default = 1;
-        description = lib.mdDoc ''
+        description = ''
           Specify the number of cores the guest is permitted to use.
           The number can be higher than the available cores on the
           host system.
@@ -132,7 +132,7 @@ in
       memorySize = lib.mkOption {
         type = lib.types.ints.positive;
         default = 1024;
-        description = lib.mdDoc ''
+        description = ''
           The memory size in megabytes of the virtual machine.
         '';
       };
@@ -140,7 +140,7 @@ in
       graphics = lib.mkOption {
         type = lib.types.bool;
         default = true;
-        description = lib.mdDoc ''
+        description = ''
           Whether to run QEMU with a graphics window, or in nographic mode.
           Serial console will be enabled on both settings, but this will
           change the preferred console.
@@ -150,7 +150,7 @@ in
       waypipe = lib.mkOption {
         type = lib.types.bool;
         default = false;
-        description = lib.mdDoc ''
+        description = ''
           Whether to use waypipe for native wayland passthrough, or not.
         '';
       };

nixosModules/clanCore/zerotier/default.nix

@@ -81,7 +81,7 @@ in
       };
     };
     settings = lib.mkOption {
-      description = lib.mdDoc "override the network config in /var/lib/zerotier/bla/$network.json";
+      description = "override the network config in /var/lib/zerotier/bla/$network.json";
       type = lib.types.submodule { freeformType = (pkgs.formats.json { }).type; };
     };
   };

pkgs/clan-cli/api.py

@@ -1,10 +1,12 @@
+import json
+
 from clan_cli.api import API
 
 
 def main() -> None:
     schema = API.to_json_schema()
     print(
-        f"""export const schema = {schema} as const;
+        f"""export const schema = {json.dumps(schema, indent=2)} as const;
 """
     )
 

pkgs/clan-cli/clan_cli/__init__.py

@@ -52,7 +52,17 @@ class AppendOptionAction(argparse.Action):
 
 
 def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(prog=prog, description="cLAN tool")
+    parser = argparse.ArgumentParser(
+        prog=prog,
+        description="The clan cli tool.",
+        epilog=(
+            """
+Online reference for the clan cli tool: https://docs.clan.lol/reference/cli/
+For more detailed information, visit: https://docs.clan.lol
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
 
     parser.add_argument(
         "--debug",
@@ -90,39 +100,185 @@ def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
         "backups",
         help="manage backups of clan machines",
         description="manage backups of clan machines",
+        epilog=(
+            """
+This subcommand provides an interface to backups that clan machines expose.
+
+Examples:
+
+  $ clan backups list [MACHINE]
+  List backups for the machine [MACHINE]
+
+  $ clan backups create [MACHINE]
+  Create a backup for the machine [MACHINE].
+
+  $ clan backups restore [MACHINE] [PROVIDER] [NAME]
+  The backup to restore for the machine [MACHINE] with the configured [PROVIDER]
+  with the name [NAME].
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/backups/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     backups.register_parser(parser_backups)
 
     parser_flake = subparsers.add_parser(
-        "flakes", help="create a clan flake inside the current directory"
+        "flakes",
+        help="create a clan flake inside the current directory",
+        description="create a clan flake inside the current directory",
+        epilog=(
+            """
+Examples:
+  $ clan flakes create [DIR]
+  Will create a new clan flake in the specified directory and create it if it
+  doesn't exist yet. The flake will be created from a default template.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     flakes.register_parser(parser_flake)
 
-    parser_config = subparsers.add_parser("config", help="set nixos configuration")
+    parser_config = subparsers.add_parser(
+        "config",
+        help="set nixos configuration",
+        description="set nixos configuration",
+        epilog=(
+            """
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     config.register_parser(parser_config)
 
-    parser_ssh = subparsers.add_parser("ssh", help="ssh to a remote machine")
+    parser_ssh = subparsers.add_parser(
+        "ssh",
+        help="ssh to a remote machine",
+        description="ssh to a remote machine",
+        epilog=(
+            """
+This subcommand allows seamless ssh access to the nixos-image builders.
+
+Examples:
+
+  $ clan ssh [ssh_args ...] --json [JSON]
+  Will ssh in to the machine based on the deployment information contained in
+  the json string. [JSON] can either be a json formatted string itself, or point
+  towards a file containing the deployment information
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/deploy
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     ssh_cli.register_parser(parser_ssh)
 
-    parser_secrets = subparsers.add_parser("secrets", help="manage secrets")
+    parser_secrets = subparsers.add_parser(
+        "secrets",
+        help="manage secrets",
+        description="manage secrets",
+        epilog=(
+            """
+This subcommand provides an interface to secret facts.
+
+Examples:
+
+  $ clan secrets list [regex]
+  Will list secrets for all managed machines.
+  It accepts an optional regex, allowing easy filtering of returned secrets.
+
+  $ clan secrets get [SECRET]
+  Will display the content of the specified secret.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/secrets/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     secrets.register_parser(parser_secrets)
 
-    parser_facts = subparsers.add_parser("facts", help="manage facts")
+    parser_facts = subparsers.add_parser(
+        "facts",
+        help="manage facts",
+        description="manage facts",
+        epilog=(
+            """
+
+This subcommand provides an interface to facts of clan machines.
+Facts are artifacts that a service can generate.
+There are public and secret facts. 
+Public facts can be referenced by other machines directly.
+Public facts can include: ip addresses, public keys.
+Secret facts can include: passwords, private keys.
+
+A service is an included clan-module that implements facts generation functionality.
+For example the zerotier module will generate private and public facts.
+In this case the public fact will be the resulting zerotier-ip of the machine.
+The secret fact will be the zerotier-identity-secret, which is used by zerotier
+to prove the machine has control of the zerotier-ip.
+
+Examples:
+
+  $ clan facts generate
+  Will generate facts for all machines.
+   
+  $ clan facts generate --service [SERVICE] --regenerate
+  Will regenerate facts, if they are already generated for a specific service.
+  This is especially useful for resetting certain passwords while leaving the rest
+  of the facts for a machine in place.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/secrets/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     facts.register_parser(parser_facts)
 
     parser_machine = subparsers.add_parser(
-        "machines", help="manage machines and their configuration"
+        "machines",
+        help="manage machines and their configuration",
+        description="manage machines and their configuration",
+        epilog=(
+            """
+This subcommand provides an interface to machines managed by clan.
+
+Examples:
+
+  $ clan machines list
+  List all the machines managed by clan.
+
+  $ clan machines update [MACHINES]
+  Will update the specified machine [MACHINE], if [MACHINE] is omitted, the command 
+  will attempt to update every configured machine.
+
+  $ clan machines install [MACHINES] [TARGET_HOST]
+  Will install the specified machine [MACHINE], to the specified [TARGET_HOST].
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/deploy
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     machines.register_parser(parser_machine)
 
-    parser_vms = subparsers.add_parser("vms", help="manage virtual machines")
+    parser_vms = subparsers.add_parser(
+        "vms", help="manage virtual machines", description="manage virtual machines"
+    )
     vms.register_parser(parser_vms)
 
-    parser_history = subparsers.add_parser("history", help="manage history")
+    parser_history = subparsers.add_parser(
+        "history",
+        help="manage history",
+        description="manage history",
+    )
     history.register_parser(parser_history)
 
     parser_flash = subparsers.add_parser(
-        "flash", help="flash machines to usb sticks or into isos"
+        "flash",
+        help="flash machines to usb sticks or into isos",
+        description="flash machines to usb sticks or into isos",
     )
     flash.register_parser(parser_flash)
 

pkgs/clan-cli/clan_cli/api/__init__.py

@@ -1,18 +1,37 @@
 from collections.abc import Callable
-from typing import Any
+from dataclasses import dataclass
+from typing import Any, Generic, Literal, TypeVar
+
+T = TypeVar("T")
+
+
+ResponseDataType = TypeVar("ResponseDataType")
+
+
+@dataclass
+class ApiError:
+    message: str
+    description: str | None
+    location: list[str] | None
+
+
+@dataclass
+class ApiResponse(Generic[ResponseDataType]):
+    status: Literal["success", "error"]
+    errors: list[ApiError] | None
+    data: ResponseDataType | None
 
 
 class _MethodRegistry:
     def __init__(self) -> None:
-        self._registry: dict[str, Callable] = {}
+        self._registry: dict[str, Callable[[Any], Any]] = {}
 
-    def register(self, fn: Callable) -> Callable:
+    def register(self, fn: Callable[..., T]) -> Callable[..., T]:
         self._registry[fn.__name__] = fn
         return fn
 
-    def to_json_schema(self) -> str:
+    def to_json_schema(self) -> dict[str, Any]:
         # Import only when needed
-        import json
         from typing import get_type_hints
 
         from clan_cli.api.util import type_to_dict
@@ -21,25 +40,51 @@ class _MethodRegistry:
             "$comment": "An object containing API methods. ",
             "type": "object",
             "additionalProperties": False,
-            "required": ["list_machines"],
+            "required": [func_name for func_name in self._registry.keys()],
             "properties": {},
         }
+
         for name, func in self._registry.items():
             hints = get_type_hints(func)
+
             serialized_hints = {
-                "argument" if key != "return" else "return": type_to_dict(
+                key: type_to_dict(
                     value, scope=name + " argument" if key != "return" else "return"
                 )
                 for key, value in hints.items()
             }
+
+            return_type = serialized_hints.pop("return")
+
             api_schema["properties"][name] = {
+                "type": "object",
+                "required": ["arguments", "return"],
+                "additionalProperties": False,
+                "properties": {
+                    "return": return_type,
+                    "arguments": {
                         "type": "object",
                         "required": [k for k in serialized_hints.keys()],
                         "additionalProperties": False,
-                "properties": {**serialized_hints},
+                        "properties": serialized_hints,
+                    },
+                },
             }
 
-        return json.dumps(api_schema, indent=2)
+        return api_schema
+
+    def get_method_argtype(self, method_name: str, arg_name: str) -> Any:
+        from inspect import signature
+
+        func = self._registry.get(method_name, None)
+        if func:
+            sig = signature(func)
+            param = sig.parameters.get(arg_name)
+            if param:
+                param_class = param.annotation
+                return param_class
+
+        return None
 
 
 API = _MethodRegistry()

pkgs/clan-cli/clan_cli/api/util.py

@@ -42,8 +42,13 @@ def type_to_dict(t: Any, scope: str = "") -> dict:
             return {"type": "array", "items": type_to_dict(t.__args__[0], scope)}
 
         elif issubclass(origin, dict):
+            value_type = t.__args__[1]
+            if value_type is Any:
+                return {"type": "object", "additionalProperties": True}
+            else:
                 return {
                     "type": "object",
+                    "additionalProperties": type_to_dict(value_type, scope),
                 }
 
         raise BaseException(f"Error api type not yet supported {t!s}")

pkgs/clan-cli/clan_cli/cmd.py

@@ -140,3 +140,23 @@ def run(
         raise ClanCmdError(cmd_out)
 
     return cmd_out
+
+
+def run_no_stdout(
+    cmd: list[str],
+    *,
+    env: dict[str, str] | None = None,
+    cwd: Path = Path.cwd(),
+    log: Log = Log.STDERR,
+    check: bool = True,
+    error_msg: str | None = None,
+) -> CmdOut:
+    """
+    Like run, but automatically suppresses stdout, if not in DEBUG log level.
+    If in DEBUG log level the stdout of commands will be shown.
+    """
+    if logging.getLogger(__name__.split(".")[0]).isEnabledFor(logging.DEBUG):
+        return run(cmd, env=env, log=log, check=check, error_msg=error_msg)
+    else:
+        log = Log.NONE
+        return run(cmd, env=env, log=log, check=check, error_msg=error_msg)

pkgs/clan-cli/clan_cli/facts/__init__.py

@@ -23,7 +23,42 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
     register_list_parser(list_parser)
 
     parser_generate = subparser.add_parser(
-        "generate", help="generate secrets for machines if they don't exist yet"
+        "generate",
+        help="generate public and secret facts for machines",
+        epilog=(
+            """
+This subcommand allows control of the generation of facts.
+Often this function will be invoked automatically on deploying machines,
+but there are situations the user may want to have more granular control,
+especially for the regeneration of certain services.
+
+A service is an included clan-module that implements facts generation functionality.
+For example the zerotier module will generate private and public facts.
+In this case the public fact will be the resulting zerotier-ip of the machine.
+The secret fact will be the zerotier-identity-secret, which is used by zerotier
+to prove the machine has control of the zerotier-ip.
+
+
+Examples:
+
+  $ clan facts generate
+  Will generate facts for all machines.
+   
+  $ clan facts generate [MACHINE]
+  Will generate facts for the specified machine.
+
+  $ clan facts generate [MACHINE] --service [SERVICE]
+  Will generate facts for the specified machine for the specified service.
+
+  $ clan facts generate --service [SERVICE] --regenerate
+  Will regenerate facts, if they are already generated for a specific service.
+  This is especially useful for resetting certain passwords while leaving the rest
+  of the facts for a machine in place.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/secrets/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     register_generate_parser(parser_generate)
 

pkgs/clan-cli/clan_cli/facts/generate.py

@@ -33,6 +33,7 @@ def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str:
 def generate_service_facts(
     machine: Machine,
     service: str,
+    regenerate: bool,
     secret_facts_store: SecretStoreBase,
     public_facts_store: FactStoreBase,
     tmpdir: Path,
@@ -42,7 +43,7 @@ def generate_service_facts(
     # check if all secrets exist and generate them if at least one is missing
     needs_regeneration = not check_secrets(machine, service=service)
     log.debug(f"{service} needs_regeneration: {needs_regeneration}")
-    if not needs_regeneration:
+    if not (needs_regeneration or regenerate):
         return False
     if not isinstance(machine.flake, Path):
         msg = f"flake is not a Path: {machine.flake}"
@@ -134,7 +135,11 @@ def prompt_func(text: str) -> str:
 
 
 def _generate_facts_for_machine(
-    machine: Machine, tmpdir: Path, prompt: Callable[[str], str] = prompt_func
+    machine: Machine,
+    service: str | None,
+    regenerate: bool,
+    tmpdir: Path,
+    prompt: Callable[[str], str] = prompt_func,
 ) -> bool:
     local_temp = tmpdir / machine.name
     local_temp.mkdir()
@@ -145,10 +150,23 @@ def _generate_facts_for_machine(
     public_facts_store = public_facts_module.FactStore(machine=machine)
 
     machine_updated = False
-    for service in machine.facts_data:
+
+    if service and service not in machine.facts_data:
+        services = list(machine.facts_data.keys())
+        raise ClanError(
+            f"Could not find service with name: {service}. The following services are available: {services}"
+        )
+
+    if service:
+        machine_service_facts = {service: machine.facts_data[service]}
+    else:
+        machine_service_facts = machine.facts_data
+
+    for service in machine_service_facts:
         machine_updated |= generate_service_facts(
             machine=machine,
             service=service,
+            regenerate=regenerate,
             secret_facts_store=secret_facts_store,
             public_facts_store=public_facts_store,
             tmpdir=local_temp,
@@ -161,7 +179,10 @@ def _generate_facts_for_machine(
 
 
 def generate_facts(
-    machines: list[Machine], prompt: Callable[[str], str] = prompt_func
+    machines: list[Machine],
+    service: str | None,
+    regenerate: bool,
+    prompt: Callable[[str], str] = prompt_func,
 ) -> bool:
     was_regenerated = False
     with TemporaryDirectory() as tmp:
@@ -170,7 +191,9 @@ def generate_facts(
         for machine in machines:
             errors = 0
             try:
-                was_regenerated |= _generate_facts_for_machine(machine, tmpdir, prompt)
+                was_regenerated |= _generate_facts_for_machine(
+                    machine, service, regenerate, tmpdir, prompt
+                )
             except Exception as exc:
                 log.error(f"Failed to generate facts for {machine.name}: {exc}")
                 errors += 1
@@ -189,7 +212,7 @@ def generate_command(args: argparse.Namespace) -> None:
         machines = get_all_machines(args.flake)
     else:
         machines = get_selected_machines(args.flake, args.machines)
-    generate_facts(machines)
+    generate_facts(machines, args.service, args.regenerate)
 
 
 def register_generate_parser(parser: argparse.ArgumentParser) -> None:
@@ -200,4 +223,17 @@ def register_generate_parser(parser: argparse.ArgumentParser) -> None:
         nargs="*",
         default=[],
     )
+    parser.add_argument(
+        "--service",
+        type=str,
+        help="service to generate facts for, if empty, generate facts for every service",
+        default=None,
+    )
+    parser.add_argument(
+        "--regenerate",
+        type=bool,
+        action=argparse.BooleanOptionalAction,
+        help="whether to regenerate facts for the specified machine",
+        default=None,
+    )
     parser.set_defaults(func=generate_command)

pkgs/clan-cli/clan_cli/facts/list.py

@@ -26,7 +26,14 @@ def get_all_facts(machine: Machine) -> dict:
 
 def get_command(args: argparse.Namespace) -> None:
     machine = Machine(name=args.machine, flake=args.flake)
-    print(json.dumps(get_all_facts(machine), indent=4))
+
+    # the raw_facts are bytestrings making them not json serializable
+    raw_facts = get_all_facts(machine)
+    facts = dict()
+    for key in raw_facts["TODO"]:
+        facts[key] = raw_facts["TODO"][key].decode("utf8")
+
+    print(json.dumps(facts, indent=4))
 
 
 def register_list_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/flakes/inspect.py

@@ -39,7 +39,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig:
     system = config["system"]
 
     # Check if the machine exists
-    machines = list_machines(flake_url)
+    machines = list_machines(flake_url, False)
     if machine_name not in machines:
         raise ClanError(
             f"Machine {machine_name} not found in {flake_url}. Available machines: {', '.join(machines)}"
@@ -53,7 +53,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig:
         gcroot_icon: Path = machine_gcroot(flake_url=str(flake_url)) / vm.machine_name
         nix_add_to_gcroots(vm.machine_icon, gcroot_icon)
 
-    # Get the cLAN name
+    # Get the Clan name
     cmd = nix_eval(
         [
             f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clanCore.clanName'
@@ -70,7 +70,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig:
     )
     res = run_cmd(cmd)
 
-    # If the icon is null, no icon is set for this cLAN
+    # If the icon is null, no icon is set for this Clan
     if res == "null":
         icon_path = None
     else:
@@ -113,7 +113,7 @@ def inspect_command(args: argparse.Namespace) -> None:
     res = inspect_flake(
         flake_url=inspect_options.flake, machine_name=inspect_options.machine
     )
-    print("cLAN name:", res.clan_name)
+    print("Clan name:", res.clan_name)
     print("Icon:", res.icon)
     print("Description:", res.description)
     print("Last updated:", res.last_updated)

pkgs/clan-cli/clan_cli/machines/__init__.py

@@ -17,7 +17,26 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
         required=True,
     )
 
-    update_parser = subparser.add_parser("update", help="Update a machine")
+    update_parser = subparser.add_parser(
+        "update",
+        help="Update a machine",
+        epilog=(
+            """
+This subcommand provides an interface to update machines managed by clan.
+
+Examples:
+
+  $ clan machines update [MACHINES]
+  Will update the specified machine [MACHINE], if [MACHINE] is omitted, the command
+  will attempt to update every configured machine.
+  To exclude machines being updated `clan.deployment.requireExplicitUpdate = true;`
+  can be set in the machine config.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/deploy
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     register_update_parser(update_parser)
 
     create_parser = subparser.add_parser("create", help="Create a machine")
@@ -26,7 +45,21 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
     delete_parser = subparser.add_parser("delete", help="Delete a machine")
     register_delete_parser(delete_parser)
 
-    list_parser = subparser.add_parser("list", help="List machines")
+    list_parser = subparser.add_parser(
+        "list",
+        help="List machines",
+        epilog=(
+            """
+This subcommand lists all machines managed by this clan.
+
+Examples:
+
+  $ clan machines list
+  Lists all the machines and their descriptions.
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     register_list_parser(list_parser)
 
     install_parser = subparser.add_parser(
@@ -37,5 +70,23 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
         The target must be a Linux based system reachable via SSH.
         Installing a machine means overwriting the target's disk.
         """,
+        epilog=(
+            """
+This subcommand provides an interface to install machines managed by clan.
+
+Examples:
+
+  $ clan machines install [MACHINE] [TARGET_HOST]
+  Will install the specified machine [MACHINE], to the specified [TARGET_HOST].
+
+  $ clan machines install [MACHINE] --json [JSON]
+  Will install the specified machine [MACHINE] to the host exposed by 
+  the deployment information of the [JSON] deployment string.
+
+For information on how to set up the installer see: https://docs.clan.lol/getting-started/installer/
+For more detailed information, visit: https://docs.clan.lol/getting-started/deploy
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     register_install_parser(install_parser)

pkgs/clan-cli/clan_cli/machines/create.py

@@ -1,13 +1,27 @@
 import argparse
 import logging
+from dataclasses import dataclass
+from pathlib import Path
 
+from clan_cli.api import API
 from clan_cli.config.machine import set_config_for_machine
 
 log = logging.getLogger(__name__)
 
 
+@dataclass
+class MachineCreateRequest:
+    name: str
+    config: dict[str, int]
+
+
+@API.register
+def create_machine(flake_dir: str | Path, machine: MachineCreateRequest) -> None:
+    set_config_for_machine(Path(flake_dir), machine.name, machine.config)
+
+
 def create_command(args: argparse.Namespace) -> None:
-    set_config_for_machine(args.flake, args.machine, dict())
+    create_machine(args.flake, MachineCreateRequest(args.machine, dict()))
 
 
 def register_create_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/machines/install.py

@@ -35,7 +35,7 @@ def install_nixos(
     target_host = f"{h.user or 'root'}@{h.host}"
     log.info(f"target host: {target_host}")
 
-    generate_facts([machine])
+    generate_facts([machine], None, False)
 
     with TemporaryDirectory() as tmpdir_:
         tmpdir = Path(tmpdir_)

pkgs/clan-cli/clan_cli/machines/list.py

@@ -1,37 +1,63 @@
 import argparse
+import dataclasses
 import json
 import logging
 from pathlib import Path
 
 from clan_cli.api import API
 
-from ..cmd import run
+from ..cmd import run_no_stdout
 from ..nix import nix_config, nix_eval
 
 log = logging.getLogger(__name__)
 
 
+@dataclasses.dataclass
+class MachineInfo:
+    machine_name: str
+    machine_description: str | None
+    machine_icon: str | None
+
+
 @API.register
-def list_machines(flake_url: Path | str) -> list[str]:
+def list_machines(flake_url: str | Path, debug: bool) -> dict[str, MachineInfo]:
     config = nix_config()
     system = config["system"]
     cmd = nix_eval(
         [
             f"{flake_url}#clanInternals.machines.{system}",
             "--apply",
-            "builtins.attrNames",
+            """builtins.mapAttrs (name: attrs: {
+  inherit (attrs.config.clanCore) machineDescription machineIcon machineName;
+})""",
             "--json",
         ]
     )
-    proc = run(cmd)
+
+    proc = run_no_stdout(cmd)
 
     res = proc.stdout.strip()
-    return json.loads(res)
+    machines_dict = json.loads(res)
+
+    return {
+        k: MachineInfo(
+            machine_name=v.get("machineName"),
+            machine_description=v.get("machineDescription", None),
+            machine_icon=v.get("machineIcon", None),
+        )
+        for k, v in machines_dict.items()
+    }
 
 
 def list_command(args: argparse.Namespace) -> None:
-    for machine in list_machines(Path(args.flake)):
-        print(machine)
+    flake_path = Path(args.flake).resolve()
+    print("Listing all machines:\n")
+    print("Source: ", flake_path)
+    print("-" * 40)
+    for name, machine in list_machines(flake_path, args.debug).items():
+        description = machine.machine_description or "[no description]"
+        print(f"{name}\n: {description}\n")
+    print("-" * 40)
 
 
 def register_list_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/machines/machines.py

@@ -10,7 +10,7 @@ from clan_cli.clan_uri import ClanURI, MachineData
 from clan_cli.dirs import vm_state_dir
 from clan_cli.qemu.qmp import QEMUMonitorProtocol
 
-from ..cmd import run
+from ..cmd import run_no_stdout
 from ..errors import ClanError
 from ..nix import nix_build, nix_config, nix_eval, nix_metadata
 from ..ssh import Host, parse_deployment_address
@@ -197,7 +197,7 @@ class Machine:
             config_json.flush()
 
             file_info = json.loads(
-                run(
+                run_no_stdout(
                     nix_eval(
                         [
                             "--impure",
@@ -247,10 +247,10 @@ class Machine:
             ]
 
         if method == "eval":
-            output = run(nix_eval(args)).stdout.strip()
+            output = run_no_stdout(nix_eval(args)).stdout.strip()
             return output
         elif method == "build":
-            outpath = run(nix_build(args)).stdout.strip()
+            outpath = run_no_stdout(nix_build(args)).stdout.strip()
             return Path(outpath)
         else:
             raise ValueError(f"Unknown method {method}")

pkgs/clan-cli/clan_cli/machines/update.py

@@ -98,7 +98,7 @@ def deploy_nixos(machines: MachineGroup) -> None:
         env = os.environ.copy()
         env["NIX_SSHOPTS"] = ssh_arg
 
-        generate_facts([machine])
+        generate_facts([machine], None, False)
         upload_secrets(machine)
 
         path = upload_sources(".", target)

pkgs/clan-cli/clan_cli/nix.py

@@ -106,7 +106,7 @@ def nix_shell(packages: list[str], cmd: list[str]) -> list[str]:
     if os.environ.get("IN_NIX_SANDBOX"):
         return cmd
     return [
-        *nix_command(["shell", "--offline", "--inputs-from", f"{nixpkgs_flake()!s}"]),
+        *nix_command(["shell", "--inputs-from", f"{nixpkgs_flake()!s}"]),
         *packages,
         "-c",
         *cmd,

pkgs/clan-cli/clan_cli/vms/run.py

@@ -69,7 +69,7 @@ def get_secrets(
     secret_facts_module = importlib.import_module(machine.secret_facts_module)
     secret_facts_store = secret_facts_module.SecretStore(machine=machine)
 
-    generate_facts([machine])
+    generate_facts([machine], None, False)
 
     secret_facts_store.upload(secrets_dir)
     return secrets_dir

pkgs/clan-cli/default.nix

@@ -83,7 +83,7 @@ let
     cp -r ${./.} $out
     chmod -R +w $out
     rm $out/clan_cli/config/jsonschema
-    ln -s ${nixpkgs'} $out/clan_cli/nixpkgs
+    ln -sf ${nixpkgs'} $out/clan_cli/nixpkgs
     cp -r ${../../lib/jsonschema} $out/clan_cli/config/jsonschema
   '';
 
@@ -101,7 +101,7 @@ let
       outputs = _inputs: { };
     }
     EOF
-    ln -s ${nixpkgs} $out/path
+    ln -sf ${nixpkgs} $out/path
     nix flake update $out \
       --store ./. \
       --extra-experimental-features 'nix-command flakes'

pkgs/clan-cli/docs.py

@@ -22,7 +22,7 @@ class Option:
         md_li += indent_next(
             f"\n{self.description.strip()}" if self.description else ""
         )
-        md_li += indent_next(f"\n{self.epilog.strip()}" if self.epilog else "")
+        # md_li += indent_next(f"\n{self.epilog.strip()}" if self.epilog else "")
 
         return md_li
 
@@ -82,13 +82,54 @@ class Category:
             md_li += indent_all(
                 f"{self.description.strip()}\n" if self.description else "", 4
             )
-            md_li += "\n"
-            md_li += indent_all(f"{self.epilog.strip()}\n" if self.epilog else "", 4)
-            md_li += "\n"
 
         return md_li
 
 
+def epilog_to_md(text: str) -> str:
+    """
+    Convert the epilog to md
+    """
+    after_examples = False
+    md = ""
+    # md += text
+    for line in text.split("\n"):
+        if line.strip() == "Examples:":
+            after_examples = True
+            md += "### Examples"
+            md += "\n"
+        else:
+            if after_examples:
+                if line.strip().startswith("$"):
+                    md += f"`{line}`"
+                    md += "\n"
+                    md += "\n"
+                else:
+                    if contains_https_link(line):
+                        line = convert_to_markdown_link(line)
+                    md += line
+                    md += "\n"
+            else:
+                md += line
+                md += "\n"
+    return md
+
+
+import re
+
+
+def contains_https_link(line: str) -> bool:
+    pattern = r"https://\S+"
+    return re.search(pattern, line) is not None
+
+
+def convert_to_markdown_link(line: str) -> str:
+    pattern = r"(https://\S+)"
+
+    # Replacement pattern to convert it to a Markdown link
+    return re.sub(pattern, r"[\1](\1)", line)
+
+
 def indent_next(text: str, indent_size: int = 4) -> str:
     """
     Indent all lines in a string except the first line.
@@ -136,7 +177,7 @@ def get_subcommands(
             continue
 
         if isinstance(action, argparse._SubParsersAction):
-            continue  # Subparsers handled sperately
+            continue  # Subparsers handled separately
 
         option_strings = ", ".join(action.option_strings)
         if option_strings:
@@ -178,7 +219,7 @@ def get_subcommands(
                     Category(
                         title=f"{parent} {name}",
                         description=subparser.description,
-                        epilog=subparser.epilog,
+                        # epilog=subparser.epilog,
                         level=level,
                         options=_options,
                         positionals=_positionals,
@@ -222,6 +263,7 @@ def collect_commands() -> list[Category]:
                         options=_options,
                         positionals=_positionals,
                         subcommands=_subcommands,
+                        epilog=subparser.epilog,
                         level=1,
                     )
                 )
@@ -280,6 +322,7 @@ def build_command_reference() -> None:
         markdown = files.get(folder / f"{filename}.md", "")
 
         markdown += f"{'#'*(cmd.level)} {cmd.title.capitalize()}\n\n"
+
         markdown += f"{cmd.description}\n\n" if cmd.description else ""
 
         # usage: clan vms run [-h] machine
@@ -320,6 +363,8 @@ def build_command_reference() -> None:
             markdown += indent_all(commands_fmt)
             markdown += "\n"
 
+        markdown += f"{epilog_to_md(cmd.epilog)}\n\n" if cmd.epilog else ""
+
         files[folder / f"{filename}.md"] = markdown
 
     for fname, content in files.items():

pkgs/clan-cli/tests/test_create_flake.py

@@ -26,6 +26,14 @@ def test_create_flake(
     cli.run(["machines", "create", "machine1"])
     capsys.readouterr()  # flush cache
 
+    # create a hardware-configuration.nix that doesn't throw an eval error
+
+    for patch_machine in ["jon", "sara"]:
+        with open(
+            flake_dir / "machines" / f"{patch_machine}/hardware-configuration.nix", "w"
+        ) as hw_config_nix:
+            hw_config_nix.write("{}")
+
     cli.run(["machines", "list"])
     assert "machine1" in capsys.readouterr().out
     flake_show = subprocess.run(

pkgs/clan-cli/tests/test_machines_cli.py

@@ -16,7 +16,10 @@ def test_machine_subcommands(
     cli.run(["--flake", str(test_flake_with_core.path), "machines", "list"])
 
     out = capsys.readouterr()
-    assert "machine1\nvm1\nvm2\n" == out.out
+
+    assert "machine1" in out.out
+    assert "vm1" in out.out
+    assert "vm2" in out.out
 
     cli.run(
         ["--flake", str(test_flake_with_core.path), "machines", "delete", "machine1"]
@@ -25,4 +28,7 @@ def test_machine_subcommands(
     capsys.readouterr()
     cli.run(["--flake", str(test_flake_with_core.path), "machines", "list"])
     out = capsys.readouterr()
-    assert "vm1\nvm2\n" == out.out
+
+    assert "machine1" not in out.out
+    assert "vm1" in out.out
+    assert "vm2" in out.out

pkgs/clan-vm-manager/clan_vm_manager/app.py

@@ -106,7 +106,7 @@ class MainApplication(Adw.Application):
     def on_activate(self, source: "MainApplication") -> None:
         if not self.window:
             self.init_style()
-            self.window = MainWindow(config=ClanConfig(initial_view="webview"))
+            self.window = MainWindow(config=ClanConfig(initial_view="list"))
             self.window.set_application(self)
 
         self.window.show()

pkgs/clan-vm-manager/clan_vm_manager/views/webview.py

@@ -1,3 +1,4 @@
+import dataclasses
 import json
 import logging
 import sys
@@ -8,6 +9,7 @@ from threading import Lock
 from typing import Any
 
 import gi
+from clan_cli.api import API
 
 gi.require_version("WebKit", "6.0")
 
@@ -22,6 +24,23 @@ site_index: Path = (
 log = logging.getLogger(__name__)
 
 
+def dataclass_to_dict(obj: Any) -> Any:
+    """
+    Utility function to convert dataclasses to dictionaries
+    It converts all nested dataclasses, lists, tuples, and dictionaries to dictionaries
+
+    It does NOT convert member functions.
+    """
+    if dataclasses.is_dataclass(obj):
+        return {k: dataclass_to_dict(v) for k, v in dataclasses.asdict(obj).items()}
+    elif isinstance(obj, list | tuple):
+        return [dataclass_to_dict(item) for item in obj]
+    elif isinstance(obj, dict):
+        return {k: dataclass_to_dict(v) for k, v in obj.items()}
+    else:
+        return obj
+
+
 class WebView:
     def __init__(self, methods: dict[str, Callable]) -> None:
         self.method_registry: dict[str, Callable] = methods
@@ -77,12 +96,35 @@ class WebView:
         self.queue_size += 1
 
     def threaded_handler(
-        self, handler_fn: Callable[[Any], Any], data: Any, method_name: str
+        self,
+        handler_fn: Callable[
+            ...,
+            Any,
+        ],
+        data: dict[str, Any] | None,
+        method_name: str,
     ) -> None:
         with self.mutex_lock:
             log.debug("Executing... ", method_name)
-            result = handler_fn(data)
-            serialized = json.dumps(result)
+            log.debug(f"{data}")
+            if data is None:
+                result = handler_fn()
+            else:
+                reconciled_arguments = {}
+                for k, v in data.items():
+                    # Some functions expect to be called with dataclass instances
+                    # But the js api returns dictionaries.
+                    # Introspect the function and create the expected dataclass from dict dynamically
+                    # Depending on the introspected argument_type
+                    arg_type = API.get_method_argtype(method_name, k)
+                    if dataclasses.is_dataclass(arg_type):
+                        reconciled_arguments[k] = arg_type(**v)
+                    else:
+                        reconciled_arguments[k] = v
+
+                result = handler_fn(**reconciled_arguments)
+
+            serialized = json.dumps(dataclass_to_dict(result))
 
             # Use idle_add to queue the response call to js on the main GTK thread
             GLib.idle_add(self.return_data_to_js, method_name, serialized)

pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py

@@ -26,7 +26,7 @@ log = logging.getLogger(__name__)
 class MainWindow(Adw.ApplicationWindow):
     def __init__(self, config: ClanConfig) -> None:
         super().__init__()
-        self.set_title("cLAN Manager")
+        self.set_title("Clan Manager")
         self.set_default_size(980, 850)
 
         overlay = ToastOverlay.use().overlay
@@ -62,8 +62,7 @@ class MainWindow(Adw.ApplicationWindow):
         stack_view.add_named(Logs(), "logs")
 
         webview = WebView(methods=API._registry)
-
-        stack_view.add_named(webview.get_webview(), "list")
+        stack_view.add_named(webview.get_webview(), "webview")
 
         stack_view.set_visible_child_name(config.initial_view)
 

pkgs/clan-vm-manager/default.nix

@@ -27,7 +27,7 @@ let
     name = "org.clan.vm-manager";
     exec = "clan-vm-manager %u";
     icon = ./clan_vm_manager/assets/clan_white.png;
-    desktopName = "cLAN Manager";
+    desktopName = "Clan Manager";
     startupWMClass = "clan";
     mimeTypes = [ "x-scheme-handler/clan" ];
   };

pkgs/editor/clan-edit-codium.nix

@@ -0,0 +1,42 @@
+{
+  lib,
+  coreutils,
+  nil,
+  nixd,
+  nixpkgs-fmt,
+  direnv,
+  vscode-extensions,
+  vscode-with-extensions,
+  vscodium,
+  writeShellApplication,
+}:
+let
+  codium = vscode-with-extensions.override {
+    vscode = vscodium;
+    vscodeExtensions = [
+      vscode-extensions.jnoortheen.nix-ide
+      vscode-extensions.mkhl.direnv
+    ];
+  };
+in
+writeShellApplication {
+  name = "clan-edit-codium";
+  runtimeInputs = [
+    coreutils
+    nil
+    nixd
+    nixpkgs-fmt
+    direnv
+  ];
+  text = ''
+    set -eux
+    DATA_DIR="''${XDG_CACHE_HOME:-$HOME/.cache}/clan-edit-codium"
+    SETTINGS="$DATA_DIR"/User/settings.json
+    ${coreutils}/bin/mkdir -p "$DATA_DIR/User"
+    cat ${./settings.json} > "$SETTINGS"
+
+    exec ${lib.getExe codium} --user-data-dir "$DATA_DIR" "$@"
+  '';
+
+  derivationArgs.passthru.completion-options = import ./completion-options.nix;
+}

pkgs/editor/completion-options.nix

@@ -0,0 +1,17 @@
+let
+  flake = builtins.getFlake "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
+  nixpkgs = flake.inputs.nixpkgs;
+  pkgs = nixpkgs.legacyPackages.${builtins.currentSystem};
+  clanCore = flake.outputs.nixosModules.clanCore;
+  clanModules = flake.outputs.clanModules;
+  allNixosModules = (import "${nixpkgs}/nixos/modules/module-list.nix") ++ [
+    "${nixpkgs}/nixos/modules/misc/assertions.nix"
+    { nixpkgs.hostPlatform = "x86_64-linux"; }
+  ];
+  clanCoreNixosModules = [
+    clanCore
+    # { clanCore.clanDir = ./.; }
+  ] ++ allNixosModules ++ (builtins.attrValues clanModules);
+  clanCoreNixos = pkgs.nixos { imports = clanCoreNixosModules; };
+in
+clanCoreNixos.options

pkgs/editor/default.nix

@@ -0,0 +1,4 @@
+{ pkgs }:
+{
+  clan-edit-codium = pkgs.callPackage ./clan-edit-codium.nix;
+}

pkgs/editor/settings.json

@@ -0,0 +1,26 @@
+{
+  "security.workspace.trust.enabled": false,
+  "nix.enableLanguageServer": true,
+  "nix.serverPath": "nixd",
+  "nix.formatterPath": "nixpkgs-fmt",
+  "nix.serverSettings": {
+    "nixd": {
+      "formatting": {
+        "command": "nixpkgs-fmt"
+      },
+      "options": {
+        "nixos": {
+          "expr": "(let pkgs = import <nixpkgs> { }; in (pkgs.lib.evalModules { modules
+         =  (import <nixpkgs/nixos/modules/module-list.nix>) ++ [ ({...}: {
+         nixpkgs.hostPlatform = builtins.currentSystem;} ) ] ; })).options"
+        },
+        "clan": {
+          "expr": "let pkgs = import <nixpkgs> { }; flake = builtins.getFlake \"https://git.clan.lol/clan/clan-core/archive/main.tar.gz\"; clanCore = flake.outputs.nixosModules.clanCore; clanModules = flake.outputs.clanModules; allNixosModules = (import <nixpkgs/nixos/modules/module-list.nix>) ++ [<nixpkgs/nixos/modules/misc/assertions.nix> {nixpkgs.hostPlatform = \"x86_64-linux\";}]; clanCoreNixosModules = [clanCore] ++ allNixosModules ++ (builtins.attrValues clanModules); clanCoreNixos = pkgs.nixos { imports = clanCoreNixosModules;}; in clanCoreNixos.options"
+        },
+	"clan-core": {
+	    "expr": "(builtins.getFlake \"/home/kenji/git/clan-projects/clan-core\").packages.x86_64-linux.editor.passthru.completion-options",
+	},
+      }
+    }
+  }
+}

pkgs/flake-module.nix

@@ -7,6 +7,7 @@
     ./installer/flake-module.nix
     ./schemas/flake-module.nix
     ./webview-ui/flake-module.nix
+    ./gui-installer/flake-module.nix
   ];
 
   perSystem =
@@ -25,6 +26,7 @@
           moonlight-sunshine-accept = pkgs.callPackage ./moonlight-sunshine-accept { };
           merge-after-ci = pkgs.callPackage ./merge-after-ci { inherit (config.packages) tea-create-pr; };
           pending-reviews = pkgs.callPackage ./pending-reviews { };
+          editor = pkgs.callPackage ./editor/clan-edit-codium.nix { };
         }
         // lib.optionalAttrs pkgs.stdenv.isLinux {
           wayland-proxy-virtwl = pkgs.callPackage ./wayland-proxy-virtwl { };

pkgs/gui-installer/flake-module.nix

@@ -0,0 +1,33 @@
+{
+  perSystem =
+    { pkgs, ... }:
+    let
+      nfpmConfig = pkgs.writeText "clan-nfpm-config.yaml" (
+
+        builtins.toJSON {
+          name = "clan-gui-installer";
+          contents = [
+            {
+              src = "${./gui-installer.sh}";
+              dst = "/usr/bin/clan-vm-manager";
+            }
+          ];
+        }
+      );
+      installerFor =
+        packager:
+        pkgs.runCommand "gui-installer" { } ''
+          mkdir build
+          cd build
+          ${pkgs.nfpm}/bin/nfpm package --config ${nfpmConfig} --packager ${packager}
+          mkdir $out
+          mv * $out/
+        '';
+    in
+    {
+      packages.gui-installer-apk = installerFor "apk";
+      packages.gui-installer-archlinux = installerFor "archlinux";
+      packages.gui-installer-deb = installerFor "deb";
+      packages.gui-installer-rpm = installerFor "rpm";
+    };
+}

pkgs/gui-installer/gui-installer.sh

@@ -0,0 +1,54 @@
+#! /bin/sh
+
+# create temp dir and ensure it is always cleaned
+trap 'clean_temp_dir' EXIT
+temp_dir=$(mktemp -d)
+
+clean_temp_dir() {
+  rm -rf "$temp_dir"
+}
+
+is_nix_installed() {
+  if [ -n "$(command -v nix)" ]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+install_nix() {
+  curl --proto '=https' --tlsv1.2 -sSf -L \
+      https://install.determinate.systems/nix \
+    > "$temp_dir"/install_nix.sh
+  NIX_INSTALLER_DIAGNOSTIC_ENDPOINT="" sh "$temp_dir"/install_nix.sh install
+}
+
+ask_then_install_nix() {
+  echo "Clan requires Nix to be installed. Would you like to install it now? (y/n)"
+  read -r response
+  if [ "$response" = "y" ]; then
+    install_nix
+  else
+    echo "Clan cannot run without Nix. Exiting."
+    exit 1
+  fi
+}
+
+ensure_nix_installed() {
+  if ! is_nix_installed; then
+    ask_then_install_nix
+  fi
+}
+
+start_clan_gui() {
+  exec nix run \
+    https://git.clan.lol/clan/clan-core/archive/main.tar.gz#clan-vm-manager \
+    --extra-experimental-features flakes nix-command -- "$@"
+}
+
+main() {
+  ensure_nix_installed
+  start_clan_gui "$@"
+}
+
+main "$@"

pkgs/webview-ui/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "typescript.tsdk": "node_modules/typescript/lib",
+    "tailwindCSS.experimental.classRegex": [["cx\\(([^)]*)\\)", "[\"'`]([^\"'`]*)[\"'`]"]],
+    "editor.wordWrap": "on"
+}

pkgs/webview-ui/app/eslint.config.mjs

@@ -0,0 +1,26 @@
+import eslint from "@eslint/js";
+import tseslint from "typescript-eslint";
+import tailwind from "eslint-plugin-tailwindcss";
+
+export default tseslint.config(
+  eslint.configs.recommended,
+  ...tseslint.configs.strict,
+  ...tseslint.configs.stylistic,
+  ...tailwind.configs["flat/recommended"],
+  {
+    rules: {
+      "tailwindcss/no-contradicting-classname": [
+        "error",
+        {
+          callees: ["cx"],
+        },
+      ],
+      "tailwindcss/no-custom-classname": [
+        "error",
+        {
+          callees: ["cx"],
+        },
+      ],
+    },
+  }
+);

pkgs/webview-ui/app/package.json

@@ -8,22 +8,29 @@
     "build": "npm run check && vite build && npm run convert-html",
     "convert-html": "node gtk.webview.js",
     "serve": "vite preview",
-    "check": "tsc --noEmit --skipLibCheck"
+    "check": "tsc --noEmit --skipLibCheck && eslint ./src"
   },
   "license": "MIT",
   "devDependencies": {
+    "@eslint/js": "^9.3.0",
+    "@types/node": "^20.12.12",
+    "@typescript-eslint/parser": "^7.10.0",
     "autoprefixer": "^10.4.19",
+    "classnames": "^2.5.1",
     "daisyui": "^4.11.1",
+    "eslint": "^8.57.0",
+    "json-schema-to-ts": "^3.1.0",
     "postcss": "^8.4.38",
+    "prettier": "^3.2.5",
     "solid-devtools": "^0.29.2",
     "tailwindcss": "^3.4.3",
-    "typescript": "^5.3.3",
+    "typescript": "^5.4.5",
+    "typescript-eslint": "^7.10.0",
     "vite": "^5.0.11",
-    "vite-plugin-solid": "^2.8.2"
+    "vite-plugin-solid": "^2.8.2",
+    "eslint-plugin-tailwindcss": "^3.17.0"
   },
   "dependencies": {
-    "@types/node": "^20.12.12",
-    "json-schema-to-ts": "^3.1.0",
     "solid-js": "^1.8.11"
   }
 }

pkgs/webview-ui/app/src/App.tsx

@@ -1,34 +1,60 @@
 import { Match, Switch, createSignal, type Component } from "solid-js";
 import { CountProvider } from "./Config";
+// import { Nested } from "./nested";
+import { Layout } from "./layout/layout";
+import cx from "classnames";
 import { Nested } from "./nested";
 
-type Route = "home" | "graph";
+type Route = "home" | "machines";
 
 const App: Component = () => {
   const [route, setRoute] = createSignal<Route>("home");
   return (
     <CountProvider>
-      <div class="w-full flex items-center flex-col gap-2 my-2">
-        <div>Clan</div>
-        <p>Current route: {route()}</p>
-
-        <div class="flex items-center">
+      <Layout>
+        <div class="col-span-1">
+          <div class={cx("text-zinc-500")}>Navigation</div>
+          <ul>
+            <li>
               <button
-            onClick={() => setRoute((o) => (o === "graph" ? "home" : "graph"))}
-            class="btn btn-link"
+                onClick={() => setRoute("home")}
+                classList={{ "bg-blue-500": route() === "home" }}
               >
-            Navigate to {route() === "home" ? "graph" : "home"}
+                Home
               </button>
+            </li>
+            <li>
+              {" "}
+              <button
+                onClick={() => setRoute("machines")}
+                classList={{ "bg-blue-500": route() === "machines" }}
+              >
+                Machines
+              </button>
+            </li>
+          </ul>
         </div>
+
+        <div class="col-span-7">
+          <div>{route()}</div>
           <Switch fallback={<p>{route()} not found</p>}>
             <Match when={route() == "home"}>
               <Nested />
             </Match>
-          <Match when={route() == "graph"}>
-            <p></p>
+            <Match when={route() == "machines"}>
+              <div class="grid grid-cols-3 gap-2">
+                <div class="h-10 w-20 bg-red-500">red</div>
+                <div class="h-10 w-20 bg-green-500">green</div>
+                <div class="h-10 w-20 bg-blue-500">blue</div>
+                <div class="h-10 w-20 bg-yellow-500">yellow</div>
+                <div class="h-10 w-20 bg-purple-500">purple</div>
+                <div class="h-10 w-20 bg-cyan-500">cyan</div>
+                <div class="h-10 w-20 bg-pink-500">pink</div>
+              </div>
             </Match>
           </Switch>
         </div>
+      </Layout>
     </CountProvider>
   );
 };

pkgs/webview-ui/app/src/Config.tsx

@@ -1,8 +1,16 @@
-import { createSignal, createContext, useContext, JSXElement } from "solid-js";
-import { pyApi } from "./message";
+import {
+  createSignal,
+  createContext,
+  useContext,
+  JSXElement,
+  createEffect,
+} from "solid-js";
+import { OperationResponse, pyApi } from "./message";
 
 export const makeCountContext = () => {
-  const [machines, setMachines] = createSignal<string[]>([]);
+  const [machines, setMachines] = createSignal<
+    OperationResponse<"list_machines">
+  >({});
   const [loading, setLoading] = createSignal(false);
 
   pyApi.list_machines.receive((machines) => {
@@ -10,13 +18,17 @@ export const makeCountContext = () => {
     setMachines(machines);
   });
 
+  createEffect(() => {
+    console.log("The count is now", machines());
+  });
+
   return [
     { loading, machines },
     {
       getMachines: () => {
         // When the gtk function sends its data the loading state will be set to false
         setLoading(true);
-        pyApi.list_machines.dispatch(".");
+        pyApi.list_machines.dispatch({ debug: true, flake_url: "." });
       },
     },
   ] as const;
@@ -25,8 +37,14 @@ export const makeCountContext = () => {
 type CountContextType = ReturnType<typeof makeCountContext>;
 
 export const CountContext = createContext<CountContextType>([
-  { loading: () => false, machines: () => [] },
   {
+    loading: () => false,
+
+    // eslint-disable-next-line
+    machines: () => ({}),
+  },
+  {
+    // eslint-disable-next-line
     getMachines: () => {},
   },
 ]);

pkgs/webview-ui/app/src/index.tsx

@@ -6,7 +6,6 @@ import App from "./App";
 
 const root = document.getElementById("app");
 
-// @ts-ignore: add the clan scope to the window object so we can register callbacks for gtk
 window.clan = window.clan || {};
 
 if (import.meta.env.DEV && !(root instanceof HTMLElement)) {
@@ -15,4 +14,5 @@ if (import.meta.env.DEV && !(root instanceof HTMLElement)) {
   );
 }
 
+// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 render(() => <App />, root!);

pkgs/webview-ui/app/src/layout/layout.tsx

@@ -0,0 +1,9 @@
+import { Component, JSXElement } from "solid-js";
+
+interface LayoutProps {
+  children: JSXElement;
+}
+
+export const Layout: Component<LayoutProps> = (props) => {
+  return <div class="grid grid-cols-8">{props.children}</div>;
+};

pkgs/webview-ui/app/src/message.ts

@@ -1,11 +1,11 @@
 import { FromSchema } from "json-schema-to-ts";
 import { schema } from "@/api";
 
-type API = FromSchema<typeof schema>;
+export type API = FromSchema<typeof schema>;
 
-type OperationNames = keyof API;
-type OperationArgs<T extends OperationNames> = API[T]["argument"];
-type OperationResponse<T extends OperationNames> = API[T]["return"];
+export type OperationNames = keyof API;
+export type OperationArgs<T extends OperationNames> = API[T]["arguments"];
+export type OperationResponse<T extends OperationNames> = API[T]["return"];
 
 declare global {
   interface Window {
@@ -15,7 +15,10 @@ declare global {
     webkit: {
       messageHandlers: {
         gtk: {
-          postMessage: (message: { method: OperationNames; data: any }) => void;
+          postMessage: (message: {
+            method: OperationNames;
+            data: OperationArgs<OperationNames>;
+          }) => void;
         };
       };
     };
@@ -31,7 +34,7 @@ function createFunctions<K extends OperationNames>(
   return {
     dispatch: (args: OperationArgs<K>) => {
       console.log(
-        `Operation: ${operationName}, Arguments: ${JSON.stringify(args)}`
+        `Operation: ${String(operationName)}, Arguments: ${JSON.stringify(args)}`
       );
       // Send the data to the gtk app
       window.webkit.messageHandlers.gtk.postMessage({
@@ -40,7 +43,7 @@ function createFunctions<K extends OperationNames>(
       });
     },
     receive: (fn: (response: OperationResponse<K>) => void) => {
-      window.clan.list_machines = deserialize(fn);
+      window.clan[operationName] = deserialize(fn);
     },
   };
 }
@@ -59,6 +62,7 @@ const deserialize =
   <T>(fn: (response: T) => void) =>
   (str: string) => {
     try {
+      console.debug("Received data: ", str);
       fn(JSON.parse(str) as T);
     } catch (e) {
       alert(`Error parsing JSON: ${e}`);
@@ -68,7 +72,10 @@ const deserialize =
 // Create the API object
 
 const pyApi: PyApi = {} as PyApi;
-operationNames.forEach((name) => {
+operationNames.forEach((opName) => {
+  const name = opName as OperationNames;
+  // @ts-expect-error - TODO: Fix this. Typescript is not recognizing the receive function correctly
   pyApi[name] = createFunctions(name);
 });
+
 export { pyApi };

pkgs/webview-ui/app/src/nested.tsx

@@ -1,24 +1,34 @@
-import { For, Match, Switch, type Component } from "solid-js";
+import { For, Match, Switch, createEffect, type Component } from "solid-js";
 import { useCountContext } from "./Config";
 
 export const Nested: Component = () => {
   const [{ machines, loading }, { getMachines }] = useCountContext();
+
+  const list = () => Object.values(machines());
+
+  createEffect(() => {
+    console.log("1", list());
+  });
+  createEffect(() => {
+    console.log("2", machines());
+  });
   return (
     <div>
       <button onClick={() => getMachines()} class="btn btn-primary">
         Get machines
       </button>
-      <hr />
+      <div></div>
       <Switch>
         <Match when={loading()}>Loading...</Match>
-        <Match when={!loading() && machines().length === 0}>
+        <Match when={!loading() && Object.entries(machines()).length === 0}>
           No machines found
         </Match>
-        <Match when={!loading() && machines().length}>
-          <For each={machines()}>
-            {(machine, i) => (
+        <Match when={!loading()}>
+          <For each={list()}>
+            {(entry, i) => (
               <li>
-                {i() + 1}: {machine}
+                {i() + 1}: {entry.machine_name}{" "}
+                {entry.machine_description || "No description"}
               </li>
             )}
           </For>

pkgs/webview-ui/default.nix

@@ -1,32 +0,0 @@
-{
-  dream2nix,
-  config,
-  src,
-  ...
-}:
-{
-  imports = [ dream2nix.modules.dream2nix.WIP-nodejs-builder-v3 ];
-
-  mkDerivation = {
-    inherit src;
-  };
-
-  deps =
-    { nixpkgs, ... }:
-    {
-      inherit (nixpkgs) stdenv;
-    };
-
-  WIP-nodejs-builder-v3 = {
-    packageLockFile = "${config.mkDerivation.src}/package-lock.json";
-    # config.groups.all.packages.${config.name}.${config.version}.
-  };
-  public.out = {
-    checkPhase = ''
-      echo "Running tests"
-      echo "Tests passed"
-    '';
-  };
-  name = "@clan/webview-ui";
-  version = "0.0.1";
-}

pkgs/webview-ui/flake-module.nix

@@ -9,10 +9,9 @@
 
         src = ./app;
 
-        # npmDepsHash = "sha256-bRD2vzijhdOOvcEi6XaG/neSqhkVQMqIX/8bxvRQkTc=";
         npmDeps = pkgs.fetchNpmDeps {
           src = ./app;
-          hash = "sha256-bRD2vzijhdOOvcEi6XaG/neSqhkVQMqIX/8bxvRQkTc=";
+          hash = "sha256-E0++hupVKnDqmLk7ljoMcqcI4w+DIMlfRYRPbKUsT2c=";
         };
         # The prepack script runs the build script, which we'd rather do in the build phase.
         npmPackFlags = [ "--ignore-scripts" ];

scripts/pre-commit

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# To install:
+# ln -sf ../../scripts/pre-commit .git/hooks/pre-commit
+set -o errexit
+set -o nounset
+set -o pipefail
+
+readarray staged < <(git diff --name-only --cached)
+[[ ${#staged[@]} = 0 ]] && exit
+unstash() {
+  local ret=$?
+  set +e
+  git stash pop -q
+  exit "$ret"
+}
+git stash push --quiet --keep-index --message "pre-commit"
+trap unstash EXIT
+nix fmt
+{
+  changed=$(git diff --name-only --exit-code);
+  status=$?;
+} || true
+if [[ $status -ne 0 ]]; then
+  exec 1>&2
+  echo Files changed by pre-commit hook:
+  echo "$changed"
+  exit $status
+fi

templates/new-clan/flake.nix

@@ -85,9 +85,9 @@
       };
     in
     {
-      # all machines managed by cLAN
+      # all machines managed by Clan
       inherit (clan) nixosConfigurations clanInternals;
-      # add the cLAN cli tool to the dev shell
+      # add the Clan cli tool to the dev shell
       devShells.${system}.default = pkgs.mkShell {
         packages = [ clan-core.packages.${system}.clan-cli ];
       };