Merge remote-tracking branch 'origin/main' into rework-installation

flake.lock

@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1720661479,
+        "lastModified": 1721417620,
-        "narHash": "sha256-nsGgA14vVn0GGiqEfomtVgviRJCuSR3UEopfP8ixW1I=",
+        "narHash": "sha256-6q9b1h8fI3hXg2DG6/vrKWCeG8c5Wj2Kvv22RCgedzg=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "786965e1b1ed3fd2018d78399984f461e2a44689",
+        "rev": "bec6e3cde912b8acb915fecdc509eda7c973fb42",
         "type": "github"
       },
       "original": {
@@ -48,11 +48,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1720659757,
+        "lastModified": 1721571445,
-        "narHash": "sha256-ltzUuCsEfPA9CYM9BAnwObBGqDyQIs2OLkbVMeOOk00=",
+        "narHash": "sha256-2MnlPVcNJZ9Nbu90kFyo7+lng366gswErP4FExfrUbc=",
         "owner": "nix-community",
         "repo": "nixos-images",
-        "rev": "5eddae0afbcfd4283af5d6676d08ad059ca04b70",
+        "rev": "accee005735844d57b411d9969c5d0aabc6a55f6",
         "type": "github"
       },
       "original": {
@@ -63,11 +63,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1720977633,
+        "lastModified": 1721571961,
-        "narHash": "sha256-if0qaFmAe8X01NsVRK5e9Asg9mEWVkHrA9WuqM5jB70=",
+        "narHash": "sha256-jfF4gpRUpTBY2OxDB0FRySsgNGOiuDckEtu7YDQom3Y=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "0af9d835c27984b3265145f8e3cbc6c153479196",
+        "rev": "4cc8b29327bed3d52b40041f810f49734298af46",
         "type": "github"
       },
       "original": {
@@ -95,11 +95,11 @@
         "nixpkgs-stable": []
       },
       "locked": {
-        "lastModified": 1720926522,
+        "lastModified": 1721531171,
-        "narHash": "sha256-eTpnrT6yu1vp8C0B5fxHXhgKxHoYMoYTEikQx///jxY=",
+        "narHash": "sha256-AsvPw7T0tBLb53xZGcUC3YPqlIpdxoSx56u8vPCr6gU=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "0703ba03fd9c1665f8ab68cc3487302475164617",
+        "rev": "909e8cfb60d83321d85c8d17209d733658a21c95",
         "type": "github"
       },
       "original": {
@@ -115,11 +115,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1720930114,
+        "lastModified": 1721458737,
-        "narHash": "sha256-VZK73b5hG5bSeAn97TTcnPjXUXtV7j/AtS4KN8ggCS0=",
+        "narHash": "sha256-wNXLQ/ATs1S4Opg1PmuNoJ+Wamqj93rgZYV3Di7kxkg=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "b92afa1501ac73f1d745526adc4f89b527595f14",
+        "rev": "888bfb10a9b091d9ed2f5f8064de8d488f7b7c97",
         "type": "github"
       },
       "original": {

lib/build-clan/default.nix

@@ -58,11 +58,11 @@ let
         {
           # { ${name} :: meta // { name, tags } }
           machines = lib.mapAttrs (
-            name: config:
+            name: machineConfig:
             (lib.attrByPath [
               "clan"
               "meta"
-            ] { } config)
+            ] { } machineConfig)
             // {
               # meta.name default is the attribute name of the machine
               name = lib.mkDefault (
@@ -70,11 +70,11 @@ let
                   "clan"
                   "meta"
                   "name"
-                ] name config
+                ] name machineConfig
               );
             }
             # tags
-            // (clanToInventory config {
+            // (clanToInventory machineConfig {
               clanPath = [
                 "clan"
                 "tags"
@@ -82,15 +82,15 @@ let
               inventoryPath = [ "tags" ];
             })
             # system
-            // (clanToInventory config {
+            // (clanToInventory machineConfig {
               clanPath = [
                 "nixpkgs"
-                "hostSystem"
+                "hostPlatform"
               ];
               inventoryPath = [ "system" ];
             })
             # deploy.targetHost
-            // (clanToInventory config {
+            // (clanToInventory machineConfig {
               clanPath = [
                 "clan"
                 "core"

nixosModules/clanCore/vars/default.nix

@@ -17,7 +17,7 @@ in
   imports = [
     ./public/in_repo.nix
     # ./public/vm.nix
-    # ./secret/password-store.nix
+    ./secret/password-store.nix
     ./secret/sops.nix
     # ./secret/vm.nix
   ];
@@ -39,7 +39,7 @@ in
     vars = {
       generators = lib.flip lib.mapAttrs config.clan.core.vars.generators (
         _name: generator: {
-          inherit (generator) dependencies finalScript;
+          inherit (generator) dependencies finalScript prompts;
           files = lib.flip lib.mapAttrs generator.files (_name: file: { inherit (file) secret; });
         }
       );

nixosModules/clanCore/vars/interface.nix

@@ -108,8 +108,9 @@ in
                 Prompts are available to the generator script as files.
                 For example, a prompt named 'prompt1' will be available via $prompts/prompt1
               '';
+              default = { };
               type = attrsOf (submodule {
-                options = {
+                options = options {
                   description = {
                     description = ''
                       The description of the prompted value

nixosModules/clanCore/vars/secret/password-store.nix

@@ -0,0 +1,12 @@
+{ config, lib, ... }:
+{
+  config.clan.core.vars.settings =
+    lib.mkIf (config.clan.core.vars.settings.secretStore == "password-store")
+      {
+        fileModule = file: {
+          path = lib.mkIf file.secret "${config.clan.core.password-store.targetDirectory}/${config.clan.core.machineName}-${file.config.generatorName}-${file.config.name}";
+        };
+        secretUploadDirectory = lib.mkDefault "/etc/secrets";
+        secretModule = "clan_cli.vars.secret_modules.password_store";
+      };
+}

nixosModules/clanCore/vm.nix

@@ -17,8 +17,13 @@ let
     imports = [
       (modulesPath + "/virtualisation/qemu-vm.nix")
       ./serial.nix
+      ./waypipe.nix
     ];
 
+    clan.services.waypipe = {
+      inherit (config.clan.core.vm.inspect.waypipe) enable command;
+    };
+
     # required for issuing shell commands via qga
     services.qemuGuest.enable = true;
 
@@ -149,12 +154,19 @@ in
         '';
       };
 
-      waypipe = lib.mkOption {
+      waypipe = {
-        type = lib.types.bool;
+        enable = lib.mkOption {
-        default = false;
+          type = lib.types.bool;
-        description = ''
+          default = false;
-          Whether to use waypipe for native wayland passthrough, or not.
+          description = ''
-        '';
+            Whether to use waypipe for native wayland passthrough, or not.
+          '';
+        };
+        command = lib.mkOption {
+          type = lib.types.listOf lib.types.str;
+          default = [ ];
+          description = "Commands that waypipe should run";
+        };
       };
     };
     # All important VM config variables needed by the vm runner
@@ -193,13 +205,22 @@ in
           whether to enable graphics for the vm
         '';
       };
-      waypipe = lib.mkOption {
+
-        type = lib.types.bool;
+      waypipe = {
-        internal = true;
+        enable = lib.mkOption {
-        readOnly = true;
+          type = lib.types.bool;
-        description = ''
+          internal = true;
-          whether to enable native wayland window passthrough with waypipe for the vm
+          readOnly = true;
-        '';
+          description = ''
+            Whether to use waypipe for native wayland passthrough, or not.
+          '';
+        };
+        command = lib.mkOption {
+          type = lib.types.listOf lib.types.str;
+          internal = true;
+          readOnly = true;
+          description = "Commands that waypipe should run";
+        };
       };
       machine_icon = lib.mkOption {
         type = lib.types.nullOr lib.types.path;
@@ -245,7 +266,12 @@ in
           initrd = "${vmConfig.config.system.build.initialRamdisk}/${vmConfig.config.system.boot.loader.initrdFile}";
           toplevel = vmConfig.config.system.build.toplevel;
           regInfo = (pkgs.closureInfo { rootPaths = vmConfig.config.virtualisation.additionalPaths; });
-          inherit (config.clan.virtualisation) memorySize cores graphics;
+          inherit (config.clan.virtualisation)
+            memorySize
+            cores
+            graphics
+            waypipe
+            ;
         }
       );
     };

nixosModules/clanCore/waypipe.nix

@@ -39,8 +39,6 @@
     # General default settings
     fonts.enableDefaultPackages = lib.mkDefault true;
     hardware.opengl.enable = lib.mkDefault true;
-    # Assume it is run inside a clan context
-    clan.virtualisation.waypipe = lib.mkDefault true;
 
     # User account
     services.getty.autologinUser = lib.mkDefault config.clan.services.waypipe.user;

pkgs/clan-app/clan_app/app.py

@@ -34,7 +34,7 @@ class MainApplication(Adw.Application):
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(
-            application_id="org.clan.clan-app",
+            application_id="org.clan.app",
             flags=Gio.ApplicationFlags.HANDLES_COMMAND_LINE,
         )
 

pkgs/clan-app/clan_app/views/webview.py

@@ -4,11 +4,10 @@ import logging
 from typing import Any
 
 import gi
-from clan_cli.api import MethodRegistry
+from clan_cli.api import MethodRegistry, dataclass_to_dict, from_dict
 
 from clan_app.api import GObjApi, GResult, ImplFunc
 from clan_app.api.file import open_file
-from clan_app.components.serializer import dataclass_to_dict, from_dict
 
 gi.require_version("WebKit", "6.0")
 from gi.repository import GLib, GObject, WebKit

pkgs/clan-app/default.nix

@@ -4,7 +4,7 @@
   setuptools,
   copyDesktopItems,
   pygobject3,
-  wrapGAppsHook,
+  wrapGAppsHook4,
   gtk4,
   adwaita-icon-theme,
   pygobject-stubs,
@@ -84,7 +84,7 @@ python3.pkgs.buildPythonApplication rec {
   nativeBuildInputs = [
     setuptools
     copyDesktopItems
-    wrapGAppsHook
+    wrapGAppsHook4
 
     gobject-introspection
   ];

pkgs/clan-app/shell.nix

@@ -1,5 +1,7 @@
 {
   lib,
+  glib,
+  gsettings-desktop-schemas,
   stdenv,
   clan-app,
   mkShell,
@@ -33,7 +35,9 @@ mkShell {
 
   buildInputs =
     [
+      glib
       ruff
+      gtk4
       gtk4.dev # has the demo called 'gtk4-widget-factory'
       libadwaita.devdoc # has the demo called 'adwaita-1-demo'
     ]
@@ -60,6 +64,9 @@ mkShell {
     # Add clan-cli to the python path so that we can import it without building it in nix first
     export PYTHONPATH="$GIT_ROOT/pkgs/clan-cli":"$PYTHONPATH"
 
+    export XDG_DATA_DIRS=${gtk4}/share/gsettings-schemas/gtk4-4.14.4:$XDG_DATA_DIRS
+    export XDG_DATA_DIRS=${gsettings-desktop-schemas}/share/gsettings-schemas/gsettings-desktop-schemas-46.0:$XDG_DATA_DIRS
+
     # Add the webview-ui to the .webui directory
     ln -nsf ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/ ./clan_app/.webui
 

pkgs/clan-cli/clan_cli/api/__init__.py

@@ -1,11 +1,141 @@
+import dataclasses
+import json
 from collections.abc import Callable
-from dataclasses import dataclass
+from dataclasses import dataclass, fields, is_dataclass
 from functools import wraps
 from inspect import Parameter, Signature, signature
-from typing import Annotated, Any, Generic, Literal, TypeVar, get_type_hints
+from pathlib import Path
+from types import UnionType
+from typing import (
+    Annotated,
+    Any,
+    Generic,
+    Literal,
+    TypeVar,
+    get_args,
+    get_origin,
+    get_type_hints,
+)
 
 from clan_cli.errors import ClanError
 
+
+def sanitize_string(s: str) -> str:
+    # Using the native string sanitizer to handle all edge cases
+    # Remove the outer quotes '"string"'
+    return json.dumps(s)[1:-1]
+
+
+def dataclass_to_dict(obj: Any) -> Any:
+    """
+    Utility function to convert dataclasses to dictionaries
+    It converts all nested dataclasses, lists, tuples, and dictionaries to dictionaries
+
+    It does NOT convert member functions.
+    """
+    if is_dataclass(obj):
+        return {
+            # Use either the original name or name
+            sanitize_string(
+                field.metadata.get("original_name", field.name)
+            ): dataclass_to_dict(getattr(obj, field.name))
+            for field in fields(obj)  # type: ignore
+        }
+    elif isinstance(obj, list | tuple):
+        return [dataclass_to_dict(item) for item in obj]
+    elif isinstance(obj, dict):
+        return {sanitize_string(k): dataclass_to_dict(v) for k, v in obj.items()}
+    elif isinstance(obj, Path):
+        return sanitize_string(str(obj))
+    elif isinstance(obj, str):
+        return sanitize_string(obj)
+    else:
+        return obj
+
+
+def is_union_type(type_hint: type) -> bool:
+    return type(type_hint) is UnionType
+
+
+def get_inner_type(type_hint: type) -> type:
+    if is_union_type(type_hint):
+        # Return the first non-None type
+        return next(t for t in get_args(type_hint) if t is not type(None))
+    return type_hint
+
+
+def get_second_type(type_hint: type[dict]) -> type:
+    """
+    Get the value type of a dictionary type hint
+    """
+    args = get_args(type_hint)
+    if len(args) == 2:
+        # Return the second argument, which should be the value type (Machine)
+        return args[1]
+
+    raise ValueError(f"Invalid type hint for dict: {type_hint}")
+
+
+def from_dict(t: type, data: dict[str, Any] | None) -> Any:
+    """
+    Dynamically instantiate a data class from a dictionary, handling nested data classes.
+    """
+    if data is None:
+        return None
+
+    try:
+        # Attempt to create an instance of the data_class
+        field_values = {}
+        for field in fields(t):
+            original_name = field.metadata.get("original_name", field.name)
+
+            field_value = data.get(original_name)
+
+            field_type = get_inner_type(field.type)  # type: ignore
+
+            if original_name in data:
+                # If the field is another dataclass, recursively instantiate it
+                if is_dataclass(field_type):
+                    field_value = from_dict(field_type, field_value)
+                elif isinstance(field_type, Path | str) and isinstance(
+                    field_value, str
+                ):
+                    field_value = (
+                        Path(field_value) if field_type == Path else field_value
+                    )
+                elif get_origin(field_type) is dict and isinstance(field_value, dict):
+                    # The field is a dictionary with a specific type
+                    inner_type = get_second_type(field_type)
+                    field_value = {
+                        k: from_dict(inner_type, v) for k, v in field_value.items()
+                    }
+                elif get_origin is list and isinstance(field_value, list):
+                    # The field is a list with a specific type
+                    inner_type = get_args(field_type)[0]
+                    field_value = [from_dict(inner_type, v) for v in field_value]
+
+            # Set the value
+            if (
+                field.default is not dataclasses.MISSING
+                or field.default_factory is not dataclasses.MISSING
+            ):
+                # Fields with default value
+                # a: Int = 1
+                # b: list = Field(default_factory=list)
+                if original_name in data or field_value is not None:
+                    field_values[field.name] = field_value
+            else:
+                # Fields without default value
+                # a: Int
+                field_values[field.name] = field_value
+
+        return t(**field_values)
+
+    except (TypeError, ValueError) as e:
+        print(f"Failed to instantiate {t.__name__}: {e} {data}")
+        return None
+
+
 T = TypeVar("T")
 
 ResponseDataType = TypeVar("ResponseDataType")

pkgs/clan-cli/clan_cli/api/modules.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 from clan_cli.cmd import run_no_stdout
 from clan_cli.errors import ClanCmdError, ClanError
-from clan_cli.inventory import Inventory, load_inventory
+from clan_cli.inventory import Inventory, load_inventory_json
 from clan_cli.nix import nix_eval
 
 from . import API
@@ -152,4 +152,4 @@ def get_module_info(
 
 @API.register
 def get_inventory(base_path: str) -> Inventory:
-    return load_inventory(base_path)
+    return load_inventory_json(base_path)

pkgs/clan-cli/clan_cli/clan/create.py

@@ -1,12 +1,11 @@
 # !/usr/bin/env python3
 import argparse
 import os
-from dataclasses import dataclass, fields
+from dataclasses import dataclass
 from pathlib import Path
 
 from clan_cli.api import API
-from clan_cli.arg_actions import AppendOptionAction
+from clan_cli.inventory import Inventory, init_inventory
-from clan_cli.inventory import Meta, load_inventory, save_inventory
 
 from ..cmd import CmdOut, run
 from ..dirs import clan_templates
@@ -27,13 +26,10 @@ class CreateClanResponse:
 @dataclass
 class CreateOptions:
     directory: Path | str
-    # Metadata for the clan
-    # Metadata can be shown with `clan show`
-    meta: Meta | None = None
     # URL to the template to use. Defaults to the "minimal" template
     template: str = "minimal"
-    setup_json_inventory: bool = True
     setup_git: bool = True
+    initial: Inventory | None = None
 
 
 def git_command(directory: Path, *args: str) -> list[str]:
@@ -67,10 +63,14 @@ def create_clan(options: CreateOptions) -> CreateClanResponse:
         ]
     )
     flake_init = run(command, cwd=directory)
+
     flake_update = run(
         nix_shell(["nixpkgs#nix"], ["nix", "flake", "update"]), cwd=directory
     )
 
+    if options.initial:
+        init_inventory(options.directory, init=options.initial)
+
     response = CreateClanResponse(
         flake_init=flake_init,
         flake_update=flake_update,
@@ -95,14 +95,6 @@ def create_clan(options: CreateOptions) -> CreateClanResponse:
             git_command(directory, "config", "user.email", "clan@example.com")
         )
 
-    # Write inventory.json file
-    if options.setup_json_inventory:
-        inventory = load_inventory(directory)
-        if options.meta is not None:
-            inventory.meta = options.meta
-        # Persist creates a commit message for each change
-        save_inventory(inventory, directory, "Init inventory")
-
     return response
 
 
@@ -115,14 +107,6 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
         default="default",
     )
 
-    parser.add_argument(
-        "--meta",
-        help=f"""Metadata to set for the clan. Available options are: {", ".join([f.name for f in fields(Meta)]) }""",
-        nargs=2,
-        metavar=("name", "value"),
-        action=AppendOptionAction,
-        default=[],
-    )
     parser.add_argument(
         "--no-git",
         help="Do not setup git",
@@ -139,7 +123,6 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
             CreateOptions(
                 directory=args.path,
                 template=args.template,
-                setup_json_inventory=False,
                 setup_git=not args.no_git,
             )
         )

pkgs/clan-cli/clan_cli/clan/update.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 
 from clan_cli.api import API
-from clan_cli.inventory import Meta, load_inventory, save_inventory
+from clan_cli.inventory import Meta, load_inventory_json, save_inventory
 
 
 @dataclass
@@ -12,7 +12,7 @@ class UpdateOptions:
 
 @API.register
 def update_clan_meta(options: UpdateOptions) -> Meta:
-    inventory = load_inventory(options.directory)
+    inventory = load_inventory_json(options.directory)
     inventory.meta = options.meta
 
     save_inventory(inventory, options.directory, "Update clan metadata")

pkgs/clan-cli/clan_cli/clan_uri.py

@@ -9,32 +9,49 @@ from .errors import ClanError
 
 @dataclass
 class FlakeId:
-    # FIXME: this is such a footgun if you accidnetally pass a string
+    loc: str | Path
-    _value: str | Path
 
     def __post_init__(self) -> None:
         assert isinstance(
-            self._value, str | Path
+            self.loc, str | Path
-        ), f"Flake {self._value} has an invalid type: {type(self._value)}"
+        ), f"Flake {self.loc} has an invalid format: {type(self.loc)}"
 
     def __str__(self) -> str:
-        return str(self._value)
+        return str(self.loc)
 
     @property
     def path(self) -> Path:
-        assert isinstance(self._value, Path), f"Flake {self._value} is not a local path"
+        assert self.is_local(), f"Flake {self.loc} is not a local path"
-        return self._value
+        return Path(self.loc)
 
     @property
     def url(self) -> str:
-        assert isinstance(self._value, str), f"Flake {self._value} is not a remote url"
+        assert self.is_remote(), f"Flake {self.loc} is not a remote url"
-        return self._value
+        return str(self.loc)
 
     def is_local(self) -> bool:
-        return isinstance(self._value, Path)
+        """
+        https://nix.dev/manual/nix/2.22/language/builtins.html?highlight=urlS#source-types
+
+        Examples:
+
+        - file:///home/eelco/nix/README.md          file        LOCAL
+        - git+file://git:github.com:NixOS/nixpkgs   git+file    LOCAL
+        - https://example.com/index.html            https       REMOTE
+        - github:nixos/nixpkgs                      github      REMOTE
+        - ftp://serv.file                           ftp         REMOTE
+        - ./.                                       ''          LOCAL
+
+        """
+        x = urllib.parse.urlparse(str(self.loc))
+        if x.scheme == "" or "file" in x.scheme:
+            # See above *file* or empty are the only local schemas
+            return True
+
+        return False
 
     def is_remote(self) -> bool:
-        return isinstance(self._value, str)
+        return not self.is_local()
 
 
 # Define the ClanURI class

pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py

@@ -48,19 +48,22 @@ class SecretStore(SecretStoreBase):
 
     def get(self, service: str, name: str) -> bytes:
         return decrypt_secret(
-            self.machine.flake_dir, f"{self.machine.name}-{name}"
+            self.machine.flake_dir,
+            sops_secrets_folder(self.machine.flake_dir) / f"{self.machine.name}-{name}",
         ).encode("utf-8")
 
     def exists(self, service: str, name: str) -> bool:
         return has_secret(
-            self.machine.flake_dir,
+            sops_secrets_folder(self.machine.flake_dir) / f"{self.machine.name}-{name}",
-            f"{self.machine.name}-{name}",
         )
 
     def upload(self, output_dir: Path) -> None:
         key_name = f"{self.machine.name}-age.key"
-        if not has_secret(self.machine.flake_dir, key_name):
+        if not has_secret(sops_secrets_folder(self.machine.flake_dir) / key_name):
             # skip uploading the secret, not managed by us
             return
-        key = decrypt_secret(self.machine.flake_dir, key_name)
+        key = decrypt_secret(
+            self.machine.flake_dir,
+            sops_secrets_folder(self.machine.flake_dir) / key_name,
+        )
         (output_dir / "key.txt").write_text(key)

pkgs/clan-cli/clan_cli/git.py

@@ -62,7 +62,7 @@ def _commit_file_to_git(
         for file_path in file_paths:
             cmd = run_cmd(
                 ["git"],
-                ["git", "-C", str(repo_dir), "add", str(file_path)],
+                ["git", "-C", str(repo_dir), "add", "--", str(file_path)],
             )
             # add the file to the git index
 
@@ -75,7 +75,7 @@ def _commit_file_to_git(
         # check if there is a diff
         cmd = run_cmd(
             ["git"],
-            ["git", "-C", str(repo_dir), "diff", "--cached", "--exit-code"]
+            ["git", "-C", str(repo_dir), "diff", "--cached", "--exit-code", "--"]
             + [str(file_path) for file_path in file_paths],
         )
         result = run(cmd, check=False, cwd=repo_dir)
@@ -93,6 +93,7 @@ def _commit_file_to_git(
                 "commit",
                 "-m",
                 commit_message,
+                "--no-verify",  # dont run pre-commit hooks
             ]
             + [str(file_path) for file_path in file_paths],
         )

pkgs/clan-cli/clan_cli/inventory/__init__.py

@@ -1,13 +1,26 @@
-import dataclasses
+"""
-import json
+All read/write operations MUST use the inventory.
-from dataclasses import fields, is_dataclass
-from pathlib import Path
-from types import UnionType
-from typing import Any, get_args, get_origin
 
-from clan_cli.errors import ClanError
+Machine data, clan data or service data can be accessed in a performant way.
+
+This file exports stable classnames for static & dynamic type safety.
+
+Utilize:
+
+- load_inventory_eval: To load the actual inventory with nix declarations merged.
+Operate on the returned inventory to make changes
+- save_inventory: To persist changes.
+"""
+
+import json
+from pathlib import Path
+
+from clan_cli.api import API, dataclass_to_dict, from_dict
+from clan_cli.errors import ClanCmdError, ClanError
 from clan_cli.git import commit_file
 
+from ..cmd import run_no_stdout
+from ..nix import nix_eval
 from .classes import (
     Inventory,
     Machine,
@@ -24,6 +37,8 @@ from .classes import (
 # Re export classes here
 # This allows to rename classes in the generated code
 __all__ = [
+    "from_dict",
+    "dataclass_to_dict",
     "Service",
     "Machine",
     "Meta",
@@ -37,121 +52,6 @@ __all__ = [
 ]
 
 
-def sanitize_string(s: str) -> str:
-    return s.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
-
-
-def dataclass_to_dict(obj: Any) -> Any:
-    """
-    Utility function to convert dataclasses to dictionaries
-    It converts all nested dataclasses, lists, tuples, and dictionaries to dictionaries
-
-    It does NOT convert member functions.
-    """
-    if is_dataclass(obj):
-        return {
-            # Use either the original name or name
-            sanitize_string(
-                field.metadata.get("original_name", field.name)
-            ): dataclass_to_dict(getattr(obj, field.name))
-            for field in fields(obj)  # type: ignore
-        }
-    elif isinstance(obj, list | tuple):
-        return [dataclass_to_dict(item) for item in obj]
-    elif isinstance(obj, dict):
-        return {sanitize_string(k): dataclass_to_dict(v) for k, v in obj.items()}
-    elif isinstance(obj, Path):
-        return str(obj)
-    elif isinstance(obj, str):
-        return sanitize_string(obj)
-    else:
-        return obj
-
-
-def is_union_type(type_hint: type) -> bool:
-    return type(type_hint) is UnionType
-
-
-def get_inner_type(type_hint: type) -> type:
-    if is_union_type(type_hint):
-        # Return the first non-None type
-        return next(t for t in get_args(type_hint) if t is not type(None))
-    return type_hint
-
-
-def get_second_type(type_hint: type[dict]) -> type:
-    """
-    Get the value type of a dictionary type hint
-    """
-    args = get_args(type_hint)
-    if len(args) == 2:
-        # Return the second argument, which should be the value type (Machine)
-        return args[1]
-
-    raise ValueError(f"Invalid type hint for dict: {type_hint}")
-
-
-def from_dict(t: type, data: dict[str, Any] | None) -> Any:
-    """
-    Dynamically instantiate a data class from a dictionary, handling nested data classes.
-    """
-    if data is None:
-        return None
-
-    try:
-        # Attempt to create an instance of the data_class
-        field_values = {}
-        for field in fields(t):
-            original_name = field.metadata.get("original_name", field.name)
-
-            field_value = data.get(original_name)
-
-            field_type = get_inner_type(field.type)  # type: ignore
-
-            if original_name in data:
-                # If the field is another dataclass, recursively instantiate it
-                if is_dataclass(field_type):
-                    field_value = from_dict(field_type, field_value)
-                elif isinstance(field_type, Path | str) and isinstance(
-                    field_value, str
-                ):
-                    field_value = (
-                        Path(field_value) if field_type == Path else field_value
-                    )
-                elif get_origin(field_type) is dict and isinstance(field_value, dict):
-                    # The field is a dictionary with a specific type
-                    inner_type = get_second_type(field_type)
-                    field_value = {
-                        k: from_dict(inner_type, v) for k, v in field_value.items()
-                    }
-                elif get_origin is list and isinstance(field_value, list):
-                    # The field is a list with a specific type
-                    inner_type = get_args(field_type)[0]
-                    field_value = [from_dict(inner_type, v) for v in field_value]
-
-            # Set the value
-            if (
-                field.default is not dataclasses.MISSING
-                or field.default_factory is not dataclasses.MISSING
-            ):
-                # Fields with default value
-                # a: Int = 1
-                # b: list = Field(default_factory=list)
-                if original_name in data or field_value is not None:
-                    field_values[field.name] = field_value
-            else:
-                # Fields without default value
-                # a: Int
-                field_values[field.name] = field_value
-
-        return t(**field_values)
-
-    except (TypeError, ValueError) as e:
-        print(f"Failed to instantiate {t.__name__}: {e} {data}")
-        return None
-        # raise ClanError(f"Failed to instantiate {t.__name__}: {e}")
-
-
 def get_path(flake_dir: str | Path) -> Path:
     """
     Get the path to the inventory file in the flake directory
@@ -165,14 +65,42 @@ default_inventory = Inventory(
 )
 
 
-def load_inventory(
+def load_inventory_eval(flake_dir: str | Path) -> Inventory:
+    """
+    Loads the actual inventory.
+    After all merge operations with eventual nix code in buildClan.
+
+    Evaluates clanInternals.inventory with nix. Which is performant.
+
+    - Contains all clan metadata
+    - Contains all machines
+    - and more
+    """
+    cmd = nix_eval(
+        [
+            f"{flake_dir}#clanInternals.inventory",
+            "--json",
+        ]
+    )
+    proc = run_no_stdout(cmd)
+
+    try:
+        res = proc.stdout.strip()
+        data = json.loads(res)
+        inventory = from_dict(Inventory, data)
+        return inventory
+    except json.JSONDecodeError as e:
+        raise ClanError(f"Error decoding inventory from flake: {e}")
+
+
+def load_inventory_json(
     flake_dir: str | Path, default: Inventory = default_inventory
 ) -> Inventory:
     """
     Load the inventory file from the flake directory
     If not file is found, returns the default inventory
     """
-    inventory = default_inventory
+    inventory = default
 
     inventory_file = get_path(flake_dir)
     if inventory_file.exists():
@@ -184,6 +112,10 @@ def load_inventory(
                 # Error decoding the inventory file
                 raise ClanError(f"Error decoding inventory file: {e}")
 
+    if not inventory_file.exists():
+        # Copy over the meta from the flake if the inventory is not initialized
+        inventory.meta = load_inventory_eval(flake_dir).meta
+
     return inventory
 
 
@@ -198,3 +130,22 @@ def save_inventory(inventory: Inventory, flake_dir: str | Path, message: str) ->
         json.dump(dataclass_to_dict(inventory), f, indent=2)
 
     commit_file(inventory_file, Path(flake_dir), commit_message=message)
+
+
+@API.register
+def init_inventory(directory: str, init: Inventory | None = None) -> None:
+    inventory = None
+    # Try reading the current flake
+    if init is None:
+        try:
+            inventory = load_inventory_eval(directory)
+        except ClanCmdError:
+            pass
+
+    if init is not None:
+        inventory = init
+
+    # Write inventory.json file
+    if inventory is not None:
+        # Persist creates a commit message for each change
+        save_inventory(inventory, directory, "Init inventory")

pkgs/clan-cli/clan_cli/machines/create.py

@@ -1,13 +1,17 @@
 import argparse
 import logging
 import re
-from pathlib import Path
 
 from ..api import API
 from ..clan_uri import FlakeId
 from ..errors import ClanError
-from ..git import commit_file
+from ..inventory import (
-from ..inventory import Machine, MachineDeploy, get_path, load_inventory, save_inventory
+    Machine,
+    MachineDeploy,
+    load_inventory_eval,
+    load_inventory_json,
+    save_inventory,
+)
 
 log = logging.getLogger(__name__)
 
@@ -20,12 +24,16 @@ def create_machine(flake: FlakeId, machine: Machine) -> None:
             "Machine name must be a valid hostname", location="Create Machine"
         )
 
-    inventory = load_inventory(flake.path)
+    inventory = load_inventory_json(flake.path)
+
+    full_inventory = load_inventory_eval(flake.path)
+
+    if machine.name in full_inventory.machines.keys():
+        raise ClanError(f"Machine with the name {machine.name} already exists")
+
     inventory.machines.update({machine.name: machine})
     save_inventory(inventory, flake.path, f"Create machine {machine.name}")
 
-    commit_file(get_path(flake.path), Path(flake.path))
-
 
 def create_command(args: argparse.Namespace) -> None:
     create_machine(

pkgs/clan-cli/clan_cli/machines/delete.py

@@ -6,12 +6,12 @@ from ..clan_uri import FlakeId
 from ..completions import add_dynamic_completer, complete_machines
 from ..dirs import specific_machine_dir
 from ..errors import ClanError
-from ..inventory import load_inventory, save_inventory
+from ..inventory import load_inventory_json, save_inventory
 
 
 @API.register
 def delete_machine(flake: FlakeId, name: str) -> None:
-    inventory = load_inventory(flake.path)
+    inventory = load_inventory_json(flake.path)
 
     machine = inventory.machines.pop(name, None)
     if machine is None:

pkgs/clan-cli/clan_cli/machines/list.py

@@ -1,31 +1,17 @@
 import argparse
-import json
 import logging
 from pathlib import Path
 
 from clan_cli.api import API
-from clan_cli.inventory import Machine, from_dict
+from clan_cli.inventory import Machine, load_inventory_eval
-
-from ..cmd import run_no_stdout
-from ..nix import nix_eval
 
 log = logging.getLogger(__name__)
 
 
 @API.register
 def list_machines(flake_url: str | Path, debug: bool = False) -> dict[str, Machine]:
-    cmd = nix_eval(
+    inventory = load_inventory_eval(flake_url)
-        [
+    return inventory.machines
-            f"{flake_url}#clanInternals.inventory.machines",
-            "--json",
-        ]
-    )
-
-    proc = run_no_stdout(cmd)
-
-    res = proc.stdout.strip()
-    data = {name: from_dict(Machine, v) for name, v in json.loads(res).items()}
-    return data
 
 
 def list_command(args: argparse.Namespace) -> None:

pkgs/clan-cli/clan_cli/secrets/groups.py

@@ -49,27 +49,27 @@ class Group:
 
 def list_groups(flake_dir: Path) -> list[Group]:
     groups: list[Group] = []
-    folder = sops_groups_folder(flake_dir)
+    groups_dir = sops_groups_folder(flake_dir)
-    if not folder.exists():
+    if not groups_dir.exists():
         return groups
 
-    for name in os.listdir(folder):
+    for group in os.listdir(groups_dir):
-        group_folder = folder / name
+        group_folder = groups_dir / group
         if not group_folder.is_dir():
             continue
-        machines_path = machines_folder(flake_dir, name)
+        machines_path = machines_folder(flake_dir, group)
         machines = []
         if machines_path.is_dir():
             for f in machines_path.iterdir():
                 if validate_hostname(f.name):
                     machines.append(f.name)
-        users_path = users_folder(flake_dir, name)
+        users_path = users_folder(flake_dir, group)
         users = []
         if users_path.is_dir():
             for f in users_path.iterdir():
                 if VALID_USER_NAME.match(f.name):
                     users.append(f.name)
-        groups.append(Group(flake_dir, name, machines, users))
+        groups.append(Group(flake_dir, group, machines, users))
     return groups
 
 
@@ -204,7 +204,9 @@ def add_group_argument(parser: argparse.ArgumentParser) -> None:
 
 def add_secret(flake_dir: Path, group: str, name: str) -> None:
     secrets.allow_member(
-        secrets.groups_folder(flake_dir, name), sops_groups_folder(flake_dir), group
+        secrets.groups_folder(sops_secrets_folder(flake_dir) / name),
+        sops_groups_folder(flake_dir),
+        group,
     )
 
 
@@ -214,7 +216,7 @@ def add_secret_command(args: argparse.Namespace) -> None:
 
 def remove_secret(flake_dir: Path, group: str, name: str) -> None:
     updated_paths = secrets.disallow_member(
-        secrets.groups_folder(flake_dir, name), group
+        secrets.groups_folder(sops_secrets_folder(flake_dir) / name), group
     )
     commit_files(
         updated_paths,

pkgs/clan-cli/clan_cli/secrets/machines.py

@@ -6,7 +6,12 @@ from ..errors import ClanError
 from ..git import commit_files
 from ..machines.types import machine_name_type, validate_hostname
 from . import secrets
-from .folders import list_objects, remove_object, sops_machines_folder
+from .folders import (
+    list_objects,
+    remove_object,
+    sops_machines_folder,
+    sops_secrets_folder,
+)
 from .secrets import update_secrets
 from .sops import read_key, write_key
 from .types import public_or_private_age_key_type, secret_name_type
@@ -56,7 +61,7 @@ def list_machines(flake_dir: Path) -> list[str]:
 
 def add_secret(flake_dir: Path, machine: str, secret: str) -> None:
     paths = secrets.allow_member(
-        secrets.machines_folder(flake_dir, secret),
+        secrets.machines_folder(sops_secrets_folder(flake_dir) / secret),
         sops_machines_folder(flake_dir),
         machine,
     )
@@ -69,7 +74,7 @@ def add_secret(flake_dir: Path, machine: str, secret: str) -> None:
 
 def remove_secret(flake_dir: Path, machine: str, secret: str) -> None:
     updated_paths = secrets.disallow_member(
-        secrets.machines_folder(flake_dir, secret), machine
+        secrets.machines_folder(sops_secrets_folder(flake_dir) / secret), machine
     )
     commit_files(
         updated_paths,

pkgs/clan-cli/clan_cli/secrets/secrets.py

@@ -82,7 +82,7 @@ def collect_keys_for_path(path: Path) -> set[str]:
 
 def encrypt_secret(
     flake_dir: Path,
-    secret: Path,
+    secret_path: Path,
     value: IO[str] | str | bytes | None,
     add_users: list[str] = [],
     add_machines: list[str] = [],
@@ -95,7 +95,7 @@ def encrypt_secret(
     for user in add_users:
         files_to_commit.extend(
             allow_member(
-                users_folder(flake_dir, secret.name),
+                users_folder(secret_path),
                 sops_users_folder(flake_dir),
                 user,
                 False,
@@ -105,7 +105,7 @@ def encrypt_secret(
     for machine in add_machines:
         files_to_commit.extend(
             allow_member(
-                machines_folder(flake_dir, secret.name),
+                machines_folder(secret_path),
                 sops_machines_folder(flake_dir),
                 machine,
                 False,
@@ -115,33 +115,33 @@ def encrypt_secret(
     for group in add_groups:
         files_to_commit.extend(
             allow_member(
-                groups_folder(flake_dir, secret.name),
+                groups_folder(secret_path),
                 sops_groups_folder(flake_dir),
                 group,
                 False,
             )
         )
 
-    keys = collect_keys_for_path(secret)
+    keys = collect_keys_for_path(secret_path)
 
     if key.pubkey not in keys:
         keys.add(key.pubkey)
         files_to_commit.extend(
             allow_member(
-                users_folder(flake_dir, secret.name),
+                users_folder(secret_path),
                 sops_users_folder(flake_dir),
                 key.username,
                 False,
             )
         )
 
-    secret_path = secret / "secret"
+    secret_path = secret_path / "secret"
     encrypt_file(secret_path, value, list(sorted(keys)))
     files_to_commit.append(secret_path)
     commit_files(
         files_to_commit,
         flake_dir,
-        f"Update secret {secret.name}",
+        f"Update secret {secret_path.name}",
     )
 
 
@@ -169,16 +169,16 @@ def add_secret_argument(parser: argparse.ArgumentParser, autocomplete: bool) ->
         add_dynamic_completer(secrets_parser, complete_secrets)
 
 
-def machines_folder(flake_dir: Path, group: str) -> Path:
+def machines_folder(secret_path: Path) -> Path:
-    return sops_secrets_folder(flake_dir) / group / "machines"
+    return secret_path / "machines"
 
 
-def users_folder(flake_dir: Path, group: str) -> Path:
+def users_folder(secret_path: Path) -> Path:
-    return sops_secrets_folder(flake_dir) / group / "users"
+    return secret_path / "users"
 
 
-def groups_folder(flake_dir: Path, group: str) -> Path:
+def groups_folder(secret_path: Path) -> Path:
-    return sops_secrets_folder(flake_dir) / group / "groups"
+    return secret_path / "groups"
 
 
 def list_directory(directory: Path) -> str:
@@ -245,8 +245,8 @@ def disallow_member(group_folder: Path, name: str) -> list[Path]:
     )
 
 
-def has_secret(flake_dir: Path, secret: str) -> bool:
+def has_secret(secret_path: Path) -> bool:
-    return (sops_secrets_folder(flake_dir) / secret / "secret").exists()
+    return (secret_path / "secret").exists()
 
 
 def list_secrets(flake_dir: Path, pattern: str | None = None) -> list[str]:
@@ -255,7 +255,7 @@ def list_secrets(flake_dir: Path, pattern: str | None = None) -> list[str]:
     def validate(name: str) -> bool:
         return (
             VALID_SECRET_NAME.match(name) is not None
-            and has_secret(flake_dir, name)
+            and has_secret(sops_secrets_folder(flake_dir) / name)
             and (pattern is None or pattern in name)
         )
 
@@ -278,16 +278,21 @@ def list_command(args: argparse.Namespace) -> None:
         print("\n".join(lst))
 
 
-def decrypt_secret(flake_dir: Path, secret: str) -> str:
+def decrypt_secret(flake_dir: Path, secret_path: Path) -> str:
     ensure_sops_key(flake_dir)
-    secret_path = sops_secrets_folder(flake_dir) / secret / "secret"
+    path = secret_path / "secret"
-    if not secret_path.exists():
+    if not path.exists():
-        raise ClanError(f"Secret '{secret}' does not exist")
+        raise ClanError(f"Secret '{secret_path!s}' does not exist")
-    return decrypt_file(secret_path)
+    return decrypt_file(path)
 
 
 def get_command(args: argparse.Namespace) -> None:
-    print(decrypt_secret(args.flake.path, args.secret), end="")
+    print(
+        decrypt_secret(
+            args.flake.path, sops_secrets_folder(args.flake.path) / args.secret
+        ),
+        end="",
+    )
 
 
 def set_command(args: argparse.Namespace) -> None:

pkgs/clan-cli/clan_cli/secrets/users.py

@@ -9,7 +9,7 @@ from ..completions import (
 from ..errors import ClanError
 from ..git import commit_files
 from . import secrets
-from .folders import list_objects, remove_object, sops_users_folder
+from .folders import list_objects, remove_object, sops_secrets_folder, sops_users_folder
 from .secrets import update_secrets
 from .sops import read_key, write_key
 from .types import (
@@ -63,7 +63,9 @@ def list_users(flake_dir: Path) -> list[str]:
 
 def add_secret(flake_dir: Path, user: str, secret: str) -> None:
     updated_paths = secrets.allow_member(
-        secrets.users_folder(flake_dir, secret), sops_users_folder(flake_dir), user
+        secrets.users_folder(sops_secrets_folder(flake_dir) / secret),
+        sops_users_folder(flake_dir),
+        user,
     )
     commit_files(
         updated_paths,
@@ -74,7 +76,7 @@ def add_secret(flake_dir: Path, user: str, secret: str) -> None:
 
 def remove_secret(flake_dir: Path, user: str, secret: str) -> None:
     updated_paths = secrets.disallow_member(
-        secrets.users_folder(flake_dir, secret), user
+        secrets.users_folder(sops_secrets_folder(flake_dir) / secret), user
     )
     commit_files(
         updated_paths,

pkgs/clan-cli/clan_cli/vars/generate.py

@@ -2,9 +2,8 @@ import argparse
 import importlib
 import logging
 import os
-import subprocess
 import sys
-from collections.abc import Callable
+from getpass import getpass
 from graphlib import TopologicalSorter
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -29,17 +28,7 @@ from .secret_modules import SecretStoreBase
 log = logging.getLogger(__name__)
 
 
-def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str:
+def bubblewrap_cmd(generator: str, tmpdir: Path) -> list[str]:
-    """
-    Read multi-line input from stdin.
-    """
-    print(prompt, flush=True)
-    proc = subprocess.run(["cat"], stdout=subprocess.PIPE, text=True)
-    log.info("Input received. Processing...")
-    return proc.stdout
-
-
-def bubblewrap_cmd(generator: str, generator_dir: Path, dep_tmpdir: Path) -> list[str]:
     # fmt: off
     return nix_shell(
         [
@@ -51,8 +40,7 @@ def bubblewrap_cmd(generator: str, generator_dir: Path, dep_tmpdir: Path) -> lis
             "--ro-bind", "/nix/store", "/nix/store",
             "--tmpfs",  "/usr/lib/systemd",
             "--dev", "/dev",
-            "--bind", str(generator_dir), str(generator_dir),
+            "--bind", str(tmpdir), str(tmpdir),
-            "--ro-bind", str(dep_tmpdir), str(dep_tmpdir),
             "--unshare-all",
             "--unshare-user",
             "--uid", "1000",
@@ -92,7 +80,7 @@ def decrypt_dependencies(
 def dependencies_as_dir(
     decrypted_dependencies: dict[str, dict[str, bytes]],
     tmpdir: Path,
-) -> Path:
+) -> None:
     for dep_generator, files in decrypted_dependencies.items():
         dep_generator_dir = tmpdir / dep_generator
         dep_generator_dir.mkdir()
@@ -102,7 +90,6 @@ def dependencies_as_dir(
             file_path.touch()
             file_path.chmod(0o600)
             file_path.write_bytes(file)
-    return tmpdir
 
 
 def execute_generator(
@@ -111,10 +98,7 @@ def execute_generator(
     regenerate: bool,
     secret_vars_store: SecretStoreBase,
     public_vars_store: FactStoreBase,
-    dep_tmpdir: Path,
-    prompt: Callable[[str], str],
 ) -> bool:
-    generator_dir = dep_tmpdir / generator_name
     # check if all secrets exist and generate them if at least one is missing
     needs_regeneration = not check_secrets(machine, generator_name=generator_name)
     log.debug(f"{generator_name} needs_regeneration: {needs_regeneration}")
@@ -124,51 +108,65 @@ def execute_generator(
         msg = f"flake is not a Path: {machine.flake}"
         msg += "fact/secret generation is only supported for local flakes"
 
-    # compatibility for old outputs.nix users
     generator = machine.vars_generators[generator_name]["finalScript"]
-    # if machine.vars_data[generator_name]["generator"]["prompt"]:
-    #     prompt_value = prompt(machine.vars_data[generator_name]["generator"]["prompt"])
-    #     env["prompt_value"] = prompt_value
 
     # build temporary file tree of dependencies
     decrypted_dependencies = decrypt_dependencies(
         machine, generator_name, secret_vars_store, public_vars_store
     )
     env = os.environ.copy()
-    generator_dir.mkdir(parents=True)
-    env["out"] = str(generator_dir)
     with TemporaryDirectory() as tmp:
-        dep_tmpdir = dependencies_as_dir(decrypted_dependencies, Path(tmp))
+        tmpdir = Path(tmp)
-        env["in"] = str(dep_tmpdir)
+        tmpdir_in = tmpdir / "in"
+        tmpdir_prompts = tmpdir / "prompts"
+        tmpdir_out = tmpdir / "out"
+        tmpdir_in.mkdir()
+        tmpdir_out.mkdir()
+        env["in"] = str(tmpdir_in)
+        env["out"] = str(tmpdir_out)
+        # populate dependency inputs
+        dependencies_as_dir(decrypted_dependencies, tmpdir_in)
+        # populate prompted values
+        # TODO: make prompts rest API friendly
+        if machine.vars_generators[generator_name]["prompts"]:
+            tmpdir_prompts.mkdir()
+            env["prompts"] = str(tmpdir_prompts)
+            for prompt_name, prompt in machine.vars_generators[generator_name][
+                "prompts"
+            ].items():
+                prompt_file = tmpdir_prompts / prompt_name
+                value = prompt_func(prompt["description"], prompt["type"])
+                prompt_file.write_text(value)
+
         if sys.platform == "linux":
-            cmd = bubblewrap_cmd(generator, generator_dir, dep_tmpdir=dep_tmpdir)
+            cmd = bubblewrap_cmd(generator, tmpdir)
         else:
             cmd = ["bash", "-c", generator]
         run(
             cmd,
             env=env,
         )
-    files_to_commit = []
+        files_to_commit = []
-    # store secrets
+        # store secrets
-    files = machine.vars_generators[generator_name]["files"]
+        files = machine.vars_generators[generator_name]["files"]
-    for file_name, file in files.items():
+        for file_name, file in files.items():
-        groups = machine.deployment["sops"]["defaultGroups"]
+            groups = machine.deployment["sops"]["defaultGroups"]
 
-        secret_file = generator_dir / file_name
+            secret_file = tmpdir_out / file_name
-        if not secret_file.is_file():
+            if not secret_file.is_file():
-            msg = f"did not generate a file for '{file_name}' when running the following command:\n"
+                msg = f"did not generate a file for '{file_name}' when running the following command:\n"
-            msg += generator
+                msg += generator
-            raise ClanError(msg)
+                raise ClanError(msg)
-        if file["secret"]:
+            if file["secret"]:
-            file_path = secret_vars_store.set(
+                file_path = secret_vars_store.set(
-                generator_name, file_name, secret_file.read_bytes(), groups
+                    generator_name, file_name, secret_file.read_bytes(), groups
-            )
+                )
-        else:
+            else:
-            file_path = public_vars_store.set(
+                file_path = public_vars_store.set(
-                generator_name, file_name, secret_file.read_bytes()
+                    generator_name, file_name, secret_file.read_bytes()
-            )
+                )
-        if file_path:
+            if file_path:
-            files_to_commit.append(file_path)
+                files_to_commit.append(file_path)
     commit_files(
         files_to_commit,
         machine.flake_dir,
@@ -177,9 +175,18 @@ def execute_generator(
     return True
 
 
-def prompt_func(text: str) -> str:
+def prompt_func(description: str, input_type: str) -> str:
-    print(f"{text}: ")
+    if input_type == "line":
-    return read_multiline_input()
+        result = input(f"Enter the value for {description}: ")
+    elif input_type == "multiline":
+        print(f"Enter the value for {description} (Finish with Ctrl-D): ")
+        result = sys.stdin.read()
+    elif input_type == "hidden":
+        result = getpass(f"Enter the value for {description} (hidden): ")
+    else:
+        raise ClanError(f"Unknown input type: {input_type} for prompt {description}")
+    log.info("Input received. Processing...")
+    return result
 
 
 def _get_subgraph(graph: dict[str, set], vertex: str) -> dict[str, set]:
@@ -197,11 +204,7 @@ def _generate_vars_for_machine(
     machine: Machine,
     generator_name: str | None,
     regenerate: bool,
-    tmpdir: Path,
-    prompt: Callable[[str], str] = prompt_func,
 ) -> bool:
-    local_temp = tmpdir / machine.name
-    local_temp.mkdir()
     secret_vars_module = importlib.import_module(machine.secret_vars_module)
     secret_vars_store = secret_vars_module.SecretStore(machine=machine)
 
@@ -216,13 +219,6 @@ def _generate_vars_for_machine(
             f"Could not find generator with name: {generator_name}. The following generators are available: {generators}"
         )
 
-    # if generator_name:
-    #     machine_generator_facts = {
-    #         generator_name: machine.vars_generators[generator_name]
-    #     }
-    # else:
-    #     machine_generator_facts = machine.vars_generators
-
     graph = {
         gen_name: set(generator["dependencies"])
         for gen_name, generator in machine.vars_generators.items()
@@ -250,8 +246,6 @@ def _generate_vars_for_machine(
             regenerate=regenerate,
             secret_vars_store=secret_vars_store,
             public_vars_store=public_vars_store,
-            dep_tmpdir=local_temp,
-            prompt=prompt,
         )
     if machine_updated:
         # flush caches to make sure the new secrets are available in evaluation
@@ -263,25 +257,21 @@ def generate_vars(
     machines: list[Machine],
     generator_name: str | None,
     regenerate: bool,
-    prompt: Callable[[str], str] = prompt_func,
 ) -> bool:
     was_regenerated = False
-    with TemporaryDirectory() as tmp:
+    for machine in machines:
-        tmpdir = Path(tmp)
+        errors = 0
-
+        try:
-        for machine in machines:
+            was_regenerated |= _generate_vars_for_machine(
-            errors = 0
+                machine, generator_name, regenerate
-            try:
+            )
-                was_regenerated |= _generate_vars_for_machine(
+        except Exception as exc:
-                    machine, generator_name, regenerate, tmpdir, prompt
+            log.error(f"Failed to generate facts for {machine.name}: {exc}")
-                )
+            errors += 1
-            except Exception as exc:
+        if errors > 0:
-                log.error(f"Failed to generate facts for {machine.name}: {exc}")
+            raise ClanError(
-                errors += 1
+                f"Failed to generate facts for {errors} hosts. Check the logs above"
-            if errors > 0:
+            )
-                raise ClanError(
-                    f"Failed to generate facts for {errors} hosts. Check the logs above"
-                )
 
     if not was_regenerated:
         print("All secrets and facts are already up to date")

pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py

@@ -13,33 +13,45 @@ class SecretStore(SecretStoreBase):
         self.machine = machine
 
     def set(
-        self, service: str, name: str, value: bytes, groups: list[str]
+        self, generator_name: str, name: str, value: bytes, groups: list[str]
     ) -> Path | None:
         subprocess.run(
             nix_shell(
                 ["nixpkgs#pass"],
-                ["pass", "insert", "-m", f"machines/{self.machine.name}/{name}"],
+                [
+                    "pass",
+                    "insert",
+                    "-m",
+                    f"machines/{self.machine.name}/{generator_name}/{name}",
+                ],
             ),
             input=value,
             check=True,
         )
         return None  # we manage the files outside of the git repo
 
-    def get(self, service: str, name: str) -> bytes:
+    def get(self, generator_name: str, name: str) -> bytes:
         return subprocess.run(
             nix_shell(
                 ["nixpkgs#pass"],
-                ["pass", "show", f"machines/{self.machine.name}/{name}"],
+                [
+                    "pass",
+                    "show",
+                    f"machines/{self.machine.name}/{generator_name}/{name}",
+                ],
             ),
             check=True,
             stdout=subprocess.PIPE,
         ).stdout
 
-    def exists(self, service: str, name: str) -> bool:
+    def exists(self, generator_name: str, name: str) -> bool:
         password_store = os.environ.get(
             "PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store"
         )
-        secret_path = Path(password_store) / f"machines/{self.machine.name}/{name}.gpg"
+        secret_path = (
+            Path(password_store)
+            / f"machines/{self.machine.name}/{generator_name}/{name}.gpg"
+        )
         return secret_path.exists()
 
     def generate_hash(self) -> bytes:

pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py

@@ -36,13 +36,20 @@ class SecretStore(SecretStoreBase):
         )
         add_machine(self.machine.flake_dir, self.machine.name, pub_key, False)
 
+    def secret_path(self, generator_name: str, secret_name: str) -> Path:
+        return (
+            self.machine.flake_dir
+            / "sops"
+            / "vars"
+            / self.machine.name
+            / generator_name
+            / secret_name
+        )
+
     def set(
         self, generator_name: str, name: str, value: bytes, groups: list[str]
     ) -> Path | None:
-        path = (
+        path = self.secret_path(generator_name, name)
-            sops_secrets_folder(self.machine.flake_dir)
-            / f"vars-{self.machine.name}-{generator_name}-{name}"
-        )
         encrypt_secret(
             self.machine.flake_dir,
             path,
@@ -54,19 +61,21 @@ class SecretStore(SecretStoreBase):
 
     def get(self, generator_name: str, name: str) -> bytes:
         return decrypt_secret(
-            self.machine.flake_dir, f"vars-{self.machine.name}-{generator_name}-{name}"
+            self.machine.flake_dir, self.secret_path(generator_name, name)
         ).encode("utf-8")
 
     def exists(self, generator_name: str, name: str) -> bool:
         return has_secret(
-            self.machine.flake_dir,
+            self.secret_path(generator_name, name),
-            f"vars-{self.machine.name}-{generator_name}-{name}",
         )
 
     def upload(self, output_dir: Path) -> None:
         key_name = f"{self.machine.name}-age.key"
-        if not has_secret(self.machine.flake_dir, key_name):
+        if not has_secret(sops_secrets_folder(self.machine.flake_dir) / key_name):
             # skip uploading the secret, not managed by us
             return
-        key = decrypt_secret(self.machine.flake_dir, key_name)
+        key = decrypt_secret(
+            self.machine.flake_dir,
+            sops_secrets_folder(self.machine.flake_dir) / key_name,
+        )
         (output_dir / "key.txt").write_text(key)

pkgs/clan-cli/clan_cli/vms/inspect.py

@@ -8,6 +8,12 @@ from ..completions import add_dynamic_completer, complete_machines
 from ..machines.machines import Machine
 
 
+@dataclass
+class WaypipeConfig:
+    enable: bool
+    command: list[str]
+
+
 @dataclass
 class VmConfig:
     machine_name: str
@@ -24,6 +30,8 @@ class VmConfig:
     def __post_init__(self) -> None:
         if isinstance(self.flake_url, str):
             self.flake_url = FlakeId(self.flake_url)
+        if isinstance(self.waypipe, dict):
+            self.waypipe = WaypipeConfig(**self.waypipe)
 
 
 def inspect_vm(machine: Machine) -> VmConfig:

pkgs/clan-cli/tests/fixtures_flakes.py

@@ -56,6 +56,7 @@ def generate_flake(
     },
     # define the machines directly including their config
     machine_configs: dict[str, dict] = {},
+    inventory: dict[str, dict] = {},
 ) -> FlakeForTest:
     """
     Creates a clan flake with the given name.
@@ -80,6 +81,12 @@ def generate_flake(
     shutil.copytree(flake_template, flake)
     sp.run(["chmod", "+w", "-R", str(flake)], check=True)
 
+    # initialize inventory
+    if inventory:
+        # check if inventory valid
+        inventory_path = flake / "inventory.json"
+        inventory_path.write_text(json.dumps(inventory, indent=2))
+
     # substitute `substitutions` in all files of the template
     for file in flake.rglob("*"):
         if file.is_file():

pkgs/clan-cli/tests/test_create_flake.py

@@ -20,12 +20,12 @@ def test_create_flake(
     cli.run(["flakes", "create", str(flake_dir), f"--url={url}"])
 
     assert (flake_dir / ".clan-flake").exists()
-
     # Replace the inputs.clan.url in the template flake.nix
     substitute(
         flake_dir / "flake.nix",
         clan_core,
     )
+    # Dont evaluate the inventory before the substitute call
 
     monkeypatch.chdir(flake_dir)
     cli.run(["machines", "create", "machine1"])

pkgs/clan-cli/tests/test_modules.py

@@ -14,7 +14,7 @@ from clan_cli.inventory import (
     ServiceBorgbackupRoleClient,
     ServiceBorgbackupRoleServer,
     ServiceMeta,
-    load_inventory,
+    load_inventory_json,
     save_inventory,
 )
 from clan_cli.machines.create import create_machine
@@ -67,7 +67,7 @@ def test_add_module_to_inventory(
         ),
     )
 
-    inventory = load_inventory(base_path)
+    inventory = load_inventory_json(base_path)
 
     inventory.services.borgbackup = {
         "borg1": ServiceBorgbackup(

pkgs/clan-cli/tests/test_vars.py

@@ -1,6 +1,8 @@
 import os
+import subprocess
 from collections import defaultdict
 from collections.abc import Callable
+from io import StringIO
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any
@@ -13,7 +15,8 @@ from root import CLAN_CORE
 
 from clan_cli.clan_uri import FlakeId
 from clan_cli.machines.machines import Machine
-from clan_cli.vars.secret_modules.sops import SecretStore
+from clan_cli.nix import nix_shell
+from clan_cli.vars.secret_modules import password_store, sops
 
 
 def def_value() -> defaultdict:
@@ -55,7 +58,8 @@ def test_dependencies_as_files() -> None:
         ),
     )
     with TemporaryDirectory() as tmpdir:
-        dep_tmpdir = dependencies_as_dir(decrypted_dependencies, Path(tmpdir))
+        dep_tmpdir = Path(tmpdir)
+        dependencies_as_dir(decrypted_dependencies, dep_tmpdir)
         assert dep_tmpdir.is_dir()
         assert (dep_tmpdir / "gen_1" / "var_1a").read_bytes() == b"var_1a"
         assert (dep_tmpdir / "gen_1" / "var_1b").read_bytes() == b"var_1b"
@@ -93,7 +97,45 @@ def test_generate_public_var(
 
 
 @pytest.mark.impure
-def test_generate_secret_var_with_default_group(
+def test_generate_secret_var_sops(
+    monkeypatch: pytest.MonkeyPatch,
+    temporary_home: Path,
+    sops_setup: SopsSetup,
+) -> None:
+    user = os.environ.get("USER", "user")
+    config = nested_dict()
+    my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
+    my_generator["files"]["my_secret"]["secret"] = True
+    my_generator["script"] = "echo hello > $out/my_secret"
+    flake = generate_flake(
+        temporary_home,
+        flake_template=CLAN_CORE / "templates" / "minimal",
+        machine_configs=dict(my_machine=config),
+    )
+    monkeypatch.chdir(flake.path)
+    cli.run(
+        [
+            "secrets",
+            "users",
+            "add",
+            "--flake",
+            str(flake.path),
+            user,
+            sops_setup.keys[0].pubkey,
+        ]
+    )
+    cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
+    var_file_path = (
+        flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret"
+    )
+    assert not var_file_path.is_file()
+    sops_store = sops.SecretStore(Machine(name="my_machine", flake=FlakeId(flake.path)))
+    assert sops_store.exists("my_generator", "my_secret")
+    assert sops_store.get("my_generator", "my_secret").decode() == "hello\n"
+
+
+@pytest.mark.impure
+def test_generate_secret_var_sops_with_default_group(
     monkeypatch: pytest.MonkeyPatch,
     temporary_home: Path,
     sops_setup: SopsSetup,
@@ -126,19 +168,75 @@ def test_generate_secret_var_with_default_group(
     assert not (
         flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret"
     ).is_file()
-    sops_store = SecretStore(Machine(name="my_machine", flake=FlakeId(flake.path)))
+    sops_store = sops.SecretStore(Machine(name="my_machine", flake=FlakeId(flake.path)))
     assert sops_store.exists("my_generator", "my_secret")
     assert (
         flake.path
         / "sops"
-        / "secrets"
+        / "vars"
-        / "vars-my_machine-my_generator-my_secret"
+        / "my_machine"
+        / "my_generator"
+        / "my_secret"
         / "groups"
         / "my_group"
     ).exists()
     assert sops_store.get("my_generator", "my_secret").decode() == "hello\n"
 
 
+@pytest.mark.impure
+def test_generate_secret_var_password_store(
+    monkeypatch: pytest.MonkeyPatch,
+    temporary_home: Path,
+) -> None:
+    config = nested_dict()
+    my_generator = config["clan"]["core"]["vars"]["settings"]["secretStore"] = (
+        "password-store"
+    )
+    my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
+    my_generator["files"]["my_secret"]["secret"] = True
+    my_generator["script"] = "echo hello > $out/my_secret"
+    flake = generate_flake(
+        temporary_home,
+        flake_template=CLAN_CORE / "templates" / "minimal",
+        machine_configs=dict(my_machine=config),
+    )
+    monkeypatch.chdir(flake.path)
+    gnupghome = temporary_home / "gpg"
+    gnupghome.mkdir(mode=0o700)
+    monkeypatch.setenv("GNUPGHOME", str(gnupghome))
+    monkeypatch.setenv("PASSWORD_STORE_DIR", str(temporary_home / "pass"))
+    gpg_key_spec = temporary_home / "gpg_key_spec"
+    gpg_key_spec.write_text(
+        """
+        Key-Type: 1
+        Key-Length: 1024
+        Name-Real: Root Superuser
+        Name-Email: test@local
+        Expire-Date: 0
+        %no-protection
+    """
+    )
+    subprocess.run(
+        nix_shell(
+            ["nixpkgs#gnupg"], ["gpg", "--batch", "--gen-key", str(gpg_key_spec)]
+        ),
+        check=True,
+    )
+    subprocess.run(
+        nix_shell(["nixpkgs#pass"], ["pass", "init", "test@local"]), check=True
+    )
+    cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
+    var_file_path = (
+        flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret"
+    )
+    assert not var_file_path.is_file()
+    store = password_store.SecretStore(
+        Machine(name="my_machine", flake=FlakeId(flake.path))
+    )
+    assert store.exists("my_generator", "my_secret")
+    assert store.get("my_generator", "my_secret").decode() == "hello\n"
+
+
 @pytest.mark.impure
 def test_generate_secret_for_multiple_machines(
     monkeypatch: pytest.MonkeyPatch,
@@ -194,8 +292,8 @@ def test_generate_secret_for_multiple_machines(
     assert machine2_var_file_path.is_file()
     assert machine2_var_file_path.read_text() == "machine2\n"
     # check if secret vars have been created correctly
-    sops_store1 = SecretStore(Machine(name="machine1", flake=FlakeId(flake.path)))
+    sops_store1 = sops.SecretStore(Machine(name="machine1", flake=FlakeId(flake.path)))
-    sops_store2 = SecretStore(Machine(name="machine2", flake=FlakeId(flake.path)))
+    sops_store2 = sops.SecretStore(Machine(name="machine2", flake=FlakeId(flake.path)))
     assert sops_store1.exists("my_generator", "my_secret")
     assert sops_store2.exists("my_generator", "my_secret")
     assert sops_store1.get("my_generator", "my_secret").decode() == "machine1\n"
@@ -232,3 +330,40 @@ def test_dependant_generators(
     )
     assert child_file_path.is_file()
     assert child_file_path.read_text() == "hello\n"
+
+
+@pytest.mark.impure
+@pytest.mark.parametrize(
+    ("prompt_type", "input_value"),
+    [
+        ("line", "my input"),
+        ("multiline", "my\nmultiline\ninput\n"),
+        # The hidden type cannot easily be tested, as getpass() reads from /dev/tty directly
+        # ("hidden", "my hidden input"),
+    ],
+)
+def test_prompt(
+    monkeypatch: pytest.MonkeyPatch,
+    temporary_home: Path,
+    prompt_type: str,
+    input_value: str,
+) -> None:
+    config = nested_dict()
+    my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
+    my_generator["files"]["my_value"]["secret"] = False
+    my_generator["prompts"]["prompt1"]["description"] = "dream2nix"
+    my_generator["prompts"]["prompt1"]["type"] = prompt_type
+    my_generator["script"] = "cat $prompts/prompt1 > $out/my_value"
+    flake = generate_flake(
+        temporary_home,
+        flake_template=CLAN_CORE / "templates" / "minimal",
+        machine_configs=dict(my_machine=config),
+    )
+    monkeypatch.chdir(flake.path)
+    monkeypatch.setattr("sys.stdin", StringIO(input_value))
+    cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
+    var_file_path = (
+        flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_value"
+    )
+    assert var_file_path.is_file()
+    assert var_file_path.read_text() == input_value

pkgs/clan-vm-manager/clan-vm-manager.code-workspace

@@ -14,6 +14,9 @@
 		},
 		{
 			"path": "../../lib/build-clan"
+		},
+		{
+			"path": "../../../democlan"
 		}
 	],
 	"settings": {

pkgs/clan-vm-manager/clan_vm_manager/components/vmobj.py

@@ -158,9 +158,15 @@ class VMObject(GObject.Object):
                 flake=uri.flake,
             )
         assert self.machine is not None
-        state_dir = vm_state_dir(
+
-            flake_url=str(self.machine.flake.url), vm_name=self.machine.name
+        if self.machine.flake.is_local():
-        )
+            state_dir = vm_state_dir(
+                flake_url=str(self.machine.flake.path), vm_name=self.machine.name
+            )
+        else:
+            state_dir = vm_state_dir(
+                flake_url=self.machine.flake.url, vm_name=self.machine.name
+            )
         self.qmp_wrap = QMPWrapper(state_dir)
         assert self.machine is not None
         yield self.machine

pkgs/clan-vm-manager/clan_vm_manager/views/webview.py

@@ -1,156 +0,0 @@
-import dataclasses
-import json
-import logging
-import sys
-import threading
-from collections.abc import Callable
-from pathlib import Path
-from threading import Lock
-from typing import Any
-
-import gi
-from clan_cli.api import API
-
-gi.require_version("WebKit", "6.0")
-
-from gi.repository import GLib, WebKit
-
-site_index: Path = (
-    Path(sys.argv[0]).absolute()
-    / Path("../..")
-    / Path("clan_vm_manager/.webui/index.html")
-).resolve()
-
-log = logging.getLogger(__name__)
-
-
-def dataclass_to_dict(obj: Any) -> Any:
-    """
-    Utility function to convert dataclasses to dictionaries
-    It converts all nested dataclasses, lists, tuples, and dictionaries to dictionaries
-
-    It does NOT convert member functions.
-    """
-    if dataclasses.is_dataclass(obj):
-        return {k: dataclass_to_dict(v) for k, v in dataclasses.asdict(obj).items()}
-    elif isinstance(obj, list | tuple):
-        return [dataclass_to_dict(item) for item in obj]
-    elif isinstance(obj, dict):
-        return {k: dataclass_to_dict(v) for k, v in obj.items()}
-    else:
-        return obj
-
-
-class WebView:
-    def __init__(self, methods: dict[str, Callable]) -> None:
-        self.method_registry: dict[str, Callable] = methods
-
-        self.webview = WebKit.WebView()
-
-        settings = self.webview.get_settings()
-        # settings.
-        settings.set_property("enable-developer-extras", True)
-        self.webview.set_settings(settings)
-
-        self.manager = self.webview.get_user_content_manager()
-        # Can be called with: window.webkit.messageHandlers.gtk.postMessage("...")
-        # Important: it seems postMessage must be given some payload, otherwise it won't trigger the event
-        self.manager.register_script_message_handler("gtk")
-        self.manager.connect("script-message-received", self.on_message_received)
-
-        self.webview.load_uri(f"file://{site_index}")
-
-        # global mutex lock to ensure functions run sequentially
-        self.mutex_lock = Lock()
-        self.queue_size = 0
-
-    def on_message_received(
-        self, user_content_manager: WebKit.UserContentManager, message: Any
-    ) -> None:
-        payload = json.loads(message.to_json(0))
-        method_name = payload["method"]
-        handler_fn = self.method_registry[method_name]
-
-        log.debug(f"Received message: {payload}")
-        log.debug(f"Queue size: {self.queue_size} (Wait)")
-
-        def threaded_wrapper() -> bool:
-            """
-            Ensures only one function is executed at a time
-
-            Wait until there is no other function acquiring the global lock.
-
-            Starts a thread with the potentially long running API function within.
-            """
-            if not self.mutex_lock.locked():
-                thread = threading.Thread(
-                    target=self.threaded_handler,
-                    args=(
-                        handler_fn,
-                        payload.get("data"),
-                        method_name,
-                    ),
-                )
-                thread.start()
-                return GLib.SOURCE_REMOVE
-
-            return GLib.SOURCE_CONTINUE
-
-        GLib.idle_add(
-            threaded_wrapper,
-        )
-        self.queue_size += 1
-
-    def threaded_handler(
-        self,
-        handler_fn: Callable[
-            ...,
-            Any,
-        ],
-        data: dict[str, Any] | None,
-        method_name: str,
-    ) -> None:
-        with self.mutex_lock:
-            log.debug("Executing... ", method_name)
-            log.debug(f"{data}")
-            if data is None:
-                result = handler_fn()
-            else:
-                reconciled_arguments = {}
-                for k, v in data.items():
-                    # Some functions expect to be called with dataclass instances
-                    # But the js api returns dictionaries.
-                    # Introspect the function and create the expected dataclass from dict dynamically
-                    # Depending on the introspected argument_type
-                    arg_type = API.get_method_argtype(method_name, k)
-                    if dataclasses.is_dataclass(arg_type):
-                        reconciled_arguments[k] = arg_type(**v)
-                    else:
-                        reconciled_arguments[k] = v
-
-                result = handler_fn(**reconciled_arguments)
-
-            serialized = json.dumps(dataclass_to_dict(result))
-
-            # Use idle_add to queue the response call to js on the main GTK thread
-            GLib.idle_add(self.return_data_to_js, method_name, serialized)
-        self.queue_size -= 1
-        log.debug(f"Done: Remaining queue size: {self.queue_size}")
-
-    def return_data_to_js(self, method_name: str, serialized: str) -> bool:
-        # This function must be run on the main GTK thread to interact with the webview
-        # result = method_fn(data) # takes very long
-        # serialized = result
-        self.webview.evaluate_javascript(
-            f"""
-            window.clan.{method_name}(`{serialized}`);
-            """,
-            -1,
-            None,
-            None,
-            None,
-        )
-        return GLib.SOURCE_REMOVE
-
-    def get_webview(self) -> WebKit.WebView:
-        return self.webview

pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py

@@ -2,7 +2,6 @@ import logging
 import threading
 
 import gi
-from clan_cli.api import API
 from clan_cli.history.list import list_history
 
 from clan_vm_manager.components.interfaces import ClanConfig
@@ -12,7 +11,6 @@ from clan_vm_manager.singletons.use_vms import ClanStore
 from clan_vm_manager.views.details import Details
 from clan_vm_manager.views.list import ClanList
 from clan_vm_manager.views.logs import Logs
-from clan_vm_manager.views.webview import WebView
 
 gi.require_version("Adw", "1")
 
@@ -61,9 +59,6 @@ class MainWindow(Adw.ApplicationWindow):
         stack_view.add_named(Details(), "details")
         stack_view.add_named(Logs(), "logs")
 
-        webview = WebView(methods=API._registry)
-        stack_view.add_named(webview.get_webview(), "webview")
-
         stack_view.set_visible_child_name(config.initial_view)
 
         view.set_content(scroll)

pkgs/clan-vm-manager/default.nix

@@ -18,7 +18,6 @@
   runCommand,
   setuptools,
   webkitgtk_6_0,
-  webview-ui,
   wrapGAppsHook,
 }:
 let
@@ -26,7 +25,7 @@ let
   desktop-file = makeDesktopItem {
     name = "org.clan.vm-manager";
     exec = "clan-vm-manager %u";
-    icon = ./clan_vm_manager/assets/clan_white.png;
+    icon = "clan-white";
     desktopName = "Clan Manager";
     startupWMClass = "clan";
     mimeTypes = [ "x-scheme-handler/clan" ];
@@ -142,10 +141,9 @@ python3.pkgs.buildPythonApplication rec {
   passthru.runtimeDependencies = runtimeDependencies;
   passthru.testDependencies = testDependencies;
 
-  # TODO: place webui in lib/python3.11/site-packages/clan_vm_manager
   postInstall = ''
-    mkdir -p $out/clan_vm_manager/.webui
+    mkdir -p $out/share/icons/hicolor
-    cp -r ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* $out/clan_vm_manager/.webui
+    cp -r ./clan_vm_manager/assets/white-favicons/* $out/share/icons/hicolor
   '';
 
   # Don't leak python packages into a devshell.

pkgs/clan-vm-manager/flake-module.nix

@@ -13,10 +13,10 @@
     else
       {
         devShells.clan-vm-manager = pkgs.callPackage ./shell.nix {
-          inherit (config.packages) clan-vm-manager webview-ui;
+          inherit (config.packages) clan-vm-manager;
         };
         packages.clan-vm-manager = pkgs.python3.pkgs.callPackage ./default.nix {
-          inherit (config.packages) clan-cli webview-ui;
+          inherit (config.packages) clan-cli;
         };
 
         checks = config.packages.clan-vm-manager.tests;

pkgs/clan-vm-manager/install-desktop.sh

@@ -1,23 +1,22 @@
 #!/usr/bin/env bash
 
-CLAN=$(nix build .#clan-vm-manager --print-out-paths)
 
 if ! command -v xdg-mime &> /dev/null; then
   echo "Warning: 'xdg-mime' is not available. The desktop file cannot be installed."
 fi
 
+ALREADY_INSTALLED=$(nix profile list --json |  jq 'has("elements") and (.elements | has("clan-vm-manager"))')
+
+if [ "$ALREADY_INSTALLED" = "true" ]; then
+  echo "Upgrading installed clan-vm-manager"
+  nix profile upgrade clan-vm-manager
+else
+  nix profile install .#clan-vm-manager --priority 4
+fi
+
+
 # install desktop file
 set -eou pipefail
 DESKTOP_FILE_NAME=org.clan.vm-manager.desktop
-DESKTOP_DST=~/.local/share/applications/"$DESKTOP_FILE_NAME"
-DESKTOP_SRC="$CLAN/share/applications/$DESKTOP_FILE_NAME"
-UI_BIN="$CLAN/bin/clan-vm-manager"
 
-cp -f "$DESKTOP_SRC" "$DESKTOP_DST"
-sleep 2
-sed -i "s|Exec=.*clan-vm-manager|Exec=$UI_BIN|" "$DESKTOP_DST"
 xdg-mime default "$DESKTOP_FILE_NAME"  x-scheme-handler/clan
-echo "==== Validating desktop file installation   ===="
-set -x
-desktop-file-validate "$DESKTOP_DST"
-set +xeou pipefail

pkgs/clan-vm-manager/shell.nix

@@ -10,7 +10,7 @@
   python3,
   gtk4,
   libadwaita,
-  webview-ui,
+
 }:
 
 let
@@ -52,11 +52,5 @@ mkShell {
 
     # Add clan-cli to the python path so that we can import it without building it in nix first
     export PYTHONPATH="$GIT_ROOT/pkgs/clan-cli":"$PYTHONPATH"
-
-    # Add the webview-ui to the .webui directory
-    rm -rf ./clan_vm_manager/.webui/*
-    mkdir -p ./clan_vm_manager/.webui
-    cp -a ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* ./clan_vm_manager/.webui
-    chmod -R +w ./clan_vm_manager/.webui
   '';
 }

pkgs/installer/flake-module.nix

@@ -32,7 +32,7 @@ let
     }
     // flashDiskoConfig;
 
-  # Important: The partition names need to be different to the clan install 
+  # Important: The partition names need to be different to the clan install
   flashDiskoConfig = {
     boot.loader.grub.efiSupport = lib.mkDefault true;
     boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;

pkgs/webview-ui/.gitignore

@@ -1 +1,3 @@
-api
+api
+
+.vite

pkgs/webview-ui/app/src/Routes.tsx

@@ -8,6 +8,7 @@ import { Flash } from "./routes/flash/view";
 import { Settings } from "./routes/settings";
 import { Welcome } from "./routes/welcome";
 import { Deploy } from "./routes/deploy";
+import { CreateMachine } from "./routes/machines/create";
 
 export type Route = keyof typeof routes;
 
@@ -22,6 +23,11 @@ export const routes = {
     label: "Machines",
     icon: "devices_other",
   },
+  "machines/add": {
+    child: CreateMachine,
+    label: "create Machine",
+    icon: "add",
+  },
   hosts: {
     child: HostList,
     label: "hosts",

pkgs/webview-ui/app/src/api.ts

@@ -67,6 +67,17 @@ function createFunctions<K extends OperationNames>(
   dispatch: (args: OperationArgs<K>) => void;
   receive: (fn: (response: OperationResponse<K>) => void, id: string) => void;
 } {
+  window.clan[operationName] = (s: string) => {
+    const f = (response: OperationResponse<K>) => {
+      // Get the correct receiver function for the op_key
+      const receiver = registry[operationName][response.op_key];
+      if (receiver) {
+        receiver(response);
+      }
+    };
+    deserialize(f)(s);
+  };
+
   return {
     dispatch: (args: OperationArgs<K>) => {
       // Send the data to the gtk app
@@ -78,15 +89,6 @@ function createFunctions<K extends OperationNames>(
     receive: (fn: (response: OperationResponse<K>) => void, id: string) => {
       // @ts-expect-error: This should work although typescript doesn't let us write
       registry[operationName][id] = fn;
-
-      window.clan[operationName] = (s: string) => {
-        const f = (response: OperationResponse<K>) => {
-          if (response.op_key === id) {
-            registry[operationName][id](response);
-          }
-        };
-        deserialize(f)(s);
-      };
     },
   };
 }
@@ -141,8 +143,9 @@ const deserialize =
       fn(r);
     } catch (e) {
       console.log("Error parsing JSON: ", e);
-      console.log({ download: () => download("error.json", str) });
+      window.localStorage.setItem("error", str);
       console.error(str);
+      console.error("See localStorage 'error'");
       alert(`Error parsing JSON: ${e}`);
     }
   };

pkgs/webview-ui/app/src/index.css

@@ -4,7 +4,6 @@
 @tailwind components;
 @tailwind utilities;
 
-
 html {
     overflow-x: hidden;
     overflow-y: scroll;

pkgs/webview-ui/app/src/routes/clan/clanDetails.tsx

@@ -42,7 +42,15 @@ export const ClanForm = () => {
     await toast.promise(
       (async () => {
         await callApi("create_clan", {
-          options: { directory: target_dir, meta, template_url },
+          options: {
+            directory: target_dir,
+            template_url,
+            initial: {
+              meta,
+              services: {},
+              machines: {},
+            },
+          },
         });
         setActiveURI(target_dir);
         setRoute("machines");

pkgs/webview-ui/app/src/routes/machines/create.tsx

@@ -0,0 +1,124 @@
+import { callApi, OperationArgs, pyApi } from "@/src/api";
+import { activeURI } from "@/src/App";
+import { createForm, required } from "@modular-forms/solid";
+import toast from "solid-toast";
+
+type CreateMachineForm = OperationArgs<"create_machine">;
+
+export function CreateMachine() {
+  const [formStore, { Form, Field }] = createForm<CreateMachineForm>({});
+
+  const handleSubmit = async (values: CreateMachineForm) => {
+    const active_dir = activeURI();
+    if (!active_dir) {
+      toast.error("Open a clan to create the machine in");
+      return;
+    }
+
+    callApi("create_machine", {
+      flake: {
+        loc: active_dir,
+      },
+      machine: {
+        name: "jon",
+        deploy: {
+          targetHost: null,
+        },
+      },
+    });
+    console.log("submit", values);
+  };
+  return (
+    <div class="px-1">
+      Create new Machine
+      <Form onSubmit={handleSubmit}>
+        <Field
+          name="machine.name"
+          validate={[required("This field is required")]}
+        >
+          {(field, props) => (
+            <>
+              <label class="input input-bordered flex items-center gap-2">
+                <input
+                  type="text"
+                  class="grow"
+                  placeholder="name"
+                  required
+                  {...props}
+                />
+              </label>
+              <div class="label">
+                {field.error && (
+                  <span class="label-text-alt font-bold text-error">
+                    {field.error}
+                  </span>
+                )}
+              </div>
+            </>
+          )}
+        </Field>
+        <Field name="machine.description">
+          {(field, props) => (
+            <>
+              <label class="input input-bordered flex items-center gap-2">
+                <input
+                  type="text"
+                  class="grow"
+                  placeholder="description"
+                  required
+                  {...props}
+                />
+              </label>
+              <div class="label">
+                {field.error && (
+                  <span class="label-text-alt font-bold text-error">
+                    {field.error}
+                  </span>
+                )}
+              </div>
+            </>
+          )}
+        </Field>
+        <Field name="machine.deploy.targetHost">
+          {(field, props) => (
+            <>
+              <label class="input input-bordered flex items-center gap-2">
+                <input
+                  type="text"
+                  class="grow"
+                  placeholder="root@flash-installer.local"
+                  required
+                  {...props}
+                />
+              </label>
+              <div class="label">
+                <span class="label-text-alt text-neutral">
+                  Must be set before deployment for the following tasks:
+                  <ul>
+                    <li>
+                      <span>Detect hardware config</span>
+                    </li>
+                    <li>
+                      <span>Detect disk layout</span>
+                    </li>
+                    <li>
+                      <span>Remote installation</span>
+                    </li>
+                  </ul>
+                </span>
+                {field.error && (
+                  <span class="label-text-alt font-bold text-error">
+                    {field.error}
+                  </span>
+                )}
+              </div>
+            </>
+          )}
+        </Field>
+        <button class="btn btn-error float-right" type="submit">
+          <span class="material-icons">add</span>Create
+        </button>
+      </Form>
+    </div>
+  );
+}

pkgs/webview-ui/app/src/routes/machines/view.tsx

@@ -7,7 +7,7 @@ import {
   createSignal,
   type Component,
 } from "solid-js";
-import { activeURI, route, setActiveURI } from "@/src/App";
+import { activeURI, route, setActiveURI, setRoute } from "@/src/App";
 import { OperationResponse, callApi, pyApi } from "@/src/api";
 import toast from "solid-toast";
 import { MachineListItem } from "@/src/components/MachineListItem";
@@ -86,6 +86,11 @@ export const MachineListView: Component = () => {
           <span class="material-icons ">refresh</span>
         </button>
       </div>
+      <div class="tooltip tooltip-bottom" data-tip="Create machine">
+        <button class="btn btn-ghost" onClick={() => setRoute("machines/add")}>
+          <span class="material-icons ">add</span>
+        </button>
+      </div>
       {/* <Show when={services()}>
         {(services) => (
           <For each={Object.values(services())}>

pkgs/webview-ui/app/src/routes/settings/index.tsx

@@ -12,7 +12,8 @@ import {
   setRoute,
   clanList,
 } from "@/src/App";
-import { For } from "solid-js";
+import { For, Show } from "solid-js";
+import { createQuery } from "@tanstack/solid-query";
 
 export const registerClan = async () => {
   try {
@@ -26,6 +27,7 @@ export const registerClan = async () => {
         const res = new Set([...s, loc.data]);
         return Array.from(res);
       });
+      setActiveURI(loc.data);
       setRoute((r) => {
         if (r === "welcome") return "machines";
         return r;
@@ -37,6 +39,87 @@ export const registerClan = async () => {
   }
 };
 
+interface ClanDetailsProps {
+  clan_dir: string;
+}
+const ClanDetails = (props: ClanDetailsProps) => {
+  const { clan_dir } = props;
+
+  const details = createQuery(() => ({
+    queryKey: [clan_dir, "meta"],
+    queryFn: async () => {
+      const result = await callApi("show_clan_meta", { uri: clan_dir });
+      if (result.status === "error") throw new Error("Failed to fetch data");
+      return result.data;
+    },
+  }));
+
+  return (
+    <div class="stat">
+      <div class="stat-figure text-primary">
+        <div class="join">
+          <button
+            class=" join-item btn-sm"
+            classList={{
+              "btn btn-ghost btn-outline": activeURI() !== clan_dir,
+              "badge badge-primary": activeURI() === clan_dir,
+            }}
+            disabled={activeURI() === clan_dir}
+            onClick={() => {
+              setActiveURI(clan_dir);
+            }}
+          >
+            {activeURI() === clan_dir ? "active" : "select"}
+          </button>
+          <button
+            class="btn btn-ghost btn-outline join-item btn-sm"
+            onClick={() => {
+              setClanList((s) =>
+                s.filter((v, idx) => {
+                  if (v == clan_dir) {
+                    setActiveURI(
+                      clanList()[idx - 1] || clanList()[idx + 1] || null
+                    );
+                    return false;
+                  }
+                  return true;
+                })
+              );
+            }}
+          >
+            Remove
+          </button>
+        </div>
+      </div>
+      <div class="stat-title">Clan URI</div>
+
+      <Show when={details.isSuccess}>
+        <div
+          class="stat-value"
+          // classList={{
+          //   "text-primary": activeURI() === clan_dir,
+          // }}
+        >
+          {details.data?.name}
+        </div>
+      </Show>
+      <Show
+        when={details.isSuccess && details.data?.description}
+        fallback={<div class="stat-desc text-lg">{clan_dir}</div>}
+      >
+        <div
+          class="stat-desc text-lg"
+          // classList={{
+          //   "text-primary": activeURI() === clan_dir,
+          // }}
+        >
+          {details.data?.description}
+        </div>
+      </Show>
+    </div>
+  );
+};
+
 export const Settings = () => {
   return (
     <div class="card card-normal">
@@ -54,60 +137,7 @@ export const Settings = () => {
         </div>
         <div class="stats stats-vertical shadow">
           <For each={clanList()}>
-            {(value) => (
+            {(value) => <ClanDetails clan_dir={value} />}
-              <div class="stat">
-                <div class="stat-figure text-primary">
-                  <div class="join">
-                    <button
-                      class=" join-item btn-sm"
-                      classList={{
-                        "btn btn-ghost btn-outline": activeURI() !== value,
-                        "badge badge-primary": activeURI() === value,
-                      }}
-                      disabled={activeURI() === value}
-                      onClick={() => {
-                        setActiveURI(value);
-                      }}
-                    >
-                      {activeURI() === value ? "active" : "select"}
-                    </button>
-                    <button
-                      class="btn btn-ghost btn-outline join-item btn-sm"
-                      onClick={() => {
-                        setClanList((s) =>
-                          s.filter((v, idx) => {
-                            if (v == value) {
-                              setActiveURI(
-                                clanList()[idx - 1] ||
-                                  clanList()[idx + 1] ||
-                                  null
-                              );
-                              return false;
-                            }
-                            return true;
-                          })
-                        );
-                        // if (activeURI() === value) {
-                        //   setActiveURI();
-                        // }
-                      }}
-                    >
-                      Remove URI
-                    </button>
-                  </div>
-                </div>
-                <div class="stat-title">Clan URI</div>
-
-                <div
-                  class="stat-desc text-lg"
-                  classList={{
-                    "text-primary": activeURI() === value,
-                  }}
-                >
-                  {value}
-                </div>
-              </div>
-            )}
           </For>
         </div>
       </div>

templates/flake-parts/flake.nix

@@ -17,7 +17,6 @@
       systems = [
         "x86_64-linux"
         "aarch64-linux"
-
         "x86_64-darwin"
         "aarch64-darwin"
       ];

templates/flake.nix

@@ -1,16 +1,14 @@
 {
-  outputs =
+  outputs = { ... }: {
-    { ... }:
+    templates = {
-    {
+      default = {
-      templates = {
+        description = "Initialize a new clan flake";
-        default = {
+        path = ./new-clan;
-          description = "Initialize a new clan flake";
+      };
-          path = ./new-clan;
+      minimal = {
-        };
+        description = "for clans managed via (G)UI";
-        minimal = {
+        path = ./minimal;
-          description = "for clans managed via (G)UI";
-          path = ./minimal;
-        };
       };
     };
+  };
 }

templates/minimal/inventory.json

@@ -0,0 +1,5 @@
+{
+  "meta": { "name": "__CHANGE_ME__" },
+  "machines": {},
+  "services": {}
+}

templates/new-clan/machines/jon/configuration.nix

@@ -24,9 +24,11 @@
 
   # IMPORTANT! Add your SSH key here
   # e.g. > cat ~/.ssh/id_ed25519.pub
-  users.users.root.openssh.authorizedKeys.keys = [''
+  users.users.root.openssh.authorizedKeys.keys = [
-    __YOUR_SSH_KEY__
+    ''
-  ''];
+      __YOUR_SSH_KEY__
+    ''
+  ];
 
   # Zerotier needs one controller to accept new nodes. Once accepted
   # the controller can be offline and routing still works.

templates/new-clan/machines/sara/configuration.nix

@@ -21,9 +21,11 @@
 
   # IMPORTANT! Add your SSH key here
   # e.g. > cat ~/.ssh/id_ed25519.pub
-  users.users.root.openssh.authorizedKeys.keys = [''
+  users.users.root.openssh.authorizedKeys.keys = [
-    __YOUR_SSH_KEY__
+    ''
-  ''];
+      __YOUR_SSH_KEY__
+    ''
+  ];
   /*
     After jon is deployed, uncomment the following line
     This will allow sara to share the VPN overlay network with jon