Merge pull request 'api: init notification queue' (#4678) from ui-notify into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4678

pkgs/clan-app/clan_app/deps/webview/webview.py

@@ -1,12 +1,14 @@
 import functools
 import json
 import logging
+import threading
 from collections.abc import Callable
 from dataclasses import dataclass, field
 from enum import IntEnum
+from time import sleep
 from typing import TYPE_CHECKING, Any
 
-from clan_lib.api import MethodRegistry
+from clan_lib.api import MethodRegistry, message_queue
 from clan_lib.api.tasks import WebThread
 from clan_lib.log_manager import LogManager
 
@@ -69,6 +71,22 @@ class Webview:
         if self.size:
             self.set_size(self.size)
 
+    def __post_init__(self) -> None:
+        self.setup_notify()  # Start the notification loop
+
+    def setup_notify(self) -> None:
+        def loop() -> None:
+            while True:
+                try:
+                    msg = message_queue.get()  # Blocks until available
+                    js_code = f"window.notifyBus({json.dumps(msg)});"
+                    self.eval(js_code)
+                except Exception as e:
+                    print("Bridge notify error:", e)
+                sleep(0.01)  # avoid busy loop
+
+        threading.Thread(target=loop, daemon=True).start()
+
     @property
     def handle(self) -> Any:
         """Get the webview handle, creating it if necessary."""
@@ -129,6 +147,7 @@ class Webview:
                 webview=self, middleware_chain=tuple(self._middleware), threads={}
             )
         self._bridge = bridge
+
         return bridge
 
     # Legacy methods for compatibility

pkgs/clan-app/clan_app/deps/webview/webview_bridge.py

@@ -25,7 +25,6 @@ class WebviewBridge(ApiBridge):
 
     def send_api_response(self, response: BackendResponse) -> None:
         """Send response back to the webview client."""
-
         serialized = json.dumps(
             dataclass_to_dict(response), indent=4, ensure_ascii=False
         )

pkgs/clan-app/ui/index.d.ts

@@ -0,0 +1,9 @@
+import { ProcessMessage } from "./src/hooks/notify";
+
+export {};
+
+declare global {
+  interface Window {
+    notifyBus: (data: ProcessMessage) => void;
+  }
+}

pkgs/clan-app/ui/src/components/Modal/Modal.module.css

@@ -1,7 +1,9 @@
 .modal_content {
-  @apply min-w-[320px] max-w-[512px];
+  @apply min-w-[320px] flex flex-col;
   @apply rounded-md overflow-hidden;
 
+  max-height: calc(100vh - 2rem);
+
   /* todo replace with a theme() color */
   box-shadow: 0.1875rem 0.1875rem 0 0 rgba(145, 172, 175, 0.32);
   @apply border border-def-3 rounded-bl-md rounded-br-md;
@@ -22,7 +24,8 @@
 }
 
 .modal_body {
-  @apply rounded-b-md p-6 pt-4 bg-def-1;
+  overflow-y: auto;
+  @apply rounded-b-md p-6 pt-4 bg-def-1 flex-grow;
 
   &[data-no-padding] {
     @apply p-0;

pkgs/clan-app/ui/src/hooks/api.ts

@@ -72,7 +72,7 @@ export const callApi = <K extends OperationNames>(
 
   const op_key = backendOpts?.op_key ?? crypto.randomUUID();
 
-  const req: BackendSendType<OperationNames> = {
+  const req: BackendSendType<K> = {
     body: args,
     header: {
       ...backendOpts,
@@ -83,11 +83,9 @@ export const callApi = <K extends OperationNames>(
   const result = (
     window as unknown as Record<
       OperationNames,
-      (
+      (args: BackendSendType<K>) => Promise<BackendReturnType<K>>
-        args: BackendSendType<OperationNames>,
-      ) => Promise<BackendReturnType<OperationNames>>
     >
-  )[method](req) as Promise<BackendReturnType<K>>;
+  )[method](req);
 
   return {
     uuid: op_key,

pkgs/clan-app/ui/src/hooks/notify.ts

@@ -0,0 +1,89 @@
+import { createSignal, onCleanup } from "solid-js";
+import { OperationNames } from "./api";
+
+export interface ProcessMessage<
+  TData = unknown,
+  TTopic extends string = string,
+> {
+  topic: TTopic;
+  data: TData;
+  origin: string | null;
+}
+
+interface Subscriber<T extends ProcessMessage> {
+  filter: (msg: T) => boolean;
+  callback: (msg: T) => void;
+}
+
+const subscribers: Subscriber<ProcessMessage>[] = [];
+
+// Declare the global function on the window type
+declare global {
+  interface Window {
+    notifyBus: (msg: ProcessMessage) => void;
+  }
+}
+
+window.notifyBus = (msg: ProcessMessage) => {
+  console.debug("notifyBus called with:", msg);
+  for (const sub of subscribers) {
+    try {
+      if (sub.filter(msg)) {
+        sub.callback(msg);
+      }
+    } catch (e) {
+      console.error("Subscriber threw an error:", e);
+    }
+  }
+};
+
+/**
+ * Subscribe to any message
+ *
+ * Returns a function to unsubsribe itself
+ *
+ * consider using useNotify for reactive usage on solidjs
+ */
+export function _subscribeNotify<T extends ProcessMessage>(
+  filter: (msg: T) => boolean,
+  callback: (msg: T) => void,
+) {
+  // Cast to shared subscriber type for storage
+  const sub: Subscriber<ProcessMessage> = {
+    filter: filter as (msg: ProcessMessage) => boolean,
+    callback: callback as (msg: ProcessMessage) => void,
+  };
+  subscribers.push(sub);
+  return () => {
+    const idx = subscribers.indexOf(sub);
+    if (idx >= 0) subscribers.splice(idx, 1);
+  };
+}
+
+/**
+ * Returns a reactive signal that tracks a message by the given filter predicate
+ * The signal has the value of the last message where filter was true
+ * null in case no message was recieved yet
+ */
+export function useNotify<T extends ProcessMessage = ProcessMessage>(
+  filter: (msg: T) => boolean = () => true as boolean,
+) {
+  const [message, setMessage] = createSignal<T | null>(null);
+
+  const unsubscribe = _subscribeNotify(filter, (msg) => setMessage(() => msg));
+
+  onCleanup(unsubscribe);
+
+  return message;
+}
+
+/**
+ * Tracks any message that was sent from this api origin
+ *
+ */
+export function useNotifyOrigin<
+  T extends ProcessMessage = ProcessMessage,
+  K extends OperationNames = OperationNames,
+>(origin: K) {
+  return useNotify<T>((m) => m.origin === origin);
+}

pkgs/clan-app/ui/src/workflows/Install/install.tsx

@@ -5,8 +5,7 @@ import {
   StepperProvider,
   useStepper,
 } from "@/src/hooks/stepper";
-import { createForm, FieldValues, SubmitHandler } from "@modular-forms/solid";
+import { Show } from "solid-js";
-import { onMount, Show } from "solid-js";
 import { Dynamic } from "solid-js/web";
 import { initialSteps } from "./steps/Initial";
 import { createInstallerSteps } from "./steps/createInstaller";
@@ -20,11 +19,12 @@ const InstallStepper = (props: InstallStepperProps) => {
   const stepSignal = useStepper<InstallSteps>();
   const [store, set] = getStepStore<InstallStoreType>(stepSignal);
 
-  onMount(() => {
+  return (
-    set("done", props.onDone);
+    <Dynamic
-  });
+      component={stepSignal.currentStep().content}
-
+      onDone={props.onDone}
-  return <Dynamic component={stepSignal.currentStep().content} />;
+    />
+  );
 };
 
 export interface InstallModalProps {
@@ -53,8 +53,8 @@ export interface InstallStoreType {
     mainDisk: string;
     // ...TODO Vars
     progress: ApiCall<"run_machine_install">;
-
     promptValues: PromptValues;
+    prepareStep: "disk" | "generators" | "install";
   };
   done: () => void;
 }
@@ -84,6 +84,7 @@ export const InstallModal = (props: InstallModalProps) => {
   return (
     <StepperProvider stepper={stepper}>
       <Modal
+        class="h-[30rem] w-screen max-w-3xl"
         mount={props.mount}
         title="Install machine"
         onClose={() => {

pkgs/clan-app/ui/src/workflows/Install/steps/Initial.tsx

@@ -12,8 +12,8 @@ const ChoiceLocalOrRemote = () => {
         <div class="flex justify-between gap-2">
           <div class="flex flex-col justify-center gap-1 px-1">
             <Typography
-              hierarchy="label"
+              hierarchy="body"
-              size="xs"
+              size="default"
               weight="bold"
               color="primary"
             >
@@ -33,8 +33,8 @@ const ChoiceLocalOrRemote = () => {
         <div class="flex justify-between gap-2">
           <div class="flex flex-col justify-center gap-1 px-1">
             <Typography
-              hierarchy="label"
+              hierarchy="body"
-              size="xs"
+              size="default"
               weight="bold"
               color="primary"
             >
@@ -64,8 +64,8 @@ const ChoiceLocalInstaller = () => {
             <div class="flex justify-between gap-2">
               <div class="flex flex-col justify-center gap-1 px-1">
                 <Typography
-                  hierarchy="label"
+                  hierarchy="body"
-                  size="xs"
+                  size="default"
                   weight="bold"
                   color="primary"
                 >
@@ -85,8 +85,8 @@ const ChoiceLocalInstaller = () => {
             <div class="flex justify-between gap-2">
               <div class="flex flex-col justify-center gap-1 px-1">
                 <Typography
-                  hierarchy="label"
+                  hierarchy="body"
-                  size="xs"
+                  size="default"
                   weight="bold"
                   color="primary"
                 >

pkgs/clan-app/ui/src/workflows/Install/steps/createInstaller.tsx

@@ -149,7 +149,7 @@ const ConfigureImage = () => {
   let content: Node;
 
   return (
-    <Form onSubmit={handleSubmit}>
+    <Form onSubmit={handleSubmit} class="h-full">
       <StepLayout
         body={
           <div
@@ -241,7 +241,7 @@ const ChooseDisk = () => {
 
   const stripId = (s: string) => s.split("-")[1] ?? s;
   return (
-    <Form onSubmit={handleSubmit}>
+    <Form onSubmit={handleSubmit} class="h-full">
       <StepLayout
         body={
           <div class="flex flex-col gap-2">
@@ -317,7 +317,7 @@ const FlashProgress = () => {
   };
 
   return (
-    <div class="flex h-60 w-full flex-col items-center justify-end bg-inv-4">
+    <div class="flex size-full h-60 flex-col items-center justify-end bg-inv-4">
       <div class="mb-6 flex w-full max-w-md flex-col items-center gap-3 fg-inv-1">
         <Typography
           hierarchy="title"
@@ -343,7 +343,7 @@ const FlashProgress = () => {
 const FlashDone = () => {
   const stepSignal = useStepper<InstallSteps>();
   return (
-    <div class="flex w-full flex-col items-center bg-inv-4">
+    <div class="flex size-full flex-col items-center justify-between bg-inv-4">
       <div class="flex w-full max-w-md flex-col items-center gap-3 py-6 fg-inv-1">
         <div class="rounded-full bg-semantic-success-4">
           <Icon icon="Checkmark" class="size-9" />
@@ -361,15 +361,15 @@ const FlashDone = () => {
           title="Remove it and plug it into the machine that you want to install."
           description=""
         />
-        <div class="mt-3 flex w-full justify-end">
+      </div>
-          <Button
+      <div class="flex w-full justify-end p-6">
-            hierarchy="primary"
+        <Button
-            endIcon="ArrowRight"
+          hierarchy="primary"
-            onClick={() => stepSignal.next()}
+          endIcon="ArrowRight"
-          >
+          onClick={() => stepSignal.next()}
-            Next
+        >
-          </Button>
+          Next
-        </div>
+        </Button>
       </div>
     </div>
   );

pkgs/clan-app/ui/src/workflows/Install/steps/installSteps.tsx

@@ -2,6 +2,7 @@ import { Typography } from "@/src/components/Typography/Typography";
 import { BackButton, NextButton, StepLayout } from "../../Steps";
 import {
   createForm,
+  FieldValues,
   getError,
   SubmitHandler,
   valiForm,
@@ -12,7 +13,7 @@ import { getStepStore, useStepper } from "@/src/hooks/stepper";
 import { InstallSteps, InstallStoreType, PromptValues } from "../install";
 import { TextInput } from "@/src/components/Form/TextInput";
 import { Alert } from "@/src/components/Alert/Alert";
-import { For, onMount, Show } from "solid-js";
+import { For, Match, Show, Switch } from "solid-js";
 import { Divider } from "@/src/components/Divider/Divider";
 import { Orienter } from "@/src/components/Form/Orienter";
 import { Button } from "@/src/components/Button/Button";
@@ -27,6 +28,7 @@ import {
 } from "@/src/hooks/queries";
 import { useClanURI } from "@/src/hooks/clan";
 import { useApiClient } from "@/src/hooks/ApiClient";
+import { ProcessMessage, useNotifyOrigin } from "@/src/hooks/notify";
 
 export const InstallHeader = (props: { machineName: string }) => {
   return (
@@ -72,7 +74,7 @@ const ConfigureAddress = () => {
   };
 
   return (
-    <Form onSubmit={handleSubmit}>
+    <Form onSubmit={handleSubmit} class="h-full">
       <StepLayout
         body={
           <div class="flex flex-col gap-2">
@@ -230,7 +232,7 @@ const ConfigureDisk = () => {
   );
 
   return (
-    <Form onSubmit={handleSubmit}>
+    <Form onSubmit={handleSubmit} class="h-full">
       <StepLayout
         body={
           <div class="flex flex-col gap-2">
@@ -302,19 +304,19 @@ const ConfigureData = () => {
   );
 };
 
-type PromptGroup = {
+interface PromptGroup {
   name: string;
   fields: {
     prompt: Prompt;
     generator: string;
     value?: string | null;
   }[];
-};
+}
 
 type Prompt = NonNullable<MachineGenerators[number]["prompts"]>[number];
-type PromptForm = {
+interface PromptForm extends FieldValues {
   promptValues: PromptValues;
-};
+}
 
 interface PromptsFieldsProps {
   generators: MachineGenerators;
@@ -365,7 +367,7 @@ const PromptsFields = (props: PromptsFieldsProps) => {
   };
 
   return (
-    <Form onSubmit={handleSubmit}>
+    <Form onSubmit={handleSubmit} class="h-full">
       <StepLayout
         body={
           <div class="flex flex-col gap-2">
@@ -449,6 +451,8 @@ const InstallSummary = () => {
     // Here you would typically trigger the installation process
     console.log("Installation started");
 
+    stepSignal.setActiveStep("install:progress");
+
     const setDisk = client.fetch("set_machine_disk_schema", {
       machine: {
         flake: {
@@ -463,6 +467,10 @@ const InstallSummary = () => {
       force: true,
     });
 
+    set("install", (s) => ({
+      ...s,
+      prepareStep: "disk",
+    }));
     const diskResult = await setDisk.result; // Wait for the disk schema to be set
     if (diskResult.status === "error") {
       console.error("Error setting disk schema:", diskResult.errors);
@@ -474,8 +482,11 @@ const InstallSummary = () => {
       base_dir: clanUri,
       machine_name: store.install.machineName,
     });
-    stepSignal.setActiveStep("install:progress");
 
+    set("install", (s) => ({
+      ...s,
+      prepareStep: "generators",
+    }));
     await runGenerators.result; // Wait for the generators to run
 
     const runInstall = client.fetch("run_machine_install", {
@@ -493,6 +504,7 @@ const InstallSummary = () => {
     });
     set("install", (s) => ({
       ...s,
+      prepareStep: "install",
       progress: runInstall,
     }));
 
@@ -533,6 +545,8 @@ const InstallSummary = () => {
   );
 };
 
+type InstallTopic = "generators" | "upload-secrets" | "nixos-anywhere";
+
 const InstallProgress = () => {
   const stepSignal = useStepper<InstallSteps>();
   const [store, get] = getStepStore<InstallStoreType>(stepSignal);
@@ -542,10 +556,14 @@ const InstallProgress = () => {
     if (progress) {
       await progress.cancel();
     }
-    store.done();
+    stepSignal.previous();
   };
+  const installState = useNotifyOrigin<ProcessMessage<unknown, InstallTopic>>(
+    "run_machine_install",
+  );
+
   return (
-    <div class="flex h-60 w-full flex-col items-center justify-end bg-inv-4">
+    <div class="flex size-full h-60 flex-col items-center justify-end bg-inv-4">
       <div class="mb-6 flex w-full max-w-md flex-col items-center gap-3 fg-inv-1">
         <Typography
           hierarchy="title"
@@ -555,6 +573,36 @@ const InstallProgress = () => {
         >
           Machine is beeing installed
         </Typography>
+        <Typography
+          hierarchy="label"
+          size="default"
+          class="py-2"
+          color="secondary"
+          inverted
+        >
+          <Switch fallback={"Waiting for preparation to start..."}>
+            <Match when={store.install.prepareStep === "disk"}>
+              Configuring disk schema ...
+            </Match>
+            <Match when={store.install.prepareStep === "generators"}>
+              Provisioning services ...
+            </Match>
+            <Match when={store.install.prepareStep === "install"}>
+              {/* Progress after the run_machine_install api call */}
+              <Switch fallback={"Waiting for installation to start..."}>
+                <Match when={installState()?.topic === "generators"}>
+                  Checking services ...
+                </Match>
+                <Match when={installState()?.topic === "upload-secrets"}>
+                  Uploading Credentials ...
+                </Match>
+                <Match when={installState()?.topic === "nixos-anywhere"}>
+                  Running nixos-anywhere ...
+                </Match>
+              </Switch>
+            </Match>
+          </Switch>
+        </Typography>
         <LoadingBar />
         <Button
           hierarchy="primary"
@@ -569,7 +617,10 @@ const InstallProgress = () => {
   );
 };
 
-const FlashDone = () => {
+interface InstallDoneProps {
+  onDone: () => void;
+}
+const InstallDone = (props: InstallDoneProps) => {
   const stepSignal = useStepper<InstallSteps>();
   const [store, get] = getStepStore<InstallStoreType>(stepSignal);
 
@@ -592,7 +643,7 @@ const FlashDone = () => {
             hierarchy="primary"
             endIcon="Close"
             size="s"
-            onClick={() => store.done()}
+            onClick={() => props.onDone()}
           >
             Close
           </Button>
@@ -639,7 +690,7 @@ export const installSteps = [
   },
   {
     id: "install:done",
-    content: FlashDone,
+    content: InstallDone,
     isSplash: true,
   },
 ] as const;

pkgs/clan-app/ui/src/workflows/Steps.tsx

@@ -9,7 +9,7 @@ interface StepLayoutProps {
 }
 export const StepLayout = (props: StepLayoutProps) => {
   return (
-    <div class="flex flex-col gap-6">
+    <div class="flex size-full grow flex-col justify-between gap-6">
       {props.body}
       {props.footer}
     </div>

pkgs/clan-cli/clan_lib/api/__init__.py

@@ -5,11 +5,13 @@ from collections.abc import Callable
 from dataclasses import dataclass
 from functools import wraps
 from inspect import Parameter, Signature, signature
+from queue import Queue
 from types import ModuleType
 from typing import (
     Annotated,
     Any,
     Literal,
+    TypedDict,
     TypeVar,
     get_type_hints,
 )
@@ -31,6 +33,30 @@ T = TypeVar("T")
 ResponseDataType = TypeVar("ResponseDataType")
 
 
+class ProcessMessage(TypedDict):
+    """
+    Represents a message to be sent to the UI.
+
+    Attributes:
+    - topic: The topic of the message, used to identify the type of message.
+    - data: The data to be sent with the message.
+    - origin: The API operation that this message is related to, if applicable.
+    """
+
+    topic: str
+    data: Any
+    origin: str | None
+
+
+message_queue: Queue[ProcessMessage] = Queue()
+"""
+A global message queue for sending messages to the UI
+This can be used to send notifications or messages to the UI. Before returning a response.
+
+The clan-app imports the queue as clan_lib.api.message_queue and subscribes to it.
+"""
+
+
 @dataclass
 class ApiError:
     message: str

pkgs/clan-cli/clan_lib/machines/install.py

@@ -9,7 +9,7 @@ from clan_cli.facts.generate import generate_facts
 from clan_cli.machines.hardware import HardwareConfig
 from clan_cli.vars.generate import generate_vars
 
-from clan_lib.api import API
+from clan_lib.api import API, message_queue
 from clan_lib.cmd import Log, RunOpts, run
 from clan_lib.machines.machines import Machine
 from clan_lib.nix import nix_config, nix_shell
@@ -22,6 +22,20 @@ log = logging.getLogger(__name__)
 BuildOn = Literal["auto", "local", "remote"]
 
 
+Step = Literal["generators", "upload-secrets", "nixos-anywhere"]
+
+
+def notify_install_step(current: Step) -> None:
+    message_queue.put(
+        {
+            "topic": current,
+            "data": None,
+            # MUST be set the to api function name, while technically you can set any origin, this is a bad idea.
+            "origin": "run_machine_install",
+        }
+    )
+
+
 @dataclass
 class InstallOptions:
     machine: Machine
@@ -64,6 +78,8 @@ def run_machine_install(opts: InstallOptions, target_host: Remote) -> None:
         ]
     )
 
+    # Notify the UI about what we are doing
+    notify_install_step("generators")
     generate_facts([machine])
     generate_vars([machine])
 
@@ -74,6 +90,9 @@ def run_machine_install(opts: InstallOptions, target_host: Remote) -> None:
         activation_secrets = base_directory / "activation_secrets"
         upload_dir = activation_secrets / machine.secrets_upload_directory.lstrip("/")
         upload_dir.mkdir(parents=True)
+
+        # Notify the UI about what we are doing
+        notify_install_step("upload-secrets")
         machine.secret_facts_store.upload(upload_dir)
         machine.secret_vars_store.populate_dir(
             machine.name, upload_dir, phases=["activation", "users", "services"]
@@ -174,4 +193,6 @@ def run_machine_install(opts: InstallOptions, target_host: Remote) -> None:
                 ["nixos-anywhere"],
                 cmd,
             )
+
+        notify_install_step("nixos-anywhere")
         run(cmd, RunOpts(log=Log.BOTH, prefix=machine.name, needs_user_terminal=True))