Merge pull request 'Serde: improve js-python bridge' (#2258) from hsjobeki/clan-core:hsjobeki-main into main

pkgs/clan-app/clan_app/views/webview.py

@@ -1,5 +1,6 @@
 import json
 import logging
+from pathlib import Path
 from typing import Any
 
 import gi
@@ -9,7 +10,7 @@ from clan_app.api import GObjApi, GResult, ImplFunc
 from clan_app.api.file import open_file
 
 gi.require_version("WebKit", "6.0")
-from gi.repository import GLib, GObject, WebKit
+from gi.repository import Gio, GLib, GObject, WebKit
 
 log = logging.getLogger(__name__)
 
@@ -122,21 +123,46 @@ class WebExecutor(GObject.Object):
 
     def on_result(self, source: ImplFunc, data: GResult) -> None:
         result = dataclass_to_dict(data.result)
-        serialized = json.dumps(result, indent=4)
+        # Important:
+        # 2. ensure_ascii = False. non-ASCII characters are correctly handled, instead of being escaped.
+        serialized = json.dumps(result, indent=4, ensure_ascii=False)
         log.debug(f"Result for {data.method_name}: {serialized}")
 
         # Use idle_add to queue the response call to js on the main GTK thread
         self.return_data_to_js(data.method_name, serialized)
 
     def return_data_to_js(self, method_name: str, serialized: str) -> bool:
+        js = f"""
+        window.clan.{method_name}({serialized});
+        """
+
+        def dump_failed_code() -> None:
+            tmp_file = Path("/tmp/clan-pyjs-bridge-error.js")
+            with tmp_file.open("w") as f:
+                f.write(js)
+            log.debug(f"Failed code dumped in JS file: {tmp_file}")
+
+        # Error handling if the JavaScript evaluation fails
+        def on_js_evaluation_finished(
+            webview: WebKit.WebView, task: Gio.AsyncResult
+        ) -> None:
+            try:
+                # Get the result of the JavaScript evaluation
+                value = webview.evaluate_javascript_finish(task)
+                if not value:
+                    log.exception("No value returned")
+                    dump_failed_code()
+            except GLib.Error:
+                log.exception("Error evaluating JS")
+                dump_failed_code()
+
         self.webview.evaluate_javascript(
-            f"""
+            js,
-            window.clan.{method_name}(`{serialized}`);
-            """,
             -1,
             None,
             None,
             None,
+            on_js_evaluation_finished,
         )
         return GLib.SOURCE_REMOVE
 

pkgs/clan-cli/clan_cli/api/modules.py

@@ -134,9 +134,10 @@ def get_roles(module_path: Path) -> None | list[str]:
 @dataclass
 class ModuleInfo:
     description: str
-    categories: list[str] | None
+    readme: str
+    categories: list[str]
     roles: list[str] | None
-    readme: str | None = None
+    features: list[str] = field(default_factory=list)
 
 
 def get_modules(base_path: str) -> dict[str, str]:
@@ -179,7 +180,7 @@ def get_module_info(
     """
     Retrieves information about a module
     """
-    if not module_path:
+    if not module_path.exists():
         msg = "Module not found"
         raise ClanError(
             msg,
@@ -205,6 +206,7 @@ def get_module_info(
         categories=frontmatter.categories,
         roles=get_roles(module_path),
         readme=readme_content,
+        features=["inventory"] if has_inventory_feature(module_path) else [],
     )
 
 

pkgs/clan-cli/clan_cli/api/serde.py

@@ -30,7 +30,6 @@ Note: This module assumes the presence of other modules and classes such as `Cla
 """
 
 import dataclasses
-import json
 from dataclasses import dataclass, fields, is_dataclass
 from enum import Enum
 from pathlib import Path
@@ -51,7 +50,8 @@ from clan_cli.errors import ClanError
 def sanitize_string(s: str) -> str:
     # Using the native string sanitizer to handle all edge cases
     # Remove the outer quotes '"string"'
-    return json.dumps(s)[1:-1]
+    # return json.dumps(s)[1:-1]
+    return s
 
 
 def dataclass_to_dict(obj: Any, *, use_alias: bool = True) -> Any:
@@ -161,7 +161,7 @@ def construct_value(
             msg = f"Expected string, got {field_value}"
             raise ClanError(msg, location=f"{loc}")
 
-        return json.loads(f'"{field_value}"')
+        return field_value
 
     if t is int and not isinstance(field_value, str):
         return int(field_value)  # type: ignore

pkgs/clan-cli/tests/test_deserializers.py

@@ -217,8 +217,8 @@ def test_none_or_string() -> None:
 
 
 def test_roundtrip_escape() -> None:
-    assert from_dict(str, "\\n") == "\n"
+    assert from_dict(str, "\n") == "\n"
-    assert dataclass_to_dict("\n") == "\\n"
+    assert dataclass_to_dict("\n") == "\n"
 
     # Test that the functions are inverses of each other
     # f(g(x)) == x

pkgs/clan-cli/tests/test_serializers.py

@@ -11,27 +11,25 @@ from clan_cli.api import (
 def test_sanitize_string() -> None:
     # Simple strings
     assert sanitize_string("Hello World") == "Hello World"
-    assert sanitize_string("Hello\nWorld") == "Hello\\nWorld"
+    assert sanitize_string("Hello\nWorld") == "Hello\nWorld"
-    assert sanitize_string("Hello\tWorld") == "Hello\\tWorld"
+    assert sanitize_string("Hello\tWorld") == "Hello\tWorld"
-    assert sanitize_string("Hello\rWorld") == "Hello\\rWorld"
+    assert sanitize_string("Hello\rWorld") == "Hello\rWorld"
-    assert sanitize_string("Hello\fWorld") == "Hello\\fWorld"
+    assert sanitize_string("Hello\fWorld") == "Hello\fWorld"
-    assert sanitize_string("Hello\vWorld") == "Hello\\u000bWorld"
+    assert sanitize_string("Hello\vWorld") == "Hello\u000bWorld"
-    assert sanitize_string("Hello\bWorld") == "Hello\\bWorld"
+    assert sanitize_string("Hello\bWorld") == "Hello\bWorld"
-    assert sanitize_string("Hello\\World") == "Hello\\\\World"
+    assert sanitize_string("Hello\\World") == "Hello\\World"
-    assert sanitize_string('Hello"World') == 'Hello\\"World'
+    assert sanitize_string('Hello"World') == 'Hello"World'
     assert sanitize_string("Hello'World") == "Hello'World"
-    assert sanitize_string("Hello\0World") == "Hello\\u0000World"
+    assert sanitize_string("Hello\0World") == "Hello\x00World"
     # Console escape characters
 
-    assert sanitize_string("\033[1mBold\033[0m") == "\\u001b[1mBold\\u001b[0m"  # Red
+    assert sanitize_string("\033[1mBold\033[0m") == "\033[1mBold\033[0m"  # Red
-    assert sanitize_string("\033[31mRed\033[0m") == "\\u001b[31mRed\\u001b[0m"  # Blue
+    assert sanitize_string("\033[31mRed\033[0m") == "\033[31mRed\033[0m"  # Blue
-    assert (
+    assert sanitize_string("\033[42mGreen\033[0m") == "\033[42mGreen\033[0m"  # Green
-        sanitize_string("\033[42mGreen\033[0m") == "\\u001b[42mGreen\\u001b[0m"
+    assert sanitize_string("\033[4mUnderline\033[0m") == "\033[4mUnderline\033[0m"
-    )  # Green
-    assert sanitize_string("\033[4mUnderline\033[0m") == "\\u001b[4mUnderline\\u001b[0m"
     assert (
         sanitize_string("\033[91m\033[1mBold Red\033[0m")
-        == "\\u001b[91m\\u001b[1mBold Red\\u001b[0m"
+        == "\033[91m\033[1mBold Red\033[0m"
     )
 
 

pkgs/webview-ui/app/package.json

@@ -45,6 +45,7 @@
     "material-icons": "^1.13.12",
     "nanoid": "^5.0.7",
     "solid-js": "^1.8.11",
+    "solid-markdown": "^2.0.13",
     "solid-toast": "^0.5.0"
   }
 }

pkgs/webview-ui/app/src/Sidebar.tsx

@@ -6,7 +6,7 @@ import { A, RouteSectionProps } from "@solidjs/router";
 import { AppRoute, routes } from "./index";
 
 export const Sidebar = (props: RouteSectionProps) => {
-  const query = createQuery(() => ({
+  const clanQuery = createQuery(() => ({
     queryKey: [activeURI(), "meta"],
     queryFn: async () => {
       const curr = activeURI();
@@ -21,8 +21,8 @@ export const Sidebar = (props: RouteSectionProps) => {
   return (
     <aside class="w-80 rounded-xl border border-slate-900 bg-slate-800  pb-10">
       <div class="m-4 flex flex-col text-center capitalize text-white">
-        <span class="text-lg">{query.data?.name}</span>
+        <span class="text-lg">{clanQuery.data?.name}</span>
-        <span class="text-sm">{query.data?.description}</span>
+        <span class="text-sm">{clanQuery.data?.description}</span>
         <RouteMenu class="menu px-4 py-2" routes={routes} />
       </div>
     </aside>

pkgs/webview-ui/app/src/api/index.ts

@@ -150,14 +150,13 @@ export const callApi = <K extends OperationNames>(
 
 const deserialize =
   <T>(fn: (response: T) => void) =>
-  (str: string) => {
+  (r: unknown) => {
     try {
-      const r = JSON.parse(str) as T;
+      fn(r as T);
-      fn(r);
     } catch (e) {
-      console.log("Error parsing JSON: ", e);
+      console.error("Error parsing JSON: ", e);
-      window.localStorage.setItem("error", str);
+      window.localStorage.setItem("error", JSON.stringify(r));
-      console.error(str);
+      console.error(r);
       console.error("See localStorage 'error'");
       alert(`Error parsing JSON: ${e}`);
     }

pkgs/webview-ui/app/src/index.tsx

@@ -15,6 +15,8 @@ import { Flash } from "./routes/flash/view";
 import { HostList } from "./routes/hosts/view";
 import { Welcome } from "./routes/welcome";
 import { Toaster } from "solid-toast";
+import { ModuleList } from "./routes/modules/list";
+import { ModuleDetails } from "./routes/modules/details";
 
 export const client = new QueryClient();
 
@@ -88,6 +90,24 @@ export const routes: AppRoute[] = [
       },
     ],
   },
+  {
+    path: "/modules",
+    label: "Modules",
+    icon: "apps",
+    children: [
+      {
+        path: "/",
+        label: "App Store",
+        component: () => <ModuleList />,
+      },
+      {
+        path: "/:id",
+        label: "Details",
+        hidden: true,
+        component: () => <ModuleDetails />,
+      },
+    ],
+  },
   {
     path: "/tools",
     label: "Tools",

pkgs/webview-ui/app/src/queries/index.ts

@@ -0,0 +1,25 @@
+import { createQuery } from "@tanstack/solid-query";
+import { callApi } from "../api";
+import toast from "solid-toast";
+
+export const createModulesQuery = (uri: string | null) =>
+  createQuery(() => ({
+    queryKey: [uri, "list_modules"],
+    placeholderData: [],
+    enabled: !!uri,
+    queryFn: async () => {
+      console.log({ uri });
+      if (uri) {
+        const response = await callApi("list_modules", {
+          base_path: uri,
+        });
+        console.log({ response });
+        if (response.status === "error") {
+          toast.error("Failed to fetch data");
+        } else {
+          return Object.entries(response.data);
+        }
+      }
+      return [];
+    },
+  }));

pkgs/webview-ui/app/src/routes/modules/details.tsx

@@ -0,0 +1,109 @@
+import { callApi, SuccessData } from "@/src/api";
+import { activeURI } from "@/src/App";
+import { BackButton } from "@/src/components/BackButton";
+import { createModulesQuery } from "@/src/queries";
+import { useParams } from "@solidjs/router";
+import { For, Match, Switch } from "solid-js";
+import { SolidMarkdown } from "solid-markdown";
+import toast from "solid-toast";
+import { ModuleInfo } from "./list";
+
+export const ModuleDetails = () => {
+  const params = useParams();
+  const modulesQuery = createModulesQuery(activeURI());
+
+  return (
+    <div class="p-1">
+      <BackButton />
+      <div class="p-2">
+        <h3 class="text-2xl">{params.id}</h3>
+        <Switch>
+          <Match when={modulesQuery.data?.find((i) => i[0] === params.id)}>
+            {(d) => <Details data={d()[1]} id={d()[0]} />}
+          </Match>
+        </Switch>
+      </div>
+    </div>
+  );
+};
+
+function deepMerge(
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  obj1: Record<string, any>,
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  obj2: Record<string, any>,
+) {
+  const result = { ...obj1 };
+
+  for (const key in obj2) {
+    if (Object.prototype.hasOwnProperty.call(obj2, key)) {
+      if (obj2[key] instanceof Object && obj1[key] instanceof Object) {
+        result[key] = deepMerge(obj1[key], obj2[key]);
+      } else {
+        result[key] = obj2[key];
+      }
+    }
+  }
+
+  return result;
+}
+
+interface DetailsProps {
+  data: ModuleInfo;
+  id: string;
+}
+const Details = (props: DetailsProps) => {
+  return (
+    <div class="flex w-full flex-col gap-2">
+      <article class="prose">{props.data.description}</article>
+      <span class="label-text">Categories</span>
+      <div>
+        <For each={props.data.categories}>
+          {(c) => <div class="badge badge-primary m-1">{c}</div>}
+        </For>
+      </div>
+      <span class="label-text">Roles</span>
+      <div>
+        <For each={props.data.roles}>
+          {(r) => <div class="badge badge-secondary m-1">{r}</div>}
+        </For>
+      </div>
+      <div class="p-2">
+        <SolidMarkdown>{props.data.readme}</SolidMarkdown>
+      </div>
+      <div class="my-2 flex w-full gap-2">
+        <button
+          class="btn btn-primary"
+          onClick={async () => {
+            const uri = activeURI();
+            if (!uri) return;
+            const res = await callApi("get_inventory", { base_path: uri });
+            if (res.status === "error") {
+              toast.error("Failed to fetch inventory");
+              return;
+            }
+            const inventory = res.data;
+            const newInventory = deepMerge(inventory, {
+              services: {
+                [props.id]: {
+                  default: {
+                    enabled: false,
+                  },
+                },
+              },
+            });
+
+            callApi("set_inventory", {
+              flake_dir: uri,
+              inventory: newInventory,
+              message: `Add module: ${props.id} in 'default' instance`,
+            });
+          }}
+        >
+          <span class="material-icons ">add</span>
+          Add to Clan
+        </button>
+      </div>
+    </div>
+  );
+};

pkgs/webview-ui/app/src/routes/modules/list.tsx

@@ -0,0 +1,45 @@
+import { callApi, SuccessData } from "@/src/api";
+import { activeURI } from "@/src/App";
+import { createModulesQuery } from "@/src/queries";
+import { A, useNavigate } from "@solidjs/router";
+import { createQuery, useQueryClient } from "@tanstack/solid-query";
+import { createEffect, For, Match, Switch } from "solid-js";
+import { SolidMarkdown } from "solid-markdown";
+
+export type ModuleInfo = SuccessData<"list_modules">[string];
+
+const ModuleListItem = (props: { name: string; info: ModuleInfo }) => {
+  const { name, info } = props;
+  const navigate = useNavigate();
+
+  return (
+    <div class="stat">
+      <div class="stat-figure text-primary">
+        <div class="join">more</div>
+      </div>
+
+      <A href={`/modules/${name}`}>
+        <div class="stat-value underline">{name}</div>
+      </A>
+
+      <div>{info.description}</div>
+    </div>
+  );
+};
+
+export const ModuleList = () => {
+  const modulesQuery = createModulesQuery(activeURI());
+  return (
+    <Switch fallback="Shit">
+      <Match when={modulesQuery.isLoading}>Loading....</Match>
+      <Match when={modulesQuery.data}>
+        <div>
+          Show Modules
+          <For each={modulesQuery.data}>
+            {([k, v]) => <ModuleListItem info={v} name={k} />}
+          </For>
+        </div>
+      </Match>
+    </Switch>
+  );
+};