Merge pull request 'feat: use process-compose to run ui dev environment' (#3646) from feat/web-ui-process-compose into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3646

devShell.nix

@@ -1,4 +1,4 @@
-{ ... }:
+{ inputs, ... }:
 {
   perSystem =
     {
@@ -39,6 +39,10 @@
         shellHook = ''
           echo -e "${ansiEscapes.green}switch to another dev-shell using: select-shell${ansiEscapes.reset}"
           export PRJ_ROOT=$(git rev-parse --show-toplevel)
+
+          # vendoring / needed for impure tests
+          ln -sfT ${self'.packages.clan-cli.nixpkgs} "$PRJ_ROOT/pkgs/clan-cli/clan_cli/nixpkgs"
+          ln -sfT ${inputs.nix-select} "$PRJ_ROOT/pkgs/clan-cli/clan_cli/select"
         '';
       };
     };

docs/nix/default.nix

@@ -45,7 +45,7 @@ pkgs.stdenv.mkDerivation {
     ln -snf ${fira-code}/share/fonts/truetype/FiraCode-VF.ttf ./site/static/
 
     # Copy icons into place
-    cp -af ../pkgs/webview-ui/app/icons ./site/static/
+    cp -af ../pkgs/ui/webview-ui/app/icons ./site/static/
   '';
 
   buildPhase = ''

formatter.nix

@@ -100,9 +100,9 @@
             extraPythonPackages = (self'.packages.clan-cli.devshellPyDeps pkgs.python3Packages);
           };
           "clan-app" = {
-            directory = "pkgs/clan-app";
+            directory = "pkgs/ui/clan-app";
             extraPythonPackages = (self'.packages.clan-app.devshellPyDeps pkgs.python3Packages);
-            extraPythonPaths = [ "../clan-cli" ];
+            extraPythonPaths = [ "../../clan-cli" ];
           };
         }
         // (

pkgs/clan-cli/flake-module.nix

@@ -35,9 +35,8 @@
     in
     {
       devShells.clan-cli = pkgs.callPackage ./shell.nix {
-        inherit (self'.packages) clan-cli;
         inherit self';
-        inherit (inputs) nix-select;
+        inherit (self'.packages) clan-cli;
       };
       packages = {
         clan-cli = pkgs.callPackage ./default.nix {

pkgs/clan-cli/shell.nix

@@ -1,7 +1,6 @@
 {
   lib,
   nix-unit,
-  nix-select,
   clan-cli,
   mkShell,
   ruff,
@@ -42,10 +41,6 @@ mkShell {
     # Add clan command to PATH
     export PATH="$PKG_ROOT/bin":"$PATH"
 
-    # Needed for impure tests
-    ln -sfT ${clan-cli.nixpkgs} "$PKG_ROOT/clan_cli/nixpkgs"
-    ln -sfT ${nix-select} "$PKG_ROOT/clan_cli/select"
-
     # Generate classes.py from inventory schema
     # This file is in .gitignore
     ${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json $PKG_ROOT/clan_lib/nix_models/inventory.py

pkgs/flake-module.nix

@@ -3,14 +3,13 @@
 {
   imports = [
     ./clan-cli/flake-module.nix
-    ./clan-app/flake-module.nix
     ./clan-vm-manager/flake-module.nix
     ./installer/flake-module.nix
-    ./webview-ui/flake-module.nix
     ./distro-packages/flake-module.nix
     ./icon-update/flake-module.nix
     ./generate-test-vars/flake-module.nix
     ./clan-core-flake/flake-module.nix
+    ./ui/flake-module.nix
   ];
 
   flake.packages.x86_64-linux =
@@ -35,7 +34,6 @@
         editor = pkgs.callPackage ./editor/clan-edit-codium.nix { };
         classgen = pkgs.callPackage ./classgen { };
         zerotierone = pkgs.callPackage ./zerotierone { };
-        webview-lib = pkgs.callPackage ./webview-lib { };
         update-clan-core-for-checks = pkgs.callPackage ./update-clan-core-for-checks { };
       };
     };

pkgs/icon-update/default.nix

@@ -13,7 +13,7 @@ pkgs.writeShellApplication {
 
   text = ''
     REPO_ROOT="$(git rev-parse --show-toplevel)"
-    OUT_DIR="$(realpath "$REPO_ROOT"/pkgs/webview-ui/app/icons)"
+    OUT_DIR="$(realpath "$REPO_ROOT"/pkgs/ui/webview-ui/app/icons)"
     export OUT_DIR
     deno run --allow-all ${src}/main.ts
   '';

pkgs/ui/.envrc

@@ -0,0 +1,7 @@
+# shellcheck shell=bash
+source_up
+
+watch_file flake-module.nix shell.nix clan-app/flake-module.nix webview-ui/flake-module.nix
+
+# Because we depend on nixpkgs sources, uploading to builders takes a long time
+use flake .#ui --builders ''

pkgs/ui/README.md

@@ -0,0 +1,48 @@
+# Clan UI
+
+## Getting started
+
+Enter the `pkgs/ui` directory and allow [direnv] to load the `ui` devshell with `direnv allow`:
+
+```console
+❯ direnv allow
+direnv: loading ~/Development/lol/clan/git/clan/clan-core/pkgs/ui/.envrc                                                                                                                                                                  
+direnv: loading ~/Development/lol/clan/git/clan/clan-core/.envrc
+direnv: using flake
+direnv: nix-direnv: Renewed cache
+switch to another dev-shell using: select-shell
+direnv: using flake .#ui --builders 
+path '/home/brian/Development/lol/clan/git/clan/clan-core/pkgs/ui' does not contain a 'flake.nix', searching up
+direnv: ([/nix/store/rjnigckx9rmga58562hxw9kr5hynavcd-direnv-2.36.0/bin/direnv export zsh]) is taking a while to execute. Use CTRL-C to give up.
+path '/home/brian/Development/lol/clan/git/clan/clan-core/pkgs/ui' does not contain a 'flake.nix', searching up
+direnv: nix-direnv: Renewed cache
+switch to another dev-shell using: select-shell
+/home/brian/.config/direnv/lib/hm-nix-direnv.sh:3858: /home/brian/Development/lol/clan/git/clan/clan-core/pkgs/ui/clan-app/.local.env: No such file or directory
+direnv: export +AR +AS +CC +CLAN_CORE_PATH +CONFIG_SHELL +CXX +DETERMINISTIC_BUILD +GETTEXTDATADIRS +GETTEXTDATADIRS_FOR_BUILD +GETTEXTDATADIRS_FOR_TARGET +GIT_ROOT +GSETTINGS_SCHEMAS_PATH +HOST_PATH +IN_NIX_SHELL +LD +NIX_BINTOOLS +NIX_BINTOOLS_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu +NIX_BUILD_CORES +NIX_CC +NIX_CC_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu +NIX_CFLAGS_COMPILE +NIX_ENFORCE_NO_NATIVE +NIX_HARDENING_ENABLE +NIX_LDFLAGS +NIX_STORE +NM +NODE_PATH +OBJCOPY +OBJDUMP +PC_CONFIG_FILES +PKG_ROOT_CLAN_APP +PKG_ROOT_UI +PKG_ROOT_WEBVIEW_UI +PRJ_ROOT +PYTHONHASHSEED +PYTHONNOUSERSITE +PYTHONPATH +RANLIB +READELF +SIZE +SOURCE_DATE_EPOCH +STRINGS +STRIP +WEBVIEW_LIB_DIR +_PYTHON_HOST_PLATFORM +_PYTHON_SYSCONFIGDATA_NAME +__structuredAttrs +buildInputs +buildPhase +builder +cmakeFlags +configureFlags +depsBuildBuild +depsBuildBuildPropagated +depsBuildTarget +depsBuildTargetPropagated +depsHostHost +depsHostHostPropagated +depsTargetTarget +depsTargetTargetPropagated +doCheck +doInstallCheck +dontAddDisableDepTrack +mesonFlags +name +nativeBuildInputs +out +outputs +patches +phases +preferLocalBuild +propagatedBuildInputs +propagatedNativeBuildInputs +shell +shellHook +stdenv +strictDeps +system ~GDK_PIXBUF_MODULE_FILE ~GI_TYPELIB_PATH ~PATH ~XDG_DATA_DIRS
+```
+
+Once that has loaded, you can run the local dev environment by running `process-compose`. 
+
+This will start a [process-compose] instance containing two processes:
+
+* `webview-ui` which is a background process running a [vite] server for `./webview-ui` in a hot-reload fashion
+* `clan-app` which is a [foreground process](https://f1bonacc1.github.io/process-compose/launcher/?h=foreground#foreground-processes),
+that is started on demand and provides the [webview] wrapper for the UI.
+
+Wait for the `webview-ui` process to enter the `Running` state, then navigate to the `clan-app` process and press `F7`. 
+This will start the [webview] window and bring `clan-app`'s terminal into the foreground, allowing for interaction with
+the debugger if required. 
+
+If you need to restart, simply enter `ctrl+c` and you will be dropped back into the `process-compose` terminal. 
+From there you can start `clan-app` again with `F7`.
+
+> **Note**
+> If you are interacting with a breakpoint, do not continue/exit with `ctrl+c` as this will introduce a quirk
+> the next time you start `clan-app` where you will be unable to see the input you are typing in a debugging session. 
+>
+> Instead, exit the debugger with `q+Enter`. 
+
+[direnv]: https://direnv.net/
+[process-compose]: https://f1bonacc1.github.io/process-compose/
+[vite]: https://vite.dev/ 
+[webview]: https://github.com/webview/webview

pkgs/ui/clan-app/README.md

@@ -9,7 +9,7 @@ Follow the instructions below to set up your development environment and start t
 
 1. **Navigate to the Webview UI Directory**
 
-   Go to the `clan-core/pkgs/webview-ui/app` directory and start the web server by executing:
+   Go to the `clan-core/pkgs/ui/webview-ui/app` directory and start the web server by executing:
 
    ```bash
    npm install
@@ -26,7 +26,6 @@ Follow the instructions below to set up your development environment and start t
 
 This will start the application in debug mode and link it to the web server running at `http://localhost:3000`.
 
-
 ### Debugging Style and Layout
 
 ```bash
@@ -86,4 +85,4 @@ Here are some important documentation links related to the Clan App:
 
 > Error dialogs should be avoided where possible, since they are disruptive.
 >
-> For simple non-critical errors, toasts can be a good alternative.
+> For simple non-critical errors, toasts can be a good alternative.

pkgs/ui/clan-app/shell.nix

@@ -3,6 +3,7 @@
   mkShell,
   ruff,
   webview-lib,
+  process-compose,
   self',
 }:
 
@@ -11,7 +12,11 @@ mkShell {
 
   inputsFrom = [ self'.devShells.default ];
 
-  inherit (clan-app) nativeBuildInputs propagatedBuildInputs;
+  inherit (clan-app) propagatedBuildInputs;
+
+  nativeBuildInputs = clan-app.nativeBuildInputs ++ [
+    process-compose
+  ];
 
   buildInputs = [
     (clan-app.pythonRuntime.withPackages (
@@ -27,7 +32,7 @@ mkShell {
 
   shellHook = ''
     export GIT_ROOT=$(git rev-parse --show-toplevel)
-    export PKG_ROOT=$GIT_ROOT/pkgs/clan-app
+    export PKG_ROOT=$GIT_ROOT/pkgs/ui/clan-app
 
     export CLAN_CORE_PATH="$GIT_ROOT"
 

pkgs/ui/flake-module.nix

@@ -0,0 +1,25 @@
+{
+  imports = [
+    ./clan-app/flake-module.nix
+    ./webview-ui/flake-module.nix
+  ];
+
+  perSystem =
+    {
+      self',
+      pkgs,
+      config,
+      ...
+    }:
+    {
+      packages = {
+        webview-lib = pkgs.callPackage ./webview-lib { };
+      };
+
+      devShells.ui = pkgs.callPackage ./shell.nix {
+        inherit self';
+        inherit (self'.packages) clan-app webview-lib webview-ui;
+        inherit (config.packages) clan-ts-api;
+      };
+    };
+}

pkgs/ui/process-compose.yaml

@@ -0,0 +1,24 @@
+version: "0.5"
+
+env_cmds:
+  PRJ_ROOT: "git rev-parse --show-toplevel"
+
+processes:
+  webview-ui:
+    command: |
+      direnv allow
+      direnv exec . npm install
+      direnv exec . vite
+    ready_log_line: "VITE"
+    working_dir: "$PKG_ROOT_WEBVIEW_UI/app"
+
+  clan-app:
+    command: |
+      direnv allow
+      direnv exec . clan-app --debug --content-uri http://localhost:3000
+    depends_on:
+      webview-ui:
+        condition: "process_log_ready"
+    is_foreground: true
+    ready_log_line: "Debug mode enabled"
+    working_dir: "$PKG_ROOT_CLAN_APP"

pkgs/ui/shell.nix

@@ -0,0 +1,90 @@
+{
+  clan-app,
+  mkShell,
+  ruff,
+  webview-lib,
+  webview-ui,
+  clan-ts-api,
+  process-compose,
+  python3,
+  json2ts,
+  self',
+}:
+
+mkShell {
+  name = "ui";
+
+  inputsFrom = [
+    self'.devShells.default
+    webview-ui
+  ];
+
+  packages = [
+    # required for reload-python-api.sh script
+    python3
+    json2ts
+  ];
+
+  inherit (clan-app) propagatedBuildInputs;
+
+  nativeBuildInputs = clan-app.nativeBuildInputs ++ [
+    process-compose
+  ];
+
+  buildInputs = [
+    (clan-app.pythonRuntime.withPackages (
+      ps:
+      with ps;
+      [
+        mypy
+      ]
+      ++ (clan-app.devshellPyDeps ps)
+    ))
+    ruff
+  ] ++ clan-app.runtimeDeps;
+
+  shellHook = ''
+    export GIT_ROOT=$(git rev-parse --show-toplevel)
+
+    export CLAN_CORE_PATH="$GIT_ROOT"
+
+    export PKG_ROOT_UI="$GIT_ROOT/pkgs/ui"
+    export PKG_ROOT_CLAN_APP="$PKG_ROOT_UI"/clan-app
+    export PKG_ROOT_WEBVIEW_UI="$PKG_ROOT_UI"/webview-ui
+
+    export NODE_PATH="$PKG_ROOT_WEBVIEW_UI/app/node_modules"
+
+    # Add current package to PYTHONPATH
+    export PYTHONPATH="$PKG_ROOT_CLAN_APP''${PYTHONPATH:+:$PYTHONPATH:}"
+
+    # Add clan-app command to PATH
+    export PATH="$PKG_ROOT_CLAN_APP/bin":"$PATH"
+
+    # Add webview-ui scripts to PATH
+    scriptsPath="$PKG_ROOT/bin"
+    export PATH="$NODE_PATH/.bin:$scriptsPath:$PATH"
+
+    # Add clan-cli to the python path so that we can import it without building it in nix first
+    export PYTHONPATH="$GIT_ROOT/pkgs/clan-cli":"$PYTHONPATH"
+
+    export XDG_DATA_DIRS=$GSETTINGS_SCHEMAS_PATH:$XDG_DATA_DIRS
+
+    export WEBVIEW_LIB_DIR=${webview-lib}/lib
+    source $PKG_ROOT_CLAN_APP/.local.env
+
+    cp -r ${self'.packages.fonts} "$PKG_ROOT_WEBVIEW_UI/app/.fonts"
+    chmod -R +w "$PKG_ROOT_WEBVIEW_UI/app/.fonts"
+
+    # Define the yellow color code
+    YELLOW='\033[1;33m'
+    # Define the reset color code
+    NC='\033[0m'
+
+    mkdir -p "$PKG_ROOT_WEBVIEW_UI/app/api"
+    cp -r ${clan-ts-api}/* "$PKG_ROOT_WEBVIEW_UI/app/api"
+    chmod -R +w "$PKG_ROOT_WEBVIEW_UI/app/api"
+
+    # configure process-compose
+    export PC_CONFIG_FILES="$PKG_ROOT_UI/process-compose.yaml"
+  '';
+}