Merge pull request 'Merge clan-app and ui environment into one' (#3661) from feat/web-ui-process-compose into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3661
Reviewed-by: brianmcgee <brian@bmcgee.ie>

.gitignore

@@ -46,3 +46,11 @@ dist
 
 # TODO: remove after bug in select is fixed
 select
+
+# Generated files
+pkgs/clan-app/ui/api/API.json
+pkgs/clan-app/ui/api/API.ts
+pkgs/clan-app/ui/api/Inventory.ts
+pkgs/clan-app/ui/api/modules_schemas.json
+pkgs/clan-app/ui/api/schema.json
+pkgs/clan-app/ui/.fonts

docs/nix/default.nix

@@ -45,7 +45,7 @@ pkgs.stdenv.mkDerivation {
     ln -snf ${fira-code}/share/fonts/truetype/FiraCode-VF.ttf ./site/static/
 
     # Copy icons into place
-    cp -af ../pkgs/ui/webview-ui/app/icons ./site/static/
+    cp -af ../pkgs/clan-app/ui/icons ./site/static/
   '';
 
   buildPhase = ''

pkgs/clan-app/.envrc

@@ -1,7 +1,7 @@
 # shellcheck shell=bash
 source_up
 
-watch_file flake-module.nix shell.nix default.nix
+watch_file flake-module.nix shell.nix webview-ui/flake-module.nix
 
 # Because we depend on nixpkgs sources, uploading to builders takes a long time
 use flake .#clan-app --builders ''

pkgs/clan-app/README.md

@@ -0,0 +1,141 @@
+# Clan App
+
+A powerful application that allows users to create and manage their own Clans.
+
+## Getting Started
+
+Enter the `pkgs/clan-app` directory and allow [direnv] to load the `clan-app` devshell with `direnv allow`:
+
+```console
+❯ direnv allow
+direnv: loading ~/Development/lol/clan/git/clan/clan-core/pkgs/ui/.envrc
+direnv: loading ~/Development/lol/clan/git/clan/clan-core/.envrc
+direnv: using flake
+direnv: nix-direnv: Renewed cache
+switch to another dev-shell using: select-shell
+direnv: using flake .#ui --builders
+path '/home/brian/Development/lol/clan/git/clan/clan-core/pkgs/ui' does not contain a 'flake.nix', searching up
+direnv: ([/nix/store/rjnigckx9rmga58562hxw9kr5hynavcd-direnv-2.36.0/bin/direnv export zsh]) is taking a while to execute. Use CTRL-C to give up.
+path '/home/brian/Development/lol/clan/git/clan/clan-core/pkgs/ui' does not contain a 'flake.nix', searching up
+direnv: nix-direnv: Renewed cache
+switch to another dev-shell using: select-shell
+/home/brian/.config/direnv/lib/hm-nix-direnv.sh:3858: /home/brian/Development/lol/clan/git/clan/clan-core/pkgs/ui/clan-app/.local.env: No such file or directory
+direnv: export +AR +AS +CC +CLAN_CORE_PATH +CONFIG_SHELL +CXX +DETERMINISTIC_BUILD +GETTEXTDATADIRS +GETTEXTDATADIRS_FOR_BUILD +GETTEXTDATADIRS_FOR_TARGET +GIT_ROOT +GSETTINGS_SCHEMAS_PATH +HOST_PATH +IN_NIX_SHELL +LD +NIX_BINTOOLS +NIX_BINTOOLS_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu +NIX_BUILD_CORES +NIX_CC +NIX_CC_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu +NIX_CFLAGS_COMPILE +NIX_ENFORCE_NO_NATIVE +NIX_HARDENING_ENABLE +NIX_LDFLAGS +NIX_STORE +NM +NODE_PATH +OBJCOPY +OBJDUMP +PC_CONFIG_FILES +PKG_ROOT_CLAN_APP +PKG_ROOT_UI +PKG_ROOT_WEBVIEW_UI +PRJ_ROOT +PYTHONHASHSEED +PYTHONNOUSERSITE +PYTHONPATH +RANLIB +READELF +SIZE +SOURCE_DATE_EPOCH +STRINGS +STRIP +WEBVIEW_LIB_DIR +_PYTHON_HOST_PLATFORM +_PYTHON_SYSCONFIGDATA_NAME +__structuredAttrs +buildInputs +buildPhase +builder +cmakeFlags +configureFlags +depsBuildBuild +depsBuildBuildPropagated +depsBuildTarget +depsBuildTargetPropagated +depsHostHost +depsHostHostPropagated +depsTargetTarget +depsTargetTargetPropagated +doCheck +doInstallCheck +dontAddDisableDepTrack +mesonFlags +name +nativeBuildInputs +out +outputs +patches +phases +preferLocalBuild +propagatedBuildInputs +propagatedNativeBuildInputs +shell +shellHook +stdenv +strictDeps +system ~GDK_PIXBUF_MODULE_FILE ~GI_TYPELIB_PATH ~PATH ~XDG_DATA_DIRS
+```
+
+Once that has loaded, you can run the local dev environment by running:
+
+```
+$ process-compose --use-uds --keep-project
+```
+
+This will start a [process-compose] instance containing two processes:
+
+* `clan-app-ui` which is a background process running a [vite] server for `./ui` in a hot-reload fashion
+* `clan-app` which is a [foreground process](https://f1bonacc1.github.io/process-compose/launcher/?h=foreground#foreground-processes),
+that is started on demand and provides the [webview] wrapper for the UI.
+
+Wait for the `clan-app-ui` process to enter the `Running` state, then navigate to the `clan-app` process and press `F7`.
+This will start the [webview] window and bring `clan-app`'s terminal into the foreground, allowing for interaction with
+the debugger if required.
+
+If you need to restart, simply enter `ctrl+c` and you will be dropped back into the `process-compose` terminal.
+From there you can start `clan-app` again with `F7`.
+
+> **Note**
+> If you are interacting with a breakpoint, do not continue/exit with `ctrl+c` as this will introduce a quirk
+> the next time you start `clan-app` where you will be unable to see the input you are typing in a debugging session.
+>
+> Instead, exit the debugger with `q+Enter`.
+
+Follow the instructions below to set up your development environment and start the application:
+
+
+## Start clan-app without process-compose
+
+
+1. **Navigate to the Webview UI Directory**
+
+   Go to the `clan-core/pkgs/clan-app/ui` directory and start the web server by executing:
+
+   ```bash
+   npm install
+   vite
+   ```
+
+2. **Start the Clan App**
+
+   In the `clan-core/pkgs/clan-app` directory, execute the following command:
+
+   ```bash
+   ./bin/clan-app --debug --content-uri http://localhost:3000
+   ```
+
+This will start the application in debug mode and link it to the web server running at `http://localhost:3000`.
+
+### Debugging Style and Layout
+
+```bash
+# Enable the GTK debugger
+gsettings set org.gtk.Settings.Debug enable-inspector-keybinding true
+
+# Start the application with the debugger attached
+GTK_DEBUG=interactive ./bin/clan-app --debug
+```
+
+Appending `--debug` flag enables debug logging printed into the console.
+
+### Profiling
+
+To activate profiling you can run
+
+```bash
+CLAN_CLI_PERF=1 ./bin/clan-app
+```
+
+### Library Components
+
+> Note:
+>
+> we recognized bugs when starting some cli-commands through the integrated vs-code terminal.
+> If encountering issues make sure to run commands in a regular os-shell.
+
+lib-Adw has a demo application showing all widgets. You can run it by executing
+
+```bash
+adwaita-1-demo
+```
+
+GTK4 has a demo application showing all widgets. You can run it by executing
+
+```bash
+gtk4-widget-factory
+```
+
+To find available icons execute
+
+```bash
+gtk4-icon-browser
+```
+
+### Links
+
+Here are some important documentation links related to the Clan App:
+
+- [GTK4 PyGobject Reference](http://lazka.github.io/pgi-docs/index.html#Gtk-4.0): This link provides the PyGObject reference documentation for GTK4, the toolkit used for building the user interface of the clan app. It includes information about GTK4 widgets, signals, and other features.
+
+- [Adw Widget Gallery](https://gnome.pages.gitlab.gnome.org/libadwaita/doc/main/widget-gallery.html): This link showcases a widget gallery for Adw, allowing you to see the available widgets and their visual appearance. It can be helpful for designing the user interface of the clan app.
+
+- [GNOME Human Interface Guidelines](https://developer.gnome.org/hig/): This link provides the GNOME Human Interface Guidelines, which offer design and usability recommendations for creating GNOME applications. It covers topics such as layout, navigation, and interaction patterns.
+
+## Error handling
+
+> Error dialogs should be avoided where possible, since they are disruptive.
+>
+> For simple non-critical errors, toasts can be a good alternative.
+
+
+[direnv]: https://direnv.net/
+[process-compose]: https://f1bonacc1.github.io/process-compose/
+[vite]: https://vite.dev/
+[webview]: https://github.com/webview/webview

pkgs/clan-app/bin/reload-python-api.sh

@@ -1,20 +1,20 @@
 #!/usr/bin/env bash
 
-script_dir=$(dirname "$(readlink -f "$0")")
+set -eux -o pipefail
 
-clan_cli="$script_dir/../../clan-cli"
+script_dir=$(dirname "$(readlink -f "$0")")
+cd "$script_dir/.."
 
 trap 'rm -rf "$tmpdir"' EXIT
 tmpdir=$(mktemp -d)
 
-set -x
-python "$clan_cli/api.py" > "$tmpdir/API.json"
+python "../clan-cli/api.py" > "$tmpdir/API.json"
 json2ts --input "$tmpdir/API.json" > "$tmpdir/API.ts"
 
 # compare sha256 sums of old and new API.ts
-old_api_hash=$(sha256sum "$script_dir/../app/api/API.ts" | cut -d ' ' -f 1)
+old_api_hash=$(sha256sum "./ui/api/API.ts" | cut -d ' ' -f 1)
 new_api_hash=$(sha256sum "$tmpdir/API.ts" | cut -d ' ' -f 1)
 if [ "$old_api_hash" != "$new_api_hash" ]; then
-  cp "$tmpdir/API.json" "$script_dir/../app/api/API.json"
-  cp "$tmpdir/API.ts" "$script_dir/../app/api/API.ts"
+  cp "$tmpdir/API.json" "./ui/api/API.json"
+  cp "$tmpdir/API.ts" "./ui/api/API.ts"
 fi

pkgs/clan-app/clan-app.code-workspace

@@ -16,10 +16,10 @@
 			"path": "../../lib/build-clan"
 		},
 		{
-			"path": "../webview-ui"
+			"path": "./ui"
 		},
 		{
-			"path": "../webview-lib"
+			"path": "./webview-lib"
 		},
 		{
 			"path": "../clan-cli/clan_lib"

pkgs/clan-app/default.nix

@@ -3,16 +3,30 @@
   copyDesktopItems,
   clan-cli,
   makeDesktopItem,
-  webview-ui,
+  clan-app-ui,
   webview-lib,
   fontconfig,
   pythonRuntime,
   wrapGAppsHook4,
   gobject-introspection,
   gtk4,
+  lib,
 }:
 let
-  source = ./.;
+  source =
+    {
+      withTests ? true,
+    }:
+    lib.fileset.toSource {
+      root = ./.;
+      fileset = lib.fileset.unions (
+        [
+          ./clan_app
+          ./pyproject.toml
+        ]
+        ++ lib.optional withTests ./tests
+      );
+    };
 
   desktop-file = makeDesktopItem {
     name = "org.clan.app";
@@ -53,7 +67,7 @@ let
 in
 pythonRuntime.pkgs.buildPythonApplication {
   name = "clan-app";
-  src = source;
+  src = source { };
   format = "pyproject";
 
   dontWrapGApps = true;
@@ -99,7 +113,7 @@ pythonRuntime.pkgs.buildPythonApplication {
             ];
           }
           ''
-            cp -r ${source} ./src
+            cp -r ${source { withTests = true; }} ./src
             chmod +w -R ./src
             cd ./src
 
@@ -132,7 +146,7 @@ pythonRuntime.pkgs.buildPythonApplication {
 
   postInstall = ''
     mkdir -p $out/${pythonRuntime.sitePackages}/clan_app/.webui
-    cp -r ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* $out/${pythonRuntime.sitePackages}/clan_app/.webui
+    cp -r ${clan-app-ui}/lib/node_modules/@clan/ui/dist/* $out/${pythonRuntime.sitePackages}/clan_app/.webui
     mkdir -p $out/share/icons/hicolor
     cp -r ./clan_app/assets/white-favicons/* $out/share/icons/hicolor
   '';

pkgs/clan-app/flake-module.nix

@@ -0,0 +1,34 @@
+{
+  perSystem =
+    {
+      self',
+      pkgs,
+      config,
+      ...
+    }:
+    {
+      packages = {
+        webview-lib = pkgs.callPackage ./webview-lib { };
+      };
+
+      devShells.clan-app = pkgs.callPackage ./shell.nix {
+        inherit self';
+        inherit (self'.packages) clan-app webview-lib clan-app-ui;
+        inherit (config.packages) clan-ts-api;
+      };
+
+      packages.clan-app = pkgs.callPackage ./default.nix {
+        inherit (config.packages) clan-cli clan-app-ui webview-lib;
+        pythonRuntime = pkgs.python3;
+      };
+
+      packages.fonts = pkgs.callPackage ./fonts.nix { };
+
+      packages.clan-app-ui = pkgs.callPackage ./ui.nix {
+        clan-ts-api = config.packages.clan-ts-api;
+        fonts = config.packages.fonts;
+      };
+
+      checks = config.packages.clan-app.tests;
+    };
+}

pkgs/clan-app/fonts.nix

@@ -0,0 +1,24 @@
+{ fetchurl, runCommand }:
+let
+  # 400 -> Regular
+  archivoRegular = fetchurl {
+    url = "https://github.com/Omnibus-Type/Archivo/raw/b5d63988ce19d044d3e10362de730af00526b672/fonts/webfonts/ArchivoSemiCondensed-Regular.woff2";
+    hash = "sha256-3PeB6tMpbYxR9JFyQ+yjpM7bAvZIjcJ4eBiHr9iV5p4=";
+  };
+  # 500 -> Medium
+  archivoMedium = fetchurl {
+    url = "https://github.com/Omnibus-Type/Archivo/raw/b5d63988ce19d044d3e10362de730af00526b672/fonts/webfonts/ArchivoSemiCondensed-Medium.woff2";
+    hash = "sha256-IKaY3YhpmjMaIVUpwKRLd6eFiIihBoAP99I/pwmyll8=";
+  };
+  # 600 -> SemiBold
+  archivoSemiBold = fetchurl {
+    url = "https://github.com/Omnibus-Type/Archivo/raw/b5d63988ce19d044d3e10362de730af00526b672/fonts/webfonts/ArchivoSemiCondensed-SemiBold.woff2";
+    hash = "sha256-fOE+b+UeTRoj+sDdUWR1pPCZVn0ABy6FEDDmXrOA4LY=";
+  };
+in
+runCommand "" { } ''
+  mkdir -p $out
+  cp ${archivoRegular} $out/ArchivoSemiCondensed-Regular.woff2
+  cp ${archivoMedium} $out/ArchivoSemiCondensed-Medium.woff2
+  cp ${archivoSemiBold} $out/ArchivoSemiCondensed-SemiBold.woff2
+''

pkgs/clan-app/process-compose.yaml

@@ -0,0 +1,19 @@
+version: "0.5"
+
+processes:
+  clan-app-ui:
+    command: |
+      cd $(git rev-parse --show-toplevel)/pkgs/clan-app/ui
+      npm install
+      vite
+    ready_log_line: "VITE"
+
+  clan-app:
+    command: |
+      cd $(git rev-parse --show-toplevel)/pkgs/clan-app
+      ./bin/clan-app --debug --content-uri http://localhost:3000
+    depends_on:
+      clan-app-ui:
+        condition: "process_log_ready"
+    is_foreground: true
+    ready_log_line: "Debug mode enabled"

pkgs/clan-app/shell.nix

@@ -0,0 +1,81 @@
+{
+  clan-app,
+  mkShell,
+  ruff,
+  webview-lib,
+  clan-app-ui,
+  clan-ts-api,
+  process-compose,
+  json2ts,
+  self',
+}:
+
+mkShell {
+  name = "ui";
+
+  inputsFrom = [
+    self'.devShells.default
+    clan-app-ui
+  ];
+
+  packages = [
+    # required for reload-python-api.sh script
+    json2ts
+  ];
+
+  inherit (clan-app) propagatedBuildInputs;
+
+  nativeBuildInputs = clan-app.nativeBuildInputs ++ [
+    process-compose
+  ];
+
+  buildInputs = [
+    (clan-app.pythonRuntime.withPackages (
+      ps:
+      with ps;
+      [
+        mypy
+      ]
+      ++ (clan-app.devshellPyDeps ps)
+    ))
+    ruff
+  ] ++ clan-app.runtimeDeps;
+
+  shellHook = ''
+    export CLAN_CORE_PATH=$(git rev-parse --show-toplevel)
+
+    ## Clan app
+    # Add clan-app command to PATH
+    pushd "$CLAN_CORE_PATH/pkgs/clan-app"
+    export PATH="$(pwd)/bin":"$PATH"
+    # Add current package to PYTHONPATH
+    export PYTHONPATH="$(pwd)/pkgs/clan-app/''${PYTHONPATH:+:$PYTHONPATH:}"
+    export XDG_DATA_DIRS=$GSETTINGS_SCHEMAS_PATH:$XDG_DATA_DIRS
+    export WEBVIEW_LIB_DIR=${webview-lib}/lib
+    # Add clan-cli to the python path so that we can import it without building it in nix first
+    export PYTHONPATH="$(pwd)":"$PYTHONPATH"
+    popd
+
+    ## Webview UI
+    # Add clan-app-ui scripts to PATH
+    pushd "$CLAN_CORE_PATH/pkgs/clan-app/ui"
+    export NODE_PATH="$(pwd)/node_modules"
+    export PATH="$NODE_PATH/.bin:$(pwd)/bin:$PATH"
+    cp -r ${self'.packages.fonts} .fonts
+    chmod -R +w .fonts
+    mkdir -p api
+    cp -r ${clan-ts-api}/* api
+    chmod -R +w api
+    popd
+
+    # configure process-compose
+    if test -f "$GIT_ROOT/pkgs/clan-app/.local.env"; then
+      source "$GIT_ROOT/pkgs/clan-app/.local.env"
+    fi
+    # Define the yellow color code
+    YELLOW='\033[1;33m'
+    # Define the reset color code
+    NC='\033[0m'
+    export PC_CONFIG_FILES="$CLAN_CORE_PATH/pkgs/clan-app/process-compose.yaml"
+  '';
+}

pkgs/clan-app/ui.nix

@@ -0,0 +1,25 @@
+{
+  buildNpmPackage,
+  nodejs_20,
+  importNpmLock,
+
+  clan-ts-api,
+  fonts,
+}:
+buildNpmPackage {
+  pname = "clan-app-ui";
+  version = "0.0.1";
+  nodejs = nodejs_20;
+  src = ./ui;
+
+  npmDeps = importNpmLock {
+    npmRoot = ./ui;
+  };
+  npmConfigHook = importNpmLock.npmConfigHook;
+
+  preBuild = ''
+    mkdir -p api
+    cp -r ${clan-ts-api}/* api
+    cp -r ${fonts} ".fonts"
+  '';
+}

pkgs/clan-app/ui/package-lock.json

@@ -1,11 +1,11 @@
 {
-  "name": "@clan/webview-ui",
+  "name": "@clan/ui",
   "version": "0.0.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
-      "name": "@clan/webview-ui",
+      "name": "@clan/ui",
       "version": "0.0.1",
       "license": "MIT",
       "dependencies": {

pkgs/clan-app/ui/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "@clan/webview-ui",
+  "name": "@clan/ui",
   "version": "0.0.1",
   "description": "",
   "type": "module",