Merge pull request 'clan-app: Fix default.nix' (#2699) from Qubasa/clan-core:Qubasa-main into main

.gitignore

@@ -14,7 +14,7 @@ example_clan
 nixos.qcow2
 **/*.glade~
 /docs/out
-
+**/.local.env
 
 # dream2nix
 .dream2nix

formatter.nix

@@ -92,13 +92,12 @@
       treefmt.programs.mypy.directories =
         {
           "clan-cli" = {
-            extraPythonPackages = self'.packages.clan-cli.testDependencies;
             directory = "pkgs/clan-cli";
+            extraPythonPackages = (self'.packages.clan-cli.devshellPyDeps pkgs.python3Packages);
           };
           "clan-app" = {
             directory = "pkgs/clan-app";
-            extraPythonPackages =
+            extraPythonPackages = (self'.packages.clan-app.devshellPyDeps pkgs.python3Packages);
-              (self'.packages.clan-app.externalTestDeps or [ ]) ++ self'.packages.clan-cli.testDependencies;
             extraPythonPaths = [ "../clan-cli" ];
           };
         }
@@ -107,8 +106,9 @@
             {
               "clan-vm-manager" = {
                 directory = "pkgs/clan-vm-manager";
-                extraPythonPackages =
+                extraPythonPackages = self'.packages.clan-vm-manager.externalTestDeps ++ [
-                  self'.packages.clan-vm-manager.externalTestDeps ++ self'.packages.clan-cli.testDependencies;
+                  (pkgs.python3.withPackages (ps: self'.packages.clan-cli.devshellPyDeps ps))
+                ];
                 extraPythonPaths = [ "../clan-cli" ];
               };
             }

pkgs/clan-app/clan_app/assets/style.css

@@ -1,63 +0,0 @@
-/* Insert custom styles here */
-
-navigation-view {
-  padding: 5px;
-  /* padding-left: 5px;
-    padding-right: 5px;
-    padding-bottom: 5px; */
-}
-
-avatar {
-  margin: 2px;
-}
-
-.trust {
-  padding-top: 25px;
-  padding-bottom: 25px;
-}
-
-.join-list {
-  margin-top: 1px;
-  margin-left: 2px;
-  margin-right: 2px;
-}
-
-.progress-bar {
-  margin-right: 25px;
-  min-width: 200px;
-}
-
-.group-list {
-  background-color: inherit;
-}
-.group-list > .activatable:hover {
-  background-color: unset;
-}
-
-.group-list > row {
-  margin-top: 12px;
-  border-bottom: unset;
-}
-
-.vm-list {
-  margin-top: 25px;
-  margin-bottom: 25px;
-}
-
-.no-shadow {
-  box-shadow: none;
-}
-
-.search-entry {
-  margin-bottom: 12px;
-}
-
-searchbar {
-  margin-bottom: 25px;
-}
-
-.log-view {
-  margin-top: 12px;
-  font-family: monospace;
-  padding: 8px;
-}

pkgs/clan-app/clan_app/deps/webview/webview.py

@@ -6,7 +6,13 @@ from collections.abc import Callable
 from enum import IntEnum
 from typing import Any
 
-from clan_cli.api import MethodRegistry, dataclass_to_dict, from_dict
+from clan_cli.api import (
+    ApiError,
+    ErrorDataClass,
+    MethodRegistry,
+    dataclass_to_dict,
+    from_dict,
+)
 
 from ._webview_ffi import _encode_c_string, _webview_lib
 
@@ -20,6 +26,11 @@ class SizeHint(IntEnum):
     FIXED = 3
 
 
+class FuncStatus(IntEnum):
+    SUCCESS = 0
+    FAILURE = 1
+
+
 class Size:
     def __init__(self, width: int, height: int, hint: SizeHint) -> None:
         self.width = width
@@ -85,44 +96,48 @@ class Webview:
                     try:
                         args = json.loads(req.decode())
 
-                        try:
+                        log.debug(f"Calling {method_name}({args[0]})")
-                            log.debug(f"Calling {method_name}({args[0]})")
+                        # Initialize dataclasses from the payload
-                            # Initialize dataclasses from the payload
+                        reconciled_arguments = {}
-                            reconciled_arguments = {}
+                        for k, v in args[0].items():
-                            for k, v in args[0].items():
+                            # Some functions expect to be called with dataclass instances
-                                # Some functions expect to be called with dataclass instances
+                            # But the js api returns dictionaries.
-                                # But the js api returns dictionaries.
+                            # Introspect the function and create the expected dataclass from dict dynamically
-                                # Introspect the function and create the expected dataclass from dict dynamically
+                            # Depending on the introspected argument_type
-                                # Depending on the introspected argument_type
+                            arg_class = api.get_method_argtype(method_name, k)
-                                arg_class = api.get_method_argtype(method_name, k)
 
-                                # TODO: rename from_dict into something like construct_checked_value
+                            # TODO: rename from_dict into something like construct_checked_value
-                                # from_dict really takes Anything and returns an instance of the type/class
+                            # from_dict really takes Anything and returns an instance of the type/class
-                                reconciled_arguments[k] = from_dict(arg_class, v)
+                            reconciled_arguments[k] = from_dict(arg_class, v)
 
-                            reconciled_arguments["op_key"] = seq.decode()
+                        reconciled_arguments["op_key"] = seq.decode()
-                            # TODO: We could remove the wrapper in the MethodRegistry
+                        # TODO: We could remove the wrapper in the MethodRegistry
-                            # and just call the method directly
+                        # and just call the method directly
-                            result = wrap_method(**reconciled_arguments)
+                        result = wrap_method(**reconciled_arguments)
-                            success = True
-                        except Exception as e:
-                            log.exception(f"Error calling {method_name}")
-                            result = str(e)
-                            success = False
 
-                        try:
+                        serialized = json.dumps(
-                            serialized = json.dumps(
+                            dataclass_to_dict(result), indent=4, ensure_ascii=False
-                                dataclass_to_dict(result), indent=4, ensure_ascii=False
+                        )
-                            )
-                        except TypeError:
-                            log.exception(f"Error serializing result for {method_name}")
-                            raise
 
                         log.debug(f"Result for {method_name}: {serialized}")
-                        self.return_(seq.decode(), 0 if success else 1, serialized)
+                        self.return_(seq.decode(), FuncStatus.SUCCESS, serialized)
                     except Exception as e:
-                        log.exception(f"Unhandled error in webview {method_name}")
+                        log.exception(f"Error while handling result of {method_name}")
-                        self.return_(seq.decode(), 1, str(e))
+                        result = ErrorDataClass(
+                            op_key=seq.decode(),
+                            status="error",
+                            errors=[
+                                ApiError(
+                                    message="An internal error occured",
+                                    description=str(e),
+                                    location=["bind_jsonschema_api", method_name],
+                                )
+                            ],
+                        )
+                        serialized = json.dumps(
+                            dataclass_to_dict(result), indent=4, ensure_ascii=False
+                        )
+                        self.return_(seq.decode(), FuncStatus.FAILURE, serialized)
 
                 thread = threading.Thread(target=thread_task)
                 thread.start()

pkgs/clan-app/default.nix

@@ -1,16 +1,16 @@
 {
-  python3Full,
   runCommand,
-  setuptools,
   copyDesktopItems,
   clan-cli,
   makeDesktopItem,
   webview-ui,
   webview-lib,
   fontconfig,
+  pythonRuntime,
 }:
 let
   source = ./.;
+
   desktop-file = makeDesktopItem {
     name = "org.clan.app";
     exec = "clan-app %u";
@@ -25,24 +25,29 @@ let
 
   ];
 
+  pyDeps = ps: [
+    ps.tkinter
+  ];
+
   # Dependencies required for running tests
   pyTestDeps =
     ps:
-    with ps;
     [
-      (python3Full.pkgs.toPythonModule pytest)
       # Testing framework
-      pytest-cov # Generate coverage reports
+      ps.pytest
-      pytest-subprocess # fake the real subprocess behavior to make your tests more independent.
+      ps.pytest-cov # Generate coverage reports
-      pytest-xdist # Run tests in parallel on multiple cores
+      ps.pytest-subprocess # fake the real subprocess behavior to make your tests more independent.
-      pytest-timeout # Add timeouts to your tests
+      ps.pytest-xdist # Run tests in parallel on multiple cores
+      ps.pytest-timeout # Add timeouts to your tests
     ]
-    ++ pytest.propagatedBuildInputs;
+    ++ ps.pytest.propagatedBuildInputs;
 
-  clan-cli-module = [ (python3Full.pkgs.toPythonModule clan-cli) ];
+  clan-cli-module = [
+    (pythonRuntime.pkgs.toPythonModule (clan-cli.override { inherit pythonRuntime; }))
+  ];
 
 in
-python3Full.pkgs.buildPythonApplication rec {
+pythonRuntime.pkgs.buildPythonApplication {
   name = "clan-app";
   src = source;
   format = "pyproject";
@@ -51,7 +56,7 @@ python3Full.pkgs.buildPythonApplication rec {
   preFixup = ''
     makeWrapperArgs+=(
       --set FONTCONFIG_FILE ${fontconfig.out}/etc/fonts/fonts.conf
-      --set WEBUI_PATH "$out/${python3Full.sitePackages}/clan_app/.webui"
+      --set WEBUI_PATH "$out/${pythonRuntime.sitePackages}/clan_app/.webui"
       --set WEBVIEW_LIB_DIR "${webview-lib}/lib"
       # This prevents problems with mixed glibc versions that might occur when the
       # cli is called through a browser built against another glibc
@@ -62,7 +67,7 @@ python3Full.pkgs.buildPythonApplication rec {
 
   # Deps needed only at build time
   nativeBuildInputs = [
-    setuptools
+    (pythonRuntime.withPackages (ps: [ ps.setuptools ]))
     copyDesktopItems
     fontconfig
   ];
@@ -71,8 +76,9 @@ python3Full.pkgs.buildPythonApplication rec {
   # same values for your Python package within Nix largely stems from ensuring
   # that all necessary dependencies are consistently available both
   # at build time and runtime,
-  buildInputs = clan-cli-module ++ runtimeDependencies;
+  propagatedBuildInputs = [
-  propagatedBuildInputs = buildInputs;
+    (pythonRuntime.withPackages (ps: clan-cli-module ++ (pyDeps ps)))
+  ] ++ runtimeDependencies;
 
   # also re-expose dependencies so we test them in CI
   passthru = {
@@ -81,7 +87,7 @@ python3Full.pkgs.buildPythonApplication rec {
         runCommand "clan-app-pytest"
           {
             buildInputs = runtimeDependencies ++ [
-              (python3Full.withPackages (ps: clan-cli-module ++ (pyTestDeps ps)))
+              (pythonRuntime.withPackages (ps: clan-cli-module ++ (pyTestDeps ps) ++ (pyDeps ps)))
               fontconfig
             ];
           }
@@ -105,7 +111,7 @@ python3Full.pkgs.buildPythonApplication rec {
             echo "STARTING ..."
             export WEBVIEW_LIB_DIR="${webview-lib}/lib"
             export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1
-            python3 -m pytest -s -m "not impure" ./tests
+            python -m pytest -s -m "not impure" ./tests
             touch $out
           '';
     };
@@ -113,11 +119,12 @@ python3Full.pkgs.buildPythonApplication rec {
 
   # Additional pass-through attributes
   passthru.desktop-file = desktop-file;
-  passthru.devshellDeps = ps: (pyTestDeps ps);
+  passthru.devshellPyDeps = ps: (pyTestDeps ps) ++ (pyDeps ps);
+  passthru.pythonRuntime = pythonRuntime;
 
   postInstall = ''
-    mkdir -p $out/${python3Full.sitePackages}/clan_app/.webui
+    mkdir -p $out/${pythonRuntime.sitePackages}/clan_app/.webui
-    cp -r ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* $out/${python3Full.sitePackages}/clan_app/.webui
+    cp -r ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* $out/${pythonRuntime.sitePackages}/clan_app/.webui
     mkdir -p $out/share/icons/hicolor
     cp -r ./clan_app/assets/white-favicons/* $out/share/icons/hicolor
   '';

pkgs/clan-app/flake-module.nix

@@ -4,23 +4,19 @@
     {
       config,
       pkgs,
-      lib,
-      system,
       self',
       ...
     }:
-    if lib.elem system lib.platforms.darwin then
+    {
-      { }
+      devShells.clan-app = pkgs.callPackage ./shell.nix {
-    else
+        inherit (config.packages) clan-app webview-lib;
-      {
+        inherit self';
-        devShells.clan-app = pkgs.callPackage ./shell.nix {
-          inherit (config.packages) clan-app webview-lib;
-          inherit self';
-        };
-        packages.clan-app = pkgs.python3.pkgs.callPackage ./default.nix {
-          inherit (config.packages) clan-cli webview-ui webview-lib;
-        };
-
-        checks = config.packages.clan-app.tests;
       };
+      packages.clan-app = pkgs.callPackage ./default.nix {
+        inherit (config.packages) clan-cli webview-ui webview-lib;
+        pythonRuntime = pkgs.python3;
+      };
+
+      checks = config.packages.clan-app.tests;
+    };
 }

pkgs/clan-app/pyproject.toml

@@ -36,6 +36,3 @@ disallow_untyped_calls = true
 disallow_untyped_defs = true
 no_implicit_optional = true
 
-[[tool.mypy.overrides]]
-module = "clan_cli.*"
-ignore_missing_imports = true

pkgs/clan-app/shell.nix

@@ -5,7 +5,6 @@
   ruff,
   gtk4,
   webview-lib,
-  python3Full,
   self',
 }:
 
@@ -14,14 +13,14 @@ mkShell {
   inputsFrom = [ self'.devShells.default ];
 
   buildInputs = [
-    (python3Full.withPackages (
+    (clan-app.pythonRuntime.withPackages (
       ps:
       with ps;
       [
         ruff
         mypy
       ]
-      ++ (clan-app.devshellDeps ps)
+      ++ (clan-app.devshellPyDeps ps)
     ))
   ];
 
@@ -42,6 +41,6 @@ mkShell {
     export XDG_DATA_DIRS=${gsettings-desktop-schemas}/share/gsettings-schemas/gsettings-desktop-schemas-46.0:$XDG_DATA_DIRS
 
     export WEBVIEW_LIB_DIR=${webview-lib}/lib
-    # export WEBVIEW_LIB_DIR=$HOME/Projects/webview/build/core
+    source $PKG_ROOT/.local.env
   '';
 }

pkgs/clan-cli/clan_cli/nix/allowed-programs.json

@@ -16,5 +16,6 @@
   "virtiofsd",
   "zbar",
   "util-linux",
-  "avahi"
+  "avahi",
+  "gnupg"
 ]

pkgs/clan-cli/default.nix

@@ -1,67 +1,53 @@
 {
   # callPackage args
-  argcomplete,
   gnupg,
   installShellFiles,
   lib,
   nix,
   pkgs,
-  pytest-cov,
-  pytest-subprocess,
-  pytest-timeout,
-  pytest-xdist,
-  pytest,
-  python3,
   runCommand,
-  setuptools,
   stdenv,
   nixVersions,
-
   # custom args
   clan-core-path,
   nixpkgs,
   includedRuntimeDeps,
-
   inventory-schema-abstract,
   classgen,
+  pythonRuntime,
 }:
 let
-  pythonDependencies = [
+  pyDeps = ps: [
-    argcomplete # Enables shell completions
+    ps.argcomplete # Enables shell completions
   ];
+  pyTestDeps =
+    ps:
+    [
+      ps.pytest
+      ps.pytest-cov
+      ps.pytest-subprocess
+      ps.pytest-xdist
+      ps.pytest-timeout
+    ]
+    ++ (pyDeps ps);
+  pythonRuntimeWithDeps = pythonRuntime.withPackages (ps: pyDeps ps);
 
   # load nixpkgs runtime dependencies from a json file
   # This file represents an allow list at the same time that is checked by the run_cmd
   #   implementation in nix.py
   allDependencies = lib.importJSON ./clan_cli/nix/allowed-programs.json;
-
   generateRuntimeDependenciesMap =
     deps:
     lib.filterAttrs (_: pkg: !pkg.meta.unsupported or false) (lib.genAttrs deps (name: pkgs.${name}));
-
   runtimeDependenciesMap = generateRuntimeDependenciesMap allDependencies;
-
   runtimeDependencies = lib.attrValues runtimeDependenciesMap;
-
   includedRuntimeDependenciesMap = generateRuntimeDependenciesMap includedRuntimeDeps;
 
-  testDependencies =
+  testDependencies = runtimeDependencies ++ [
-    runtimeDependencies
+    gnupg
-    ++ [
+    stdenv.cc # Compiler used for certain native extensions
-      gnupg
+    (pythonRuntime.withPackages (ps: (pyTestDeps ps) ++ (pyDeps ps)))
-      stdenv.cc # Compiler used for certain native extensions
+  ];
-    ]
-    ++ pythonDependencies
-    ++ [
-      pytest
-      pytest-cov # Generate coverage reports
-      pytest-subprocess # fake the real subprocess behavior to make your tests more independent.
-      pytest-xdist # Run tests in parallel on multiple cores
-      pytest-timeout # Add timeouts to your tests
-    ];
-
-  # Setup Python environment with all dependencies for running tests
-  pythonWithTestDeps = python3.withPackages (_ps: testDependencies);
 
   source = runCommand "clan-cli-source" { } ''
     cp -r ${./.} $out
@@ -73,7 +59,6 @@ let
   '';
 
   # Create a custom nixpkgs for use within the project
-
   nixpkgs' =
     runCommand "nixpkgs"
       {
@@ -101,7 +86,7 @@ let
           --extra-experimental-features 'nix-command flakes'
       '';
 in
-python3.pkgs.buildPythonApplication {
+pythonRuntime.pkgs.buildPythonApplication {
   name = "clan-cli";
   src = source;
   format = "pyproject";
@@ -123,30 +108,29 @@ python3.pkgs.buildPythonApplication {
   ];
 
   nativeBuildInputs = [
-    setuptools
+    (pythonRuntime.withPackages (ps: [ ps.setuptools ]))
     installShellFiles
   ];
 
-  propagatedBuildInputs = pythonDependencies;
+  propagatedBuildInputs = [ pythonRuntimeWithDeps ] ++ runtimeDependencies;
 
   # Define and expose the tests and checks to run in CI
   passthru.tests = (lib.mapAttrs' (n: lib.nameValuePair "clan-dep-${n}") runtimeDependenciesMap) // {
     clan-pytest-without-core =
-      runCommand "clan-pytest-without-core"
+      runCommand "clan-pytest-without-core" { nativeBuildInputs = testDependencies; }
-        { nativeBuildInputs = [ pythonWithTestDeps ] ++ testDependencies; }
         ''
           cp -r ${source} ./src
           chmod +w -R ./src
           cd ./src
 
           export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 PYTHONWARNINGS=error
-          ${pythonWithTestDeps}/bin/python -m pytest -m "not impure and not with_core" ./tests
+          python -m pytest -m "not impure and not with_core" ./tests
           touch $out
         '';
     clan-pytest-with-core =
       runCommand "clan-pytest-with-core"
         {
-          nativeBuildInputs = [ pythonWithTestDeps ] ++ testDependencies;
+          nativeBuildInputs = testDependencies;
           buildInputs = [
             pkgs.bash
             pkgs.coreutils
@@ -177,25 +161,25 @@ python3.pkgs.buildPythonApplication {
           mkdir -p "$CLAN_TEST_STORE/nix/store"
           xargs cp --recursive --target "$CLAN_TEST_STORE/nix/store"  < "$closureInfo/store-paths"
           nix-store --load-db --store "$CLAN_TEST_STORE" < "$closureInfo/registration"
-          ${pythonWithTestDeps}/bin/python -m pytest -m "not impure and with_core" ./tests
+          python -m pytest -m "not impure and with_core" ./tests
           touch $out
         '';
   };
 
   passthru.nixpkgs = nixpkgs';
-  passthru.testDependencies = testDependencies;
+  passthru.devshellPyDeps = ps: (pyTestDeps ps) ++ (pyDeps ps);
-  passthru.pythonWithTestDeps = pythonWithTestDeps;
+  passthru.pythonRuntime = pythonRuntime;
   passthru.runtimeDependencies = runtimeDependencies;
   passthru.runtimeDependenciesMap = runtimeDependenciesMap;
 
   postInstall = ''
-    cp -r ${nixpkgs'} $out/${python3.sitePackages}/clan_cli/nixpkgs
+    cp -r ${nixpkgs'} $out/${pythonRuntime.sitePackages}/clan_cli/nixpkgs
     installShellCompletion --bash --name clan \
-      <(${argcomplete}/bin/register-python-argcomplete --shell bash clan)
+      <(${pythonRuntimeWithDeps.pkgs.argcomplete}/bin/register-python-argcomplete --shell bash clan)
     installShellCompletion --fish --name clan.fish \
-      <(${argcomplete}/bin/register-python-argcomplete --shell fish clan)
+      <(${pythonRuntimeWithDeps.pkgs.argcomplete}/bin/register-python-argcomplete --shell fish clan)
     installShellCompletion --zsh --name _clan \
-      <(${argcomplete}/bin/register-python-argcomplete --shell zsh clan)
+      <(${pythonRuntimeWithDeps.pkgs.argcomplete}/bin/register-python-argcomplete --shell zsh clan)
   '';
 
   # Clean up after the package to avoid leaking python packages into a devshell

pkgs/clan-cli/flake-module.nix

@@ -103,21 +103,23 @@
         inherit self';
       };
       packages = {
-        clan-cli = pkgs.python3.pkgs.callPackage ./default.nix {
+        clan-cli = pkgs.callPackage ./default.nix {
           inherit (inputs) nixpkgs;
           inherit (self'.packages) classgen;
           inherit (self'.legacyPackages.schemas) inventory-schema-abstract;
+          pythonRuntime = pkgs.python3;
           clan-core-path = clanCoreWithVendoredDeps;
           includedRuntimeDeps = [
             "age"
             "git"
           ];
         };
-        clan-cli-full = pkgs.python3.pkgs.callPackage ./default.nix {
+        clan-cli-full = pkgs.callPackage ./default.nix {
           inherit (inputs) nixpkgs;
           inherit (self'.packages) classgen;
           inherit (self'.legacyPackages.schemas) inventory-schema-abstract;
           clan-core-path = clanCoreWithVendoredDeps;
+          pythonRuntime = pkgs.python3;
           includedRuntimeDeps = lib.importJSON ./clan_cli/nix/allowed-programs.json;
         };
         clan-cli-docs = pkgs.stdenv.mkDerivation {

pkgs/clan-cli/pyproject.toml

@@ -45,19 +45,4 @@ disallow_untyped_defs = true
 no_implicit_optional = true
 exclude = "clan_cli.nixpkgs"
 
-[[tool.mypy.overrides]]
-module = "argcomplete.*"
-ignore_missing_imports = true
-
-[[tool.mypy.overrides]]
-module = "ipdb.*"
-ignore_missing_imports = true
-
-[[tool.mypy.overrides]]
-module = "pytest.*"
-ignore_missing_imports = true
-
-[[tool.mypy.overrides]]
-module = "setuptools.*"
-ignore_missing_imports = true
 

pkgs/clan-cli/shell.nix

@@ -5,25 +5,22 @@
   clan-cli-full,
   mkShell,
   ruff,
-  python3,
   self',
 }:
-let
+
-  devshellTestDeps =
-    clan-cli.passthru.testDependencies
-    ++ (with python3.pkgs; [
-      rope
-      setuptools
-      wheel
-      webcolors
-      pip
-    ]);
-in
 mkShell {
   buildInputs = [
+    (clan-cli.pythonRuntime.withPackages (
+      ps:
+      with ps;
+      [
+        ruff
+        mypy
+      ]
+      ++ (clan-cli.devshellPyDeps ps)
+    ))
     nix-unit
-    ruff
+  ] ++ clan-cli.runtimeDependencies;
-  ] ++ devshellTestDeps;
 
   inputsFrom = [ self'.devShells.default ];