diff --git a/.gitignore b/.gitignore index f97691bc0..08c012b5c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ example_clan nixos.qcow2 **/*.glade~ /docs/out - +**/.local.env # dream2nix .dream2nix diff --git a/formatter.nix b/formatter.nix index c88a555fd..a1cd9a1d9 100644 --- a/formatter.nix +++ b/formatter.nix @@ -92,13 +92,12 @@ treefmt.programs.mypy.directories = { "clan-cli" = { - extraPythonPackages = self'.packages.clan-cli.testDependencies; directory = "pkgs/clan-cli"; + extraPythonPackages = (self'.packages.clan-cli.devshellPyDeps pkgs.python3Packages); }; "clan-app" = { directory = "pkgs/clan-app"; - extraPythonPackages = - (self'.packages.clan-app.externalTestDeps or [ ]) ++ self'.packages.clan-cli.testDependencies; + extraPythonPackages = (self'.packages.clan-app.devshellPyDeps pkgs.python3Packages); extraPythonPaths = [ "../clan-cli" ]; }; } @@ -107,8 +106,9 @@ { "clan-vm-manager" = { directory = "pkgs/clan-vm-manager"; - extraPythonPackages = - self'.packages.clan-vm-manager.externalTestDeps ++ self'.packages.clan-cli.testDependencies; + extraPythonPackages = self'.packages.clan-vm-manager.externalTestDeps ++ [ + (pkgs.python3.withPackages (ps: self'.packages.clan-cli.devshellPyDeps ps)) + ]; extraPythonPaths = [ "../clan-cli" ]; }; } diff --git a/pkgs/clan-app/clan_app/assets/penguin.jpeg b/pkgs/clan-app/clan_app/assets/penguin.jpeg deleted file mode 100644 index 72f8e503c..000000000 Binary files a/pkgs/clan-app/clan_app/assets/penguin.jpeg and /dev/null differ diff --git a/pkgs/clan-app/clan_app/assets/placeholder.jpeg b/pkgs/clan-app/clan_app/assets/placeholder.jpeg deleted file mode 100644 index f3fa915bf..000000000 Binary files a/pkgs/clan-app/clan_app/assets/placeholder.jpeg and /dev/null differ diff --git a/pkgs/clan-app/clan_app/assets/style.css b/pkgs/clan-app/clan_app/assets/style.css deleted file mode 100644 index 772b9ad7c..000000000 --- a/pkgs/clan-app/clan_app/assets/style.css +++ /dev/null @@ -1,63 +0,0 @@ -/* Insert custom styles here */ - -navigation-view { - padding: 5px; - /* padding-left: 5px; - padding-right: 5px; - padding-bottom: 5px; */ -} - -avatar { - margin: 2px; -} - -.trust { - padding-top: 25px; - padding-bottom: 25px; -} - -.join-list { - margin-top: 1px; - margin-left: 2px; - margin-right: 2px; -} - -.progress-bar { - margin-right: 25px; - min-width: 200px; -} - -.group-list { - background-color: inherit; -} -.group-list > .activatable:hover { - background-color: unset; -} - -.group-list > row { - margin-top: 12px; - border-bottom: unset; -} - -.vm-list { - margin-top: 25px; - margin-bottom: 25px; -} - -.no-shadow { - box-shadow: none; -} - -.search-entry { - margin-bottom: 12px; -} - -searchbar { - margin-bottom: 25px; -} - -.log-view { - margin-top: 12px; - font-family: monospace; - padding: 8px; -} diff --git a/pkgs/clan-app/clan_app/deps/webview/webview.py b/pkgs/clan-app/clan_app/deps/webview/webview.py index f422066eb..9a6338703 100644 --- a/pkgs/clan-app/clan_app/deps/webview/webview.py +++ b/pkgs/clan-app/clan_app/deps/webview/webview.py @@ -6,7 +6,13 @@ from collections.abc import Callable from enum import IntEnum from typing import Any -from clan_cli.api import MethodRegistry, dataclass_to_dict, from_dict +from clan_cli.api import ( + ApiError, + ErrorDataClass, + MethodRegistry, + dataclass_to_dict, + from_dict, +) from ._webview_ffi import _encode_c_string, _webview_lib @@ -20,6 +26,11 @@ class SizeHint(IntEnum): FIXED = 3 +class FuncStatus(IntEnum): + SUCCESS = 0 + FAILURE = 1 + + class Size: def __init__(self, width: int, height: int, hint: SizeHint) -> None: self.width = width @@ -85,44 +96,48 @@ class Webview: try: args = json.loads(req.decode()) - try: - log.debug(f"Calling {method_name}({args[0]})") - # Initialize dataclasses from the payload - reconciled_arguments = {} - for k, v in args[0].items(): - # Some functions expect to be called with dataclass instances - # But the js api returns dictionaries. - # Introspect the function and create the expected dataclass from dict dynamically - # Depending on the introspected argument_type - arg_class = api.get_method_argtype(method_name, k) + log.debug(f"Calling {method_name}({args[0]})") + # Initialize dataclasses from the payload + reconciled_arguments = {} + for k, v in args[0].items(): + # Some functions expect to be called with dataclass instances + # But the js api returns dictionaries. + # Introspect the function and create the expected dataclass from dict dynamically + # Depending on the introspected argument_type + arg_class = api.get_method_argtype(method_name, k) - # TODO: rename from_dict into something like construct_checked_value - # from_dict really takes Anything and returns an instance of the type/class - reconciled_arguments[k] = from_dict(arg_class, v) + # TODO: rename from_dict into something like construct_checked_value + # from_dict really takes Anything and returns an instance of the type/class + reconciled_arguments[k] = from_dict(arg_class, v) - reconciled_arguments["op_key"] = seq.decode() - # TODO: We could remove the wrapper in the MethodRegistry - # and just call the method directly - result = wrap_method(**reconciled_arguments) - success = True - except Exception as e: - log.exception(f"Error calling {method_name}") - result = str(e) - success = False + reconciled_arguments["op_key"] = seq.decode() + # TODO: We could remove the wrapper in the MethodRegistry + # and just call the method directly + result = wrap_method(**reconciled_arguments) - try: - serialized = json.dumps( - dataclass_to_dict(result), indent=4, ensure_ascii=False - ) - except TypeError: - log.exception(f"Error serializing result for {method_name}") - raise + serialized = json.dumps( + dataclass_to_dict(result), indent=4, ensure_ascii=False + ) log.debug(f"Result for {method_name}: {serialized}") - self.return_(seq.decode(), 0 if success else 1, serialized) + self.return_(seq.decode(), FuncStatus.SUCCESS, serialized) except Exception as e: - log.exception(f"Unhandled error in webview {method_name}") - self.return_(seq.decode(), 1, str(e)) + log.exception(f"Error while handling result of {method_name}") + result = ErrorDataClass( + op_key=seq.decode(), + status="error", + errors=[ + ApiError( + message="An internal error occured", + description=str(e), + location=["bind_jsonschema_api", method_name], + ) + ], + ) + serialized = json.dumps( + dataclass_to_dict(result), indent=4, ensure_ascii=False + ) + self.return_(seq.decode(), FuncStatus.FAILURE, serialized) thread = threading.Thread(target=thread_task) thread.start() diff --git a/pkgs/clan-app/default.nix b/pkgs/clan-app/default.nix index e518ef71d..4c5ca67ac 100644 --- a/pkgs/clan-app/default.nix +++ b/pkgs/clan-app/default.nix @@ -1,16 +1,16 @@ { - python3Full, runCommand, - setuptools, copyDesktopItems, clan-cli, makeDesktopItem, webview-ui, webview-lib, fontconfig, + pythonRuntime, }: let source = ./.; + desktop-file = makeDesktopItem { name = "org.clan.app"; exec = "clan-app %u"; @@ -25,24 +25,29 @@ let ]; + pyDeps = ps: [ + ps.tkinter + ]; + # Dependencies required for running tests pyTestDeps = ps: - with ps; [ - (python3Full.pkgs.toPythonModule pytest) # Testing framework - pytest-cov # Generate coverage reports - pytest-subprocess # fake the real subprocess behavior to make your tests more independent. - pytest-xdist # Run tests in parallel on multiple cores - pytest-timeout # Add timeouts to your tests + ps.pytest + ps.pytest-cov # Generate coverage reports + ps.pytest-subprocess # fake the real subprocess behavior to make your tests more independent. + ps.pytest-xdist # Run tests in parallel on multiple cores + ps.pytest-timeout # Add timeouts to your tests ] - ++ pytest.propagatedBuildInputs; + ++ ps.pytest.propagatedBuildInputs; - clan-cli-module = [ (python3Full.pkgs.toPythonModule clan-cli) ]; + clan-cli-module = [ + (pythonRuntime.pkgs.toPythonModule (clan-cli.override { inherit pythonRuntime; })) + ]; in -python3Full.pkgs.buildPythonApplication rec { +pythonRuntime.pkgs.buildPythonApplication { name = "clan-app"; src = source; format = "pyproject"; @@ -51,7 +56,7 @@ python3Full.pkgs.buildPythonApplication rec { preFixup = '' makeWrapperArgs+=( --set FONTCONFIG_FILE ${fontconfig.out}/etc/fonts/fonts.conf - --set WEBUI_PATH "$out/${python3Full.sitePackages}/clan_app/.webui" + --set WEBUI_PATH "$out/${pythonRuntime.sitePackages}/clan_app/.webui" --set WEBVIEW_LIB_DIR "${webview-lib}/lib" # This prevents problems with mixed glibc versions that might occur when the # cli is called through a browser built against another glibc @@ -62,7 +67,7 @@ python3Full.pkgs.buildPythonApplication rec { # Deps needed only at build time nativeBuildInputs = [ - setuptools + (pythonRuntime.withPackages (ps: [ ps.setuptools ])) copyDesktopItems fontconfig ]; @@ -71,8 +76,9 @@ python3Full.pkgs.buildPythonApplication rec { # same values for your Python package within Nix largely stems from ensuring # that all necessary dependencies are consistently available both # at build time and runtime, - buildInputs = clan-cli-module ++ runtimeDependencies; - propagatedBuildInputs = buildInputs; + propagatedBuildInputs = [ + (pythonRuntime.withPackages (ps: clan-cli-module ++ (pyDeps ps))) + ] ++ runtimeDependencies; # also re-expose dependencies so we test them in CI passthru = { @@ -81,7 +87,7 @@ python3Full.pkgs.buildPythonApplication rec { runCommand "clan-app-pytest" { buildInputs = runtimeDependencies ++ [ - (python3Full.withPackages (ps: clan-cli-module ++ (pyTestDeps ps))) + (pythonRuntime.withPackages (ps: clan-cli-module ++ (pyTestDeps ps) ++ (pyDeps ps))) fontconfig ]; } @@ -105,7 +111,7 @@ python3Full.pkgs.buildPythonApplication rec { echo "STARTING ..." export WEBVIEW_LIB_DIR="${webview-lib}/lib" export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 - python3 -m pytest -s -m "not impure" ./tests + python -m pytest -s -m "not impure" ./tests touch $out ''; }; @@ -113,11 +119,12 @@ python3Full.pkgs.buildPythonApplication rec { # Additional pass-through attributes passthru.desktop-file = desktop-file; - passthru.devshellDeps = ps: (pyTestDeps ps); + passthru.devshellPyDeps = ps: (pyTestDeps ps) ++ (pyDeps ps); + passthru.pythonRuntime = pythonRuntime; postInstall = '' - mkdir -p $out/${python3Full.sitePackages}/clan_app/.webui - cp -r ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* $out/${python3Full.sitePackages}/clan_app/.webui + mkdir -p $out/${pythonRuntime.sitePackages}/clan_app/.webui + cp -r ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* $out/${pythonRuntime.sitePackages}/clan_app/.webui mkdir -p $out/share/icons/hicolor cp -r ./clan_app/assets/white-favicons/* $out/share/icons/hicolor ''; diff --git a/pkgs/clan-app/flake-module.nix b/pkgs/clan-app/flake-module.nix index 72c098ce7..adfdc486d 100644 --- a/pkgs/clan-app/flake-module.nix +++ b/pkgs/clan-app/flake-module.nix @@ -4,23 +4,19 @@ { config, pkgs, - lib, - system, self', ... }: - if lib.elem system lib.platforms.darwin then - { } - else - { - devShells.clan-app = pkgs.callPackage ./shell.nix { - inherit (config.packages) clan-app webview-lib; - inherit self'; - }; - packages.clan-app = pkgs.python3.pkgs.callPackage ./default.nix { - inherit (config.packages) clan-cli webview-ui webview-lib; - }; - - checks = config.packages.clan-app.tests; + { + devShells.clan-app = pkgs.callPackage ./shell.nix { + inherit (config.packages) clan-app webview-lib; + inherit self'; }; + packages.clan-app = pkgs.callPackage ./default.nix { + inherit (config.packages) clan-cli webview-ui webview-lib; + pythonRuntime = pkgs.python3; + }; + + checks = config.packages.clan-app.tests; + }; } diff --git a/pkgs/clan-app/pyproject.toml b/pkgs/clan-app/pyproject.toml index 33c8654cf..16191469e 100644 --- a/pkgs/clan-app/pyproject.toml +++ b/pkgs/clan-app/pyproject.toml @@ -36,6 +36,3 @@ disallow_untyped_calls = true disallow_untyped_defs = true no_implicit_optional = true -[[tool.mypy.overrides]] -module = "clan_cli.*" -ignore_missing_imports = true diff --git a/pkgs/clan-app/shell.nix b/pkgs/clan-app/shell.nix index 9d0cac805..43253f255 100644 --- a/pkgs/clan-app/shell.nix +++ b/pkgs/clan-app/shell.nix @@ -5,7 +5,6 @@ ruff, gtk4, webview-lib, - python3Full, self', }: @@ -14,14 +13,14 @@ mkShell { inputsFrom = [ self'.devShells.default ]; buildInputs = [ - (python3Full.withPackages ( + (clan-app.pythonRuntime.withPackages ( ps: with ps; [ ruff mypy ] - ++ (clan-app.devshellDeps ps) + ++ (clan-app.devshellPyDeps ps) )) ]; @@ -42,6 +41,6 @@ mkShell { export XDG_DATA_DIRS=${gsettings-desktop-schemas}/share/gsettings-schemas/gsettings-desktop-schemas-46.0:$XDG_DATA_DIRS export WEBVIEW_LIB_DIR=${webview-lib}/lib - # export WEBVIEW_LIB_DIR=$HOME/Projects/webview/build/core + source $PKG_ROOT/.local.env ''; } diff --git a/pkgs/clan-cli/clan_cli/nix/allowed-programs.json b/pkgs/clan-cli/clan_cli/nix/allowed-programs.json index 12ffcf8f8..f49856f9d 100644 --- a/pkgs/clan-cli/clan_cli/nix/allowed-programs.json +++ b/pkgs/clan-cli/clan_cli/nix/allowed-programs.json @@ -16,5 +16,6 @@ "virtiofsd", "zbar", "util-linux", - "avahi" + "avahi", + "gnupg" ] diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 3ff032641..9a5feeff5 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -1,67 +1,53 @@ { # callPackage args - argcomplete, gnupg, installShellFiles, lib, nix, pkgs, - pytest-cov, - pytest-subprocess, - pytest-timeout, - pytest-xdist, - pytest, - python3, runCommand, - setuptools, stdenv, nixVersions, - # custom args clan-core-path, nixpkgs, includedRuntimeDeps, - inventory-schema-abstract, classgen, + pythonRuntime, }: let - pythonDependencies = [ - argcomplete # Enables shell completions + pyDeps = ps: [ + ps.argcomplete # Enables shell completions ]; + pyTestDeps = + ps: + [ + ps.pytest + ps.pytest-cov + ps.pytest-subprocess + ps.pytest-xdist + ps.pytest-timeout + ] + ++ (pyDeps ps); + pythonRuntimeWithDeps = pythonRuntime.withPackages (ps: pyDeps ps); # load nixpkgs runtime dependencies from a json file # This file represents an allow list at the same time that is checked by the run_cmd # implementation in nix.py allDependencies = lib.importJSON ./clan_cli/nix/allowed-programs.json; - generateRuntimeDependenciesMap = deps: lib.filterAttrs (_: pkg: !pkg.meta.unsupported or false) (lib.genAttrs deps (name: pkgs.${name})); - runtimeDependenciesMap = generateRuntimeDependenciesMap allDependencies; - runtimeDependencies = lib.attrValues runtimeDependenciesMap; - includedRuntimeDependenciesMap = generateRuntimeDependenciesMap includedRuntimeDeps; - testDependencies = - runtimeDependencies - ++ [ - gnupg - stdenv.cc # Compiler used for certain native extensions - ] - ++ pythonDependencies - ++ [ - pytest - pytest-cov # Generate coverage reports - pytest-subprocess # fake the real subprocess behavior to make your tests more independent. - pytest-xdist # Run tests in parallel on multiple cores - pytest-timeout # Add timeouts to your tests - ]; - - # Setup Python environment with all dependencies for running tests - pythonWithTestDeps = python3.withPackages (_ps: testDependencies); + testDependencies = runtimeDependencies ++ [ + gnupg + stdenv.cc # Compiler used for certain native extensions + (pythonRuntime.withPackages (ps: (pyTestDeps ps) ++ (pyDeps ps))) + ]; source = runCommand "clan-cli-source" { } '' cp -r ${./.} $out @@ -73,7 +59,6 @@ let ''; # Create a custom nixpkgs for use within the project - nixpkgs' = runCommand "nixpkgs" { @@ -101,7 +86,7 @@ let --extra-experimental-features 'nix-command flakes' ''; in -python3.pkgs.buildPythonApplication { +pythonRuntime.pkgs.buildPythonApplication { name = "clan-cli"; src = source; format = "pyproject"; @@ -123,30 +108,29 @@ python3.pkgs.buildPythonApplication { ]; nativeBuildInputs = [ - setuptools + (pythonRuntime.withPackages (ps: [ ps.setuptools ])) installShellFiles ]; - propagatedBuildInputs = pythonDependencies; + propagatedBuildInputs = [ pythonRuntimeWithDeps ] ++ runtimeDependencies; # Define and expose the tests and checks to run in CI passthru.tests = (lib.mapAttrs' (n: lib.nameValuePair "clan-dep-${n}") runtimeDependenciesMap) // { clan-pytest-without-core = - runCommand "clan-pytest-without-core" - { nativeBuildInputs = [ pythonWithTestDeps ] ++ testDependencies; } + runCommand "clan-pytest-without-core" { nativeBuildInputs = testDependencies; } '' cp -r ${source} ./src chmod +w -R ./src cd ./src export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 PYTHONWARNINGS=error - ${pythonWithTestDeps}/bin/python -m pytest -m "not impure and not with_core" ./tests + python -m pytest -m "not impure and not with_core" ./tests touch $out ''; clan-pytest-with-core = runCommand "clan-pytest-with-core" { - nativeBuildInputs = [ pythonWithTestDeps ] ++ testDependencies; + nativeBuildInputs = testDependencies; buildInputs = [ pkgs.bash pkgs.coreutils @@ -177,25 +161,25 @@ python3.pkgs.buildPythonApplication { mkdir -p "$CLAN_TEST_STORE/nix/store" xargs cp --recursive --target "$CLAN_TEST_STORE/nix/store" < "$closureInfo/store-paths" nix-store --load-db --store "$CLAN_TEST_STORE" < "$closureInfo/registration" - ${pythonWithTestDeps}/bin/python -m pytest -m "not impure and with_core" ./tests + python -m pytest -m "not impure and with_core" ./tests touch $out ''; }; passthru.nixpkgs = nixpkgs'; - passthru.testDependencies = testDependencies; - passthru.pythonWithTestDeps = pythonWithTestDeps; + passthru.devshellPyDeps = ps: (pyTestDeps ps) ++ (pyDeps ps); + passthru.pythonRuntime = pythonRuntime; passthru.runtimeDependencies = runtimeDependencies; passthru.runtimeDependenciesMap = runtimeDependenciesMap; postInstall = '' - cp -r ${nixpkgs'} $out/${python3.sitePackages}/clan_cli/nixpkgs + cp -r ${nixpkgs'} $out/${pythonRuntime.sitePackages}/clan_cli/nixpkgs installShellCompletion --bash --name clan \ - <(${argcomplete}/bin/register-python-argcomplete --shell bash clan) + <(${pythonRuntimeWithDeps.pkgs.argcomplete}/bin/register-python-argcomplete --shell bash clan) installShellCompletion --fish --name clan.fish \ - <(${argcomplete}/bin/register-python-argcomplete --shell fish clan) + <(${pythonRuntimeWithDeps.pkgs.argcomplete}/bin/register-python-argcomplete --shell fish clan) installShellCompletion --zsh --name _clan \ - <(${argcomplete}/bin/register-python-argcomplete --shell zsh clan) + <(${pythonRuntimeWithDeps.pkgs.argcomplete}/bin/register-python-argcomplete --shell zsh clan) ''; # Clean up after the package to avoid leaking python packages into a devshell diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index 5aadcaca2..f16928d53 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -103,21 +103,23 @@ inherit self'; }; packages = { - clan-cli = pkgs.python3.pkgs.callPackage ./default.nix { + clan-cli = pkgs.callPackage ./default.nix { inherit (inputs) nixpkgs; inherit (self'.packages) classgen; inherit (self'.legacyPackages.schemas) inventory-schema-abstract; + pythonRuntime = pkgs.python3; clan-core-path = clanCoreWithVendoredDeps; includedRuntimeDeps = [ "age" "git" ]; }; - clan-cli-full = pkgs.python3.pkgs.callPackage ./default.nix { + clan-cli-full = pkgs.callPackage ./default.nix { inherit (inputs) nixpkgs; inherit (self'.packages) classgen; inherit (self'.legacyPackages.schemas) inventory-schema-abstract; clan-core-path = clanCoreWithVendoredDeps; + pythonRuntime = pkgs.python3; includedRuntimeDeps = lib.importJSON ./clan_cli/nix/allowed-programs.json; }; clan-cli-docs = pkgs.stdenv.mkDerivation { diff --git a/pkgs/clan-cli/pyproject.toml b/pkgs/clan-cli/pyproject.toml index cdd7369c9..df77c8166 100644 --- a/pkgs/clan-cli/pyproject.toml +++ b/pkgs/clan-cli/pyproject.toml @@ -45,19 +45,4 @@ disallow_untyped_defs = true no_implicit_optional = true exclude = "clan_cli.nixpkgs" -[[tool.mypy.overrides]] -module = "argcomplete.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "ipdb.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pytest.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "setuptools.*" -ignore_missing_imports = true diff --git a/pkgs/clan-cli/shell.nix b/pkgs/clan-cli/shell.nix index 88ccd6feb..1d4a8f17c 100644 --- a/pkgs/clan-cli/shell.nix +++ b/pkgs/clan-cli/shell.nix @@ -5,25 +5,22 @@ clan-cli-full, mkShell, ruff, - python3, self', }: -let - devshellTestDeps = - clan-cli.passthru.testDependencies - ++ (with python3.pkgs; [ - rope - setuptools - wheel - webcolors - pip - ]); -in + mkShell { buildInputs = [ + (clan-cli.pythonRuntime.withPackages ( + ps: + with ps; + [ + ruff + mypy + ] + ++ (clan-cli.devshellPyDeps ps) + )) nix-unit - ruff - ] ++ devshellTestDeps; + ] ++ clan-cli.runtimeDependencies; inputsFrom = [ self'.devShells.default ];