diff --git a/formatter.nix b/formatter.nix index c88a555fd..a1cd9a1d9 100644 --- a/formatter.nix +++ b/formatter.nix @@ -92,13 +92,12 @@ treefmt.programs.mypy.directories = { "clan-cli" = { - extraPythonPackages = self'.packages.clan-cli.testDependencies; directory = "pkgs/clan-cli"; + extraPythonPackages = (self'.packages.clan-cli.devshellPyDeps pkgs.python3Packages); }; "clan-app" = { directory = "pkgs/clan-app"; - extraPythonPackages = - (self'.packages.clan-app.externalTestDeps or [ ]) ++ self'.packages.clan-cli.testDependencies; + extraPythonPackages = (self'.packages.clan-app.devshellPyDeps pkgs.python3Packages); extraPythonPaths = [ "../clan-cli" ]; }; } @@ -107,8 +106,9 @@ { "clan-vm-manager" = { directory = "pkgs/clan-vm-manager"; - extraPythonPackages = - self'.packages.clan-vm-manager.externalTestDeps ++ self'.packages.clan-cli.testDependencies; + extraPythonPackages = self'.packages.clan-vm-manager.externalTestDeps ++ [ + (pkgs.python3.withPackages (ps: self'.packages.clan-cli.devshellPyDeps ps)) + ]; extraPythonPaths = [ "../clan-cli" ]; }; } diff --git a/pkgs/clan-app/default.nix b/pkgs/clan-app/default.nix index e518ef71d..4c5ca67ac 100644 --- a/pkgs/clan-app/default.nix +++ b/pkgs/clan-app/default.nix @@ -1,16 +1,16 @@ { - python3Full, runCommand, - setuptools, copyDesktopItems, clan-cli, makeDesktopItem, webview-ui, webview-lib, fontconfig, + pythonRuntime, }: let source = ./.; + desktop-file = makeDesktopItem { name = "org.clan.app"; exec = "clan-app %u"; @@ -25,24 +25,29 @@ let ]; + pyDeps = ps: [ + ps.tkinter + ]; + # Dependencies required for running tests pyTestDeps = ps: - with ps; [ - (python3Full.pkgs.toPythonModule pytest) # Testing framework - pytest-cov # Generate coverage reports - pytest-subprocess # fake the real subprocess behavior to make your tests more independent. - pytest-xdist # Run tests in parallel on multiple cores - pytest-timeout # Add timeouts to your tests + ps.pytest + ps.pytest-cov # Generate coverage reports + ps.pytest-subprocess # fake the real subprocess behavior to make your tests more independent. + ps.pytest-xdist # Run tests in parallel on multiple cores + ps.pytest-timeout # Add timeouts to your tests ] - ++ pytest.propagatedBuildInputs; + ++ ps.pytest.propagatedBuildInputs; - clan-cli-module = [ (python3Full.pkgs.toPythonModule clan-cli) ]; + clan-cli-module = [ + (pythonRuntime.pkgs.toPythonModule (clan-cli.override { inherit pythonRuntime; })) + ]; in -python3Full.pkgs.buildPythonApplication rec { +pythonRuntime.pkgs.buildPythonApplication { name = "clan-app"; src = source; format = "pyproject"; @@ -51,7 +56,7 @@ python3Full.pkgs.buildPythonApplication rec { preFixup = '' makeWrapperArgs+=( --set FONTCONFIG_FILE ${fontconfig.out}/etc/fonts/fonts.conf - --set WEBUI_PATH "$out/${python3Full.sitePackages}/clan_app/.webui" + --set WEBUI_PATH "$out/${pythonRuntime.sitePackages}/clan_app/.webui" --set WEBVIEW_LIB_DIR "${webview-lib}/lib" # This prevents problems with mixed glibc versions that might occur when the # cli is called through a browser built against another glibc @@ -62,7 +67,7 @@ python3Full.pkgs.buildPythonApplication rec { # Deps needed only at build time nativeBuildInputs = [ - setuptools + (pythonRuntime.withPackages (ps: [ ps.setuptools ])) copyDesktopItems fontconfig ]; @@ -71,8 +76,9 @@ python3Full.pkgs.buildPythonApplication rec { # same values for your Python package within Nix largely stems from ensuring # that all necessary dependencies are consistently available both # at build time and runtime, - buildInputs = clan-cli-module ++ runtimeDependencies; - propagatedBuildInputs = buildInputs; + propagatedBuildInputs = [ + (pythonRuntime.withPackages (ps: clan-cli-module ++ (pyDeps ps))) + ] ++ runtimeDependencies; # also re-expose dependencies so we test them in CI passthru = { @@ -81,7 +87,7 @@ python3Full.pkgs.buildPythonApplication rec { runCommand "clan-app-pytest" { buildInputs = runtimeDependencies ++ [ - (python3Full.withPackages (ps: clan-cli-module ++ (pyTestDeps ps))) + (pythonRuntime.withPackages (ps: clan-cli-module ++ (pyTestDeps ps) ++ (pyDeps ps))) fontconfig ]; } @@ -105,7 +111,7 @@ python3Full.pkgs.buildPythonApplication rec { echo "STARTING ..." export WEBVIEW_LIB_DIR="${webview-lib}/lib" export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 - python3 -m pytest -s -m "not impure" ./tests + python -m pytest -s -m "not impure" ./tests touch $out ''; }; @@ -113,11 +119,12 @@ python3Full.pkgs.buildPythonApplication rec { # Additional pass-through attributes passthru.desktop-file = desktop-file; - passthru.devshellDeps = ps: (pyTestDeps ps); + passthru.devshellPyDeps = ps: (pyTestDeps ps) ++ (pyDeps ps); + passthru.pythonRuntime = pythonRuntime; postInstall = '' - mkdir -p $out/${python3Full.sitePackages}/clan_app/.webui - cp -r ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* $out/${python3Full.sitePackages}/clan_app/.webui + mkdir -p $out/${pythonRuntime.sitePackages}/clan_app/.webui + cp -r ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* $out/${pythonRuntime.sitePackages}/clan_app/.webui mkdir -p $out/share/icons/hicolor cp -r ./clan_app/assets/white-favicons/* $out/share/icons/hicolor ''; diff --git a/pkgs/clan-app/flake-module.nix b/pkgs/clan-app/flake-module.nix index 72c098ce7..b74948dba 100644 --- a/pkgs/clan-app/flake-module.nix +++ b/pkgs/clan-app/flake-module.nix @@ -17,8 +17,9 @@ inherit (config.packages) clan-app webview-lib; inherit self'; }; - packages.clan-app = pkgs.python3.pkgs.callPackage ./default.nix { + packages.clan-app = pkgs.callPackage ./default.nix { inherit (config.packages) clan-cli webview-ui webview-lib; + pythonRuntime = pkgs.python3; }; checks = config.packages.clan-app.tests; diff --git a/pkgs/clan-app/pyproject.toml b/pkgs/clan-app/pyproject.toml index 33c8654cf..16191469e 100644 --- a/pkgs/clan-app/pyproject.toml +++ b/pkgs/clan-app/pyproject.toml @@ -36,6 +36,3 @@ disallow_untyped_calls = true disallow_untyped_defs = true no_implicit_optional = true -[[tool.mypy.overrides]] -module = "clan_cli.*" -ignore_missing_imports = true diff --git a/pkgs/clan-app/shell.nix b/pkgs/clan-app/shell.nix index 9d0cac805..3c7a6bf15 100644 --- a/pkgs/clan-app/shell.nix +++ b/pkgs/clan-app/shell.nix @@ -5,7 +5,6 @@ ruff, gtk4, webview-lib, - python3Full, self', }: @@ -14,14 +13,14 @@ mkShell { inputsFrom = [ self'.devShells.default ]; buildInputs = [ - (python3Full.withPackages ( + (clan-app.pythonRuntime.withPackages ( ps: with ps; [ ruff mypy ] - ++ (clan-app.devshellDeps ps) + ++ (clan-app.devshellPyDeps ps) )) ]; diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 3ff032641..9a5feeff5 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -1,67 +1,53 @@ { # callPackage args - argcomplete, gnupg, installShellFiles, lib, nix, pkgs, - pytest-cov, - pytest-subprocess, - pytest-timeout, - pytest-xdist, - pytest, - python3, runCommand, - setuptools, stdenv, nixVersions, - # custom args clan-core-path, nixpkgs, includedRuntimeDeps, - inventory-schema-abstract, classgen, + pythonRuntime, }: let - pythonDependencies = [ - argcomplete # Enables shell completions + pyDeps = ps: [ + ps.argcomplete # Enables shell completions ]; + pyTestDeps = + ps: + [ + ps.pytest + ps.pytest-cov + ps.pytest-subprocess + ps.pytest-xdist + ps.pytest-timeout + ] + ++ (pyDeps ps); + pythonRuntimeWithDeps = pythonRuntime.withPackages (ps: pyDeps ps); # load nixpkgs runtime dependencies from a json file # This file represents an allow list at the same time that is checked by the run_cmd # implementation in nix.py allDependencies = lib.importJSON ./clan_cli/nix/allowed-programs.json; - generateRuntimeDependenciesMap = deps: lib.filterAttrs (_: pkg: !pkg.meta.unsupported or false) (lib.genAttrs deps (name: pkgs.${name})); - runtimeDependenciesMap = generateRuntimeDependenciesMap allDependencies; - runtimeDependencies = lib.attrValues runtimeDependenciesMap; - includedRuntimeDependenciesMap = generateRuntimeDependenciesMap includedRuntimeDeps; - testDependencies = - runtimeDependencies - ++ [ - gnupg - stdenv.cc # Compiler used for certain native extensions - ] - ++ pythonDependencies - ++ [ - pytest - pytest-cov # Generate coverage reports - pytest-subprocess # fake the real subprocess behavior to make your tests more independent. - pytest-xdist # Run tests in parallel on multiple cores - pytest-timeout # Add timeouts to your tests - ]; - - # Setup Python environment with all dependencies for running tests - pythonWithTestDeps = python3.withPackages (_ps: testDependencies); + testDependencies = runtimeDependencies ++ [ + gnupg + stdenv.cc # Compiler used for certain native extensions + (pythonRuntime.withPackages (ps: (pyTestDeps ps) ++ (pyDeps ps))) + ]; source = runCommand "clan-cli-source" { } '' cp -r ${./.} $out @@ -73,7 +59,6 @@ let ''; # Create a custom nixpkgs for use within the project - nixpkgs' = runCommand "nixpkgs" { @@ -101,7 +86,7 @@ let --extra-experimental-features 'nix-command flakes' ''; in -python3.pkgs.buildPythonApplication { +pythonRuntime.pkgs.buildPythonApplication { name = "clan-cli"; src = source; format = "pyproject"; @@ -123,30 +108,29 @@ python3.pkgs.buildPythonApplication { ]; nativeBuildInputs = [ - setuptools + (pythonRuntime.withPackages (ps: [ ps.setuptools ])) installShellFiles ]; - propagatedBuildInputs = pythonDependencies; + propagatedBuildInputs = [ pythonRuntimeWithDeps ] ++ runtimeDependencies; # Define and expose the tests and checks to run in CI passthru.tests = (lib.mapAttrs' (n: lib.nameValuePair "clan-dep-${n}") runtimeDependenciesMap) // { clan-pytest-without-core = - runCommand "clan-pytest-without-core" - { nativeBuildInputs = [ pythonWithTestDeps ] ++ testDependencies; } + runCommand "clan-pytest-without-core" { nativeBuildInputs = testDependencies; } '' cp -r ${source} ./src chmod +w -R ./src cd ./src export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 PYTHONWARNINGS=error - ${pythonWithTestDeps}/bin/python -m pytest -m "not impure and not with_core" ./tests + python -m pytest -m "not impure and not with_core" ./tests touch $out ''; clan-pytest-with-core = runCommand "clan-pytest-with-core" { - nativeBuildInputs = [ pythonWithTestDeps ] ++ testDependencies; + nativeBuildInputs = testDependencies; buildInputs = [ pkgs.bash pkgs.coreutils @@ -177,25 +161,25 @@ python3.pkgs.buildPythonApplication { mkdir -p "$CLAN_TEST_STORE/nix/store" xargs cp --recursive --target "$CLAN_TEST_STORE/nix/store" < "$closureInfo/store-paths" nix-store --load-db --store "$CLAN_TEST_STORE" < "$closureInfo/registration" - ${pythonWithTestDeps}/bin/python -m pytest -m "not impure and with_core" ./tests + python -m pytest -m "not impure and with_core" ./tests touch $out ''; }; passthru.nixpkgs = nixpkgs'; - passthru.testDependencies = testDependencies; - passthru.pythonWithTestDeps = pythonWithTestDeps; + passthru.devshellPyDeps = ps: (pyTestDeps ps) ++ (pyDeps ps); + passthru.pythonRuntime = pythonRuntime; passthru.runtimeDependencies = runtimeDependencies; passthru.runtimeDependenciesMap = runtimeDependenciesMap; postInstall = '' - cp -r ${nixpkgs'} $out/${python3.sitePackages}/clan_cli/nixpkgs + cp -r ${nixpkgs'} $out/${pythonRuntime.sitePackages}/clan_cli/nixpkgs installShellCompletion --bash --name clan \ - <(${argcomplete}/bin/register-python-argcomplete --shell bash clan) + <(${pythonRuntimeWithDeps.pkgs.argcomplete}/bin/register-python-argcomplete --shell bash clan) installShellCompletion --fish --name clan.fish \ - <(${argcomplete}/bin/register-python-argcomplete --shell fish clan) + <(${pythonRuntimeWithDeps.pkgs.argcomplete}/bin/register-python-argcomplete --shell fish clan) installShellCompletion --zsh --name _clan \ - <(${argcomplete}/bin/register-python-argcomplete --shell zsh clan) + <(${pythonRuntimeWithDeps.pkgs.argcomplete}/bin/register-python-argcomplete --shell zsh clan) ''; # Clean up after the package to avoid leaking python packages into a devshell diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index 5aadcaca2..f16928d53 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -103,21 +103,23 @@ inherit self'; }; packages = { - clan-cli = pkgs.python3.pkgs.callPackage ./default.nix { + clan-cli = pkgs.callPackage ./default.nix { inherit (inputs) nixpkgs; inherit (self'.packages) classgen; inherit (self'.legacyPackages.schemas) inventory-schema-abstract; + pythonRuntime = pkgs.python3; clan-core-path = clanCoreWithVendoredDeps; includedRuntimeDeps = [ "age" "git" ]; }; - clan-cli-full = pkgs.python3.pkgs.callPackage ./default.nix { + clan-cli-full = pkgs.callPackage ./default.nix { inherit (inputs) nixpkgs; inherit (self'.packages) classgen; inherit (self'.legacyPackages.schemas) inventory-schema-abstract; clan-core-path = clanCoreWithVendoredDeps; + pythonRuntime = pkgs.python3; includedRuntimeDeps = lib.importJSON ./clan_cli/nix/allowed-programs.json; }; clan-cli-docs = pkgs.stdenv.mkDerivation { diff --git a/pkgs/clan-cli/pyproject.toml b/pkgs/clan-cli/pyproject.toml index cdd7369c9..df77c8166 100644 --- a/pkgs/clan-cli/pyproject.toml +++ b/pkgs/clan-cli/pyproject.toml @@ -45,19 +45,4 @@ disallow_untyped_defs = true no_implicit_optional = true exclude = "clan_cli.nixpkgs" -[[tool.mypy.overrides]] -module = "argcomplete.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "ipdb.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pytest.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "setuptools.*" -ignore_missing_imports = true diff --git a/pkgs/clan-cli/shell.nix b/pkgs/clan-cli/shell.nix index 88ccd6feb..1d4a8f17c 100644 --- a/pkgs/clan-cli/shell.nix +++ b/pkgs/clan-cli/shell.nix @@ -5,25 +5,22 @@ clan-cli-full, mkShell, ruff, - python3, self', }: -let - devshellTestDeps = - clan-cli.passthru.testDependencies - ++ (with python3.pkgs; [ - rope - setuptools - wheel - webcolors - pip - ]); -in + mkShell { buildInputs = [ + (clan-cli.pythonRuntime.withPackages ( + ps: + with ps; + [ + ruff + mypy + ] + ++ (clan-cli.devshellPyDeps ps) + )) nix-unit - ruff - ] ++ devshellTestDeps; + ] ++ clan-cli.runtimeDependencies; inputsFrom = [ self'.devShells.default ];