From 8ea35b2b9919558597d705da2e96cc43f64202bf Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 27 May 2025 19:11:00 +0200 Subject: [PATCH 1/6] Tests(clan_lib): enable clan_lib pytests not marked with_core --- pkgs/clan-cli/default.nix | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index debddc29e..85ae453f9 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -179,6 +179,35 @@ pythonRuntime.pkgs.buildPythonApplication { python -m pytest -m "not impure and not with_core" -n $jobs ./clan_cli/tests touch $out ''; + clan-lib-pytest-without-core = + runCommand "clan-lib-pytest-without-core" + { + nativeBuildInputs = testDependencies; + closureInfo = pkgs.closureInfo { + rootPaths = [ + templateDerivation + ]; + }; + } + '' + set -euo pipefail + cp -r ${sourceWithTests} ./src + chmod +w -R ./src + cd ./src + + export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 PYTHONWARNINGS=error + + # required to prevent concurrent 'nix flake lock' operations + export CLAN_TEST_STORE=$TMPDIR/store + export LOCK_NIX=$TMPDIR/nix_lock + mkdir -p "$CLAN_TEST_STORE/nix/store" + + # limit build cores to 16 + jobs="$((NIX_BUILD_CORES>16 ? 16 : NIX_BUILD_CORES))" + + python -m pytest -m "not impure and not with_core" -n $jobs ./clan_lib + touch $out + ''; } // lib.optionalAttrs (!stdenv.isDarwin) { # disabled on macOS until we fix all remaining issues From a49466e1a91f5888f9bc26b2b8216c92fcca3efb Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 27 May 2025 19:11:21 +0200 Subject: [PATCH 2/6] Feat(inventoryStore): add possibility to turn off output filtering --- pkgs/clan-cli/clan_lib/persist/inventory_store.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_lib/persist/inventory_store.py b/pkgs/clan-cli/clan_lib/persist/inventory_store.py index f46767baa..898638c8a 100644 --- a/pkgs/clan-cli/clan_lib/persist/inventory_store.py +++ b/pkgs/clan-cli/clan_lib/persist/inventory_store.py @@ -132,7 +132,10 @@ class InventoryStore: - and more """ raw_value = self._flake.select("clanInternals.inventoryClass.inventory") - filtered = {k: v for k, v in raw_value.items() if k in self._keys} + if self._keys: + filtered = {k: v for k, v in raw_value.items() if k in self._keys} + else: + filtered = raw_value sanitized = sanitize(filtered, self._allowed_path_transforms, []) return sanitized From 4c6063c75e96879bb67c9c17bdcceda5e21f7a28 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 27 May 2025 19:12:03 +0200 Subject: [PATCH 3/6] Tests(inventoryStore): fixup tests to run in ci --- pkgs/clan-cli/clan_lib/persist/inventory_store_test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py b/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py index 581651a11..06e2bd7e6 100644 --- a/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py +++ b/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py @@ -71,6 +71,7 @@ class MockFlake: folder_path = Path(__file__).parent.resolve() +@pytest.mark.with_core def test_simple_read_write() -> None: entry_file = "1.nix" inventory_file = entry_file.replace(".nix", ".json") @@ -90,7 +91,9 @@ def test_simple_read_write() -> None: store = InventoryStore( flake=MockFlake(Path(tmp) / entry_file), inventory_file_name=inventory_file, + _keys=[], # disable toplevel filtering ) + store._flake.invalidate_cache() data: dict = store.read() # type: ignore assert data == {"foo": "bar", "protected": "protected"} @@ -118,6 +121,7 @@ def test_simple_read_write() -> None: store.write(data, "test", commit=False) # type: ignore +@pytest.mark.with_core def test_read_deferred() -> None: entry_file = "deferred.nix" inventory_file = entry_file.replace(".nix", ".json") @@ -138,6 +142,7 @@ def test_read_deferred() -> None: flake=MockFlake(Path(tmp) / entry_file), inventory_file_name=inventory_file, _allowed_path_transforms=["foo.*"], + _keys=[], # disable toplevel filtering ) data = store.read() From 84622222d8b23b7e8566b87bbd452b899be69e00 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 28 May 2025 10:42:38 +0200 Subject: [PATCH 4/6] Fix(tests): fix failing pytests --- .../clan_lib/bwrap/tests/test_bwrap.py | 7 ----- .../clan_lib/persist/inventory_store_test.py | 2 ++ pkgs/clan-cli/default.nix | 31 +------------------ 3 files changed, 3 insertions(+), 37 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/bwrap/tests/test_bwrap.py b/pkgs/clan-cli/clan_lib/bwrap/tests/test_bwrap.py index b376e3266..270a69bb8 100644 --- a/pkgs/clan-cli/clan_lib/bwrap/tests/test_bwrap.py +++ b/pkgs/clan-cli/clan_lib/bwrap/tests/test_bwrap.py @@ -7,10 +7,3 @@ from clan_lib.bwrap import bubblewrap_works @pytest.mark.skipif(sys.platform != "linux", reason="bubblewrap only works on linux") def test_bubblewrap_works_on_linux() -> None: assert bubblewrap_works() is True - - -@pytest.mark.skipif( - sys.platform == "linux", reason="bubblewrap does not work on non-linux" -) -def test_bubblewrap_detection_non_linux() -> None: - assert bubblewrap_works() is False diff --git a/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py b/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py index 06e2bd7e6..79f87a8ef 100644 --- a/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py +++ b/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py @@ -39,6 +39,8 @@ class MockFlake: output = subprocess.run( [ "nix", + "--extra-experimental-features", + "nix-command flakes", "eval", "--impure", "--json", diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 85ae453f9..1d05db7d0 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -176,36 +176,7 @@ pythonRuntime.pkgs.buildPythonApplication { # limit build cores to 16 jobs="$((NIX_BUILD_CORES>16 ? 16 : NIX_BUILD_CORES))" - python -m pytest -m "not impure and not with_core" -n $jobs ./clan_cli/tests - touch $out - ''; - clan-lib-pytest-without-core = - runCommand "clan-lib-pytest-without-core" - { - nativeBuildInputs = testDependencies; - closureInfo = pkgs.closureInfo { - rootPaths = [ - templateDerivation - ]; - }; - } - '' - set -euo pipefail - cp -r ${sourceWithTests} ./src - chmod +w -R ./src - cd ./src - - export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 PYTHONWARNINGS=error - - # required to prevent concurrent 'nix flake lock' operations - export CLAN_TEST_STORE=$TMPDIR/store - export LOCK_NIX=$TMPDIR/nix_lock - mkdir -p "$CLAN_TEST_STORE/nix/store" - - # limit build cores to 16 - jobs="$((NIX_BUILD_CORES>16 ? 16 : NIX_BUILD_CORES))" - - python -m pytest -m "not impure and not with_core" -n $jobs ./clan_lib + python -m pytest -m "not impure and not with_core" -n $jobs ./clan_cli ./clan_lib touch $out ''; } From 3e160d63923c5d63d19eaf09cc57af7d705339ce Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 28 May 2025 11:02:14 +0200 Subject: [PATCH 5/6] Fix(clan_cli): filter tests files from source --- pkgs/clan-cli/default.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 1d05db7d0..c10142b0b 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -83,7 +83,9 @@ let include = [ ( _root: path: _type: - (builtins.match "test_.*\.py" path) == null + (builtins.match ".*/test_[^/]+\.py" path) == null + && (builtins.match ".*/[^/]+_test\.py" path) == null + # && (builtins.match ".*/tests/.+" path) == null ) ]; } From db31b91401537b4fca17949f890203f77a0c5206 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 28 May 2025 11:27:13 +0200 Subject: [PATCH 6/6] Fix(inventoryStore): use 'nix_eval' wrapper to set the store correct --- .../clan-cli/clan_lib/persist/inventory_store_test.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py b/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py index 79f87a8ef..ee8d09496 100644 --- a/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py +++ b/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py @@ -10,6 +10,7 @@ from typing import Any import pytest from clan_lib.errors import ClanError +from clan_lib.nix import nix_eval from clan_lib.persist.inventory_store import InventoryStore from clan_lib.persist.util import delete_by_path, set_value_by_path @@ -36,14 +37,9 @@ class MockFlake: assert select, "NIX_SELECT environment variable is not set" assert clan_core_path, "CLAN_CORE_PATH environment variable is not set" - output = subprocess.run( + cmd = nix_eval( [ - "nix", - "--extra-experimental-features", - "nix-command flakes", - "eval", "--impure", - "--json", "--expr", f""" let @@ -56,6 +52,9 @@ class MockFlake: select "{selector}" result """, ], + ) + output = subprocess.run( + cmd, capture_output=True, ) res_str = output.stdout.decode()