diff --git a/lib/introspection/default.nix b/lib/introspection/default.nix index e0ade12b4..ef5c49b68 100644 --- a/lib/introspection/default.nix +++ b/lib/introspection/default.nix @@ -58,12 +58,15 @@ let getPrios { options = submoduleEval.options; } else # Nested attrsOf - (lib.optionalAttrs (type.nestedTypes.elemType.name == "attrsOf") ( - prioPerValue { - type = type.nestedTypes.elemType; - defs = zipDefs defs.${attrName}; - } prioSet.value - )) + (lib.optionalAttrs + (type.nestedTypes.elemType.name == "attrsOf" || type.nestedTypes.elemType.name == "lazyAttrsOf") + ( + prioPerValue { + type = type.nestedTypes.elemType; + defs = zipDefs defs.${attrName}; + } prioSet.value + ) + ) ) ); @@ -79,7 +82,7 @@ let in if opt ? type && opt.type.name == "submodule" then (prio) // submodulePrios - else if opt ? type && opt.type.name == "attrsOf" then + else if opt ? type && (opt.type.name == "attrsOf" || opt.type.name == "lazyAttrsOf") then prio // (prioPerValue { type = opt.type; diff --git a/pkgs/clan-cli/clan_lib/persist/fixtures/1.json b/pkgs/clan-cli/clan_lib/persist/fixtures/1.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/pkgs/clan-cli/clan_lib/persist/fixtures/1.json @@ -0,0 +1 @@ +{} diff --git a/pkgs/clan-cli/clan_lib/persist/fixtures/1.nix b/pkgs/clan-cli/clan_lib/persist/fixtures/1.nix new file mode 100644 index 000000000..6918b0e37 --- /dev/null +++ b/pkgs/clan-cli/clan_lib/persist/fixtures/1.nix @@ -0,0 +1,29 @@ +{ clanLib, lib, ... }: +let + eval = lib.evalModules { + modules = [ + { + # Trying to write into the default + options.foo = lib.mkOption { + type = lib.types.str; + default = "bar"; + }; + options.protected = lib.mkOption { + type = lib.types.str; + }; + } + { + # Cannot write into the default set prio + protected = "protected"; + } + # Merge the "inventory.json" + (builtins.fromJSON (builtins.readFile ./1.json)) + ]; + }; +in +{ + clanInternals.inventoryClass.inventory = eval.config; + clanInternals.inventoryClass.introspection = clanLib.introspection.getPrios { + options = eval.options; + }; +} diff --git a/pkgs/clan-cli/clan_lib/persist/inventory_store.py b/pkgs/clan-cli/clan_lib/persist/inventory_store.py index 29286a614..51a3f3900 100644 --- a/pkgs/clan-cli/clan_lib/persist/inventory_store.py +++ b/pkgs/clan-cli/clan_lib/persist/inventory_store.py @@ -1,8 +1,9 @@ import json from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol from clan_lib.errors import ClanError -from clan_lib.flake import Flake from clan_lib.git import commit_file from clan_lib.nix_models.inventory import Inventory @@ -21,13 +22,23 @@ class WriteInfo: data_disk: Inventory +class FlakeInterface(Protocol): + def select( + self, + selector: str, + nix_options: list[str] | None = None, + ) -> Any: ... + + @property + def path(self) -> Path: ... + + class InventoryStore: def __init__( - self, - flake: Flake, + self, flake: FlakeInterface, inventory_file_name: str = "inventory.json" ) -> None: self._flake = flake - self.inventory_file = self._flake.path / "inventory.json" + self.inventory_file = self._flake.path / inventory_file_name def _load_merged_inventory(self) -> Inventory: """ @@ -49,7 +60,6 @@ class InventoryStore: """ # TODO: make this configurable - if not self.inventory_file.exists(): return {} with self.inventory_file.open() as f: @@ -110,13 +120,11 @@ class InventoryStore: """ return self._load_merged_inventory() - def delete(self, delete_set: set[str]) -> None: + def delete(self, delete_set: set[str], commit: bool = True) -> None: """ Delete keys from the inventory """ - write_info = self._write_info() - - data_disk = dict(write_info.data_disk) + data_disk = dict(self._get_persisted()) for delete_path in delete_set: delete_by_path(data_disk, delete_path) @@ -124,11 +132,12 @@ class InventoryStore: with self.inventory_file.open("w") as f: json.dump(data_disk, f, indent=2) - commit_file( - self.inventory_file, - self._flake.path, - commit_message=f"Delete inventory keys {delete_set}", - ) + if commit: + commit_file( + self.inventory_file, + self._flake.path, + commit_message=f"Delete inventory keys {delete_set}", + ) def write(self, update: Inventory, message: str, commit: bool = True) -> None: """ @@ -157,8 +166,7 @@ class InventoryStore: for patch_path, data in patchset.items(): apply_patch(persisted, patch_path, data) - for delete_path in delete_set: - delete_by_path(persisted, delete_path) + self.delete(delete_set, commit=False) with self.inventory_file.open("w") as f: json.dump(persisted, f, indent=2) diff --git a/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py b/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py index e69de29bb..bfa6c2f54 100644 --- a/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py +++ b/pkgs/clan-cli/clan_lib/persist/inventory_store_test.py @@ -0,0 +1,110 @@ +# ruff: noqa: SLF001 +import json +import os +import shutil +import subprocess +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +import pytest + +from clan_lib.errors import ClanError +from clan_lib.persist.inventory_store import InventoryStore + + +class MockFlake: + def __init__(self, default: Path) -> None: + f = default + assert f.exists(), f"File {f} does not exist" + self._file = f + + def select( + self, + selector: str, + nix_options: list[str] | None = None, + ) -> Any: + nixpkgs = os.environ.get("NIXPKGS") + select = os.environ.get("NIX_SELECT") + clan_core_path = os.environ.get("CLAN_CORE_PATH") + + assert nixpkgs, "NIXPKGS environment variable is not set" + assert select, "NIX_SELECT environment variable is not set" + assert clan_core_path, "CLAN_CORE_PATH environment variable is not set" + + output = subprocess.run( + [ + "nix", + "eval", + "--impure", + "--json", + "--expr", + f""" + let + pkgs = import {nixpkgs} {{}}; + inherit (pkgs) lib; + clanLib = import {Path(clan_core_path)}/lib {{ inherit lib; self = null; nixpkgs = {nixpkgs}; }}; + select = (import {select}/select.nix).select; + result = import {self._file} {{ inherit pkgs lib clanLib; }}; + in + select "{selector}" result + """, + ], + capture_output=True, + ) + res_str = output.stdout.decode() + + if output.returncode != 0: + msg = f"Failed to evaluate {selector} in {self._file}: {output.stderr.decode()}" + raise ClanError(msg) + return json.loads(res_str) + + @property + def path(self) -> Path: + return self._file.parent + + +folder_path = Path(__file__).parent.resolve() + + +def test_for_johannes() -> None: + nix_file = folder_path / "fixtures/1.nix" + json_file = folder_path / "fixtures/1.json" + with TemporaryDirectory() as tmp: + shutil.copyfile( + str(nix_file), + str(Path(tmp) / "1.nix"), + ) + shutil.copyfile( + str(json_file), + str(Path(tmp) / "1.json"), + ) + + store = InventoryStore( + flake=MockFlake(Path(tmp) / "1.nix"), + inventory_file_name="1.json", + ) + assert store.read() == {"foo": "bar", "protected": "protected"} + + data = {"foo": "foo"} + store.write(data, "test", commit=False) # type: ignore + # Default method to access the inventory + assert store.read() == {"foo": "foo", "protected": "protected"} + + # Test the data is actually persisted + assert store._get_persisted() == data + + # clan_lib.errors.ClanError: Key 'protected' is not writeable. + invalid_data = {"protected": "foo"} + with pytest.raises(ClanError) as e: + store.write(invalid_data, "test", commit=False) # type: ignore + assert str(e.value) == "Key 'protected' is not writeable." + + # Test the data is not touched + assert store.read() == {"foo": "foo", "protected": "protected"} + assert store._get_persisted() == data + + # Remove the foo key from the persisted data + # Technically data = { } should also work + data = {"protected": "protected"} + store.write(data, "test", commit=False) # type: ignore diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 18fcca1a0..debddc29e 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -274,6 +274,10 @@ pythonRuntime.pkgs.buildPythonApplication { xargs cp --recursive --target "$CLAN_TEST_STORE/nix/store" < "$closureInfo/store-paths" nix-store --load-db --store "$CLAN_TEST_STORE" < "$closureInfo/registration" + # used for tests without flakes + export NIXPKGS=${nixpkgs} + export NIX_SELECT=${nix-select} + # limit build cores to 4 jobs="$((NIX_BUILD_CORES>4 ? 4 : NIX_BUILD_CORES))" diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index 7ff498cc2..95a42ab0f 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -35,7 +35,7 @@ in { devShells.clan-cli = pkgs.callPackage ./shell.nix { - inherit self'; + inherit self' self; inherit (self'.packages) clan-cli; }; packages = { diff --git a/pkgs/clan-cli/shell.nix b/pkgs/clan-cli/shell.nix index f5a76fdcd..7057c4423 100644 --- a/pkgs/clan-cli/shell.nix +++ b/pkgs/clan-cli/shell.nix @@ -4,6 +4,7 @@ clan-cli, mkShell, ruff, + self, self', }: @@ -35,6 +36,10 @@ mkShell { export CLAN_CORE_PATH="$GIT_ROOT" + # used for tests without flakes + export NIXPKGS=${self.inputs.nixpkgs.outPath} + export NIX_SELECT=${self.inputs.nix-select.outPath} + # Add current package to PYTHONPATH export PYTHONPATH="$PKG_ROOT''${PYTHONPATH:+:$PYTHONPATH:}"