Merge pull request 'Test(InventoryPersistence): add persist integration tests' (#3736) from hsjobeki/clan-core:fix-2 into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3736

lib/introspection/default.nix

@@ -58,12 +58,15 @@ let
                 getPrios { options = submoduleEval.options; }
               else
                 # Nested attrsOf
-                (lib.optionalAttrs (type.nestedTypes.elemType.name == "attrsOf") (
+                (lib.optionalAttrs
+                  (type.nestedTypes.elemType.name == "attrsOf" || type.nestedTypes.elemType.name == "lazyAttrsOf")
+                  (
                     prioPerValue {
                       type = type.nestedTypes.elemType;
                       defs = zipDefs defs.${attrName};
                     } prioSet.value
-                ))
+                  )
+                )
             )
           );
 
@@ -79,7 +82,7 @@ let
       in
       if opt ? type && opt.type.name == "submodule" then
         (prio) // submodulePrios
-      else if opt ? type && opt.type.name == "attrsOf" then
+      else if opt ? type && (opt.type.name == "attrsOf" || opt.type.name == "lazyAttrsOf") then
         prio
         // (prioPerValue {
           type = opt.type;

pkgs/clan-cli/clan_lib/persist/fixtures/1.json

@@ -0,0 +1 @@
+{}

pkgs/clan-cli/clan_lib/persist/fixtures/1.nix

@@ -0,0 +1,29 @@
+{ clanLib, lib, ... }:
+let
+  eval = lib.evalModules {
+    modules = [
+      {
+        # Trying to write into the default
+        options.foo = lib.mkOption {
+          type = lib.types.str;
+          default = "bar";
+        };
+        options.protected = lib.mkOption {
+          type = lib.types.str;
+        };
+      }
+      {
+        # Cannot write into the default set prio
+        protected = "protected";
+      }
+      # Merge the "inventory.json"
+      (builtins.fromJSON (builtins.readFile ./1.json))
+    ];
+  };
+in
+{
+  clanInternals.inventoryClass.inventory = eval.config;
+  clanInternals.inventoryClass.introspection = clanLib.introspection.getPrios {
+    options = eval.options;
+  };
+}

pkgs/clan-cli/clan_lib/persist/inventory_store.py

@@ -1,8 +1,9 @@
 import json
 from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Protocol
 
 from clan_lib.errors import ClanError
-from clan_lib.flake import Flake
 from clan_lib.git import commit_file
 from clan_lib.nix_models.inventory import Inventory
 
@@ -21,13 +22,23 @@ class WriteInfo:
     data_disk: Inventory
 
 
+class FlakeInterface(Protocol):
+    def select(
+        self,
+        selector: str,
+        nix_options: list[str] | None = None,
+    ) -> Any: ...
+
+    @property
+    def path(self) -> Path: ...
+
+
 class InventoryStore:
     def __init__(
-        self,
-        flake: Flake,
+        self, flake: FlakeInterface, inventory_file_name: str = "inventory.json"
     ) -> None:
         self._flake = flake
-        self.inventory_file = self._flake.path / "inventory.json"
+        self.inventory_file = self._flake.path / inventory_file_name
 
     def _load_merged_inventory(self) -> Inventory:
         """
@@ -49,7 +60,6 @@ class InventoryStore:
         """
 
         # TODO: make this configurable
-
         if not self.inventory_file.exists():
             return {}
         with self.inventory_file.open() as f:
@@ -110,13 +120,11 @@ class InventoryStore:
         """
         return self._load_merged_inventory()
 
-    def delete(self, delete_set: set[str]) -> None:
+    def delete(self, delete_set: set[str], commit: bool = True) -> None:
         """
         Delete keys from the inventory
         """
-        write_info = self._write_info()
-
-        data_disk = dict(write_info.data_disk)
+        data_disk = dict(self._get_persisted())
 
         for delete_path in delete_set:
             delete_by_path(data_disk, delete_path)
@@ -124,6 +132,7 @@ class InventoryStore:
         with self.inventory_file.open("w") as f:
             json.dump(data_disk, f, indent=2)
 
+        if commit:
             commit_file(
                 self.inventory_file,
                 self._flake.path,
@@ -157,8 +166,7 @@ class InventoryStore:
         for patch_path, data in patchset.items():
             apply_patch(persisted, patch_path, data)
 
-        for delete_path in delete_set:
-            delete_by_path(persisted, delete_path)
+        self.delete(delete_set, commit=False)
 
         with self.inventory_file.open("w") as f:
             json.dump(persisted, f, indent=2)

pkgs/clan-cli/clan_lib/persist/inventory_store_test.py

@@ -0,0 +1,110 @@
+# ruff: noqa: SLF001
+import json
+import os
+import shutil
+import subprocess
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any
+
+import pytest
+
+from clan_lib.errors import ClanError
+from clan_lib.persist.inventory_store import InventoryStore
+
+
+class MockFlake:
+    def __init__(self, default: Path) -> None:
+        f = default
+        assert f.exists(), f"File {f} does not exist"
+        self._file = f
+
+    def select(
+        self,
+        selector: str,
+        nix_options: list[str] | None = None,
+    ) -> Any:
+        nixpkgs = os.environ.get("NIXPKGS")
+        select = os.environ.get("NIX_SELECT")
+        clan_core_path = os.environ.get("CLAN_CORE_PATH")
+
+        assert nixpkgs, "NIXPKGS environment variable is not set"
+        assert select, "NIX_SELECT environment variable is not set"
+        assert clan_core_path, "CLAN_CORE_PATH environment variable is not set"
+
+        output = subprocess.run(
+            [
+                "nix",
+                "eval",
+                "--impure",
+                "--json",
+                "--expr",
+                f"""
+            let
+                pkgs = import {nixpkgs} {{}};
+                inherit (pkgs) lib;
+                clanLib = import {Path(clan_core_path)}/lib {{ inherit lib; self = null; nixpkgs = {nixpkgs}; }};
+                select = (import {select}/select.nix).select;
+                result = import {self._file} {{ inherit pkgs lib clanLib; }};
+            in
+                select "{selector}" result
+            """,
+            ],
+            capture_output=True,
+        )
+        res_str = output.stdout.decode()
+
+        if output.returncode != 0:
+            msg = f"Failed to evaluate {selector} in {self._file}: {output.stderr.decode()}"
+            raise ClanError(msg)
+        return json.loads(res_str)
+
+    @property
+    def path(self) -> Path:
+        return self._file.parent
+
+
+folder_path = Path(__file__).parent.resolve()
+
+
+def test_for_johannes() -> None:
+    nix_file = folder_path / "fixtures/1.nix"
+    json_file = folder_path / "fixtures/1.json"
+    with TemporaryDirectory() as tmp:
+        shutil.copyfile(
+            str(nix_file),
+            str(Path(tmp) / "1.nix"),
+        )
+        shutil.copyfile(
+            str(json_file),
+            str(Path(tmp) / "1.json"),
+        )
+
+        store = InventoryStore(
+            flake=MockFlake(Path(tmp) / "1.nix"),
+            inventory_file_name="1.json",
+        )
+        assert store.read() == {"foo": "bar", "protected": "protected"}
+
+        data = {"foo": "foo"}
+        store.write(data, "test", commit=False)  # type: ignore
+        # Default method to access the inventory
+        assert store.read() == {"foo": "foo", "protected": "protected"}
+
+        # Test the data is actually persisted
+        assert store._get_persisted() == data
+
+        # clan_lib.errors.ClanError: Key 'protected' is not writeable.
+        invalid_data = {"protected": "foo"}
+        with pytest.raises(ClanError) as e:
+            store.write(invalid_data, "test", commit=False)  # type: ignore
+        assert str(e.value) == "Key 'protected' is not writeable."
+
+        # Test the data is not touched
+        assert store.read() == {"foo": "foo", "protected": "protected"}
+        assert store._get_persisted() == data
+
+        # Remove the foo key from the persisted data
+        # Technically data = { } should also work
+        data = {"protected": "protected"}
+        store.write(data, "test", commit=False)  # type: ignore

pkgs/clan-cli/default.nix

@@ -274,6 +274,10 @@ pythonRuntime.pkgs.buildPythonApplication {
             xargs cp --recursive --target "$CLAN_TEST_STORE/nix/store"  < "$closureInfo/store-paths"
             nix-store --load-db --store "$CLAN_TEST_STORE" < "$closureInfo/registration"
 
+            # used for tests without flakes
+            export NIXPKGS=${nixpkgs}
+            export NIX_SELECT=${nix-select}
+
             # limit build cores to 4
             jobs="$((NIX_BUILD_CORES>4 ? 4 : NIX_BUILD_CORES))"
 

pkgs/clan-cli/flake-module.nix

@@ -35,7 +35,7 @@
     in
     {
       devShells.clan-cli = pkgs.callPackage ./shell.nix {
-        inherit self';
+        inherit self' self;
         inherit (self'.packages) clan-cli;
       };
       packages = {

pkgs/clan-cli/shell.nix

@@ -4,6 +4,7 @@
   clan-cli,
   mkShell,
   ruff,
+  self,
   self',
 }:
 
@@ -35,6 +36,10 @@ mkShell {
 
     export CLAN_CORE_PATH="$GIT_ROOT"
 
+     # used for tests without flakes
+    export NIXPKGS=${self.inputs.nixpkgs.outPath}
+    export NIX_SELECT=${self.inputs.nix-select.outPath}
+
     # Add current package to PYTHONPATH
     export PYTHONPATH="$PKG_ROOT''${PYTHONPATH:+:$PYTHONPATH:}"