yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'persistence: generate properties for data by traversing data' (#5502) from persistence_attributes into main

Browse Source 
Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/5502
This commit is contained in:
hsjobeki
2025-10-14 15:27:10 +00:00
parent 4cfe866079 8609538756
commit 8803343ae1
7 changed files with 137 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkgs/clan-cli/clan_cli/completions.py
View File

@@ -425,7 +425,7 @@ def complete_tags(
flake = "."
inventory_store = InventoryStore(Flake(str(flake)))
inventory = inventory_store.get_readonly_raw()
inventory = inventory_store.get_readonly_raw(inventory_store.default_keys())
if "tags" in inventory:
tags.extend(inventory["tags"].keys())
@@ -442,7 +442,7 @@ def complete_tags(
else:
flake = "."
inventory_store = InventoryStore(Flake(str(flake)))
inventory = inventory_store.get_readonly_raw()
inventory = inventory_store.get_readonly_raw(inventory_store.default_keys())
machine_tags_result = inventory.get("machines")
if machine_tags_result is None:
return

18
pkgs/clan-cli/clan_lib/persist/inventory_store.py
View File

@@ -11,7 +11,6 @@ from clan_lib.nix_models.clan import (
InventoryInstancesType,
InventoryMachinesType,
InventoryMetaType,
InventoryTagsType,
)
from clan_lib.persist.patch_engine import calc_patches
from clan_lib.persist.path_utils import (
@@ -20,7 +19,7 @@ from clan_lib.persist.path_utils import (
path_match,
set_value_by_path_tuple,
)
from clan_lib.persist.write_rules import AttributeMap, compute_attribute_map
from clan_lib.persist.write_rules import AttributeMap, compute_attribute_persistence
def unwrap_known_unknown(value: Any) -> Any:
@@ -102,7 +101,6 @@ class InventorySnapshot(TypedDict):
machines: NotRequired[InventoryMachinesType]
instances: NotRequired[InventoryInstancesType]
meta: NotRequired[InventoryMetaType]
tags: NotRequired[InventoryTagsType]
class InventoryStore:
@@ -111,7 +109,7 @@ class InventoryStore:
flake: FlakeInterface,
inventory_file_name: str = "inventory.json",
_allowed_path_transforms: list[str] | None = None,
_keys: list[str] | None = None,
_keys: set[str] | None = None,
) -> None:
"""InventoryStore constructor
@@ -134,8 +132,8 @@ class InventoryStore:
self._keys = _keys
@classmethod
def default_keys(cls) -> list[str]:
return list(InventorySnapshot.__annotations__.keys())
def default_keys(cls) -> set[str]:
return set(InventorySnapshot.__annotations__.keys())
@classmethod
def default_selectors(cls) -> list[str]:
@@ -154,7 +152,7 @@ class InventoryStore:
- Contains all machines
- and more
"""
raw_value = self.get_readonly_raw()
raw_value = self.get_readonly_raw(self._keys)
if self._keys:
filtered = cast(
"InventorySnapshot",
@@ -164,8 +162,8 @@ class InventoryStore:
filtered = cast("InventorySnapshot", raw_value)
return sanitize(filtered, self._allowed_path_transforms, [])
def get_readonly_raw(self) -> Inventory:
attrs = "{" + ",".join(self._keys) + "}"
def get_readonly_raw(self, keys: set[str]) -> Inventory:
attrs = "{" + ",".join(keys) + "}"
return self._flake.select(f"clanInternals.inventoryClass.inventory.{attrs}")
def _get_persisted(self) -> InventorySnapshot:
@@ -216,7 +214,7 @@ class InventoryStore:
data_eval: InventorySnapshot = self._load_merged_inventory()
data_disk: InventorySnapshot = self._get_persisted()
write_map = compute_attribute_map(
write_map = compute_attribute_persistence(
current_priority,
dict(data_eval),
dict(data_disk),

8
pkgs/clan-cli/clan_lib/persist/inventory_store_test.py
View File

@@ -102,7 +102,7 @@ def test_simple_read_write(setup_test_files: Path) -> None:
store = InventoryStore(
flake=MockFlake(nix_file),
inventory_file_name=json_file.name,
_keys=["foo", "protected"],
_keys={"foo", "protected"},
)
store._flake.invalidate_cache()
data = store.read() # type: ignore[assignment]
@@ -147,7 +147,7 @@ def test_simple_deferred(setup_test_files: Path) -> None:
inventory_file_name=json_file.name,
# Needed to allow auto-transforming deferred modules
_allowed_path_transforms=["foo.*"],
_keys=["foo"], # disable toplevel filtering
_keys={"foo"}, # disable toplevel filtering
)
data = store.read()
@@ -228,7 +228,7 @@ def test_manipulate_list(setup_test_files: Path) -> None:
store = InventoryStore(
flake=MockFlake(nix_file),
inventory_file_name=json_file.name,
_keys=["empty", "predefined"],
_keys={"empty", "predefined"},
)
data = store.read()
@@ -270,7 +270,7 @@ def test_static_list_items(setup_test_files: Path) -> None:
store = InventoryStore(
flake=MockFlake(nix_file),
inventory_file_name=json_file.name,
_keys=["empty", "predefined"],
_keys={"empty", "predefined"},
)
data = store.read()

101
pkgs/clan-cli/clan_lib/persist/patch_engine_test.py
View File

@@ -14,7 +14,10 @@ from clan_lib.persist.path_utils import (
set_value_by_path,
set_value_by_path_tuple,
)
from clan_lib.persist.write_rules import PersistenceAttribute, compute_attribute_map
from clan_lib.persist.write_rules import (
PersistenceAttribute,
compute_attribute_persistence,
)
# --- calculate_static_data ---
@@ -219,7 +222,7 @@ def test_update_simple() -> None:
data_disk: dict = {}
attribute_props = compute_attribute_map(prios, data_eval, data_disk)
attribute_props = compute_attribute_persistence(prios, data_eval, data_disk)
assert attribute_props == {
("foo",): {PersistenceAttribute.WRITE},
@@ -256,7 +259,7 @@ def test_update_add_empty_dict() -> None:
data_disk: dict = {}
writeables = compute_attribute_map(prios, data_eval, data_disk)
writeables = compute_attribute_persistence(prios, data_eval, data_disk)
update = deepcopy(data_eval)
@@ -297,7 +300,7 @@ def test_update_many() -> None:
data_disk = {"foo": {"bar": "baz", "nested": {"x": "x"}}}
attribute_props = compute_attribute_map(prios, data_eval, data_disk)
attribute_props = compute_attribute_persistence(prios, data_eval, data_disk)
assert attribute_props == {
("foo",): {PersistenceAttribute.WRITE},
@@ -356,7 +359,7 @@ def test_update_parent_non_writeable() -> None:
},
}
attribute_props = compute_attribute_map(prios, data_eval, data_disk)
attribute_props = compute_attribute_persistence(prios, data_eval, data_disk)
assert attribute_props == {
("foo",): {PersistenceAttribute.READONLY},
@@ -380,7 +383,8 @@ def test_update_parent_non_writeable() -> None:
# def test_remove_non_writable_attrs() -> None:
# prios = {
# "foo": {
# "__prio": 100, # <- writeable: "foo"
# "__this": {"total": True, "prio": 100},
# # We cannot delete children because "foo" is total
# },
# }
@@ -388,7 +392,7 @@ def test_update_parent_non_writeable() -> None:
# data_disk: dict = {}
# writeables = compute_write_map(prios, data_eval, data_disk)
# attribute_props = compute_attribute_persistence(prios, data_eval, data_disk)
# update: dict = {
# "foo": {
@@ -398,7 +402,9 @@ def test_update_parent_non_writeable() -> None:
# }
# with pytest.raises(ClanError) as error:
# calc_patches(data_disk, update, all_values=data_eval, writeables=writeables)
# calc_patches(
# data_disk, update, all_values=data_eval, attribute_props=attribute_props
# )
# assert "Cannot delete path 'foo.baz'" in str(error.value)
@@ -417,7 +423,7 @@ def test_update_list() -> None:
data_disk = {"foo": ["B"]}
attribute_props = compute_attribute_map(prios, data_eval, data_disk)
attribute_props = compute_attribute_persistence(prios, data_eval, data_disk)
assert attribute_props == {
("foo",): {PersistenceAttribute.WRITE},
@@ -464,7 +470,7 @@ def test_update_list_duplicates() -> None:
data_disk = {"foo": ["B"]}
attribute_props = compute_attribute_map(prios, data_eval, data_disk)
attribute_props = compute_attribute_persistence(prios, data_eval, data_disk)
assert attribute_props == {
("foo",): {PersistenceAttribute.WRITE},
@@ -483,19 +489,26 @@ def test_update_list_duplicates() -> None:
def test_dont_persist_defaults() -> None:
"""Default values should not be persisted to disk if not explicitly requested by the user."""
prios = {
introspection = {
"enabled": {"__prio": 1500},
"config": {"__prio": 100},
"config": {
"__prio": 100,
"foo": { # <- default in the 'config' submodule
"__prio": 1500,
},
},
}
data_eval = {
"enabled": True,
"config": {"foo": "bar"},
}
data_disk: dict[str, Any] = {}
attribute_props = compute_attribute_map(prios, data_eval, data_disk)
attribute_props = compute_attribute_persistence(introspection, data_eval, data_disk)
assert attribute_props == {
("enabled",): {PersistenceAttribute.WRITE},
("config",): {PersistenceAttribute.WRITE},
("config", "foo"): {PersistenceAttribute.WRITE, PersistenceAttribute.DELETE},
}
update = deepcopy(data_eval)
@@ -526,8 +539,8 @@ def test_set_null() -> None:
data_disk,
update,
all_values=data_eval,
attribute_props=compute_attribute_map(
{"__prio": 100, "foo": {"__prio": 100}},
attribute_props=compute_attribute_persistence(
{"__prio": 100, "foo": {"__prio": 100}, "bar": {"__prio": 100}},
data_eval,
data_disk,
),
@@ -549,9 +562,24 @@ def test_machine_delete() -> None:
}
data_disk = data_eval
attribute_props = compute_attribute_map(prios, data_eval, data_disk)
attribute_props = compute_attribute_persistence(prios, data_eval, data_disk)
assert attribute_props == {
("machines",): {PersistenceAttribute.WRITE},
("machines", "foo"): {PersistenceAttribute.WRITE, PersistenceAttribute.DELETE},
("machines", "foo", "name"): {
PersistenceAttribute.WRITE,
PersistenceAttribute.DELETE,
},
("machines", "bar"): {PersistenceAttribute.WRITE, PersistenceAttribute.DELETE},
("machines", "bar", "name"): {
PersistenceAttribute.WRITE,
PersistenceAttribute.DELETE,
},
("machines", "naz"): {PersistenceAttribute.WRITE, PersistenceAttribute.DELETE},
("machines", "naz", "name"): {
PersistenceAttribute.WRITE,
PersistenceAttribute.DELETE,
},
}
# Delete machine "bar" from the inventory
@@ -580,7 +608,7 @@ def test_update_mismatching_update_type() -> None:
data_disk: dict = {}
attribute_props = compute_attribute_map(prios, data_eval, data_disk)
attribute_props = compute_attribute_persistence(prios, data_eval, data_disk)
assert attribute_props == {
("foo",): {PersistenceAttribute.WRITE},
@@ -611,10 +639,11 @@ def test_delete_key() -> None:
data_disk = data_eval
attribute_props = compute_attribute_map(prios, data_eval, data_disk)
attribute_props = compute_attribute_persistence(prios, data_eval, data_disk)
assert attribute_props == {
("foo",): {PersistenceAttribute.WRITE},
("foo", "bar"): {PersistenceAttribute.WRITE, PersistenceAttribute.DELETE},
}
# remove all keys from foo
@@ -652,10 +681,36 @@ def test_delete_key_intermediate() -> None:
data_disk = data_eval
attribute_props = compute_attribute_map(prios, data_eval, data_disk)
attribute_props = compute_attribute_persistence(prios, data_eval, data_disk)
assert attribute_props == {
("foo",): {PersistenceAttribute.WRITE},
("foo", "bar"): {PersistenceAttribute.WRITE, PersistenceAttribute.DELETE},
("foo", "bar", "name"): {
PersistenceAttribute.WRITE,
PersistenceAttribute.DELETE,
},
("foo", "bar", "info"): {
PersistenceAttribute.WRITE,
PersistenceAttribute.DELETE,
},
("foo", "bar", "other"): {
PersistenceAttribute.WRITE,
PersistenceAttribute.DELETE,
},
("foo", "other"): {PersistenceAttribute.WRITE, PersistenceAttribute.DELETE},
("foo", "other", "name"): {
PersistenceAttribute.WRITE,
PersistenceAttribute.DELETE,
},
("foo", "other", "info"): {
PersistenceAttribute.WRITE,
PersistenceAttribute.DELETE,
},
("foo", "other", "other"): {
PersistenceAttribute.WRITE,
PersistenceAttribute.DELETE,
},
}
# remove all keys from foo
@@ -687,10 +742,16 @@ def test_delete_key_non_writeable() -> None:
data_disk = data_eval
attribute_props = compute_attribute_map(prios, data_eval, data_disk)
attribute_props = compute_attribute_persistence(prios, data_eval, data_disk)
# TOOD: Collapse these paths, by early stopping the recursion in
# compute_attribute_persistence
assert attribute_props == {
("foo",): {PersistenceAttribute.READONLY},
("foo", "bar"): {PersistenceAttribute.READONLY},
("foo", "bar", "name"): {PersistenceAttribute.READONLY},
("foo", "bar", "info"): {PersistenceAttribute.READONLY},
("foo", "bar", "other"): {PersistenceAttribute.READONLY},
}
# remove all keys from foo

29
pkgs/clan-cli/clan_lib/persist/write_rules.py
View File

@@ -139,10 +139,17 @@ def _determine_props_recursive(
if results is None:
results = {}
for key, value in priorities.items():
if not isinstance(all_values, dict):
# Nothing to do for non-dict values
# they are handled at parent level
return results
for key in all_values:
value = priorities.get(key, {})
# Skip metadata keys
if key in {"__this", "__list", "__prio"}:
continue
# if key in {"__this", "__list", "__prio"}:
# continue
path = (*current_path, key)
@@ -170,7 +177,7 @@ def _determine_props_recursive(
if force_non_writeable:
results.setdefault(path, set()).clear()
results.setdefault(path, set()).add(PersistenceAttribute.READONLY)
# All children are also non-writeable
# All children are also non-writeable, we are done here.
if isinstance(value, dict):
_determine_props_recursive(
value,
@@ -199,6 +206,8 @@ def _determine_props_recursive(
results.setdefault(path, set()).add(PersistenceAttribute.READONLY)
# Recurse into children
# TODO: Dont need to recurse?
# if the current value is READONLY all children are readonly as well
if isinstance(value, dict):
_determine_props_recursive(
value,
@@ -214,7 +223,7 @@ def _determine_props_recursive(
return results
def compute_attribute_map(
def compute_attribute_persistence(
priorities: dict[str, Any], all_values: dict[str, Any], persisted: dict[str, Any]
) -> AttributeMap:
"""Determine writeability for all paths based on priorities and current data.
@@ -232,4 +241,14 @@ def compute_attribute_map(
Dict with sets of writeable and non-writeable paths using tuple keys
"""
persistence_unsupported = set(all_values.keys()) - set(priorities.keys())
if persistence_unsupported:
msg = (
f"Persistence priorities are not defined for top-level keys: "
f"{', '.join(sorted(persistence_unsupported))}. "
f"Either remove them from the python inventory model or extend support of"
f" persistence properties in 'clanInternals.inventoryClass.introspection'."
)
raise ClanError(msg)
return _determine_props_recursive(priorities, all_values, persisted)

34
pkgs/clan-cli/clan_lib/persist/write_rules_test.py
View File

@@ -5,7 +5,10 @@ import pytest
from clan_lib.flake.flake import Flake
from clan_lib.persist.inventory_store import InventoryStore
from clan_lib.persist.write_rules import PersistenceAttribute, compute_attribute_map
from clan_lib.persist.write_rules import (
PersistenceAttribute,
compute_attribute_persistence,
)
if TYPE_CHECKING:
from clan_lib.nix_models.clan import Clan
@@ -21,17 +24,13 @@ def test_write_integration(clan_flake: Callable[..., Flake]) -> None:
data_eval = cast("dict", inventory_store.read())
prios = flake.select("clanInternals.inventoryClass.introspection")
res = compute_attribute_map(prios, data_eval, {})
res = compute_attribute_persistence(prios, data_eval, {})
# We should be able to write to these top-level keys
assert PersistenceAttribute.WRITE in res[("machines",)]
assert PersistenceAttribute.WRITE in res[("instances",)]
assert PersistenceAttribute.WRITE in res[("meta",)]
# # Managed by nix
assert PersistenceAttribute.WRITE not in res[("assertions",)]
assert PersistenceAttribute.READONLY in res[("assertions",)]
# New style __this.prio
@@ -47,9 +46,9 @@ def test_write_simple() -> None:
"foo.bar": {"__this": {"prio": 1000}},
}
default: dict = {"foo": {}}
data: dict = {}
res = compute_attribute_map(prios, default, data)
all_values: dict = {"foo": {"bar": "baz"}, "foo.bar": {}}
persisted: dict = {}
res = compute_attribute_persistence(prios, all_values, persisted)
assert res == {
("foo",): {PersistenceAttribute.WRITE},
@@ -74,8 +73,8 @@ def test_write_inherited() -> None:
},
}
data: dict = {}
res = compute_attribute_map(prios, {"foo": {"bar": {}}}, data)
persisted: dict = {}
res = compute_attribute_persistence(prios, {"foo": {"bar": {"baz": {}}}}, persisted)
assert res == {
("foo",): {PersistenceAttribute.WRITE},
@@ -99,7 +98,7 @@ def test_non_write_inherited() -> None:
}
data: dict = {}
res = compute_attribute_map(prios, {}, data)
res = compute_attribute_persistence(prios, {"foo": {"bar": {"baz": {}}}}, data)
assert res == {
("foo",): {PersistenceAttribute.READONLY},
@@ -122,7 +121,7 @@ def test_write_list() -> None:
"b",
], # <- writeable: because lists are merged. Filtering out nix-values comes later
}
res = compute_attribute_map(prios, default, data)
res = compute_attribute_persistence(prios, default, data)
assert res == {
("foo",): {PersistenceAttribute.WRITE},
@@ -144,9 +143,10 @@ def test_write_because_written() -> None:
},
}
data_eval: dict = {"foo": {"bar": {"baz": 1, "foobar": 1}}}
# Given the following data. {}
# Check that the non-writeable paths are correct.
res = compute_attribute_map(prios, {"foo": {"bar": {}}}, {})
res = compute_attribute_persistence(prios, data_eval, {})
assert res == {
("foo",): {PersistenceAttribute.WRITE},
@@ -165,7 +165,7 @@ def test_write_because_written() -> None:
},
},
}
res = compute_attribute_map(prios, {}, data)
res = compute_attribute_persistence(prios, data_eval, data)
assert res[("foo", "bar", "baz")] == {
PersistenceAttribute.WRITE,
@@ -210,7 +210,7 @@ def test_static_object() -> None:
data_eval: dict = {"foo": {"a": {"c": {"bar": 1}}}}
persisted: dict = {"foo": {"a": {"c": {"bar": 1}}}}
res = compute_attribute_map(introspection, data_eval, persisted)
res = compute_attribute_persistence(introspection, data_eval, persisted)
assert res == {
# We can extend "foo", "foo.a", "foo.a.c"
# That means the user could define "foo.b"
@@ -246,7 +246,7 @@ def test_attributes_totality() -> None:
data_eval: dict = {"foo": {"a": {}}}
persisted: dict = {"foo": {"a": {}}}
res = compute_attribute_map(introspection, data_eval, persisted)
res = compute_attribute_persistence(introspection, data_eval, persisted)
assert res == {
("foo",): {PersistenceAttribute.WRITE},

2
pkgs/clan-cli/clan_lib/tags/list.py
View File

@@ -41,7 +41,7 @@ def list_tags(flake: Flake) -> TagList:
for tag in role_tags:
tags.add(tag)
global_tags = inventory.get("tags", {})
global_tags = inventory_store.get_readonly_raw({"tags"}).get("tags", {})
for tag in global_tags:
if tag not in tags: