yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'Feat(persist): add support for deferredModule read/write' (#3752) from deferredModule-serde into main

Browse Source 
Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3752
This commit is contained in:
hsjobeki
2025-05-26 14:18:53 +00:00
parent d389c4dc5d 8aed51c63e
commit d7b111ef0a
11 changed files with 425 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pkgs/clan-cli/clan_lib/persist/fixtures/deferred.json Normal file
View File

@@ -0,0 +1 @@
{}

28
pkgs/clan-cli/clan_lib/persist/fixtures/deferred.nix Normal file
View File

@@ -0,0 +1,28 @@
{ clanLib, lib, ... }:
let
eval = lib.evalModules {
modules = [
{
# Trying to write into the default
options.foo = lib.mkOption {
type = lib.types.attrsOf clanLib.types.uniqueDeferredSerializableModule;
};
}
{
foo = {
a = { };
b = { };
};
}
# Merge the "inventory.json"
(builtins.fromJSON (builtins.readFile ./deferred.json))
];
};
in
{
clanInternals.inventoryClass.inventory = eval.config;
clanInternals.inventoryClass.introspection = clanLib.introspection.getPrios {
options = eval.options;
};
}

100
pkgs/clan-cli/clan_lib/persist/inventory_store.py
View File

@@ -12,9 +12,66 @@ from .util import (
calc_patches,
delete_by_path,
determine_writeability,
path_match,
)
def unwrap_known_unknown(value: Any) -> Any:
"""
Helper untility to unwrap our custom deferred module. (uniqueDeferredSerializableModule)
This works because we control ClanLib.type.uniqueDeferredSerializableModule
If value is a dict with the form:
{
"imports": [
{
"_file": <any>,
"imports": [<actual_value>]
}
]
}
then return the actual_value.
Otherwise, return the value unchanged.
"""
if (
isinstance(value, dict)
and "imports" in value
and isinstance(value["imports"], list)
and len(value["imports"]) == 1
and isinstance(value["imports"][0], dict)
and "_file" in value["imports"][0]
and "imports" in value["imports"][0]
and isinstance(value["imports"][0]["imports"], list)
and len(value["imports"][0]["imports"]) == 1
):
return value["imports"][0]["imports"][0]
return value
def sanitize(data: Any, whitelist_paths: list[str], current_path: list[str]) -> Any:
"""
Recursively walks dicts only, unwraps matching values only on whitelisted paths.
Throws error if a value would be transformed on non-whitelisted path.
"""
if isinstance(data, dict):
sanitized = {}
for k, v in data.items():
new_path = [*current_path, k]
unwrapped_v = unwrap_known_unknown(v)
if unwrapped_v is not v: # means unwrap will happen
# check whitelist
wl_paths_split = [wp.split(".") for wp in whitelist_paths]
if not path_match(new_path, wl_paths_split):
msg = f"Unwrap attempted at disallowed path: {'.'.join(new_path)}"
raise ValueError(msg)
sanitized[k] = unwrapped_v
else:
sanitized[k] = sanitize(v, whitelist_paths, new_path)
return sanitized
return data
@dataclass
class WriteInfo:
writeables: dict[str, set[str]]
@@ -35,10 +92,31 @@ class FlakeInterface(Protocol):
class InventoryStore:
def __init__(
self, flake: FlakeInterface, inventory_file_name: str = "inventory.json"
self,
flake: FlakeInterface,
inventory_file_name: str = "inventory.json",
_allowed_path_transforms: list[str] | None = None,
_keys: list[str] | None = None,
) -> None:
"""
InventoryStore constructor
:param flake: The flake to use
:param inventory_file_name: The name of the inventory file
:param _allowed_path_transforms: The paths where deferredModules are allowed to be transformed
"""
self._flake = flake
self.inventory_file = self._flake.path / inventory_file_name
if _allowed_path_transforms is None:
_allowed_path_transforms = [
"instances.*.settings",
"instances.*.machines.*.settings",
]
self._allowed_path_transforms = _allowed_path_transforms
if _keys is None:
_keys = ["machines", "instances", "meta", "services"]
self._keys = _keys
def _load_merged_inventory(self) -> Inventory:
"""
@@ -51,7 +129,11 @@ class InventoryStore:
- Contains all machines
- and more
"""
return self._flake.select("clanInternals.inventoryClass.inventory")
raw_value = self._flake.select("clanInternals.inventoryClass.inventory")
filtered = {k: v for k, v in raw_value.items() if k in self._keys}
sanitized = sanitize(filtered, self._allowed_path_transforms, [])
return sanitized
def _get_persisted(self) -> Inventory:
"""
@@ -146,27 +228,19 @@ class InventoryStore:
"""
write_info = self._write_info()
# Remove internals from the inventory
update.pop("tags", None) # type: ignore
update.pop("options", None) # type: ignore
update.pop("assertions", None) # type: ignore
# Remove instances until the 'settings' deferred module is properly supported.
update.pop("instances", None)
patchset, delete_set = calc_patches(
dict(write_info.data_disk),
dict(update),
dict(write_info.data_eval),
write_info.writeables,
)
persisted = dict(write_info.data_disk)
persisted = dict(write_info.data_disk)
for patch_path, data in patchset.items():
apply_patch(persisted, patch_path, data)
self.delete(delete_set, commit=False)
for delete_path in delete_set:
delete_by_path(persisted, delete_path)
with self.inventory_file.open("w") as f:
json.dump(persisted, f, indent=2)

77
pkgs/clan-cli/clan_lib/persist/inventory_store_test.py
View File

@@ -11,6 +11,7 @@ import pytest
from clan_lib.errors import ClanError
from clan_lib.persist.inventory_store import InventoryStore
from clan_lib.persist.util import apply_patch, delete_by_path
class MockFlake:
@@ -67,32 +68,36 @@ class MockFlake:
folder_path = Path(__file__).parent.resolve()
def test_for_johannes() -> None:
nix_file = folder_path / "fixtures/1.nix"
json_file = folder_path / "fixtures/1.json"
def test_simple_read_write() -> None:
entry_file = "1.nix"
inventory_file = entry_file.replace(".nix", ".json")
nix_file = folder_path / f"fixtures/{entry_file}"
json_file = folder_path / f"fixtures/{inventory_file}"
with TemporaryDirectory() as tmp:
shutil.copyfile(
str(nix_file),
str(Path(tmp) / "1.nix"),
str(Path(tmp) / entry_file),
)
shutil.copyfile(
str(json_file),
str(Path(tmp) / "1.json"),
str(Path(tmp) / inventory_file),
)
store = InventoryStore(
flake=MockFlake(Path(tmp) / "1.nix"),
inventory_file_name="1.json",
flake=MockFlake(Path(tmp) / entry_file),
inventory_file_name=inventory_file,
)
assert store.read() == {"foo": "bar", "protected": "protected"}
data: dict = store.read() # type: ignore
assert data == {"foo": "bar", "protected": "protected"}
data = {"foo": "foo"}
apply_patch(data, "foo", "foo") # type: ignore
store.write(data, "test", commit=False) # type: ignore
# Default method to access the inventory
assert store.read() == {"foo": "foo", "protected": "protected"}
# Test the data is actually persisted
assert store._get_persisted() == data
assert store._get_persisted() == {"foo": "foo"}
# clan_lib.errors.ClanError: Key 'protected' is not writeable.
invalid_data = {"protected": "foo"}
@@ -101,10 +106,58 @@ def test_for_johannes() -> None:
assert str(e.value) == "Key 'protected' is not writeable."
# Test the data is not touched
assert store.read() == {"foo": "foo", "protected": "protected"}
assert store._get_persisted() == data
assert store.read() == data
assert store._get_persisted() == {"foo": "foo"}
# Remove the foo key from the persisted data
# Technically data = { } should also work
data = {"protected": "protected"}
store.write(data, "test", commit=False) # type: ignore
def test_read_deferred() -> None:
entry_file = "deferred.nix"
inventory_file = entry_file.replace(".nix", ".json")
nix_file = folder_path / f"fixtures/{entry_file}"
json_file = folder_path / f"fixtures/{inventory_file}"
with TemporaryDirectory() as tmp:
shutil.copyfile(
str(nix_file),
str(Path(tmp) / entry_file),
)
shutil.copyfile(
str(json_file),
str(Path(tmp) / inventory_file),
)
store = InventoryStore(
flake=MockFlake(Path(tmp) / entry_file),
inventory_file_name=inventory_file,
_allowed_path_transforms=["foo.*"],
)
data = store.read()
assert data == {"foo": {"a": {}, "b": {}}}
# Create a new "deferredModule" "C"
apply_patch(data, "foo.c", {})
store.write(data, "test", commit=False) # type: ignore
assert store.read() == {"foo": {"a": {}, "b": {}, "c": {}}}
# Remove the "deferredModule" "C"
delete_by_path(data, "foo.c") # type: ignore
store.write(data, "test", commit=False)
assert store.read() == {"foo": {"a": {}, "b": {}}}
# Write settings into a new "deferredModule" "C" and read them back
apply_patch(data, "foo.c", {"timeout": "1s"})
store.write(data, "test", commit=False) # type: ignore
assert store.read() == {"foo": {"a": {}, "b": {}, "c": {"timeout": "1s"}}}
# Remove the "deferredModule" "C" along with its settings
delete_by_path(data, "foo.c") # type: ignore
store.write(data, "test", commit=False)
assert store.read() == {"foo": {"a": {}, "b": {}}}

136
pkgs/clan-cli/clan_lib/persist/util.py
View File

@@ -9,6 +9,33 @@ from typing import Any
from clan_lib.errors import ClanError
def path_match(path: list[str], whitelist_paths: list[list[str]]) -> bool:
"""
Returns True if path matches any whitelist path with "*" wildcards.
I.e.:
whitelist_paths = [["a.b.*"]]
path = ["a", "b", "c"]
path_match(path, whitelist_paths) == True
whitelist_paths = ["a.b.c", "a.b.*"]
path = ["a", "b", "d"]
path_match(path, whitelist_paths) == False
"""
for wp in whitelist_paths:
if len(path) != len(wp):
continue
match = True
for p, w in zip(path, wp, strict=False):
if w != "*" and p != w:
match = False
break
if match:
return True
return False
def flatten_data(data: dict, parent_key: str = "", separator: str = ".") -> dict:
"""
Recursively flattens a nested dictionary structure where keys are joined by the separator.
@@ -28,7 +55,11 @@ def flatten_data(data: dict, parent_key: str = "", separator: str = ".") -> dict
if isinstance(value, dict):
# Recursively flatten the nested dictionary
flattened.update(flatten_data(value, new_key, separator))
if value:
flattened.update(flatten_data(value, new_key, separator))
else:
# If the value is an empty dictionary, add it to the flattened dict
flattened[new_key] = {}
else:
flattened[new_key] = value
@@ -58,7 +89,7 @@ def find_duplicates(string_list: list[str]) -> list[str]:
def find_deleted_paths(
persisted: dict[str, Any], update: dict[str, Any], parent_key: str = ""
curr: dict[str, Any], update: dict[str, Any], parent_key: str = ""
) -> set[str]:
"""
Recursively find keys (at any nesting level) that exist in persisted but do not
@@ -72,7 +103,7 @@ def find_deleted_paths(
deleted_paths = set()
# Iterate over keys in persisted
for key, p_value in persisted.items():
for key, p_value in curr.items():
current_path = f"{parent_key}.{key}" if parent_key else key
# Check if this key exists in update
if key not in update:
@@ -103,6 +134,16 @@ def find_deleted_paths(
return deleted_paths
def parent_is_dict(key: str, data: dict[str, Any]) -> bool:
parts = key.split(".")
while len(parts) > 1:
parts.pop()
parent_key = ".".join(parts)
if parent_key in data:
return isinstance(data[parent_key], dict)
return False
def calc_patches(
persisted: dict[str, Any],
update: dict[str, Any],
@@ -114,7 +155,7 @@ def calc_patches(
Given its current state and the update to apply.
Filters out nix-values so it doesnt matter if the anyone sends them.
Filters out nix-values so it doesn't matter if the anyone sends them.
: param persisted: The current state of the inventory.
: param update: The update to apply.
@@ -124,9 +165,9 @@ def calc_patches(
Returns a tuple with the SET and DELETE patches.
"""
persisted_flat = flatten_data(persisted)
update_flat = flatten_data(update)
all_values_flat = flatten_data(all_values)
data_all = flatten_data(all_values)
data_all_updated = flatten_data(update)
data_dyn = flatten_data(persisted)
def is_writeable_key(key: str) -> bool:
"""
@@ -146,49 +187,66 @@ def calc_patches(
msg = f"Cannot determine writeability for key '{key}'"
raise ClanError(msg, description="F001")
all_keys = set(data_all) | set(data_all_updated)
patchset = {}
for update_key, update_data in update_flat.items():
if not is_writeable_key(update_key):
if update_data != all_values_flat.get(update_key):
msg = f"Key '{update_key}' is not writeable."
raise ClanError(msg)
continue
if is_writeable_key(update_key):
prev_value = all_values_flat.get(update_key)
if prev_value and type(update_data) is not type(prev_value):
msg = f"Type mismatch for key '{update_key}'. Cannot update {type(all_values_flat.get(update_key))} with {type(update_data)}"
delete_set = find_deleted_paths(all_values, update, parent_key="")
for key in all_keys:
# Get the old and new values
old = data_all.get(key, None)
new = data_all_updated.get(key, None)
# Some kind of change
if old != new:
# If there is a change, check if the key is writeable
if not is_writeable_key(key):
msg = f"Key '{key}' is not writeable."
raise ClanError(msg)
# Handle list separation
if isinstance(update_data, list):
duplicates = find_duplicates(update_data)
if any(key.startswith(d) for d in delete_set):
# Skip this key if it or any of its parent paths are marked for deletion
continue
if old is not None and type(old) is not type(new):
if new is None:
# If this is a deleted key, they are handled by 'find_deleted_paths'
continue
msg = f"Type mismatch for key '{key}'. Cannot update {type(old)} with {type(new)}"
description = f"""
Previous_value is of type '{type(old)}' this operation would change it to '{type(new)}'.
Prev: {old}
->
After: {new}
"""
raise ClanError(msg, description=description)
if isinstance(new, list):
duplicates = find_duplicates(new)
if duplicates:
msg = f"Key '{update_key}' contains duplicates: {duplicates}. This not supported yet."
msg = f"Key '{key}' contains duplicates: {duplicates}. This not supported yet."
raise ClanError(msg)
# List of current values
persisted_data = persisted_flat.get(update_key, [])
persisted_data = data_dyn.get(key, [])
# List including nix values
all_list = all_values_flat.get(update_key, [])
all_list = data_all.get(key, [])
nix_list = unmerge_lists(all_list, persisted_data)
if update_data != all_list:
patchset[update_key] = unmerge_lists(update_data, nix_list)
if new != all_list:
patchset[key] = unmerge_lists(new, nix_list)
else:
patchset[key] = new
elif update_data != persisted_flat.get(update_key, None):
patchset[update_key] = update_data
continue
msg = f"Cannot determine writeability for key '{update_key}'"
# Ensure not inadvertently patching something already marked for deletion
conflicts = {key for d in delete_set for key in patchset if key.startswith(d)}
if conflicts:
conflict_list = ", ".join(sorted(conflicts))
msg = (
f"The following keys are marked for deletion but also have update values: {conflict_list}. "
"You cannot delete and patch the same key and its subkeys."
)
raise ClanError(msg)
delete_set = find_deleted_paths(persisted, update)
for delete_key in delete_set:
if not is_writeable_key(delete_key):
msg = f"Cannot delete: Key '{delete_key}' is not writeable."
raise ClanError(msg)
return patchset, delete_set

92
pkgs/clan-cli/clan_lib/persist/util_test.py
View File

@@ -1,4 +1,5 @@
# Functions to test
from copy import deepcopy
from typing import Any
import pytest
@@ -9,10 +10,58 @@ from clan_lib.persist.util import (
calc_patches,
delete_by_path,
determine_writeability,
path_match,
unmerge_lists,
)
@pytest.mark.parametrize(
("path", "whitelist", "expected"),
[
# Exact matches
(["a", "b", "c"], [["a", "b", "c"]], True),
(["a", "b"], [["a", "b"]], True),
([], [[]], True),
# Wildcard matches
(["a", "b", "c"], [["a", "*", "c"]], True),
(["a", "x", "c"], [["a", "*", "c"]], True),
(["a", "b", "c"], [["*", "b", "c"]], True),
(["a", "b", "c"], [["a", "b", "*"]], True),
(["a", "b", "c"], [["*", "*", "*"]], True),
# Multiple patterns - one matches
(["a", "b", "c"], [["x", "y", "z"], ["a", "*", "c"]], True),
(["x", "y", "z"], [["a", "*", "c"], ["x", "y", "z"]], True),
# Length mismatch
(["a", "b", "c"], [["a", "b"]], False),
(["a", "b"], [["a", "b", "c"]], False),
# Non-matching
(["a", "b", "c"], [["a", "b", "x"]], False),
(["a", "b", "c"], [["a", "x", "x"]], False),
(["a", "b", "c"], [["x", "x", "x"]], False),
# Empty whitelist
(["a"], [], False),
# Wildcards and exact mixed
(
["instances", "inst1", "roles", "roleA", "settings"],
[["instances", "*", "roles", "*", "settings"]],
True,
),
# Partial wildcard - length mismatch should fail
(
["instances", "inst1", "roles", "roleA"],
[["instances", "*", "roles", "*", "settings"]],
False,
),
# Empty path, no patterns
([], [], False),
],
)
def test_path_match(
path: list[str], whitelist: list[list[str]], expected: bool
) -> None:
assert path_match(path, whitelist) == expected
# --------- Patching tests ---------
def test_patch_nested() -> None:
orig = {"a": 1, "b": {"a": 2.1, "b": 2.2}, "c": 3}
@@ -205,12 +254,37 @@ def test_update_simple() -> None:
assert patchset == {"foo.bar": "new value"}
def test_update_add_empty_dict() -> None:
prios = {
"foo": {
"__prio": 100, # <- writeable: "foo"
"nix": {"__prio": 100}, # <- non writeable: "foo.nix" (defined in nix)
},
}
data_eval: dict = {"foo": {"nix": {}}}
data_disk: dict = {}
writeables = determine_writeability(prios, data_eval, data_disk)
update = deepcopy(data_eval)
apply_patch(update, "foo.mimi", {})
patchset, _ = calc_patches(
data_disk, update, all_values=data_eval, writeables=writeables
)
assert patchset == {"foo.mimi": {}} # this is what gets persisted
def test_update_many() -> None:
prios = {
"foo": {
"__prio": 100, # <- writeable: "foo"
"bar": {"__prio": 100}, # <-
"nix": {"__prio": 100}, # <- non writeable: "foo.bar" (defined in nix)
"nix": {"__prio": 100}, # <- non writeable: "foo.nix" (defined in nix)
"nested": {
"__prio": 100,
"x": {"__prio": 100}, # <- writeable: "foo.nested.x"
@@ -377,7 +451,9 @@ def test_dont_persist_defaults() -> None:
writeables = determine_writeability(prios, data_eval, data_disk)
assert writeables == {"writeable": {"config", "enabled"}, "non_writeable": set()}
update = {"config": {"foo": "foo"}}
update = deepcopy(data_eval)
apply_patch(update, "config.foo", "foo")
patchset, delete_set = calc_patches(
data_disk, update, all_values=data_eval, writeables=writeables
)
@@ -402,7 +478,9 @@ def test_machine_delete() -> None:
assert writeables == {"writeable": {"machines"}, "non_writeable": set()}
# Delete machine "bar" from the inventory
update = {"machines": {"foo": {"name": "foo"}, "naz": {"name": "naz"}}}
update = deepcopy(data_eval)
delete_by_path(update, "machines.bar")
patchset, delete_set = calc_patches(
data_disk, update, all_values=data_eval, writeables=writeables
)
@@ -433,8 +511,8 @@ def test_update_mismatching_update_type() -> None:
calc_patches(data_disk, update, all_values=data_eval, writeables=writeables)
assert (
str(error.value)
== "Type mismatch for key 'foo'. Cannot update <class 'list'> with <class 'int'>"
"Type mismatch for key 'foo'. Cannot update <class 'list'> with <class 'int'>"
in str(error.value)
)
@@ -460,7 +538,7 @@ def test_delete_key() -> None:
data_disk, update, all_values=data_eval, writeables=writeables
)
assert patchset == {}
assert patchset == {"foo": {}}
assert delete_set == {"foo.bar"}
@@ -524,7 +602,7 @@ def test_delete_key_non_writeable() -> None:
with pytest.raises(ClanError) as error:
calc_patches(data_disk, update, all_values=data_eval, writeables=writeables)
assert "Cannot delete" in str(error.value)
assert "is not writeable" in str(error.value)
def test_delete_atom() -> None: