From 75121767d39b0579440d725b2f9c748b1daaf94f Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 24 Sep 2025 17:22:15 +0200 Subject: [PATCH 1/2] lib/introspect: use valueMeta to expose more information --- lib/introspection/default.nix | 128 +++--- lib/introspection/test.nix | 400 ++++++++++++++++-- pkgs/clan-cli/clan_lib/persist/write_rules.py | 5 + .../clan_lib/persist/write_rules_test.py | 45 +- 4 files changed, 479 insertions(+), 99 deletions(-) diff --git a/lib/introspection/default.nix b/lib/introspection/default.nix index ef5c49b68..6cfd4a648 100644 --- a/lib/introspection/default.nix +++ b/lib/introspection/default.nix @@ -10,6 +10,31 @@ let ] ); + /** + Takes a set of options as returned by `configuration` + + Returns a recursive structure that contains '__this' along with attribute names that map to the same structure. + + Within the reserved attribute '__this' the following attributes are available: + + - prio: The highest priority this option was defined with + - files: A list of files this option was defined in + - type: The type of this option (e.g. "string", "attrsOf + - total: Whether this is a total object. Meaning all attributes are fixed. No additional attributes can be added. Or one of them removed. + + Example Result: + { + foo = { + __this = { ... }; + bar = { + __this = { ... }; + }; + baz = { + __this = { ... }; + }; + }; + } + */ getPrios = { options, @@ -20,76 +45,59 @@ let lib.mapAttrs ( _: opt: let - prio = { - __prio = opt.highestPrio; + definitionInfo = { + __this = { + prio = opt.highestPrio or null; + files = opt.files or [ ]; + type = opt.type.name or null; + total = opt.type.name or null == "submodule"; + }; }; - filteredSubOptions = filterOptions (opt.type.getSubOptions opt.loc); - zipDefs = builtins.zipAttrsWith (_: vs: vs); + # TODO: respect freeformType + submodulePrios = getPrios { options = filterOptions opt.valueMeta.configuration.options; }; - prioPerValue = - { type, defs }: - lib.mapAttrs ( - attrName: prioSet: - let - # Evaluate the submodule - # Remove once: https://github.com/NixOS/nixpkgs/pull/391544 lands - # This is currently a workaround to get the submodule options - # It also has a certain loss of information, on nested attrsOf, which is rare, but not ideal. - options = filteredSubOptions; - modules = ( - [ - { - inherit options; - _file = ""; - } - ] - ++ map (config: { inherit config; }) defs.${attrName} - ); - submoduleEval = lib.evalModules { - inherit modules; - }; - in - (lib.optionalAttrs (prioSet ? highestPrio) { - __prio = prioSet.highestPrio; - }) - // ( - if type.nestedTypes.elemType.name == "submodule" then - getPrios { options = submoduleEval.options; } - else - # Nested attrsOf - (lib.optionalAttrs - (type.nestedTypes.elemType.name == "attrsOf" || type.nestedTypes.elemType.name == "lazyAttrsOf") - ( - prioPerValue { - type = type.nestedTypes.elemType; - defs = zipDefs defs.${attrName}; - } prioSet.value - ) - ) - ) - ); + /** + Maps attrsOf and lazyAttrsOf + */ + handleAttrsOf = attrs: lib.mapAttrs (_: handleMeta) attrs; - submodulePrios = + /** + Maps attrsOf and lazyAttrsOf + */ + handleListOf = list: { __list = lib.map handleMeta list; }; + + /** + Unwraps the valueMeta of an option based on its type + */ + handleMeta = + meta: let - modules = (opt.definitions ++ opt.type.getSubModules); - submoduleEval = lib.evalModules { - inherit modules; - }; + hasType = meta ? _internal.type; + type = meta._internal.type; in - getPrios { options = filterOptions submoduleEval.options; }; - + if !hasType then + { } + else if type.name == "submodule" then + # TODO: handle types + getPrios { options = filterOptions meta.configuration.options; } + else if type.name == "attrsOf" || type.name == "lazyAttrsOf" then + handleAttrsOf meta.attrs + # TODO: Add index support in nixpkgs first + # else if type.name == "listOf" then + # handleListOf meta.list + else + throw "Yet Unsupported type: ${type.name}"; in if opt ? type && opt.type.name == "submodule" then - (prio) // submodulePrios + (definitionInfo) // submodulePrios else if opt ? type && (opt.type.name == "attrsOf" || opt.type.name == "lazyAttrsOf") then - prio - // (prioPerValue { - type = opt.type; - defs = zipDefs opt.definitions; - } (lib.modules.mergeAttrDefinitionsWithPrio opt)) + definitionInfo // (handleAttrsOf opt.valueMeta.attrs) + # TODO: Add index support in nixpkgs, otherwise we cannot + else if opt ? type && (opt.type.name == "listOf") then + definitionInfo // (handleListOf opt.valueMeta.list) else if opt ? type && opt._type == "option" then - prio + definitionInfo else getPrios { options = opt; } ) filteredOptions; diff --git a/lib/introspection/test.nix b/lib/introspection/test.nix index f6cc5bb22..0145f10ec 100644 --- a/lib/introspection/test.nix +++ b/lib/introspection/test.nix @@ -29,8 +29,15 @@ in ]).options; }; expected = { - foo.bar = { - __prio = 1500; + foo = { + bar = { + __this = { + files = [ "" ]; + prio = 1500; + total = false; + type = "bool"; + }; + }; }; }; }; @@ -46,8 +53,15 @@ in ]).options; }; expected = { - foo.bar = { - __prio = 9999; + foo = { + bar = { + __this = { + files = [ ]; + prio = 9999; + total = false; + type = "bool"; + }; + }; }; }; }; @@ -71,11 +85,20 @@ in }; expected = { foo = { - # Prio of the submodule itself - __prio = 9999; - - # Prio of the bar option within the submodule - bar.__prio = 9999; + __this = { + files = [ ]; + prio = 9999; + total = true; + type = "submodule"; + }; + bar = { + __this = { + files = [ ]; + prio = 9999; + total = false; + type = "bool"; + }; + }; }; }; }; @@ -87,6 +110,7 @@ in { options.foo = lib.mkOption { type = lib.types.submodule { + _file = "option"; options = { normal = lib.mkOption { type = lib.types.bool; @@ -106,9 +130,11 @@ in }; } { + _file = "default"; foo.default = lib.mkDefault true; } { + _file = "normal"; foo.normal = false; } ] @@ -121,11 +147,47 @@ in }; expected = { foo = { - __prio = 100; - normal.__prio = 100; # Set via other module - default.__prio = 1000; - optionDefault.__prio = 1500; - unset.__prio = 9999; + __this = { + files = [ + "normal" + "default" + ]; + prio = 100; + total = true; + type = "submodule"; + }; + default = { + __this = { + files = [ "default" ]; + prio = 1000; + total = false; + type = "bool"; + }; + }; + normal = { + __this = { + files = [ "normal" ]; + prio = 100; + total = false; + type = "bool"; + }; + }; + optionDefault = { + __this = { + files = [ "option" ]; + prio = 1500; + total = false; + type = "bool"; + }; + }; + unset = { + __this = { + files = [ ]; + prio = 9999; + total = false; + type = "bool"; + }; + }; }; }; }; @@ -160,8 +222,20 @@ in }; expected = { foo = { - __prio = 100; - bar.__prio = 100; # Set via other module + __this = { + files = [ "" ]; + prio = 100; + total = true; + type = "submodule"; + }; + bar = { + __this = { + files = [ "" ]; + prio = 100; + total = false; + type = "bool"; + }; + }; }; }; }; @@ -230,13 +304,34 @@ in { expr = slib.getPrios { options = evaluated.options; }; expected = { - foo.__prio = 100; - - foo.nested.__prio = 100; - foo.other.__prio = 100; - - foo.nested.bar.__prio = 100; - foo.other.bar.__prio = 50; + foo = { + __this = { + files = [ "" ]; + prio = 100; + total = false; + type = "attrsOf"; + }; + nested = { + bar = { + __this = { + files = [ "" ]; + prio = 100; + total = false; + type = "int"; + }; + }; + }; + other = { + bar = { + __this = { + files = [ "" ]; + prio = 50; + total = false; + type = "int"; + }; + }; + }; + }; }; }; test_attrsOf_attrsOf_submodule = @@ -278,21 +373,254 @@ in inherit evaluated; expr = slib.getPrios { options = evaluated.options; }; expected = { - foo.__prio = 100; + foo = { + __this = { + files = [ "" ]; + prio = 100; + total = false; + type = "attrsOf"; + }; + a = { + b = { + bar = { + __this = { + files = [ "" ]; + prio = 100; + total = false; + type = "int"; + }; + }; + }; + c = { + bar = { + __this = { + files = [ "" ]; + prio = 100; + total = false; + type = "int"; + }; + }; + }; + }; + x = { + y = { + bar = { + __this = { + files = [ "" ]; + prio = 100; + total = false; + type = "int"; + }; + }; + }; + z = { + bar = { + __this = { + files = [ "" ]; + prio = 100; + total = false; + type = "int"; + }; + }; + }; + }; + }; + }; + }; - # Sub A - foo.a.__prio = 100; - # a.b doesnt have a prio - # a.c doesnt have a prio - foo.a.b.bar.__prio = 100; - foo.a.c.bar.__prio = 100; + test_attrsOf_submodule_default = + let + evaluated = eval [ + { + options.machines = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule { + options = { + prim = lib.mkOption { + type = lib.types.int; + default = 2; + }; + settings = lib.mkOption { + type = lib.types.submodule { }; + default = { }; + }; + fludl = lib.mkOption { + type = lib.types.submodule { }; + default = { }; + }; + }; + } + ); + }; + } + ({ + _file = "inventory.json"; + machines.jon = { + prim = 3; + }; + }) + ({ + # _file = "clan.nix"; + machines.jon = { }; + }) - # Sub X - foo.x.__prio = 100; - # x.y doesnt have a prio - # x.z doesnt have a prio - foo.x.y.bar.__prio = 100; - foo.x.z.bar.__prio = 100; + ]; + in + { + inherit evaluated; + expr = slib.getPrios { options = evaluated.options; }; + expected = { + machines = { + __this = { + files = [ + "" + "inventory.json" + ]; + prio = 100; + total = false; + type = "attrsOf"; + }; + jon = { + fludl = { + __this = { + files = [ "" ]; + prio = 1500; + total = true; + type = "submodule"; + }; + }; + prim = { + __this = { + files = [ "inventory.json" ]; + prio = 100; + total = false; + type = "int"; + }; + }; + settings = { + __this = { + files = [ "" ]; + prio = 1500; + total = true; + type = "submodule"; + }; + }; + }; + }; + }; + }; + test_listOf_submodule_default = + let + evaluated = eval [ + { + options.machines = lib.mkOption { + type = lib.types.listOf ( + lib.types.submodule { + options = { + prim = lib.mkOption { + type = lib.types.int; + default = 2; + }; + settings = lib.mkOption { + type = lib.types.submodule { }; + default = { }; + }; + fludl = lib.mkOption { + type = lib.types.submodule { }; + default = { }; + }; + }; + } + ); + }; + } + ({ + _file = "inventory.json"; + machines = [ + { + prim = 10; + } + ]; + }) + ({ + _file = "clan.nix"; + machines = [ + { + prim = 3; + } + ]; + }) + ]; + in + { + inherit evaluated; + expr = slib.getPrios { options = evaluated.options; }; + expected = { + machines = { + __list = [ + { + fludl = { + __this = { + files = [ "" ]; + prio = 1500; + total = true; + type = "submodule"; + }; + }; + prim = { + __this = { + files = [ "clan.nix" ]; + prio = 100; + total = false; + type = "int"; + }; + }; + settings = { + __this = { + files = [ "" ]; + prio = 1500; + total = true; + type = "submodule"; + }; + }; + } + { + fludl = { + __this = { + files = [ "" ]; + prio = 1500; + total = true; + type = "submodule"; + }; + }; + prim = { + __this = { + files = [ "inventory.json" ]; + prio = 100; + total = false; + type = "int"; + }; + }; + settings = { + __this = { + files = [ "" ]; + prio = 1500; + total = true; + type = "submodule"; + }; + }; + } + ]; + __this = { + files = [ + "clan.nix" + "inventory.json" + ]; + prio = 100; + total = false; + type = "listOf"; + }; + }; }; }; } diff --git a/pkgs/clan-cli/clan_lib/persist/write_rules.py b/pkgs/clan-cli/clan_lib/persist/write_rules.py index edcebb3ee..f8904848d 100644 --- a/pkgs/clan-cli/clan_lib/persist/write_rules.py +++ b/pkgs/clan-cli/clan_lib/persist/write_rules.py @@ -51,6 +51,8 @@ def get_priority(value: Any) -> int | None: """Extract priority from a value, handling both dict and non-dict cases.""" if isinstance(value, dict) and "__prio" in value: return value["__prio"] + if isinstance(value, dict) and "__this" in value: + return value["__this"]["prio"] return None @@ -110,6 +112,9 @@ def _determine_writeability_recursive( for key, value in priorities.items(): # Skip metadata keys + if key == "__this": + continue + # Backwards compatibility if key == "__prio": continue diff --git a/pkgs/clan-cli/clan_lib/persist/write_rules_test.py b/pkgs/clan-cli/clan_lib/persist/write_rules_test.py index 4ce9b3bfc..af0516956 100644 --- a/pkgs/clan-cli/clan_lib/persist/write_rules_test.py +++ b/pkgs/clan-cli/clan_lib/persist/write_rules_test.py @@ -1,13 +1,49 @@ +from collections.abc import Callable +from typing import TYPE_CHECKING, cast + +import pytest + +from clan_lib.flake.flake import Flake +from clan_lib.persist.inventory_store import InventoryStore from clan_lib.persist.write_rules import compute_write_map +if TYPE_CHECKING: + from clan_lib.nix_models.clan import Clan + + +# Integration test +@pytest.mark.with_core +def test_write_integration(clan_flake: Callable[..., Flake]) -> None: + clan_nix: Clan = {} + flake = clan_flake(clan_nix) + inventory_store = InventoryStore(flake) + # downcast into a dict + data_eval = cast("dict", inventory_store.read()) + prios = flake.select("clanInternals.inventoryClass.introspection") + + res = compute_write_map(prios, data_eval, {}) + + # We should be able to write to these top-level keys + assert ("machines",) in res["writeable"] + assert ("instances",) in res["writeable"] + assert ("meta",) in res["writeable"] + + # Managed by nix + assert ("assertions",) in res["non_writeable"] + + +# New style __this.prio + def test_write_simple() -> None: prios = { "foo": { - "__prio": 100, # <- writeable: "foo" - "bar": {"__prio": 1000}, # <- writeable: mkDefault "foo.bar" + "__this": { + "prio": 100, # <- writeable: "foo" + }, + "bar": {"__this": {"prio": 1000}}, # <- writeable: mkDefault "foo.bar" }, - "foo.bar": {"__prio": 1000}, + "foo.bar": {"__this": {"prio": 1000}}, } default: dict = {"foo": {}} @@ -20,6 +56,9 @@ def test_write_simple() -> None: } +# Compatibility test for old __prio style + + def test_write_inherited() -> None: prios = { "foo": { From bcadf6b0fba3c99cb10629e6a4bc350c23a364b8 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 24 Sep 2025 18:34:09 +0200 Subject: [PATCH 2/2] clan_lib: write_rules complete internal keys --- pkgs/clan-cli/clan_lib/persist/write_rules.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/persist/write_rules.py b/pkgs/clan-cli/clan_lib/persist/write_rules.py index f8904848d..67307fe91 100644 --- a/pkgs/clan-cli/clan_lib/persist/write_rules.py +++ b/pkgs/clan-cli/clan_lib/persist/write_rules.py @@ -112,10 +112,7 @@ def _determine_writeability_recursive( for key, value in priorities.items(): # Skip metadata keys - if key == "__this": - continue - # Backwards compatibility - if key == "__prio": + if key in {"__this", "__list", "__prio"}: continue path = (*current_path, key)