lib/introspect: use valueMeta to expose more information

lib/introspection/default.nix

@@ -10,6 +10,31 @@ let
     ]
   );
 
+  /**
+    Takes a set of options as returned by `configuration`
+
+    Returns a recursive structure that contains '__this' along with attribute names that map to the same structure.
+
+    Within the reserved attribute '__this' the following attributes are available:
+
+    - prio: The highest priority this option was defined with
+    - files: A list of files this option was defined in
+    - type: The type of this option (e.g. "string", "attrsOf
+    - total: Whether this is a total object. Meaning all attributes are fixed. No additional attributes can be added. Or one of them removed.
+
+    Example Result:
+    {
+      foo = {
+        __this = { ... };
+        bar = {
+          __this = { ... };
+        };
+        baz = {
+          __this = { ... };
+        };
+      };
+    }
+  */
   getPrios =
     {
       options,
@@ -20,76 +45,59 @@ let
     lib.mapAttrs (
       _: opt:
       let
-        prio = {
+        definitionInfo = {
-          __prio = opt.highestPrio;
+          __this = {
+            prio = opt.highestPrio or null;
+            files = opt.files or [ ];
+            type = opt.type.name or null;
+            total = opt.type.name or null == "submodule";
+          };
         };
-        filteredSubOptions = filterOptions (opt.type.getSubOptions opt.loc);
 
-        zipDefs = builtins.zipAttrsWith (_: vs: vs);
+        # TODO: respect freeformType
+        submodulePrios = getPrios { options = filterOptions opt.valueMeta.configuration.options; };
 
-        prioPerValue =
+        /**
-          { type, defs }:
+          Maps attrsOf and lazyAttrsOf
-          lib.mapAttrs (
+        */
-            attrName: prioSet:
+        handleAttrsOf = attrs: lib.mapAttrs (_: handleMeta) attrs;
-            let
-              # Evaluate the submodule
-              # Remove once: https://github.com/NixOS/nixpkgs/pull/391544 lands
-              # This is currently a workaround to get the submodule options
-              # It also has a certain loss of information, on nested attrsOf, which is rare, but not ideal.
-              options = filteredSubOptions;
-              modules = (
-                [
-                  {
-                    inherit options;
-                    _file = "<artifical submodule>";
-                  }
-                ]
-                ++ map (config: { inherit config; }) defs.${attrName}
-              );
-              submoduleEval = lib.evalModules {
-                inherit modules;
-              };
-            in
-            (lib.optionalAttrs (prioSet ? highestPrio) {
-              __prio = prioSet.highestPrio;
-            })
-            // (
-              if type.nestedTypes.elemType.name == "submodule" then
-                getPrios { options = submoduleEval.options; }
-              else
-                # Nested attrsOf
-                (lib.optionalAttrs
-                  (type.nestedTypes.elemType.name == "attrsOf" || type.nestedTypes.elemType.name == "lazyAttrsOf")
-                  (
-                    prioPerValue {
-                      type = type.nestedTypes.elemType;
-                      defs = zipDefs defs.${attrName};
-                    } prioSet.value
-                  )
-                )
-            )
-          );
 
-        submodulePrios =
+        /**
+          Maps attrsOf and lazyAttrsOf
+        */
+        handleListOf = list: { __list = lib.map handleMeta list; };
+
+        /**
+          Unwraps the valueMeta of an option based on its type
+        */
+        handleMeta =
+          meta:
           let
-            modules = (opt.definitions ++ opt.type.getSubModules);
+            hasType = meta ? _internal.type;
-            submoduleEval = lib.evalModules {
+            type = meta._internal.type;
-              inherit modules;
-            };
           in
-          getPrios { options = filterOptions submoduleEval.options; };
+          if !hasType then
-
+            { }
+          else if type.name == "submodule" then
+            # TODO: handle types
+            getPrios { options = filterOptions meta.configuration.options; }
+          else if type.name == "attrsOf" || type.name == "lazyAttrsOf" then
+            handleAttrsOf meta.attrs
+          # TODO: Add index support in nixpkgs first
+          # else if type.name == "listOf" then
+          #   handleListOf meta.list
+          else
+            throw "Yet Unsupported type: ${type.name}";
       in
       if opt ? type && opt.type.name == "submodule" then
-        (prio) // submodulePrios
+        (definitionInfo) // submodulePrios
       else if opt ? type && (opt.type.name == "attrsOf" || opt.type.name == "lazyAttrsOf") then
-        prio
+        definitionInfo // (handleAttrsOf opt.valueMeta.attrs)
-        // (prioPerValue {
+      # TODO: Add index support in nixpkgs, otherwise we cannot
-          type = opt.type;
+      else if opt ? type && (opt.type.name == "listOf") then
-          defs = zipDefs opt.definitions;
+        definitionInfo // (handleListOf opt.valueMeta.list)
-        } (lib.modules.mergeAttrDefinitionsWithPrio opt))
       else if opt ? type && opt._type == "option" then
-        prio
+        definitionInfo
       else
         getPrios { options = opt; }
     ) filteredOptions;

lib/introspection/test.nix

@@ -29,8 +29,15 @@ in
         ]).options;
     };
     expected = {
-      foo.bar = {
+      foo = {
-        __prio = 1500;
+        bar = {
+          __this = {
+            files = [ "<unknown-file>" ];
+            prio = 1500;
+            total = false;
+            type = "bool";
+          };
+        };
       };
     };
   };
@@ -46,8 +53,15 @@ in
         ]).options;
     };
     expected = {
-      foo.bar = {
+      foo = {
-        __prio = 9999;
+        bar = {
+          __this = {
+            files = [ ];
+            prio = 9999;
+            total = false;
+            type = "bool";
+          };
+        };
       };
     };
   };
@@ -71,11 +85,20 @@ in
     };
     expected = {
       foo = {
-        # Prio of the submodule itself
+        __this = {
-        __prio = 9999;
+          files = [ ];
-
+          prio = 9999;
-        # Prio of the bar option within the submodule
+          total = true;
-        bar.__prio = 9999;
+          type = "submodule";
+        };
+        bar = {
+          __this = {
+            files = [ ];
+            prio = 9999;
+            total = false;
+            type = "bool";
+          };
+        };
       };
     };
   };
@@ -87,6 +110,7 @@ in
           {
             options.foo = lib.mkOption {
               type = lib.types.submodule {
+                _file = "option";
                 options = {
                   normal = lib.mkOption {
                     type = lib.types.bool;
@@ -106,9 +130,11 @@ in
             };
           }
           {
+            _file = "default";
             foo.default = lib.mkDefault true;
           }
           {
+            _file = "normal";
             foo.normal = false;
           }
         ]
@@ -121,11 +147,47 @@ in
       };
       expected = {
         foo = {
-          __prio = 100;
+          __this = {
-          normal.__prio = 100; # Set via other module
+            files = [
-          default.__prio = 1000;
+              "normal"
-          optionDefault.__prio = 1500;
+              "default"
-          unset.__prio = 9999;
+            ];
+            prio = 100;
+            total = true;
+            type = "submodule";
+          };
+          default = {
+            __this = {
+              files = [ "default" ];
+              prio = 1000;
+              total = false;
+              type = "bool";
+            };
+          };
+          normal = {
+            __this = {
+              files = [ "normal" ];
+              prio = 100;
+              total = false;
+              type = "bool";
+            };
+          };
+          optionDefault = {
+            __this = {
+              files = [ "option" ];
+              prio = 1500;
+              total = false;
+              type = "bool";
+            };
+          };
+          unset = {
+            __this = {
+              files = [ ];
+              prio = 9999;
+              total = false;
+              type = "bool";
+            };
+          };
         };
       };
     };
@@ -160,8 +222,20 @@ in
       };
       expected = {
         foo = {
-          __prio = 100;
+          __this = {
-          bar.__prio = 100; # Set via other module
+            files = [ "<unknown-file>" ];
+            prio = 100;
+            total = true;
+            type = "submodule";
+          };
+          bar = {
+            __this = {
+              files = [ "<unknown-file>" ];
+              prio = 100;
+              total = false;
+              type = "bool";
+            };
+          };
         };
       };
     };
@@ -230,13 +304,34 @@ in
     {
       expr = slib.getPrios { options = evaluated.options; };
       expected = {
-        foo.__prio = 100;
+        foo = {
-
+          __this = {
-        foo.nested.__prio = 100;
+            files = [ "<unknown-file>" ];
-        foo.other.__prio = 100;
+            prio = 100;
-
+            total = false;
-        foo.nested.bar.__prio = 100;
+            type = "attrsOf";
-        foo.other.bar.__prio = 50;
+          };
+          nested = {
+            bar = {
+              __this = {
+                files = [ "<unknown-file>" ];
+                prio = 100;
+                total = false;
+                type = "int";
+              };
+            };
+          };
+          other = {
+            bar = {
+              __this = {
+                files = [ "<unknown-file>" ];
+                prio = 50;
+                total = false;
+                type = "int";
+              };
+            };
+          };
+        };
       };
     };
   test_attrsOf_attrsOf_submodule =
@@ -278,21 +373,254 @@ in
       inherit evaluated;
       expr = slib.getPrios { options = evaluated.options; };
       expected = {
-        foo.__prio = 100;
+        foo = {
+          __this = {
+            files = [ "<unknown-file>" ];
+            prio = 100;
+            total = false;
+            type = "attrsOf";
+          };
+          a = {
+            b = {
+              bar = {
+                __this = {
+                  files = [ "<unknown-file>" ];
+                  prio = 100;
+                  total = false;
+                  type = "int";
+                };
+              };
+            };
+            c = {
+              bar = {
+                __this = {
+                  files = [ "<unknown-file>" ];
+                  prio = 100;
+                  total = false;
+                  type = "int";
+                };
+              };
+            };
+          };
+          x = {
+            y = {
+              bar = {
+                __this = {
+                  files = [ "<unknown-file>" ];
+                  prio = 100;
+                  total = false;
+                  type = "int";
+                };
+              };
+            };
+            z = {
+              bar = {
+                __this = {
+                  files = [ "<unknown-file>" ];
+                  prio = 100;
+                  total = false;
+                  type = "int";
+                };
+              };
+            };
+          };
+        };
+      };
+    };
 
-        # Sub A
+  test_attrsOf_submodule_default =
-        foo.a.__prio = 100;
+    let
-        # a.b doesnt have a prio
+      evaluated = eval [
-        # a.c doesnt have a prio
+        {
-        foo.a.b.bar.__prio = 100;
+          options.machines = lib.mkOption {
-        foo.a.c.bar.__prio = 100;
+            type = lib.types.attrsOf (
+              lib.types.submodule {
+                options = {
+                  prim = lib.mkOption {
+                    type = lib.types.int;
+                    default = 2;
+                  };
+                  settings = lib.mkOption {
+                    type = lib.types.submodule { };
+                    default = { };
+                  };
+                  fludl = lib.mkOption {
+                    type = lib.types.submodule { };
+                    default = { };
+                  };
+                };
+              }
+            );
+          };
+        }
+        ({
+          _file = "inventory.json";
+          machines.jon = {
+            prim = 3;
+          };
+        })
+        ({
+          # _file = "clan.nix";
+          machines.jon = { };
+        })
 
-        # Sub X
+      ];
-        foo.x.__prio = 100;
+    in
-        # x.y doesnt have a prio
+    {
-        # x.z doesnt have a prio
+      inherit evaluated;
-        foo.x.y.bar.__prio = 100;
+      expr = slib.getPrios { options = evaluated.options; };
-        foo.x.z.bar.__prio = 100;
+      expected = {
+        machines = {
+          __this = {
+            files = [
+              "<unknown-file>"
+              "inventory.json"
+            ];
+            prio = 100;
+            total = false;
+            type = "attrsOf";
+          };
+          jon = {
+            fludl = {
+              __this = {
+                files = [ "<unknown-file>" ];
+                prio = 1500;
+                total = true;
+                type = "submodule";
+              };
+            };
+            prim = {
+              __this = {
+                files = [ "inventory.json" ];
+                prio = 100;
+                total = false;
+                type = "int";
+              };
+            };
+            settings = {
+              __this = {
+                files = [ "<unknown-file>" ];
+                prio = 1500;
+                total = true;
+                type = "submodule";
+              };
+            };
+          };
+        };
+      };
+    };
+  test_listOf_submodule_default =
+    let
+      evaluated = eval [
+        {
+          options.machines = lib.mkOption {
+            type = lib.types.listOf (
+              lib.types.submodule {
+                options = {
+                  prim = lib.mkOption {
+                    type = lib.types.int;
+                    default = 2;
+                  };
+                  settings = lib.mkOption {
+                    type = lib.types.submodule { };
+                    default = { };
+                  };
+                  fludl = lib.mkOption {
+                    type = lib.types.submodule { };
+                    default = { };
+                  };
+                };
+              }
+            );
+          };
+        }
+        ({
+          _file = "inventory.json";
+          machines = [
+            {
+              prim = 10;
+            }
+          ];
+        })
+        ({
+          _file = "clan.nix";
+          machines = [
+            {
+              prim = 3;
+            }
+          ];
+        })
+      ];
+    in
+    {
+      inherit evaluated;
+      expr = slib.getPrios { options = evaluated.options; };
+      expected = {
+        machines = {
+          __list = [
+            {
+              fludl = {
+                __this = {
+                  files = [ "<unknown-file>" ];
+                  prio = 1500;
+                  total = true;
+                  type = "submodule";
+                };
+              };
+              prim = {
+                __this = {
+                  files = [ "clan.nix" ];
+                  prio = 100;
+                  total = false;
+                  type = "int";
+                };
+              };
+              settings = {
+                __this = {
+                  files = [ "<unknown-file>" ];
+                  prio = 1500;
+                  total = true;
+                  type = "submodule";
+                };
+              };
+            }
+            {
+              fludl = {
+                __this = {
+                  files = [ "<unknown-file>" ];
+                  prio = 1500;
+                  total = true;
+                  type = "submodule";
+                };
+              };
+              prim = {
+                __this = {
+                  files = [ "inventory.json" ];
+                  prio = 100;
+                  total = false;
+                  type = "int";
+                };
+              };
+              settings = {
+                __this = {
+                  files = [ "<unknown-file>" ];
+                  prio = 1500;
+                  total = true;
+                  type = "submodule";
+                };
+              };
+            }
+          ];
+          __this = {
+            files = [
+              "clan.nix"
+              "inventory.json"
+            ];
+            prio = 100;
+            total = false;
+            type = "listOf";
+          };
+        };
       };
     };
 }

pkgs/clan-cli/clan_lib/persist/write_rules.py

@@ -51,6 +51,8 @@ def get_priority(value: Any) -> int | None:
     """Extract priority from a value, handling both dict and non-dict cases."""
     if isinstance(value, dict) and "__prio" in value:
         return value["__prio"]
+    if isinstance(value, dict) and "__this" in value:
+        return value["__this"]["prio"]
     return None
 
 
@@ -110,6 +112,9 @@ def _determine_writeability_recursive(
 
     for key, value in priorities.items():
         # Skip metadata keys
+        if key == "__this":
+            continue
+        # Backwards compatibility
         if key == "__prio":
             continue
 

pkgs/clan-cli/clan_lib/persist/write_rules_test.py

@@ -1,13 +1,49 @@
+from collections.abc import Callable
+from typing import TYPE_CHECKING, cast
+
+import pytest
+
+from clan_lib.flake.flake import Flake
+from clan_lib.persist.inventory_store import InventoryStore
 from clan_lib.persist.write_rules import compute_write_map
 
+if TYPE_CHECKING:
+    from clan_lib.nix_models.clan import Clan
+
+
+# Integration test
+@pytest.mark.with_core
+def test_write_integration(clan_flake: Callable[..., Flake]) -> None:
+    clan_nix: Clan = {}
+    flake = clan_flake(clan_nix)
+    inventory_store = InventoryStore(flake)
+    # downcast into a dict
+    data_eval = cast("dict", inventory_store.read())
+    prios = flake.select("clanInternals.inventoryClass.introspection")
+
+    res = compute_write_map(prios, data_eval, {})
+
+    # We should be able to write to these top-level keys
+    assert ("machines",) in res["writeable"]
+    assert ("instances",) in res["writeable"]
+    assert ("meta",) in res["writeable"]
+
+    # Managed by nix
+    assert ("assertions",) in res["non_writeable"]
+
+
+# New style __this.prio
+
 
 def test_write_simple() -> None:
     prios = {
         "foo": {
-            "__prio": 100,  # <- writeable: "foo"
+            "__this": {
-            "bar": {"__prio": 1000},  # <- writeable: mkDefault "foo.bar"
+                "prio": 100,  # <- writeable: "foo"
+            },
+            "bar": {"__this": {"prio": 1000}},  # <- writeable: mkDefault "foo.bar"
         },
-        "foo.bar": {"__prio": 1000},
+        "foo.bar": {"__this": {"prio": 1000}},
     }
 
     default: dict = {"foo": {}}
@@ -20,6 +56,9 @@ def test_write_simple() -> None:
     }
 
 
+# Compatibility test for old __prio style
+
+
 def test_write_inherited() -> None:
     prios = {
         "foo": {