Docs: refactor all option documentation to use tree representations

Using a tree instead of a list leads to better representation of options
In the future this could also enable better disvocerability by applying tree-specific filters and views
The OptionList should only be used as an exchange format between nix and rendering tools

docs/nix/render_options/__init__.py

@@ -25,6 +25,7 @@
 
 import json
 import os
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
 
@@ -82,19 +83,39 @@ def join_lines_with_indentation(lines: list[str], indent: int = 4) -> str:
     return "\n".join(indent_str + line for line in lines)
 
 
+def sanitize_anchor(text: str) -> str:
+    parts = text.split(".")
+    res = []
+    for part in parts:
+        if "<" in part:
+            continue
+        res.append(part)
+
+    return ".".join(res)
+
+
 def render_option(
-    name: str, option: dict[str, Any], level: int = 3, short_head: str | None = None
+    name: str,
+    option: dict[str, Any],
+    level: int = 3,
+    short_head: str | None = None,
 ) -> str:
     read_only = option.get("readOnly")
 
     res = f"""
-{"#" * level} {sanitize(name) if short_head is None else sanitize(short_head)}
+{"#" * level} {sanitize(name) if short_head is None else sanitize(short_head)} {"{: #"+sanitize_anchor(name)+"}" if level > 1 else ""}
 
+"""
+
+    res += f"""
 {f"**Attribute: `{name}`**" if short_head is not None else ""}
 
 {"**Readonly**" if read_only else ""}
 
-{option.get("description", "No description available.")}
+{option.get("description", "")}
+"""
+    if option.get("type"):
+        res += f"""
 
 **Type**: `{option["type"]}`
 
@@ -104,7 +125,7 @@ def render_option(
 **Default**:
 
 ```nix
-{option["default"]["text"] if option.get("default") else "No default set."}
+{option.get("default",{}).get("text") if option.get("default") else "No default set."}
 ```
         """
     example = option.get("example", {}).get("text")
@@ -131,18 +152,23 @@ def render_option(
         res += f"""
 :simple-git: [{name}]({source_path})
 """
-        res += "\n"
+        res += "\n\n"
 
     return res
 
 
-def print_options(options_file: str, head: str, no_options: str) -> str:
+def print_options(
+    options_file: str, head: str, no_options: str, replace_prefix: str | None = None
+) -> str:
     res = ""
     with (Path(options_file) / "share/doc/nixos/options.json").open() as f:
         options: dict[str, dict[str, Any]] = json.load(f)
 
         res += head if len(options.items()) else no_options
         for option_name, info in options.items():
+            if replace_prefix:
+                option_name = option_name.replace(replace_prefix + ".", "")
+
             res += render_option(option_name, info, 4)
     return res
 
@@ -223,10 +249,13 @@ Every clan module has a `frontmatter` section within its readme. It provides mac
 This provides an overview of the available attributes of the `frontmatter` within the `README.md` of a clan module.
 
 """
-        for option_name, info in options.items():
+        # for option_name, info in options.items():
-            if option_name == "_module.args":
+        #     if option_name == "_module.args":
-                continue
+        #         continue
-            output += render_option(option_name, info)
+        #     output += render_option(option_name, info)
+        root = options_to_tree(options, debug=True)
+        for option in root.suboptions:
+            output += options_docs_from_tree(option, init_level=2)
 
         outfile = Path(OUT) / "clanModules/frontmatter/index.md"
         outfile.parent.mkdir(
@@ -251,28 +280,48 @@ def produce_clan_core_docs() -> None:
     with CLAN_CORE_DOCS.open() as f:
         options: dict[str, dict[str, Any]] = json.load(f)
         module_name = "clan-core"
-        for option_name, info in options.items():
-            outfile = f"{module_name}/index.md"
 
-            # Create separate files for nested options
+        transform = {n.replace("clan.core.", ""): v for n, v in options.items()}
-            if len(option_name.split(".")) <= 3:
+        split = split_options_by_root(transform)
-                # i.e. clan-core.clanDir
+
-                output = core_outputs.get(
+        # Prepopulate the index file header
-                    outfile,
+        indexfile = f"{module_name}/index.md"
-                    module_header(module_name) + clan_core_descr + options_head,
+        core_outputs[indexfile] = (
+            module_header(module_name) + clan_core_descr + options_head
+        )
+
+        for submodule_name, split_options in split.items():
+            outfile = f"{module_name}/{submodule_name}.md"
+            print(
+                f"[clan_core.{submodule_name}] Rendering option of: {submodule_name}... {outfile}"
+            )
+            init_level = 1
+            root = options_to_tree(split_options, debug=True)
+
+            print(f"Submodule {submodule_name} - suboptions", len(root.suboptions))
+
+            module = root.suboptions[0]
+            print("type", module.info.get("type"))
+
+            module_type = module.info.get("type")
+
+            if module_type is not None and "submodule" not in module_type:
+                outfile = indexfile
+                init_level = 2
+
+            output = ""
+            for option in root.suboptions:
+                output += options_docs_from_tree(
+                    option,
+                    init_level=init_level,
+                    prefix=["clan", "core"],
                 )
-                output += render_option(option_name, info)
+
-                # Update the content
+            # Append the content
+            if outfile not in core_outputs:
                 core_outputs[outfile] = output
             else:
-                # Clan sub-options
+                core_outputs[outfile] += output
-                [_, sub] = option_name.split(".")[1:3]
-                outfile = f"{module_name}/{sub}.md"
-                # Get the content or write the header
-                output = core_outputs.get(outfile, render_option_header(sub))
-                output += render_option(option_name, info)
-                # Update the content
-                core_outputs[outfile] = output
 
         for outfile, output in core_outputs.items():
             (Path(OUT) / outfile).parent.mkdir(parents=True, exist_ok=True)
@@ -386,7 +435,7 @@ def produce_clan_modules_docs() -> None:
         links: dict[str, str] = json.load(f)
 
     for module_name, options_file in links.items():
-        print(f"Rendering {module_name}")
+        print(f"Rendering ClanModule: {module_name}")
         readme_file = CLAN_CORE_PATH / "clanModules" / module_name / "README.md"
         with readme_file.open() as f:
             readme = f.read()
@@ -445,7 +494,12 @@ def produce_clan_modules_docs() -> None:
 
 The following options are available when using the `{role}` role.
 """
-                output += print_options(role_options_file, heading, no_options)
+                output += print_options(
+                    role_options_file,
+                    heading,
+                    no_options,
+                    replace_prefix=f"clan.{module_name}",
+                )
         else:
             # No roles means no inventory usage
             output += """## Usage via Inventory
@@ -463,7 +517,6 @@ The following options are available when using the `{role}` role.
 **This module cannot be imported directly in your nixos configuration.**
 
 """
-
         else:
             output += module_nix_usage(module_name)
             no_options = "** This module doesnt require any options to be set.**"
@@ -562,7 +615,9 @@ Each attribute is documented below
 """
     with Path(BUILD_CLAN_PATH).open() as f:
         options: dict[str, dict[str, Any]] = json.load(f)
-        for option_name, info in options.items():
+
+        split = split_options_by_root(options)
+        for option_name, options in split.items():
             # Skip underscore options
             if option_name.startswith("_"):
                 continue
@@ -571,8 +626,11 @@ Each attribute is documented below
             if option_name.startswith("inventory."):
                 continue
 
-            print(f"Rendering option of {option_name}...")
+            print(f"[build_clan_docs] Rendering option of {option_name}...")
-            output += render_option(option_name, info)
+            root = options_to_tree(options)
+
+            for option in root.suboptions:
+                output += options_docs_from_tree(option, init_level=2)
 
     outfile = Path(OUT) / "nix-api/buildclan.md"
     outfile.parent.mkdir(parents=True, exist_ok=True)
@@ -580,6 +638,43 @@ Each attribute is documented below
         of.write(output)
 
 
+def split_options_by_root(options: dict[str, Any]) -> dict[str, dict[str, Any]]:
+    """
+    Split the flat dictionary of options into a dict of which each entry will construct complete option trees.
+    {
+        "a": { Data }
+        "a.b": { Data }
+        "c": { Data }
+    }
+    ->
+    {
+        "a": {
+            "a": { Data },
+            "a.b": { Data }
+        }
+        "c": {
+            "c": { Data }
+        }
+    }
+    """
+    res: dict[str, dict[str, Any]] = {}
+    for key, value in options.items():
+        parts = key.split(".")
+        root = parts[0]
+        if root not in res:
+            res[root] = {}
+        res[root][key] = value
+    return res
+
+
+@dataclass
+class Option:
+    name: str
+    path: list[str]
+    info: dict[str, Any]
+    suboptions: list["Option"] = field(default_factory=list)
+
+
 def produce_inventory_docs() -> None:
     if not BUILD_CLAN_PATH:
         msg = f"Environment variables are not set correctly: BUILD_CLAN_PATH={BUILD_CLAN_PATH}. Expected a path to the optionsJSON"
@@ -600,47 +695,21 @@ It can be set via the `inventory` attribute of the [`buildClan`](./buildclan.md#
     with Path(BUILD_CLAN_PATH).open() as f:
         options: dict[str, dict[str, Any]] = json.load(f)
 
-        def by_cat(item: tuple[str, dict[str, Any]]) -> Any:
+        clan_root_option = options_to_tree(options)
-            name, _info = item
+        # Find the inventory options
-            parts = name.split(".") if "." in name else ["root", "sub"]
+        inventory_opt: None | Option = None
+        for opt in clan_root_option.suboptions:
+            if opt.name == "inventory":
+                inventory_opt = opt
+                break
 
-            # Make everything fixed length.
+        if not inventory_opt:
-            remain = 10 - len(parts)
+            print("No inventory options found.")
-            parts.extend(["A"] * remain)
+            exit(1)
-            category = parts[1]
+        # Render the inventory options
-            # Sort by category,
+        # This for loop excludes the root node
-            # then by length of the option,
+        for option in inventory_opt.suboptions:
-            # then by the rest of the options
+            output += options_docs_from_tree(option, init_level=2)
-            comparator = (category, -remain, parts[2:9])
-            return comparator
-
-        seen_categories = set()
-        for option_name, info in sorted(options.items(), key=by_cat):
-            # Skip underscore options
-            if option_name.startswith("_"):
-                continue
-
-            # Skip non inventory sub options
-            if not option_name.startswith("inventory."):
-                continue
-
-            category = option_name.split(".")[1]
-
-            heading_level = 3
-            if category not in seen_categories:
-                heading_level = 2
-                seen_categories.add(category)
-
-            parts = option_name.split(".")
-            short_name = ""
-            for part in parts[1:]:
-                if "<" in part:
-                    continue
-                short_name += ("." + part) if short_name else part
-
-            output += render_option(
-                option_name, info, level=heading_level, short_head=short_name
-            )
 
     outfile = Path(OUT) / "nix-api/inventory.md"
     outfile.parent.mkdir(parents=True, exist_ok=True)
@@ -648,11 +717,117 @@ It can be set via the `inventory` attribute of the [`buildClan`](./buildclan.md#
         of.write(output)
 
 
+def option_short_name(option_name: str) -> str:
+    parts = option_name.split(".")
+    short_name = ""
+    for part in parts[1:]:
+        if "<" in part:
+            continue
+        short_name += ("." + part) if short_name else part
+    return short_name
+
+
+def options_to_tree(options: dict[str, Any], debug: bool = False) -> Option:
+    """
+    Convert the options dictionary to a tree structure.
+    """
+
+    # Helper function to create nested structure
+    def add_to_tree(path_parts: list[str], info: Any, current_node: Option) -> None:
+        if not path_parts:
+            return
+
+        name = path_parts[0]
+        remaining_path = path_parts[1:]
+
+        # Look for an existing suboption
+        for sub in current_node.suboptions:
+            if sub.name == name:
+                add_to_tree(remaining_path, info, sub)
+                return
+
+        # If no existing suboption is found, create a new one
+        new_option = Option(
+            name=name,
+            path=[*current_node.path, name],
+            info={},  # Populate info only at the final leaf
+        )
+        current_node.suboptions.append(new_option)
+
+        # If it's a leaf node, populate info
+        if not remaining_path:
+            new_option.info = info
+        else:
+            add_to_tree(remaining_path, info, new_option)
+
+    # Create the root option
+    root = Option(name="<root>", path=[], info={})
+
+    # Process each key-value pair in the dictionary
+    for key, value in options.items():
+        path_parts = key.split(".")
+        add_to_tree(path_parts, value, root)
+
+    def print_tree(option: Option, level: int = 0) -> None:
+        print("  " * level + option.name + ":", option.path)
+        for sub in option.suboptions:
+            print_tree(sub, level + 1)
+
+    # Example usage
+    if debug:
+        print("Options tree:")
+        print_tree(root)
+
+    return root
+
+
+def options_docs_from_tree(
+    root: Option, init_level: int = 1, prefix: list[str] | None = None
+) -> str:
+    """
+    Render the options from the tree structure.
+
+    Args:
+    root (Option): The root option node.
+    init_level (int): The initial level of indentation.
+    prefix (list str): Will be printed as common prefix of all attribute names.
+    """
+
+    def render_tree(option: Option, level: int = init_level) -> str:
+        output = ""
+
+        should_render = not option.name.startswith("<") and not option.name.startswith(
+            "_"
+        )
+        if should_render:
+            # short_name = option_short_name(option.name)
+            path = ".".join(prefix + option.path) if prefix else ".".join(option.path)
+            output += render_option(
+                path,
+                option.info,
+                level=level,
+                short_head=option.name,
+            )
+
+        for sub in option.suboptions:
+            h_increment = 1 if should_render else 0
+
+            if "_module" in sub.path:
+                continue
+            output += render_tree(sub, level + h_increment)
+
+        return output
+
+    md = render_tree(root)
+    return md
+
+
 if __name__ == "__main__":  #
+    produce_clan_core_docs()
+
     produce_build_clan_docs()
     produce_inventory_docs()
 
-    produce_clan_core_docs()
     produce_clan_modules_docs()
 
     produce_clan_modules_frontmatter_docs()