From cbbc23557099ae950c0037a8e18a3a67d6a27e82 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 20 Aug 2025 19:46:28 +0200 Subject: [PATCH 1/4] api/modules: rename Frontmatter -> ModulesFrontmatter to make room for other disk templates metadata --- docs/nix/render_options/__init__.py | 13 ++++++------ pkgs/clan-cli/clan_lib/services/modules.py | 23 ++++++++++++++-------- pkgs/clan-cli/clan_lib/templates/disk.py | 6 +++--- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/docs/nix/render_options/__init__.py b/docs/nix/render_options/__init__.py index eb4cd0e53..c49097599 100644 --- a/docs/nix/render_options/__init__.py +++ b/docs/nix/render_options/__init__.py @@ -32,7 +32,7 @@ from typing import Any from clan_lib.errors import ClanError from clan_lib.services.modules import ( CategoryInfo, - Frontmatter, + ModuleFrontmatter, ) # Get environment variables @@ -176,9 +176,8 @@ def print_options( return res -def module_header(module_name: str, has_inventory_feature: bool = False) -> str: - indicator = " 🔹" if has_inventory_feature else "" - return f"# {module_name}{indicator}\n\n" +def module_header(module_name: str) -> str: + return f"# {module_name}\n\n" clan_core_descr = """ @@ -385,7 +384,7 @@ Learn how to use `clanServices` in practice in the [Using clanServices guide](.. # output += f"`clan.modules.{module_name}`\n" output += f"*{module_info['manifest']['description']}*\n" - fm = Frontmatter("") + fm = ModuleFrontmatter("") # output += "## Categories\n\n" output += render_categories( module_info["manifest"]["categories"], fm.categories_info @@ -417,7 +416,7 @@ Learn how to use `clanServices` in practice in the [Using clanServices guide](.. of.write(output) -def build_option_card(module_name: str, frontmatter: Frontmatter) -> str: +def build_option_card(module_name: str, frontmatter: ModuleFrontmatter) -> str: """ Build the overview index card for each reference target option. """ @@ -431,7 +430,7 @@ def build_option_card(module_name: str, frontmatter: Frontmatter) -> str: indented_text = indent + ("\n" + indent).join(lines) return indented_text - def to_md_li(module_name: str, frontmatter: Frontmatter) -> str: + def to_md_li(module_name: str, frontmatter: ModuleFrontmatter) -> str: md_li = ( f"""- **[{module_name}](./{"-".join(module_name.split(" "))}.md)**\n\n""" ) diff --git a/pkgs/clan-cli/clan_lib/services/modules.py b/pkgs/clan-cli/clan_lib/services/modules.py index 7955fc1df..c2e3587a5 100644 --- a/pkgs/clan-cli/clan_lib/services/modules.py +++ b/pkgs/clan-cli/clan_lib/services/modules.py @@ -2,7 +2,7 @@ import re import tomllib from dataclasses import dataclass, field from pathlib import Path -from typing import Any, TypedDict +from typing import Any, TypedDict, TypeVar from clan_lib.api import API from clan_lib.errors import ClanError @@ -22,7 +22,7 @@ class CategoryInfo(TypedDict): @dataclass -class Frontmatter: +class ModuleFrontmatter: description: str categories: list[str] = field(default_factory=lambda: ["Uncategorized"]) features: list[str] = field(default_factory=list) @@ -87,14 +87,19 @@ def parse_frontmatter(readme_content: str) -> tuple[dict[str, Any] | None, str]: raise ClanError( msg, description="Invalid TOML frontmatter", - location="extract_frontmatter", + location="parse_frontmatter", ) from e return frontmatter_parsed, remaining_content return None, readme_content -def extract_frontmatter(readme_content: str, err_scope: str) -> tuple[Frontmatter, str]: +T = TypeVar("T") + + +def extract_frontmatter[T]( + readme_content: str, err_scope: str, fm_class: type[T] +) -> tuple[T, str]: """ Extracts TOML frontmatter from a README file content. @@ -111,13 +116,13 @@ def extract_frontmatter(readme_content: str, err_scope: str) -> tuple[Frontmatte frontmatter_raw, remaining_content = parse_frontmatter(readme_content) if frontmatter_raw: - return Frontmatter(**frontmatter_raw), remaining_content + return fm_class(**frontmatter_raw), remaining_content # If no frontmatter is found, raise an error msg = "Invalid README: Frontmatter not found." raise ClanError( msg, - location="extract_frontmatter", + location="extract_module_frontmatter", description=f"{err_scope} does not contain valid frontmatter.", ) @@ -128,7 +133,9 @@ def has_inventory_feature(module_path: Path) -> bool: return False with readme_file.open() as f: readme = f.read() - frontmatter, _ = extract_frontmatter(readme, f"{module_path}") + frontmatter, _ = extract_frontmatter( + readme, f"{module_path}", fm_class=ModuleFrontmatter + ) return "inventory" in frontmatter.features @@ -338,7 +345,7 @@ def get_module_info( with module_readme.open() as f: readme = f.read() frontmatter, readme_content = extract_frontmatter( - readme, f"{module_path}/README.md" + readme, f"{module_path}/README.md", fm_class=ModuleFrontmatter ) return LegacyModuleInfo( diff --git a/pkgs/clan-cli/clan_lib/templates/disk.py b/pkgs/clan-cli/clan_lib/templates/disk.py index 49a897bc0..95346c004 100644 --- a/pkgs/clan-cli/clan_lib/templates/disk.py +++ b/pkgs/clan-cli/clan_lib/templates/disk.py @@ -11,7 +11,7 @@ from clan_lib.errors import ClanError from clan_lib.git import commit_file from clan_lib.machines.hardware import HardwareConfig, get_machine_hardware_config from clan_lib.machines.machines import Machine -from clan_lib.services.modules import Frontmatter, extract_frontmatter +from clan_lib.services.modules import ModuleFrontmatter, extract_frontmatter log = logging.getLogger(__name__) @@ -57,7 +57,7 @@ class Placeholder: class DiskSchema: name: str readme: str - frontmatter: Frontmatter + frontmatter: ModuleFrontmatter placeholders: dict[str, Placeholder] @@ -128,7 +128,7 @@ def get_machine_disk_schemas( raw_readme = (disk_template / "README.md").read_text() frontmatter, readme = extract_frontmatter( - raw_readme, f"{disk_template}/README.md" + raw_readme, f"{disk_template}/README.md", fm_class=ModuleFrontmatter ) disk_schemas[schema_name] = DiskSchema( From f4d6edc50108f6f2e46e90181d49051df56fb159 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 20 Aug 2025 20:13:35 +0200 Subject: [PATCH 2/4] api/modules: unify frontmatter with module manifest --- pkgs/clan-cli/clan_cli/tests/test_modules.py | 17 --- pkgs/clan-cli/clan_lib/services/modules.py | 111 ++++--------------- pkgs/clan-cli/clan_lib/templates/disk.py | 11 +- pkgs/clan-cli/clan_lib/tests/test_create.py | 3 +- 4 files changed, 33 insertions(+), 109 deletions(-) delete mode 100644 pkgs/clan-cli/clan_cli/tests/test_modules.py diff --git a/pkgs/clan-cli/clan_cli/tests/test_modules.py b/pkgs/clan-cli/clan_cli/tests/test_modules.py deleted file mode 100644 index 2ba3f3bcd..000000000 --- a/pkgs/clan-cli/clan_cli/tests/test_modules.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import TYPE_CHECKING - -import pytest -from clan_cli.tests.fixtures_flakes import FlakeForTest -from clan_lib.flake import Flake -from clan_lib.services.modules import list_service_modules - -if TYPE_CHECKING: - pass - - -@pytest.mark.with_core -def test_list_modules(test_flake_with_core: FlakeForTest) -> None: - base_path = test_flake_with_core.path - modules_info = list_service_modules(Flake(str(base_path))) - - assert "modules" in modules_info diff --git a/pkgs/clan-cli/clan_lib/services/modules.py b/pkgs/clan-cli/clan_lib/services/modules.py index c2e3587a5..2627c128f 100644 --- a/pkgs/clan-cli/clan_lib/services/modules.py +++ b/pkgs/clan-cli/clan_lib/services/modules.py @@ -1,7 +1,6 @@ import re import tomllib -from dataclasses import dataclass, field -from pathlib import Path +from dataclasses import dataclass, field, fields from typing import Any, TypedDict, TypeVar from clan_lib.api import API @@ -22,11 +21,11 @@ class CategoryInfo(TypedDict): @dataclass -class ModuleFrontmatter: +class ModuleManifest: + name: str description: str categories: list[str] = field(default_factory=lambda: ["Uncategorized"]) features: list[str] = field(default_factory=list) - constraints: dict[str, Any] = field(default_factory=dict) @property def categories_info(self) -> dict[str, CategoryInfo]: @@ -59,6 +58,15 @@ class ModuleFrontmatter: msg = f"Invalid category: {category}" raise ValueError(msg) + @classmethod + def from_dict(cls, data: dict) -> "ModuleManifest": + """ + Create an instance of ModuleFrontmatter from a dictionary. + Drops any keys that are not defined in the dataclass. + """ + valid = {f.name for f in fields(cls)} + return cls(**{k: v for k, v in data.items() if k in valid}) + def parse_frontmatter(readme_content: str) -> tuple[dict[str, Any] | None, str]: """ @@ -127,40 +135,6 @@ def extract_frontmatter[T]( ) -def has_inventory_feature(module_path: Path) -> bool: - readme_file = module_path / "README.md" - if not readme_file.exists(): - return False - with readme_file.open() as f: - readme = f.read() - frontmatter, _ = extract_frontmatter( - readme, f"{module_path}", fm_class=ModuleFrontmatter - ) - return "inventory" in frontmatter.features - - -def get_roles(module_path: Path) -> None | list[str]: - if not has_inventory_feature(module_path): - return None - - roles_dir = module_path / "roles" - if not roles_dir.exists() or not roles_dir.is_dir(): - return [] - - return [ - role.stem # filename without .nix extension - for role in roles_dir.iterdir() - if role.is_file() and role.suffix == ".nix" - ] - - -class ModuleManifest(TypedDict): - name: str - description: str - categories: list[str] - features: dict[str, bool] - - @dataclass class ModuleInfo(TypedDict): manifest: ModuleManifest @@ -178,7 +152,18 @@ def list_service_modules(flake: Flake) -> ModuleList: """ modules = flake.select("clanInternals.inventoryClass.modulesPerSource") - return ModuleList({"modules": modules}) + res: dict[str, dict[str, ModuleInfo]] = {} + for input_name, module_set in modules.items(): + res[input_name] = {} + + for module_name, module_info in module_set.items(): + # breakpoint() + res[input_name][module_name] = ModuleInfo( + manifest=ModuleManifest.from_dict(module_info.get("manifest")), + roles=module_info.get("roles", {}), + ) + + return ModuleList(modules=res) @API.register @@ -308,51 +293,3 @@ def create_service_instance( ) return - - -@dataclass -class LegacyModuleInfo: - description: str - categories: list[str] - roles: None | list[str] - readme: str - features: list[str] - constraints: dict[str, Any] - - -def get_module_info( - module_name: str, - module_path: Path, -) -> LegacyModuleInfo: - """ - Retrieves information about a module - """ - if not module_path.exists(): - msg = "Module not found" - raise ClanError( - msg, - location=f"show_module_info {module_name}", - description="Module does not exist", - ) - module_readme = module_path / "README.md" - if not module_readme.exists(): - msg = "Module not found" - raise ClanError( - msg, - location=f"show_module_info {module_name}", - description="Module does not exist or doesn't have any README.md file", - ) - with module_readme.open() as f: - readme = f.read() - frontmatter, readme_content = extract_frontmatter( - readme, f"{module_path}/README.md", fm_class=ModuleFrontmatter - ) - - return LegacyModuleInfo( - description=frontmatter.description, - categories=frontmatter.categories, - roles=get_roles(module_path), - readme=readme_content, - features=["inventory"] if has_inventory_feature(module_path) else [], - constraints=frontmatter.constraints, - ) diff --git a/pkgs/clan-cli/clan_lib/templates/disk.py b/pkgs/clan-cli/clan_lib/templates/disk.py index 95346c004..84b538eec 100644 --- a/pkgs/clan-cli/clan_lib/templates/disk.py +++ b/pkgs/clan-cli/clan_lib/templates/disk.py @@ -11,11 +11,16 @@ from clan_lib.errors import ClanError from clan_lib.git import commit_file from clan_lib.machines.hardware import HardwareConfig, get_machine_hardware_config from clan_lib.machines.machines import Machine -from clan_lib.services.modules import ModuleFrontmatter, extract_frontmatter +from clan_lib.services.modules import extract_frontmatter log = logging.getLogger(__name__) +@dataclass +class DiskManifest: + description: str + + def disk_in_facter_report(hw_report: dict) -> bool: return "hardware" in hw_report and "disk" in hw_report["hardware"] @@ -57,7 +62,7 @@ class Placeholder: class DiskSchema: name: str readme: str - frontmatter: ModuleFrontmatter + frontmatter: DiskManifest placeholders: dict[str, Placeholder] @@ -128,7 +133,7 @@ def get_machine_disk_schemas( raw_readme = (disk_template / "README.md").read_text() frontmatter, readme = extract_frontmatter( - raw_readme, f"{disk_template}/README.md", fm_class=ModuleFrontmatter + raw_readme, f"{disk_template}/README.md", fm_class=DiskManifest ) disk_schemas[schema_name] = DiskSchema( diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index dcef373f4..13e8af74b 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -204,8 +204,7 @@ def test_clan_create_api( modules = list_service_modules(clan_dir_flake) assert ( - modules["modules"]["clan-core"]["admin"]["manifest"]["name"] - == "clan-core/admin" + modules["modules"]["clan-core"]["admin"]["manifest"].name == "clan-core/admin" ) set_value_by_path(inventory, "instances", inventory_conf.instances) From a83f301e59ede061406be13012c60d4dfdd5a552 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 20 Aug 2025 20:19:49 +0200 Subject: [PATCH 3/4] docs/render: remove dead code --- docs/nix/render_options/__init__.py | 74 ----------------------------- 1 file changed, 74 deletions(-) diff --git a/docs/nix/render_options/__init__.py b/docs/nix/render_options/__init__.py index c49097599..a20dfffb6 100644 --- a/docs/nix/render_options/__init__.py +++ b/docs/nix/render_options/__init__.py @@ -270,54 +270,6 @@ def produce_clan_core_docs() -> None: of.write(output) -def render_roles(roles: list[str] | None, module_name: str) -> str: - if roles: - roles_list = "\n".join([f"- `{r}`" for r in roles]) - return ( - f""" -### Roles - -This module can be used via predefined roles - -{roles_list} -""" - """ -Every role has its own configuration options, which are each listed below. - -For more information, see the [inventory guide](../../concepts/inventory.md). - -??? Example - For example the `admin` module adds the following options globally to all machines where it is used. - - `clan.admin.allowedkeys` - - ```nix - clan-core.lib.clan { - inventory.services = { - admin.me = { - roles.default.machines = [ "jon" ]; - config.allowedkeys = [ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD..." ]; - }; - }; - }; - ``` -""" - ) - return "" - - -clan_modules_descr = """ -Clan modules are [NixOS modules](https://wiki.nixos.org/wiki/NixOS_modules) -which have been enhanced with additional features provided by Clan, with -certain option types restricted to enable configuration through a graphical -interface. - -!!! note "🔹" - Modules with this indicator support the [inventory](../../concepts/inventory.md) feature. - -""" - - def render_categories( categories: list[str], categories_info: dict[str, CategoryInfo] ) -> str: @@ -416,32 +368,6 @@ Learn how to use `clanServices` in practice in the [Using clanServices guide](.. of.write(output) -def build_option_card(module_name: str, frontmatter: ModuleFrontmatter) -> str: - """ - Build the overview index card for each reference target option. - """ - - def indent_all(text: str, indent_size: int = 4) -> str: - """ - Indent all lines in a string. - """ - indent = " " * indent_size - lines = text.split("\n") - indented_text = indent + ("\n" + indent).join(lines) - return indented_text - - def to_md_li(module_name: str, frontmatter: ModuleFrontmatter) -> str: - md_li = ( - f"""- **[{module_name}](./{"-".join(module_name.split(" "))}.md)**\n\n""" - ) - md_li += f"""{indent_all("---", 4)}\n\n""" - fmd = f"\n{frontmatter.description.strip()}" if frontmatter.description else "" - md_li += f"""{indent_all(fmd, 4)}""" - return md_li - - return f"{to_md_li(module_name, frontmatter)}\n\n" - - def split_options_by_root(options: dict[str, Any]) -> dict[str, dict[str, Any]]: """ Split the flat dictionary of options into a dict of which each entry will construct complete option trees. From 1f2f71ab03b8c9b70d3b72620bfc20ebcf0191e0 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 21 Aug 2025 14:41:03 +0200 Subject: [PATCH 4/4] lib/modules: make categories class method --- docs/nix/render_options/__init__.py | 5 ++--- pkgs/clan-cli/clan_lib/services/modules.py | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/nix/render_options/__init__.py b/docs/nix/render_options/__init__.py index a20dfffb6..f5331110c 100644 --- a/docs/nix/render_options/__init__.py +++ b/docs/nix/render_options/__init__.py @@ -32,7 +32,7 @@ from typing import Any from clan_lib.errors import ClanError from clan_lib.services.modules import ( CategoryInfo, - ModuleFrontmatter, + ModuleManifest, ) # Get environment variables @@ -336,10 +336,9 @@ Learn how to use `clanServices` in practice in the [Using clanServices guide](.. # output += f"`clan.modules.{module_name}`\n" output += f"*{module_info['manifest']['description']}*\n" - fm = ModuleFrontmatter("") # output += "## Categories\n\n" output += render_categories( - module_info["manifest"]["categories"], fm.categories_info + module_info["manifest"]["categories"], ModuleManifest.categories_info() ) output += f"{module_info['manifest']['readme']}\n" diff --git a/pkgs/clan-cli/clan_lib/services/modules.py b/pkgs/clan-cli/clan_lib/services/modules.py index 2627c128f..1989ef0a3 100644 --- a/pkgs/clan-cli/clan_lib/services/modules.py +++ b/pkgs/clan-cli/clan_lib/services/modules.py @@ -27,8 +27,8 @@ class ModuleManifest: categories: list[str] = field(default_factory=lambda: ["Uncategorized"]) features: list[str] = field(default_factory=list) - @property - def categories_info(self) -> dict[str, CategoryInfo]: + @classmethod + def categories_info(cls) -> dict[str, CategoryInfo]: category_map: dict[str, CategoryInfo] = { "AudioVideo": { "color": "#AEC6CF", @@ -54,14 +54,14 @@ class ModuleManifest: def __post_init__(self) -> None: for category in self.categories: - if category not in self.categories_info: + if category not in ModuleManifest.categories_info(): msg = f"Invalid category: {category}" raise ValueError(msg) @classmethod def from_dict(cls, data: dict) -> "ModuleManifest": """ - Create an instance of ModuleFrontmatter from a dictionary. + Create an instance of this class from a dictionary. Drops any keys that are not defined in the dataclass. """ valid = {f.name for f in fields(cls)}