From b41029ea48d7fbec8778c45efeab1d5e596907dd Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Fri, 4 Jul 2025 12:09:16 +0200 Subject: [PATCH 1/3] clan_lib/openapi: add openapi rendering --- pkgs/clan-cli/flake-module.nix | 14 +++ pkgs/clan-cli/open_api.py | 191 +++++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 pkgs/clan-cli/open_api.py diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index 608523f6a..cb2cb132c 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -100,6 +100,20 @@ cp ${self'.legacyPackages.schemas.inventory}/* $out ''; }; + clan-lib-openapi = pkgs.stdenv.mkDerivation { + name = "clan-lib-openapi"; + src = ./.; + + buildInputs = [ + pkgs.python3 + ]; + + installPhase = '' + export INPUT_PATH=${self'.packages.clan-ts-api}/API.json + python open_api.py + cp openapi.json $out + ''; + }; default = self'.packages.clan-cli; }; diff --git a/pkgs/clan-cli/open_api.py b/pkgs/clan-cli/open_api.py new file mode 100644 index 000000000..1f0a159c4 --- /dev/null +++ b/pkgs/clan-cli/open_api.py @@ -0,0 +1,191 @@ +import json +import os +from copy import deepcopy +from pathlib import Path + +# !!! IMPORTANT !!! +# AVOID VERBS NOT IN THIS LIST +# We might restrict this even further to build a consistent and easy to use API +COMMON_VERBS = { + "get", + "list", + "show", + "set", + "create", + "update", + "delete", + "generate", + "maybe", + "open", + "flash", + "install", + "deploy", + "check", + "cancel", +} + + +def is_verb(word: str) -> bool: + return word in COMMON_VERBS + + +def singular(word: str) -> str: + if word.endswith("ies"): + return word[:-3] + "y" + if word.endswith("ses"): + return word[:-2] + if word.endswith("s") and not word.endswith("ss"): + return word[:-1] + return word + + +def normalize_tag(parts: list[str]) -> list[str]: + # parts contains [ VERB NOUN NOUN ... ] + # Where each NOUN is a SUB-RESOURCE + verb = parts[0] + + nouns = parts[1:] + if not nouns: + nouns = ["misc"] + # msg = "Operation names MUST have at least one NOUN" + # raise Error(msg) + nouns = [singular(p).capitalize() for p in nouns] + return [verb, *nouns] + + +def operation_to_tag(op_name: str) -> str: + def check_operation_name(verb: str, resource_nouns: list[str]): + if not is_verb(verb): + print( + f"""⚠️ WARNING: Verb '{op_name}' of API operation {op_name} is not allowed. +Use one of: {", ".join(COMMON_VERBS)} +""" + ) + + parts = op_name.lower().split("_") + normalized = normalize_tag(parts) + + check_operation_name(verb=normalized[0], resource_nouns=normalized[1:]) + + return " / ".join(normalized[1:]) + + +def fix_nullables(schema: dict) -> dict: + if isinstance(schema, dict): + # If 'oneOf' present + if "oneOf" in schema and isinstance(schema["oneOf"], list): + # Filter out 'type:null' schemas + non_nulls = [s for s in schema["oneOf"] if s.get("type") != "null"] + if len(non_nulls) == 1: + # Only one non-null schema remains - convert to that + nullable:true + new_schema = deepcopy(non_nulls[0]) + new_schema["nullable"] = True + # Merge any other keys from original schema except oneOf + for k, v in schema.items(): + if k != "oneOf": + new_schema[k] = v + return fix_nullables(new_schema) + # More than one non-null, keep oneOf without nulls + schema["oneOf"] = non_nulls + return {k: fix_nullables(v) for k, v in schema.items()} + # Recursively fix nested schemas + return {k: fix_nullables(v) for k, v in schema.items()} + if isinstance(schema, list): + return [fix_nullables(i) for i in schema] + return schema + + +def fix_error_refs(schema: dict) -> None: + if isinstance(schema, dict): + for key, value in schema.items(): + if key == "$ref" and value == "#/$defs/error": + schema[key] = "#/components/schemas/error" + else: + fix_error_refs(value) + elif isinstance(schema, list): + for item in schema: + fix_error_refs(item) + + +# === Helper to make reusable schema names === +def make_schema_name(func_name: str, part: str) -> str: + return f"{func_name}_{part}" + + +def main() -> None: + INPUT_PATH = Path(os.environ["INPUT_PATH"]) + + # === Load input JSON Schema === + with INPUT_PATH.open() as f: + schema = json.load(f) + + defs = schema.get("$defs", {}) + functions = schema["properties"] + + # === Start OpenAPI 3.0 spec in JSON === + openapi = { + "openapi": "3.0.3", + "info": { + "title": "Function-Based API", + "version": "1.0.0", + "description": "Auto-generated OpenAPI 3.0 spec from custom JSON Schema", + }, + "paths": {}, + "components": {"schemas": {}}, + } + + # === Convert each function === + for func_name, func_schema in functions.items(): + args_schema = fix_nullables(deepcopy(func_schema["properties"]["arguments"])) + return_schema = fix_nullables(deepcopy(func_schema["properties"]["return"])) + fix_error_refs(return_schema) + # Register schemas under components + args_name = make_schema_name(func_name, "args") + return_name = make_schema_name(func_name, "return") + openapi["components"]["schemas"][args_name] = args_schema + openapi["components"]["schemas"][return_name] = return_schema + tag = operation_to_tag(func_name) + # Create a POST endpoint for the function + openapi["paths"][f"/{func_name}"] = { + "post": { + "summary": func_name, + "operationId": func_name, + "tags": [tag], + "requestBody": { + "required": True, + "content": { + "application/json": { + "schema": {"$ref": f"#/components/schemas/{args_name}"} + } + }, + }, + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "$ref": f"#/components/schemas/{return_name}" + } + } + }, + } + }, + } + } + + # === Add global definitions from $defs === + for def_name, def_schema in defs.items(): + fixed_schema = fix_nullables(deepcopy(def_schema)) + fix_error_refs(fixed_schema) + openapi["components"]["schemas"][def_name] = fixed_schema + + # === Write to output JSON === + with Path("openapi.json").open("w") as f: + json.dump(openapi, f, indent=2) + + print("✅ OpenAPI 3.0 JSON written to openapi.json") + + +if __name__ == "__main__": + main() From 62c1db976958ce8677993fbe10640e380b82179a Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Fri, 4 Jul 2025 12:43:01 +0200 Subject: [PATCH 2/3] Docs: init redoc internal rest inspired docs --- docs/.gitignore | 1 + docs/mkdocs.yml | 4 ++++ docs/nix/default.nix | 6 ++++++ docs/nix/flake-module.nix | 7 ++++++- docs/site/intern/api.md | 7 +++++++ docs/site/intern/index.md | 25 +++++++++++++++++++++++++ pkgs/clan-cli/open_api.py | 20 ++++++++++---------- 7 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 docs/site/intern/api.md create mode 100644 docs/site/intern/index.md diff --git a/docs/.gitignore b/docs/.gitignore index 07d1f16a1..d79d037d4 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,4 +1,5 @@ /site/reference /site/static /site/options-page +/site/openapi.json !/site/static/extra.css diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 89ed12540..ce47a30c7 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -181,6 +181,9 @@ nav: - 05-deployment-parameters: decisions/05-deployment-parameters.md - Template: decisions/_template.md - Options: options.md + - Developer: + - Introduction: intern/index.md + - API: intern/api.md docs_dir: site site_dir: out @@ -238,3 +241,4 @@ extra: plugins: - search - macros + - redoc-tag diff --git a/docs/nix/default.nix b/docs/nix/default.nix index 474c72ae1..143b38146 100644 --- a/docs/nix/default.nix +++ b/docs/nix/default.nix @@ -3,6 +3,7 @@ pkgs, module-docs, clan-cli-docs, + clan-lib-openapi, asciinema-player-js, asciinema-player-css, roboto, @@ -29,6 +30,7 @@ pkgs.stdenv.mkDerivation { mkdocs mkdocs-material mkdocs-macros + mkdocs-redoc-tag ]); configurePhase = '' pushd docs @@ -36,6 +38,10 @@ pkgs.stdenv.mkDerivation { mkdir -p ./site/reference/cli cp -af ${module-docs}/* ./site/reference/ cp -af ${clan-cli-docs}/* ./site/reference/cli/ + + mkdir -p ./site/reference/internal + cp -af ${clan-lib-openapi} ./site/openapi.json + chmod -R +w ./site/reference echo "Generated API documentation in './site/reference/' " diff --git a/docs/nix/flake-module.nix b/docs/nix/flake-module.nix index 3624bbd5c..87986bb04 100644 --- a/docs/nix/flake-module.nix +++ b/docs/nix/flake-module.nix @@ -127,7 +127,12 @@ packages = { docs = pkgs.python3.pkgs.callPackage ./default.nix { clan-core = self; - inherit (self'.packages) clan-cli-docs docs-options inventory-api-docs; + inherit (self'.packages) + clan-cli-docs + docs-options + inventory-api-docs + clan-lib-openapi + ; inherit (inputs) nixpkgs; inherit module-docs; inherit asciinema-player-js; diff --git a/docs/site/intern/api.md b/docs/site/intern/api.md new file mode 100644 index 000000000..97b2beafb --- /dev/null +++ b/docs/site/intern/api.md @@ -0,0 +1,7 @@ +--- +template: options.html +hide: + - navigation + - toc +--- + \ No newline at end of file diff --git a/docs/site/intern/index.md b/docs/site/intern/index.md new file mode 100644 index 000000000..cbf897802 --- /dev/null +++ b/docs/site/intern/index.md @@ -0,0 +1,25 @@ +# Developer Documentation + +!!! Danger + This documentation is **not** intended for external users. It may contain low-level details and internal-only interfaces.* + +Welcome to the internal developer documentation. + +This section is intended for contributors, engineers, and internal stakeholders working directly with our system, tooling, and APIs. It provides a technical overview of core components, internal APIs, conventions, and patterns that support the platform. + +Our goal is to make the internal workings of the system **transparent, discoverable, and consistent** — helping you contribute confidently, troubleshoot effectively, and build faster. + +## What's Here? + +!!! note "docs migration ongoing" + +- [ ] **API Reference**: 🚧🚧🚧 Detailed documentation of internal API functions, inputs, and expected outputs. 🚧🚧🚧 +- [ ] **System Concepts**: Architectural overviews and domain-specific guides. +- [ ] **Development Guides**: How to test, extend, or integrate with key components. +- [ ] **Design Notes**: Rationales behind major design decisions or patterns. + +## Who is This For? + +* Developers contributing to the platform +* Engineers debugging or extending internal systems +* Anyone needing to understand **how** and **why** things work under the hood diff --git a/pkgs/clan-cli/open_api.py b/pkgs/clan-cli/open_api.py index 1f0a159c4..069fc244b 100644 --- a/pkgs/clan-cli/open_api.py +++ b/pkgs/clan-cli/open_api.py @@ -54,7 +54,7 @@ def normalize_tag(parts: list[str]) -> list[str]: def operation_to_tag(op_name: str) -> str: - def check_operation_name(verb: str, resource_nouns: list[str]): + def check_operation_name(verb: str, _resource_nouns: list[str]) -> None: if not is_verb(verb): print( f"""⚠️ WARNING: Verb '{op_name}' of API operation {op_name} is not allowed. @@ -65,7 +65,7 @@ Use one of: {", ".join(COMMON_VERBS)} parts = op_name.lower().split("_") normalized = normalize_tag(parts) - check_operation_name(verb=normalized[0], resource_nouns=normalized[1:]) + check_operation_name(normalized[0], normalized[1:]) return " / ".join(normalized[1:]) @@ -113,10 +113,10 @@ def make_schema_name(func_name: str, part: str) -> str: def main() -> None: - INPUT_PATH = Path(os.environ["INPUT_PATH"]) + input_path = Path(os.environ["INPUT_PATH"]) # === Load input JSON Schema === - with INPUT_PATH.open() as f: + with input_path.open() as f: schema = json.load(f) defs = schema.get("$defs", {}) @@ -126,9 +126,9 @@ def main() -> None: openapi = { "openapi": "3.0.3", "info": { - "title": "Function-Based API", + "title": "Function-Based Python API", "version": "1.0.0", - "description": "Auto-generated OpenAPI 3.0 spec from custom JSON Schema", + "description": "!!! INTERNAL USE ONLY !!! We don't provide a world usable API yet.\nThis prototype maps python function calls to POST Requests because we are planning towards RESTfull API in the future.", }, "paths": {}, "components": {"schemas": {}}, @@ -142,11 +142,11 @@ def main() -> None: # Register schemas under components args_name = make_schema_name(func_name, "args") return_name = make_schema_name(func_name, "return") - openapi["components"]["schemas"][args_name] = args_schema - openapi["components"]["schemas"][return_name] = return_schema + openapi["components"]["schemas"][args_name] = args_schema # type: ignore + openapi["components"]["schemas"][return_name] = return_schema # type: ignore tag = operation_to_tag(func_name) # Create a POST endpoint for the function - openapi["paths"][f"/{func_name}"] = { + openapi["paths"][f"/{func_name}"] = { # type: ignore "post": { "summary": func_name, "operationId": func_name, @@ -178,7 +178,7 @@ def main() -> None: for def_name, def_schema in defs.items(): fixed_schema = fix_nullables(deepcopy(def_schema)) fix_error_refs(fixed_schema) - openapi["components"]["schemas"][def_name] = fixed_schema + openapi["components"]["schemas"][def_name] = fixed_schema # type: ignore # === Write to output JSON === with Path("openapi.json").open("w") as f: From ba0397242feec6e3b60cefebb1f50b63ff887d2f Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Fri, 4 Jul 2025 13:40:59 +0200 Subject: [PATCH 3/3] api: rename script to openapi.py --- pkgs/clan-cli/flake-module.nix | 2 +- pkgs/clan-cli/{open_api.py => openapi.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename pkgs/clan-cli/{open_api.py => openapi.py} (100%) diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index cb2cb132c..a38cbe951 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -110,7 +110,7 @@ installPhase = '' export INPUT_PATH=${self'.packages.clan-ts-api}/API.json - python open_api.py + python openapi.py cp openapi.json $out ''; }; diff --git a/pkgs/clan-cli/open_api.py b/pkgs/clan-cli/openapi.py similarity index 100% rename from pkgs/clan-cli/open_api.py rename to pkgs/clan-cli/openapi.py