docs: generate api docs

docs/nix/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+{ pkgs, module-docs, ... }:
 pkgs.stdenv.mkDerivation {
   name = "clan-documentation";
 
@@ -10,6 +10,11 @@ pkgs.stdenv.mkDerivation {
       mkdocs
       mkdocs-material
     ]);
+  configurePhase = ''
+    mkdir -p ./site/reference
+    cp -af ${module-docs}/* ./site/reference/
+
+  '';
 
   buildPhase = ''
     mkdocs build --strict

docs/nix/flake-module.nix

@@ -1,4 +1,4 @@
-{ inputs, ... }:
+{ inputs, self, ... }:
 {
   perSystem =
     {
@@ -7,11 +7,61 @@
       pkgs,
       ...
     }:
+    let
+      # Simply evaluated options (JSON)
+      # { clanCore = «derivation JSON»; clanModules = { ${name} = «derivation JSON» }; }
+      jsonDocs = import ./get-module-docs.nix {
+        inherit (inputs) nixpkgs;
+        inherit pkgs;
+        inherit (self.nixosModules) clanCore;
+        inherit (self) clanModules;
+      };
+
+      clanModulesFileInfo = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModules);
+
+      # Simply evaluated options (JSON)
+      renderOptions =
+        pkgs.runCommand "renderOptions.py"
+          {
+            # TODO: ruff does not splice properly in nativeBuildInputs
+            depsBuildBuild = [ pkgs.ruff ];
+            nativeBuildInputs = [
+              pkgs.python3
+              pkgs.mypy
+            ];
+          }
+          ''
+            install ${./scripts/renderOptions.py} $out
+            patchShebangs --build $out
+
+            ruff format --check --diff $out
+            ruff --line-length 88 $out
+            mypy --strict $out
+          '';
+
+      module-docs = pkgs.runCommand "rendered" { nativeBuildInputs = [ pkgs.python3 ]; } ''
+        export CLAN_CORE=${jsonDocs.clanCore}/share/doc/nixos/options.json 
+        # A file that contains the links to all clanModule docs
+        export CLAN_MODULES=${clanModulesFileInfo}
+
+        mkdir $out
+
+        # The python script will place mkDocs files in the output directory
+        python3 ${renderOptions}
+      '';
+    in
     {
-      devShells.docs = pkgs.callPackage ./shell.nix { inherit (self'.packages) docs; };
+      devShells.docs = pkgs.callPackage ./shell.nix {
+        inherit (self'.packages) docs;
+        inherit module-docs;
+      };
       packages = {
-        docs = pkgs.python3.pkgs.callPackage ./default.nix { inherit (inputs) nixpkgs; };
+        docs = pkgs.python3.pkgs.callPackage ./default.nix {
+          inherit (inputs) nixpkgs;
+          inherit module-docs;
+        };
         deploy-docs = pkgs.callPackage ./deploy-docs.nix { inherit (config.packages) docs; };
+        inherit module-docs;
       };
     };
 }

docs/nix/get-module-docs.nix

@@ -0,0 +1,45 @@
+{
+  nixpkgs,
+  pkgs,
+  clanCore,
+  clanModules,
+}:
+let
+  allNixosModules = (import "${nixpkgs}/nixos/modules/module-list.nix") ++ [
+    "${nixpkgs}/nixos/modules/misc/assertions.nix"
+    { nixpkgs.hostPlatform = "x86_64-linux"; }
+  ];
+
+  clanCoreNixosModules = [
+    clanCore
+    { clanCore.clanDir = ./.; }
+  ] ++ allNixosModules;
+
+  # TODO: optimally we would not have to evaluate all nixos modules for every page
+  #   but some of our module options secretly depend on nixos modules.
+  #   We would have to get rid of these implicit dependencies and make them explicit
+  clanCoreNixos = pkgs.nixos { imports = clanCoreNixosModules; };
+
+  # using extendModules here instead of re-evaluating nixos every time
+  #   improves eval performance slightly (10%)
+  getOptions = modules: (clanCoreNixos.extendModules { inherit modules; }).options;
+
+  evalDocs =
+    options:
+    pkgs.nixosOptionsDoc {
+      options = options;
+      warningsAreErrors = false;
+    };
+
+  # clanModules docs
+  clanModulesDocs = builtins.mapAttrs (
+    name: module: (evalDocs ((getOptions [ module ]).clan.${name} or { })).optionsJSON
+  ) clanModules;
+
+  # clanCore docs
+  clanCoreDocs = (evalDocs (getOptions [ ]).clanCore).optionsJSON;
+in
+{
+  clanCore = clanCoreDocs;
+  clanModules = clanModulesDocs;
+}

docs/nix/scripts/renderOptions.py

@@ -0,0 +1,159 @@
+# Options are available in the following format:
+# https://github.com/nixos/nixpkgs/blob/master/nixos/lib/make-options-doc/default.nix
+#
+# ```json
+# {
+# ...
+# "fileSystems.<name>.options": {
+#     "declarations": ["nixos/modules/tasks/filesystems.nix"],
+#     "default": {
+#       "_type": "literalExpression",
+#       "text": "[\n  \"defaults\"\n]"
+#     },
+#     "description": "Options used to mount the file system.",
+#     "example": {
+#       "_type": "literalExpression",
+#       "text": "[\n  \"data=journal\"\n]"
+#     },
+#     "loc": ["fileSystems", "<name>", "options"],
+#     "readOnly": false,
+#     "type": "non-empty (list of string (with check: non-empty))"
+#     "relatedPackages": "- [`pkgs.tmux`](\n    https://search.nixos.org/packages?show=tmux&sort=relevance&query=tmux\n  )\n",
+# }
+# }
+# ```
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+# Get environment variables
+CLAN_CORE = os.getenv("CLAN_CORE")
+CLAN_MODULES = os.environ.get("CLAN_MODULES")
+OUT = os.environ.get("out")
+
+
+def sanitize(text: str) -> str:
+    return text.replace(">", "\\>")
+
+
+def replace_store_path(text: str) -> Path:
+    res = text
+    if text.startswith("/nix/store/"):
+        res = "https://git.clan.lol/clan/clan-core/src/branch/main/" + str(
+            Path(*Path(text).parts[4:])
+        )
+    return Path(res)
+
+
+def render_option(name: str, option: dict[str, Any]) -> str:
+    read_only = option.get("readOnly")
+
+    res = f"""
+## {sanitize(name)}
+{"Readonly" if read_only else ""}
+{option.get("description", "No description available.")}
+
+**Type**: `{option["type"]}`
+
+"""
+    if option.get("default"):
+        res += f"""
+**Default**:
+
+```nix
+{option["default"]["text"] if option.get("default") else "No default set."}
+```
+        """
+    example = option.get("example", {}).get("text")
+    if example:
+        res += f"""
+
+??? example
+
+    ```nix  
+    {example}
+    ```
+"""
+    if option.get("relatedPackages"):
+        res += f"""
+### Related Packages
+
+{option["relatedPackages"]}
+"""
+
+    decls = option.get("declarations", [])
+    source_path = replace_store_path(decls[0])
+    res += f"""
+:simple-git: [{source_path.name}]({source_path})
+"""
+    res += "\n"
+
+    return res
+
+
+def module_header(module_name: str) -> str:
+    return f"""# {module_name}
+ 
+To use this module, import it like this:
+
+```nix
+  {{config, lib, inputs, ...}}: {{
+    imports = [ inputs.clan-core.clanModules.{module_name} ];
+    # ...
+  }}
+```
+"""
+
+
+def produce_clan_core_docs() -> None:
+    if not CLAN_CORE:
+        raise ValueError(
+            f"Environment variables are not set correctly: $CLAN_CORE={CLAN_CORE}"
+        )
+
+    if not OUT:
+        raise ValueError(f"Environment variables are not set correctly: $out={OUT}")
+
+    with open(CLAN_CORE) as f:
+        options: dict[str, dict[str, Any]] = json.load(f)
+        module_name = "clan-core"
+        output = module_header(module_name)
+        for option_name, info in options.items():
+            output += render_option(option_name, info)
+
+        outfile = Path(OUT) / f"{module_name}.md"
+        with open(outfile, "w") as of:
+            of.write(output)
+
+
+def produce_clan_modules_docs() -> None:
+    if not CLAN_MODULES:
+        raise ValueError(
+            f"Environment variables are not set correctly: $CLAN_MODULES={CLAN_MODULES}"
+        )
+
+    if not OUT:
+        raise ValueError(f"Environment variables are not set correctly: $out={OUT}")
+
+    with open(CLAN_MODULES) as f:
+        links: dict[str, str] = json.load(f)
+
+    # {'borgbackup': '/nix/store/hi17dwgy7963ddd4ijh81fv0c9sbh8sw-options.json', ... }
+    for module_name, options_file in links.items():
+        with open(Path(options_file) / "share/doc/nixos/options.json") as f:
+            options: dict[str, dict[str, Any]] = json.load(f)
+            print(f"Rendering options for {module_name}...")
+            output = module_header(module_name)
+            for option_name, info in options.items():
+                output += render_option(option_name, info)
+
+            outfile = Path(OUT) / f"{module_name}.md"
+            with open(outfile, "w") as of:
+                of.write(output)
+
+
+if __name__ == "__main__":
+    produce_clan_core_docs()
+    produce_clan_modules_docs()

docs/nix/shell.nix

@@ -1 +1,16 @@
-{ docs, pkgs, ... }: pkgs.mkShell { inputsFrom = [ docs ]; }
+{
+  docs,
+  pkgs,
+  module-docs,
+  ...
+}:
+pkgs.mkShell {
+  inputsFrom = [ docs ];
+  shellHook = ''
+    mkdir -p ./site/reference
+    cp -af ${module-docs}/* ./site/reference/
+    chmod +w ./site/reference/*
+
+    echo "Generated API documentation in './site/reference/' "
+  '';
+}