Merge pull request 'docs: generate api docs' (#1233) from hsjobeki-tutorials into main

docs/.envrc

@@ -1,6 +1,6 @@
 source_up
 
-watch_file nix/flake-module.nix nix/shell.nix nix/default.nix
+watch_file $(find ./nix -name "*.nix" -printf '"%p" ')
 
 # Because we depend on nixpkgs sources, uploading to builders takes a long time
 use flake .#docs --builders ''

docs/.gitignore

@@ -0,0 +1 @@
+/site/reference

docs/mkdocs.yml

@@ -10,6 +10,11 @@ validation:
   unrecognized_links: warn
 
 markdown_extensions:
+  - attr_list
+  - pymdownx.emoji:
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
+
   - pymdownx.tasklist:
       custom_checkbox: true
   - pymdownx.superfences
@@ -40,6 +45,28 @@ nav:
       - Backup & Restore: getting-started/backups.md
       - Flake-parts: getting-started/flake-parts.md
   - Templates: templates/index.md
+  - Reference:
+      - ClanCore: reference/clan-core.md
+      - ClanModules:
+          - reference/borgbackup.md
+          - reference/deltachat.md
+          - reference/diskLayouts.md
+          - reference/ergochat.md
+          - reference/graphical.md
+          - reference/localbackup.md
+          - reference/localsend.md
+          - reference/matrix-synapse.md
+          - reference/moonlight.md
+          - reference/root-password.md
+          - reference/sshd.md
+          - reference/sunshine.md
+          - reference/syncthing.md
+          - reference/thelounge.md
+          - reference/vm-user.md
+          - reference/waypipe.md
+          - reference/xfce-vm.md
+          - reference/xfce.md
+          - reference/zt-tcp-relay.md
   - Contributing: contributing/contributing.md
 
 docs_dir: site

docs/modules/default.nix

@@ -1,72 +0,0 @@
-{
-  self,
-  lib,
-  inputs,
-  ...
-}:
-{
-  imports = [ ./zola-pages.nix ];
-
-  perSystem =
-    { pkgs, ... }:
-    let
-
-      allNixosModules = (import "${inputs.nixpkgs}/nixos/modules/module-list.nix") ++ [
-        "${inputs.nixpkgs}/nixos/modules/misc/assertions.nix"
-        { nixpkgs.hostPlatform = "x86_64-linux"; }
-      ];
-
-      clanCoreNixosModules = [
-        self.nixosModules.clanCore
-        { clanCore.clanDir = ./.; }
-      ] ++ allNixosModules;
-
-      # TODO: optimally we would not have to evaluate all nixos modules for every page
-      #   but some of our module options secretly depend on nixos modules.
-      #   We would have to get rid of these implicit dependencies and make them explicit
-      clanCoreNixos = pkgs.nixos { imports = clanCoreNixosModules; };
-
-      # using extendModules here instead of re-evaluating nixos every time
-      #   improves eval performance slightly (10%)
-      options = modules: (clanCoreNixos.extendModules { inherit modules; }).options;
-
-      docs =
-        options:
-        pkgs.nixosOptionsDoc {
-          options = options;
-          warningsAreErrors = false;
-          # transform each option so that the declaration link points to git.clan.lol
-          #   and not to the /nix/store
-          transformOptions =
-            opt:
-            opt
-            // {
-              declarations = lib.forEach opt.declarations (
-                decl:
-                if lib.hasPrefix "${self}" decl then
-                  let
-                    subpath = lib.removePrefix "${self}" decl;
-                  in
-                  {
-                    url = "https://git.clan.lol/clan/clan-core/src/branch/main/" + subpath;
-                    name = subpath;
-                  }
-                else
-                  decl
-              );
-            };
-        };
-
-      outputsFor = name: docs: { packages."docs-md-${name}" = docs.optionsCommonMark; };
-
-      clanModulesPages = lib.flip lib.mapAttrsToList self.clanModules (
-        name: module: outputsFor "module-${name}" (docs ((options [ module ]).clan.${name} or { }))
-      );
-    in
-    {
-      imports = clanModulesPages ++ [
-        # renders all clanCore options in a single page
-        (outputsFor "core-options" (docs (options [ ]).clanCore))
-      ];
-    };
-}

docs/modules/zola-pages.nix

@@ -1,88 +0,0 @@
-{
-  perSystem =
-    {
-      lib,
-      pkgs,
-      self',
-      ...
-    }:
-    let
-
-      getMdPages =
-        prefix:
-        let
-          mdDocs' = lib.filterAttrs (name: _: lib.hasPrefix prefix name) self'.packages;
-          mdDocs = lib.mapAttrs' (name: pkg: lib.nameValuePair (lib.removePrefix prefix name) pkg) mdDocs';
-        in
-        if mdDocs != { } then
-          mdDocs
-        else
-          throw ''
-            Error: no markdown files found in clan-core.packages' with prefix "${prefix}"
-          '';
-
-      makeZolaIndexMd =
-        title: weight:
-        pkgs.writeText "_index.md" ''
-          +++
-          title = "${title}"
-          template = "docs/section.html"
-          weight = ${toString weight}
-          sort_by = "title"
-          draft = false
-          +++
-        '';
-
-      makeZolaPages =
-        {
-          sectionTitle,
-          files,
-          makeIntro ? _name: "",
-          weight ? 9999,
-        }:
-        pkgs.runCommand "zola-pages-clan-core" { } ''
-          mkdir $out
-          # create new section via _index.md
-          cp ${makeZolaIndexMd sectionTitle weight} $out/_index.md
-          # generate zola compatible md files for each nixos options md
-          ${lib.concatStringsSep "\n" (
-            lib.flip lib.mapAttrsToList files (
-              name: md: ''
-                # generate header for zola with title, template, weight
-                title="${name}"
-                echo -e "+++\ntitle = \"$title\"\ntemplate = \"docs/page.html\"\n+++" > "$out/${name}.md"
-                cat <<EOF >> "$out/${name}.md"
-                ${makeIntro name}
-                EOF
-                # append everything from the nixpkgs generated md file
-                cat "${md}" >> "$out/${name}.md"
-              ''
-            )
-          )}
-        '';
-    in
-    {
-      packages.docs-zola-pages-core = makeZolaPages {
-        sectionTitle = "cLAN Core Reference";
-        files = getMdPages "docs-md-core-";
-        weight = 20;
-      };
-
-      packages.docs-zola-pages-modules = makeZolaPages {
-        sectionTitle = "cLAN Modules Reference";
-        files = getMdPages "docs-md-module-";
-        weight = 25;
-        makeIntro = name: ''
-          To use this module, import it like this:
-
-          \`\`\`nix
-            {config, lib, inputs, ...}: {
-              imports = [inputs.clan-core.clanModules.${name}];
-              # ...
-            }
-          \`\`\`
-
-        '';
-      };
-    };
-}

docs/nix/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+{ pkgs, module-docs, ... }:
 pkgs.stdenv.mkDerivation {
   name = "clan-documentation";
 
@@ -10,6 +10,11 @@ pkgs.stdenv.mkDerivation {
       mkdocs
       mkdocs-material
     ]);
+  configurePhase = ''
+    mkdir -p ./site/reference
+    cp -af ${module-docs}/* ./site/reference/
+
+  '';
 
   buildPhase = ''
     mkdocs build --strict

docs/nix/flake-module.nix

@@ -1,4 +1,4 @@
-{ inputs, ... }:
+{ inputs, self, ... }:
 {
   perSystem =
     {
@@ -7,11 +7,61 @@
       pkgs,
       ...
     }:
+    let
+      # Simply evaluated options (JSON)
+      # { clanCore = «derivation JSON»; clanModules = { ${name} = «derivation JSON» }; }
+      jsonDocs = import ./get-module-docs.nix {
+        inherit (inputs) nixpkgs;
+        inherit pkgs;
+        inherit (self.nixosModules) clanCore;
+        inherit (self) clanModules;
+      };
+
+      clanModulesFileInfo = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModules);
+
+      # Simply evaluated options (JSON)
+      renderOptions =
+        pkgs.runCommand "renderOptions.py"
           {
-      devShells.docs = pkgs.callPackage ./shell.nix { inherit (self'.packages) docs; };
+            # TODO: ruff does not splice properly in nativeBuildInputs
+            depsBuildBuild = [ pkgs.ruff ];
+            nativeBuildInputs = [
+              pkgs.python3
+              pkgs.mypy
+            ];
+          }
+          ''
+            install ${./scripts/renderOptions.py} $out
+            patchShebangs --build $out
+
+            ruff format --check --diff $out
+            ruff --line-length 88 $out
+            mypy --strict $out
+          '';
+
+      module-docs = pkgs.runCommand "rendered" { nativeBuildInputs = [ pkgs.python3 ]; } ''
+        export CLAN_CORE=${jsonDocs.clanCore}/share/doc/nixos/options.json 
+        # A file that contains the links to all clanModule docs
+        export CLAN_MODULES=${clanModulesFileInfo}
+
+        mkdir $out
+
+        # The python script will place mkDocs files in the output directory
+        python3 ${renderOptions}
+      '';
+    in
+    {
+      devShells.docs = pkgs.callPackage ./shell.nix {
+        inherit (self'.packages) docs;
+        inherit module-docs;
+      };
       packages = {
-        docs = pkgs.python3.pkgs.callPackage ./default.nix { inherit (inputs) nixpkgs; };
+        docs = pkgs.python3.pkgs.callPackage ./default.nix {
+          inherit (inputs) nixpkgs;
+          inherit module-docs;
+        };
         deploy-docs = pkgs.callPackage ./deploy-docs.nix { inherit (config.packages) docs; };
+        inherit module-docs;
       };
     };
 }

docs/nix/get-module-docs.nix

@@ -0,0 +1,45 @@
+{
+  nixpkgs,
+  pkgs,
+  clanCore,
+  clanModules,
+}:
+let
+  allNixosModules = (import "${nixpkgs}/nixos/modules/module-list.nix") ++ [
+    "${nixpkgs}/nixos/modules/misc/assertions.nix"
+    { nixpkgs.hostPlatform = "x86_64-linux"; }
+  ];
+
+  clanCoreNixosModules = [
+    clanCore
+    { clanCore.clanDir = ./.; }
+  ] ++ allNixosModules;
+
+  # TODO: optimally we would not have to evaluate all nixos modules for every page
+  #   but some of our module options secretly depend on nixos modules.
+  #   We would have to get rid of these implicit dependencies and make them explicit
+  clanCoreNixos = pkgs.nixos { imports = clanCoreNixosModules; };
+
+  # using extendModules here instead of re-evaluating nixos every time
+  #   improves eval performance slightly (10%)
+  getOptions = modules: (clanCoreNixos.extendModules { inherit modules; }).options;
+
+  evalDocs =
+    options:
+    pkgs.nixosOptionsDoc {
+      options = options;
+      warningsAreErrors = false;
+    };
+
+  # clanModules docs
+  clanModulesDocs = builtins.mapAttrs (
+    name: module: (evalDocs ((getOptions [ module ]).clan.${name} or { })).optionsJSON
+  ) clanModules;
+
+  # clanCore docs
+  clanCoreDocs = (evalDocs (getOptions [ ]).clanCore).optionsJSON;
+in
+{
+  clanCore = clanCoreDocs;
+  clanModules = clanModulesDocs;
+}

docs/nix/scripts/renderOptions.py

@@ -0,0 +1,159 @@
+# Options are available in the following format:
+# https://github.com/nixos/nixpkgs/blob/master/nixos/lib/make-options-doc/default.nix
+#
+# ```json
+# {
+# ...
+# "fileSystems.<name>.options": {
+#     "declarations": ["nixos/modules/tasks/filesystems.nix"],
+#     "default": {
+#       "_type": "literalExpression",
+#       "text": "[\n  \"defaults\"\n]"
+#     },
+#     "description": "Options used to mount the file system.",
+#     "example": {
+#       "_type": "literalExpression",
+#       "text": "[\n  \"data=journal\"\n]"
+#     },
+#     "loc": ["fileSystems", "<name>", "options"],
+#     "readOnly": false,
+#     "type": "non-empty (list of string (with check: non-empty))"
+#     "relatedPackages": "- [`pkgs.tmux`](\n    https://search.nixos.org/packages?show=tmux&sort=relevance&query=tmux\n  )\n",
+# }
+# }
+# ```
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+# Get environment variables
+CLAN_CORE = os.getenv("CLAN_CORE")
+CLAN_MODULES = os.environ.get("CLAN_MODULES")
+OUT = os.environ.get("out")
+
+
+def sanitize(text: str) -> str:
+    return text.replace(">", "\\>")
+
+
+def replace_store_path(text: str) -> Path:
+    res = text
+    if text.startswith("/nix/store/"):
+        res = "https://git.clan.lol/clan/clan-core/src/branch/main/" + str(
+            Path(*Path(text).parts[4:])
+        )
+    return Path(res)
+
+
+def render_option(name: str, option: dict[str, Any]) -> str:
+    read_only = option.get("readOnly")
+
+    res = f"""
+## {sanitize(name)}
+{"Readonly" if read_only else ""}
+{option.get("description", "No description available.")}
+
+**Type**: `{option["type"]}`
+
+"""
+    if option.get("default"):
+        res += f"""
+**Default**:
+
+```nix
+{option["default"]["text"] if option.get("default") else "No default set."}
+```
+        """
+    example = option.get("example", {}).get("text")
+    if example:
+        res += f"""
+
+??? example
+
+    ```nix  
+    {example}
+    ```
+"""
+    if option.get("relatedPackages"):
+        res += f"""
+### Related Packages
+
+{option["relatedPackages"]}
+"""
+
+    decls = option.get("declarations", [])
+    source_path = replace_store_path(decls[0])
+    res += f"""
+:simple-git: [{source_path.name}]({source_path})
+"""
+    res += "\n"
+
+    return res
+
+
+def module_header(module_name: str) -> str:
+    return f"""# {module_name}
+ 
+To use this module, import it like this:
+
+```nix
+  {{config, lib, inputs, ...}}: {{
+    imports = [ inputs.clan-core.clanModules.{module_name} ];
+    # ...
+  }}
+```
+"""
+
+
+def produce_clan_core_docs() -> None:
+    if not CLAN_CORE:
+        raise ValueError(
+            f"Environment variables are not set correctly: $CLAN_CORE={CLAN_CORE}"
+        )
+
+    if not OUT:
+        raise ValueError(f"Environment variables are not set correctly: $out={OUT}")
+
+    with open(CLAN_CORE) as f:
+        options: dict[str, dict[str, Any]] = json.load(f)
+        module_name = "clan-core"
+        output = module_header(module_name)
+        for option_name, info in options.items():
+            output += render_option(option_name, info)
+
+        outfile = Path(OUT) / f"{module_name}.md"
+        with open(outfile, "w") as of:
+            of.write(output)
+
+
+def produce_clan_modules_docs() -> None:
+    if not CLAN_MODULES:
+        raise ValueError(
+            f"Environment variables are not set correctly: $CLAN_MODULES={CLAN_MODULES}"
+        )
+
+    if not OUT:
+        raise ValueError(f"Environment variables are not set correctly: $out={OUT}")
+
+    with open(CLAN_MODULES) as f:
+        links: dict[str, str] = json.load(f)
+
+    # {'borgbackup': '/nix/store/hi17dwgy7963ddd4ijh81fv0c9sbh8sw-options.json', ... }
+    for module_name, options_file in links.items():
+        with open(Path(options_file) / "share/doc/nixos/options.json") as f:
+            options: dict[str, dict[str, Any]] = json.load(f)
+            print(f"Rendering options for {module_name}...")
+            output = module_header(module_name)
+            for option_name, info in options.items():
+                output += render_option(option_name, info)
+
+            outfile = Path(OUT) / f"{module_name}.md"
+            with open(outfile, "w") as of:
+                of.write(output)
+
+
+if __name__ == "__main__":
+    produce_clan_core_docs()
+    produce_clan_modules_docs()

docs/nix/shell.nix

@@ -1 +1,16 @@
-{ docs, pkgs, ... }: pkgs.mkShell { inputsFrom = [ docs ]; }
+{
+  docs,
+  pkgs,
+  module-docs,
+  ...
+}:
+pkgs.mkShell {
+  inputsFrom = [ docs ];
+  shellHook = ''
+    mkdir -p ./site/reference
+    cp -af ${module-docs}/* ./site/reference/
+    chmod +w ./site/reference/*
+
+    echo "Generated API documentation in './site/reference/' "
+  '';
+}