Merge pull request 'Inventory: docs improvements' (#2132) from hsjobeki/clan-core:hsjobeki-inventory-docs into main

clanModules/admin/README.md

@@ -1,4 +1,5 @@
 ---
 description = "Convenient Administration for the Clan App"
-categories = ["administration"]
+categories = ["Utility"]
+features = [ "inventory" ]
 ---

clanModules/admin/roles/default.nix

@@ -1 +0,0 @@
-{ }

clanModules/borgbackup-static/README.md

@@ -1,6 +1,11 @@
 ---
 description = "Statically configure borgbackup with sane defaults."
 ---
+!!! Danger "Deprecated"
+    Use [borgbackup](borgbackup.md) instead.
+
+    Don't use borgbackup-static through [inventory](../../guides/inventory.md).
+
 This module implements the `borgbackup` backend and implements sane defaults
 for backup management through `borgbackup` for members of the clan.
 

clanModules/borgbackup/README.md

@@ -1,6 +1,7 @@
 ---
 description = "Efficient, deduplicating backup program with optional compression and secure encryption."
-categories = ["backup"]
+categories = ["System"]
+features = [ "inventory" ]
 ---
 BorgBackup (short: Borg) gives you:
 

clanModules/disk-id/README.md

@@ -1,3 +1,4 @@
 ---
 description = "Generates a uuid for use in disk device naming"
+features = [ "inventory" ]
 ---

clanModules/disk-id/roles/default.nix

@@ -1 +0,0 @@
-{ }

clanModules/iwd/README.md

@@ -1,5 +1,6 @@
 ---
 description = "Automatically provisions wifi credentials"
+features = [ "inventory" ]
 ---
 
 !!! Warning

clanModules/iwd/roles/default.nix

@@ -1 +0,0 @@
-{ }

clanModules/machine-id/README.md

@@ -1,3 +1,4 @@
 ---
 description = "Sets the /etc/machine-id and exposes it as a nix option"
+features = [ "inventory" ]
 ---

clanModules/machine-id/roles/default.nix

@@ -1 +0,0 @@
-{ }

clanModules/mumble/README.md

@@ -1,6 +1,6 @@
 ---
 description = "Open Source, Low Latency, High Quality Voice Chat."
-categories = ["chat", "voice"]
+categories = ["Audio", "Social"]
 ---
 The mumble clan module gives you:
 
@@ -10,5 +10,5 @@ The mumble clan module gives you:
 - Backed by a large and active open-source community.
 
 This all set up in a way that allows peer-to-peer hosting.
-Every machine inside the clan can be a host for mumble, 
+Every machine inside the clan can be a host for mumble,
 and thus it doesn't matter who in the network is online - as long as two people are online they are able to chat with each other.

clanModules/packages/README.md

@@ -1,4 +1,5 @@
 ---
 description = "Define package sets from nixpkgs and install them on one or more machines"
-categories = ["packages"]
+categories = ["System"]
+features = [ "inventory" ]
 ---

clanModules/packages/roles/default.nix

@@ -1 +0,0 @@
-{ }

clanModules/single-disk/README.md

@@ -1,6 +1,7 @@
 ---
 description = "Configures partitioning of the main disk"
-categories = ["disk-layout"]
+categories = ["System"]
+features = [ "inventory" ]
 ---
 # Primary Disk Layout
 

clanModules/single-disk/roles/default.nix

@@ -1 +0,0 @@
-{ }

clanModules/state-version/README.md

@@ -1,12 +1,13 @@
 ---
 description = "Automatically generate the state version of the nixos installation."
+features = [ "inventory" ]
 ---
 
 This module generates the `system.stateVersion` of the nixos installation automatically.
 
 Options: [system.stateVersion](https://search.nixos.org/options?channel=unstable&show=system.stateVersion&from=0&size=50&sort=relevance&type=packages&query=stateVersion)
 
-Migration: 
+Migration:
 If you are already setting `system.stateVersion`, then import the module and then either let the automatic generation happen, or trigger the generation manually for the machine. The module will take the specified version, if one is already supplied through the config.
 To manually generate the version for a specified machine run:
 

clanModules/state-version/roles/default.nix

@@ -1 +0,0 @@
-{ }

docs/mkdocs.yml

@@ -51,10 +51,12 @@ nav:
       - Flake-parts: getting-started/flake-parts.md
   - Guides:
       - guides/index.md
+      - Adding Machines: guides/add-machines.md
       - Inventory: guides/inventory.md
   - Reference:
       - reference/index.md
       - Clan Modules:
+          - reference/clanModules/index.md
           - reference/clanModules/admin.md
           - reference/clanModules/borgbackup-static.md
           - reference/clanModules/borgbackup.md
@@ -65,7 +67,6 @@ nav:
           - reference/clanModules/garage.md
           - reference/clanModules/golem-provider.md
           - reference/clanModules/heisenbridge.md
-          - reference/clanModules/index.md
           - reference/clanModules/iwd.md
           - reference/clanModules/localbackup.md
           - reference/clanModules/localsend.md

docs/nix/render_options/__init__.py

@@ -28,7 +28,7 @@ import os
 from pathlib import Path
 from typing import Any
 
-from clan_cli.api.modules import Frontmatter, extract_frontmatter
+from clan_cli.api.modules import Frontmatter, extract_frontmatter, get_roles
 from clan_cli.errors import ClanError
 
 # Get environment variables
@@ -125,14 +125,15 @@ def render_option(name: str, option: dict[str, Any], level: int = 3) -> str:
     return res
 
 
-def module_header(module_name: str) -> str:
+def module_header(module_name: str, has_inventory_feature: bool = False) -> str:
-    return f"# {module_name}\n\n"
+    indicator = " 🔹" if has_inventory_feature else ""
+    return f"# {module_name}{indicator}\n\n"
 
 
 def module_usage(module_name: str) -> str:
     return f"""## Usage
 
-To use this module, import it like this:
+To use this module, import it like th:
 
 ```nix
 {{config, lib, inputs, ...}}: {{
@@ -198,25 +199,13 @@ def render_roles(roles: list[str] | None, module_name: str) -> str:
     if roles:
         roles_list = "\n".join([f"    - `{r}`" for r in roles])
         return f"""
-???+ tip "Inventory usage"
+## Inventory Roles
 
-    Predefined roles:
+Predefined roles
 
 {roles_list}
 
-    Usage:
+For more information, see the [inventory guide](../../guides/inventory.md).
-
-    ```{{.nix hl_lines="4"}}
-    buildClan {{
-      inventory.services = {{
-        {module_name}.instance_1 = {{
-            roles.{roles[0]}.machines = [ "sara_machine" ];
-            # ...
-        }};
-      }};
-    }}
-    ```
-
 """
     return ""
 
@@ -226,6 +215,21 @@ clan_modules_descr = """Clan modules are [NixOS modules](https://wiki.nixos.org/
 """
 
 
+def render_categories(categories: list[str], frontmatter: Frontmatter) -> str:
+    cat_info = frontmatter.categories_info
+    res = """<div style="display: grid; grid-template-columns: repeat(3, 1fr); gap: 10px;">"""
+    for cat in categories:
+        color = cat_info[cat]["color"]
+        # description = cat_info[cat]["description"]
+        res += f"""
+    <div style="background-color: {color}; color: white; padding: 10px; border-radius: 20px; text-align: center;">
+        {cat}
+    </div>
+"""
+    res += "</div>"
+    return res
+
+
 def produce_clan_modules_docs() -> None:
     if not CLAN_MODULES:
         msg = f"Environment variables are not set correctly: $out={CLAN_MODULES}"
@@ -270,16 +274,21 @@ def produce_clan_modules_docs() -> None:
         with (Path(options_file) / "share/doc/nixos/options.json").open() as f:
             options: dict[str, dict[str, Any]] = json.load(f)
             print(f"Rendering options for {module_name}...")
-            output = module_header(module_name)
+            output = module_header(module_name, "inventory" in frontmatter.features)
 
             if frontmatter.description:
                 output += f"**{frontmatter.description}**\n\n"
+
+            output += "## Categories\n\n"
+            output += render_categories(frontmatter.categories, frontmatter)
+            output += "\n---\n\n"
+
             output += f"{readme_content}\n"
 
             # get_roles(str) -> list[str] | None
-            # roles = get_roles(CLAN_CORE_PATH / "clanModules" / module_name)
+            roles = get_roles(CLAN_CORE_PATH / "clanModules" / module_name)
-            # if roles:
+            if roles:
-            #     output += render_roles(roles, module_name)
+                output += render_roles(roles, module_name)
 
             output += module_usage(module_name)
 

docs/site/guides/add-machines.md

@@ -0,0 +1,50 @@
+# How to add machines
+
+Clan has two general methods of adding machines
+
+- **Automatic**: Detects every folder in the `machines` folder.
+- **Declarative**: Explicit declarations in nix.
+
+## Automatic register
+
+Every machine of the form `machines/{machineName}` will be registered automatically.
+
+Automatically imported:
+
+- [x] ``machines/{machineName}/configuration.nix`
+- [x] ``machines/{machineName}/hardware-configuration.nix`
+- [x] ``machines/{machineName}/facter.json` Automatically configured, for further information see [nixos-facter](../blog/posts/nixos-facter.md)
+
+## Manual declaration
+
+Machines can also be added manually under `buildClan`, `clan.*` in flake-parts or via [`inventory`](../guides/inventory.md).
+
+!!! Note
+    It is possible to use `inventory` and `buildClan` together at the same time.
+
+=== "**Individual Machine Configuration**"
+
+    ```{.nix}
+    buildClan {
+        machines = {
+            "jon" = {
+                # Any valid nixos config
+            };
+        };
+    }
+    ```
+
+=== "**Inventory Configuration**"
+
+    ```{.nix}
+    buildClan {
+        inventory = {
+            machines = {
+                "jon" = {
+                    # Inventory machines can set tags
+                    tags = [ "zone1" ];
+                };
+            };
+        };
+    }
+    ```

docs/site/guides/index.md

@@ -2,4 +2,5 @@
 
 Detailed guides into the following subtopics:
 
+- [Adding Machines](./add-machines.md): How to add machines.
 - [Inventory](./inventory.md): Configuring Services across machine boundaries

docs/site/guides/inventory.md

@@ -4,73 +4,36 @@
 
 See [Inventory API Documentation](../reference/nix-api/inventory.md)
 
-This guide will walk you through setting up a backup-service, where the inventory becomes useful.
+This guide will walk you through setting up a backup service, where the inventory becomes useful.
 
-## Prerequisites Meta (optional)
+!!! example "Experimental status"
+    The inventory implementation is not considered stable yet.
+    We are actively soliciting feedback from users.
 
-Metadata about the clan, will be displayed upfront in the upcomming clan-app, make sure to choose a unique name.
+    Stabilizing the API is a priority.
 
-Make sure to set `name` either via `inventory.meta` OR via `clan.meta`.
+## Categories
 
-```{.nix hl_lines="3-8"}
+ <div style="display: grid; grid-template-columns: repeat(3, 1fr); gap: 10px;">
-buildClan {
+    <!-- Chip 1 -->
-    inventory = {
+    <div style="background-color: #F5B7B1; color: white; padding: 10px; border-radius: 20px; text-align: center;">
-        meta = {
+        Light Coral
-            name = "Superclan"
+    </div>
-            description = "Awesome backups and family stuff"
+    <!-- Chip 2 -->
-        };
+    <div style="background-color: #85C1E9; color: white; padding: 10px; border-radius: 20px; text-align: center;">
-    };
+        Baby Blue
-}
+    </div>
-```
+</div>
 
-## How to add machines
+## Prerequisites
 
-Every machine of the form `machines/{machineName}/configuration.nix` will be registered automatically.
+- [x] [Add machines](./add-machines.md) to your clan.
-
-Machines can also be manually added under `inventory.machines` OR via `buildClan` directly.
-
-!!! Note
-    It doesn't matter where the machine gets introduced to buildClan - All delarations are valid, duplications are merged.
-
-    However the clan-app (UI) will create machines in the inventory, because it cannot create arbitrary nix code or nixos configs.
-
-In the following example `backup_server` is one machine - it may specify parts of its configuration in different places.
-
-```{.nix hl_lines="3-5 12-20"}
-buildClan {
-    machines = {
-        "backup_server" = {
-            # Any valid nixos config
-        };
-        "jon" = {
-            # Any valid nixos config
-        };
-    };
-    inventory = {
-        machines = {
-            "backup_server" = {
-                # Don't include any nixos config here.
-                # See the Inventory API Docs for the available attributes.
-            };
-            "jon" = {
-                # Same as above
-            };
-        };
-    };
-}
-```
 
 ## Services
 
-### Available clanModules
+The inventory defines `services`. Membership of `machines` is defined via roles exclusively.
 
-Currently the inventory interface is implemented by the following clanModules
+See the each [module documentation](../reference/clanModules/index.md) for available roles.
-
-- [borgbackup](../reference/clanModules/borgbackup.md)
-- [packages](../reference/clanModules/packages.md)
-- [single-disk](../reference/clanModules/single-disk.md)
-
-See the respective module documentation for available roles.
 
 !!! Note
     It is possible to use any [clanModule](../reference/clanModules/index.md) in the inventory and add machines via
@@ -95,21 +58,12 @@ Each service can still be customized and configured according to the modules opt
 
     See also: [Multiple Service Instances](#multiple-service-instances)
 
-    ```{.nix hl_lines="14-17"}
+    ```{.nix hl_lines="6-7"}
     buildClan {
         inventory = {
-            machines = {
-                "backup_server" = {
-                    # Don't include any nixos config here
-                    # See inventory.Machines for available options
-                };
-                "jon" = {
-                    # Don't include any nixos config here
-                    # See inventory.Machines for available options
-                };
-            };
             services = {
                 borgbackup.instance_1 = {
+                    # Machines can be added here.
                     roles.client.machines = [ "jon" ];
                     roles.server.machines = [ "backup_server" ];
                 };
@@ -120,24 +74,23 @@ Each service can still be customized and configured according to the modules opt
 
 ### Scaling the Backup
 
-It is possible to add services to multiple machines via tags. The service instance gets added in the specified role. In this case `role = "client"`
+The inventory allows machines to set Tags
+
+It is possible to add services to multiple machines via tags as shown
 
 !!! Example "Tags Example"
 
-    ```{.nix hl_lines="9 12 17"}
+    ```{.nix hl_lines="5 8 14"}
     buildClan {
         inventory = {
             machines = {
-                "backup_server" = {
-                    # Don't include any nixos config here
-                    # See inventory.Machines for available options
-                };
                 "jon" = {
                     tags = [ "backup" ];
                 };
                 "sara" = {
                     tags = [ "backup" ];
                 };
+                # ...
             };
             services = {
                 borgbackup.instance_1 = {
@@ -172,7 +125,7 @@ It is possible to add services to multiple machines via tags. The service instan
                     roles.client.machines = [ "jon" ];
                     roles.server.machines = [ "backup_server" ];
                 };
-                borgbackup.instance_1 = {
+                borgbackup.instance_2 = {
                     roles.client.machines = [ "backup_server" ];
                     roles.server.machines = [ "backup_backup_server" ];
                 };

lib/build-clan/module.nix

@@ -68,7 +68,7 @@ let
           hwConfig = "${directory}/machines/${name}/hardware-configuration.nix";
 
           facterModules = lib.optionals (builtins.pathExists facterJson) [
-            "${clan-core.inputs.nixos-facter-modules}/modules/nixos/facter.nix"
+            clan-core.inputs.nixos-facter-modules.nixosModules.facter
             { config.facter.reportPath = facterJson; }
           ];
         in
@@ -231,6 +231,9 @@ in
         )
       );
     }
+    {
+      inventory.machines = lib.mapAttrs (_n: _: { }) config.machines;
+    }
     # Merge the meta attributes from the buildClan function
     { inventory.meta = if config.meta != null then config.meta else { }; }
   ];

lib/description.nix

@@ -12,7 +12,7 @@ rec {
     in
     readmeContents;
 
-  getShortDescription =
+  getFrontmatter =
     modulename:
     let
       content = getReadme modulename;
@@ -21,14 +21,21 @@ rec {
       parsed = builtins.partition ({ index, ... }: if index >= 2 then false else true) (
         lib.filter ({ index, ... }: index != 0) (lib.imap0 (index: part: { inherit index part; }) parts)
       );
-
-      # Use this if the content is needed
-      # readmeContent = lib.concatMapStrings (v: "---" + v.part) parsed.wrong;
-
       meta = builtins.fromTOML (builtins.head parsed.right).part;
     in
     if (builtins.length parts >= 3) then
-      meta.description
+      meta
     else
-      throw "Short description delimiter `---` not found in README.md for module ${modulename}";
+      throw ''
+        TOML Frontmatter not found in README.md for module ${modulename}
+
+        Please add the following to the top of your README.md:
+
+        ---
+        description = "Your description here"
+        categories = [ "Your categories here" ]
+        features = [ "inventory" ]
+        ---
+        ...rest of your README.md...
+      '';
 }

lib/inventory/build-inventory/assertions.nix

@@ -74,4 +74,5 @@
       ) [ ] config.services;
     }
   ];
+
 }

lib/inventory/build-inventory/default.nix

@@ -37,44 +37,18 @@ let
         ) [ ] members.tags or [ ]);
     };
 
+  checkService =
+    serviceName:
+    let
+      frontmatter = clan-core.lib.modules.getFrontmatter serviceName;
+    in
+    if builtins.elem "inventory" frontmatter.features or [ ] then true else false;
+
   /*
     Returns a NixOS configuration for every machine in the inventory.
 
     machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration }
   */
-
-  # { client_1_machine = { tags = [ "backup" ]; }; client_2_machine = { tags = [ "backup" ]; }; not_used_machine = { }; }
-  getAllMachines =
-    inventory:
-    lib.foldlAttrs (
-      res: serviceName: serviceConfigs:
-      (lib.foldlAttrs (
-        res: instanceName: serviceConfig:
-        lib.foldlAttrs (
-          res: roleName: members:
-          let
-            resolved = resolveTags {
-              inherit
-                serviceName
-                instanceName
-                roleName
-                inventory
-                members
-                ;
-            };
-          in
-          res
-          // builtins.listToAttrs (
-            builtins.map (m: {
-              name = m;
-              value = { };
-            }) resolved.machines
-          )
-        ) res serviceConfig.roles
-      ) res serviceConfigs)
-    ) { } (inventory.services or { })
-    // inventory.machines or { };
-
   buildInventory =
     { inventory, directory }:
     # For every machine in the inventory, build a NixOS configuration
@@ -178,7 +152,7 @@ let
             ]
           else
             acc2
-        ) [ ] serviceConfigs)
+        ) [ ] (serviceConfigs))
       ) [ ] inventory.services
       # Append each machine config
       ++ [
@@ -188,9 +162,35 @@ let
         (lib.optionalAttrs (machineConfig.deploy.targetHost or null != null) {
           config.clan.core.networking.targetHost = machineConfig.deploy.targetHost;
         })
+        {
+          assertions = lib.foldlAttrs (
+            acc: serviceName: _:
+            acc
+            ++ [
+              {
+                assertion = checkService serviceName;
+                message = ''
+                  Service ${serviceName} cannot be used in inventory. It does not declare the 'inventory' feature.
+
+
+                          To allow it add the following to the beginning of the README.md of the module:
+
+                            ---
+                            ...
+
+                            features = [ "inventory" ]
+                            ---
+
+                          Also make sure to test the module with the 'inventory' feature enabled.
+
+                '';
+              }
+            ]
+          ) [ ] inventory.services;
+        }
       ]
-    ) (getAllMachines inventory);
+    ) (inventory.machines or { });
 in
 {
-  inherit buildInventory getAllMachines;
+  inherit buildInventory;
 }

lib/inventory/tests/default.nix

@@ -1,37 +1,8 @@
 { inventory, clan-core, ... }:
 let
-  inherit (inventory) buildInventory getAllMachines;
+  inherit (inventory) buildInventory;
 in
 {
-  test_get_all_used_machines = {
-    # Test that all machines are returned
-    expr = getAllMachines {
-      machines = {
-        machine_3 = {
-          tags = [ "tag_3" ];
-        };
-      };
-      services = {
-        borgbackup.instance_1 = {
-          roles.server.machines = [ "backup_server" ];
-          roles.client.machines = [
-            "client_1_machine"
-            "client_2_machine"
-          ];
-          roles.client.tags = [ "tag_3" ];
-        };
-      };
-    };
-    expected = {
-      backup_server = { };
-      client_1_machine = { };
-      client_2_machine = { };
-      machine_3 = {
-        tags = [ "tag_3" ];
-      };
-    };
-  };
-
   test_inventory_empty = {
     # Empty inventory should return an empty module
     expr = buildInventory {
@@ -117,9 +88,9 @@ in
         not_used_machine = builtins.length configs.not_used_machine;
       };
       expected = {
-        client_1_machine = 5;
+        client_1_machine = 6;
-        client_2_machine = 5;
+        client_2_machine = 6;
-        not_used_machine = 2;
+        not_used_machine = 3;
       };
     };
 

pkgs/clan-cli/clan_cli/api/modules.py

@@ -1,9 +1,9 @@
 import json
 import re
 import tomllib
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, get_args, get_type_hints
+from typing import Any, TypedDict, get_args, get_type_hints
 
 from clan_cli.cmd import run_no_stdout
 from clan_cli.errors import ClanCmdError, ClanError
@@ -15,10 +15,47 @@ from . import API
 from .serde import from_dict
 
 
+class CategoryInfo(TypedDict):
+    color: str
+    description: str
+
+
 @dataclass
 class Frontmatter:
     description: str
-    categories: list[str] | None = None
+    categories: list[str] = field(default_factory=lambda: ["Uncategorized"])
+    features: list[str] = field(default_factory=list)
+
+    @property
+    def categories_info(self) -> dict[str, CategoryInfo]:
+        category_map: dict[str, CategoryInfo] = {
+            "AudioVideo": {
+                "color": "#AEC6CF",
+                "description": "Applications for presenting, creating, or processing multimedia (audio/video)",
+            },
+            "Audio": {"color": "#CFCFC4", "description": "Audio"},
+            "Video": {"color": "#FFD1DC", "description": "Video"},
+            "Development": {"color": "#F49AC2", "description": "Development"},
+            "Education": {"color": "#B39EB5", "description": "Education"},
+            "Game": {"color": "#FFB347", "description": "Game"},
+            "Graphics": {"color": "#FF6961", "description": "Graphics"},
+            "Social": {"color": "#76D7C4", "description": "Social"},
+            "Network": {"color": "#77DD77", "description": "Network"},
+            "Office": {"color": "#85C1E9", "description": "Office"},
+            "Science": {"color": "#779ECB", "description": "Science"},
+            "System": {"color": "#F5C3C0", "description": "System"},
+            "Settings": {"color": "#03C03C", "description": "Settings"},
+            "Utility": {"color": "#B19CD9", "description": "Utility"},
+            "Uncategorized": {"color": "#C23B22", "description": "Uncategorized"},
+        }
+
+        return category_map
+
+    def __post_init__(self) -> None:
+        for category in self.categories:
+            if category not in self.categories_info:
+                msg = f"Invalid category: {category}"
+                raise ValueError(msg)
 
 
 def extract_frontmatter(readme_content: str, err_scope: str) -> tuple[Frontmatter, str]:

pkgs/schemas/flake-module.nix

@@ -23,11 +23,14 @@
         modulename: _: jsonLib.parseOptions (optionsFromModule modulename) { }
       ) clanModules;
 
-      clanModuleFunctionSchemas = lib.mapAttrsFlatten (modulename: _: {
+      clanModuleFunctionSchemas = lib.mapAttrsFlatten (
-        name = modulename;
+        modulename: _:
-        description = self.lib.modules.getShortDescription modulename;
+        (self.lib.modules.getFrontmatter modulename)
-        parameters = jsonLib.parseOptions (optionsFromModule modulename) { };
+        // {
-      }) clanModules;
+          name = modulename;
+          parameters = jsonLib.parseOptions (optionsFromModule modulename) { };
+        }
+      ) clanModules;
     in
     rec {
       checks = {