From 25fb899f649ea22a3312730631e2982782de8da0 Mon Sep 17 00:00:00 2001
From: Johannes Kirschbauer <hsjobeki@gmail.com>
Date: Thu, 21 Nov 2024 11:45:31 +0100
Subject: [PATCH 1/7] Inventory: init external modules support

---
 lib/build-clan/interface.nix                |  1 +
 lib/build-clan/module.nix                   |  3 ++-
 lib/default.nix                             |  2 +-
 lib/eval-clan-modules/default.nix           |  6 ++---
 lib/frontmatter/default.nix                 | 28 ++++++++++-----------
 lib/inventory/build-inventory/default.nix   | 16 ++++++------
 lib/inventory/build-inventory/interface.nix |  8 ++++++
 7 files changed, 37 insertions(+), 27 deletions(-)

diff --git a/lib/build-clan/interface.nix b/lib/build-clan/interface.nix
index 8dd3abf74..1ed9b0314 100644
--- a/lib/build-clan/interface.nix
+++ b/lib/build-clan/interface.nix
@@ -102,6 +102,7 @@ in
           # We don't specify the type here, for better performance.
           inventory = lib.mkOption { type = lib.types.raw; };
           inventoryFile = lib.mkOption { type = lib.types.raw; };
+          serviceConfigs = lib.mkOption { type = lib.types.raw; };
           clanModules = lib.mkOption { type = lib.types.raw; };
           source = lib.mkOption { type = lib.types.raw; };
           meta = lib.mkOption { type = lib.types.raw; };
diff --git a/lib/build-clan/module.nix b/lib/build-clan/module.nix
index 3951b2085..74deadfb5 100644
--- a/lib/build-clan/module.nix
+++ b/lib/build-clan/module.nix
@@ -159,7 +159,7 @@ in
       inventory.machines = lib.mapAttrs (_n: _: { }) config.machines;
     }
     # Merge the meta attributes from the buildClan function
-    #
+    { inventory.modules = clan-core.clanModules; }
     # config.inventory.meta <- config.meta
     { inventory.meta = config.meta; }
     # Set default for computed tags
@@ -169,6 +169,7 @@ in
   inherit nixosConfigurations;
 
   clanInternals = {
+    inherit serviceConfigs;
     inherit (clan-core) clanModules;
     inherit inventoryFile;
     inventory = config.inventory;
diff --git a/lib/default.nix b/lib/default.nix
index e2c1f01d7..4b88491cb 100644
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -16,5 +16,5 @@ in
   facts = import ./facts.nix { inherit lib; };
   inventory = import ./inventory { inherit lib clan-core; };
   jsonschema = import ./jsonschema { inherit lib; };
-  modules = import ./frontmatter { inherit clan-core lib; };
+  modules = import ./frontmatter { inherit lib; };
 }
diff --git a/lib/eval-clan-modules/default.nix b/lib/eval-clan-modules/default.nix
index 291a0d242..393d3cab8 100644
--- a/lib/eval-clan-modules/default.nix
+++ b/lib/eval-clan-modules/default.nix
@@ -53,7 +53,7 @@ let
     }
   */
   evalClanModulesWithRoles =
-    clanModules:
+    allModules:
     let
       res = builtins.mapAttrs (
         moduleName: module:
@@ -62,7 +62,7 @@ let
           roles =
             if builtins.elem "inventory" frontmatter.features or [ ] then
               assert lib.isPath module;
-              clan-core.lib.modules.getRoles moduleName
+              clan-core.lib.modules.getRoles allModules moduleName
             else
               [ ];
         in
@@ -83,7 +83,7 @@ let
               }).options.clan.${moduleName} or { };
           }) roles
         )
-      ) clanModules;
+      ) allModules;
     in
     res;
 in
diff --git a/lib/frontmatter/default.nix b/lib/frontmatter/default.nix
index c91c94fb9..e200f0b01 100644
--- a/lib/frontmatter/default.nix
+++ b/lib/frontmatter/default.nix
@@ -1,4 +1,4 @@
-{ clan-core, lib }:
+{ lib }:
 let
   # Trim the .nix extension from a filename
   trimExtension = name: builtins.substring 0 (builtins.stringLength name - 4) name;
@@ -8,18 +8,20 @@ let
       moduleName,
       instanceName,
       resolvedRoles,
+      allModules,
     }:
     lib.evalModules {
       specialArgs = {
         inherit moduleName resolvedRoles instanceName;
-        allRoles = getRoles moduleName;
+        allRoles = getRoles allModules moduleName;
       };
       modules = [
-        (getFrontmatter moduleName)
+        (getFrontmatter allModules.${moduleName} moduleName)
         ./interface.nix
       ];
     };
 
+  # For Documentation purposes only
   frontmatterOptions =
     (lib.evalModules {
       specialArgs = {
@@ -32,26 +34,24 @@ let
     }).options;
 
   getRoles =
-    serviceName:
+    allModules: serviceName:
     lib.mapAttrsToList (name: _value: trimExtension name) (
       lib.filterAttrs (name: type: type == "regular" && lib.hasSuffix ".nix" name) (
         builtins.readDir (
-          if clan-core.clanModules ? ${serviceName} then
-            clan-core.clanModules.${serviceName} + "/roles"
+          if allModules ? ${serviceName} then
+            allModules.${serviceName} + "/roles"
           else
             throw "ClanModule not found: '${serviceName}'. Make sure the module is added in the 'clanModules' attribute of clan-core."
         )
       )
     );
 
-  getConstraints = modulename: (getFrontmatter modulename).constraints;
-
   checkConstraints = args: (evalFrontmatter args).config.constraints.assertions;
 
   getReadme =
-    modulename:
+    modulepath: modulename:
     let
-      readme = "${clan-core}/clanModules/${modulename}/README.md";
+      readme = modulepath + "/README.md";
       readmeContents =
         if (builtins.pathExists readme) then
           (builtins.readFile readme)
@@ -61,9 +61,9 @@ let
     readmeContents;
 
   getFrontmatter =
-    modulename:
+    modulepath: modulename:
     let
-      content = getReadme modulename;
+      content = getReadme modulepath modulename;
       parts = lib.splitString "---" content;
       # Partition the parts into the first part (the readme content) and the rest (the metadata)
       parsed = builtins.partition ({ index, ... }: if index >= 2 then false else true) (
@@ -89,12 +89,10 @@ let
 in
 {
   inherit
-    evalFrontmatter
     frontmatterOptions
 
     getFrontmatter
-    getReadme
-    getConstraints
+
     checkConstraints
     getRoles
     ;
diff --git a/lib/inventory/build-inventory/default.nix b/lib/inventory/build-inventory/default.nix
index 9fb5f965d..217dbf174 100644
--- a/lib/inventory/build-inventory/default.nix
+++ b/lib/inventory/build-inventory/default.nix
@@ -38,8 +38,9 @@ let
     };
 
   checkService =
-    serviceName:
-    builtins.elem "inventory" (clan-core.lib.modules.getFrontmatter serviceName).features or [ ];
+    modulepath: serviceName:
+    builtins.elem "inventory"
+      (clan-core.lib.modules.getFrontmatter modulepath serviceName).features or [ ];
 
   extendMachine =
     { machineConfig, inventory }:
@@ -53,7 +54,7 @@ let
           acc
           ++ [
             {
-              assertion = checkService serviceName;
+              assertion = checkService inventory.modules.${serviceName} serviceName;
               message = ''
                 Service ${serviceName} cannot be used in inventory. It does not declare the 'inventory' feature.
 
@@ -94,7 +95,7 @@ let
         acc2: instanceName: serviceConfig:
 
         let
-          roles = clan-core.lib.modules.getRoles serviceName;
+          roles = clan-core.lib.modules.getRoles inventory.modules serviceName;
 
           resolvedRoles = lib.genAttrs roles (
             roleName:
@@ -129,11 +130,11 @@ let
           # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy
           roleModules = builtins.map (
             role:
-            if builtins.elem role roles && clan-core.clanModules ? ${serviceName} then
-              clan-core.clanModules.${serviceName} + "/roles/${role}.nix"
+            if builtins.elem role roles && inventory.modules ? ${serviceName} then
+              inventory.modules.${serviceName} + "/roles/${role}.nix"
             else
               throw "Module ${serviceName} doesn't have role: '${role}'. Role: ${
-                clan-core.clanModules.${serviceName}
+                inventory.modules.${serviceName}
               }/roles/${role}.nix not found."
           ) machineRoles;
 
@@ -151,6 +152,7 @@ let
 
           constraintAssertions = clan-core.lib.modules.checkConstraints {
             moduleName = serviceName;
+            allModules = inventory.modules;
             inherit resolvedRoles instanceName;
           };
         in
diff --git a/lib/inventory/build-inventory/interface.nix b/lib/inventory/build-inventory/interface.nix
index 4e23ba8b3..cad9ad055 100644
--- a/lib/inventory/build-inventory/interface.nix
+++ b/lib/inventory/build-inventory/interface.nix
@@ -92,6 +92,14 @@ in
     ./assertions.nix
   ];
   options = {
+    modules = lib.mkOption {
+      type = types.attrsOf types.path;
+      internal = true;
+      visible = false;
+      default = { };
+      defaultText = "clanModules of clan-core";
+    };
+
     assertions = lib.mkOption {
       type = types.listOf types.unspecified;
       internal = true;

From db5350d3dc1c88b76d2e2e1b45f47c0963915330 Mon Sep 17 00:00:00 2001
From: Johannes Kirschbauer <hsjobeki@gmail.com>
Date: Thu, 21 Nov 2024 12:57:49 +0100
Subject: [PATCH 2/7] Inventory: improve check service message

---
 lib/inventory/build-inventory/default.nix | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/inventory/build-inventory/default.nix b/lib/inventory/build-inventory/default.nix
index 217dbf174..88b952d5e 100644
--- a/lib/inventory/build-inventory/default.nix
+++ b/lib/inventory/build-inventory/default.nix
@@ -59,15 +59,15 @@ let
                 Service ${serviceName} cannot be used in inventory. It does not declare the 'inventory' feature.
 
 
-                        To allow it add the following to the beginning of the README.md of the module:
+                To allow it add the following to the beginning of the README.md of the module:
 
-                          ---
-                          ...
+                  ---
+                  ...
 
-                          features = [ "inventory" ]
-                          ---
+                  features = [ "inventory" ]
+                  ---
 
-                        Also make sure to test the module with the 'inventory' feature enabled.
+                Also make sure to test the module with the 'inventory' feature enabled.
 
               '';
             }

From d6b8716e00d06819fe8f8e15e67d7b8c3b8a6673 Mon Sep 17 00:00:00 2001
From: Johannes Kirschbauer <hsjobeki@gmail.com>
Date: Thu, 21 Nov 2024 12:59:19 +0100
Subject: [PATCH 3/7] Inventory/constraints: improve assertion messages

---
 lib/constraints/default.nix | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/constraints/default.nix b/lib/constraints/default.nix
index 7fc080624..ed2764c8a 100644
--- a/lib/constraints/default.nix
+++ b/lib/constraints/default.nix
@@ -24,8 +24,8 @@ in
             "${moduleName}.${instanceName}.roles.${roleName}.min" = {
               assertion = memberCount >= roleConstraints.min;
               message = ''
-                The ${moduleName} module requires at least ${builtins.toString roleConstraints.min} '${roleName}'s
-                but found '${builtins.toString memberCount}' within instance '${instanceName}':
+                The ${moduleName} module requires at least ${builtins.toString roleConstraints.min} members of the '${roleName}' role
+                but found '${builtins.toString memberCount}' members within instance '${instanceName}':
 
                 ${lib.concatLines members}
               '';
@@ -36,8 +36,8 @@ in
             "${moduleName}.${instanceName}.roles.${roleName}.max" = {
               assertion = memberCount <= roleConstraints.max;
               message = ''
-                The ${moduleName} module allows at most for ${builtins.toString roleConstraints.max} '${roleName}'s
-                but found '${builtins.toString memberCount}' within instance '${instanceName}':
+                The ${moduleName} module allows at most for ${builtins.toString roleConstraints.max} members of the '${roleName}' role
+                but found '${builtins.toString memberCount}' members within instance '${instanceName}':
 
                 ${lib.concatLines members}
               '';

From 36d094501d8c4771ae9d1b3c65d76e7b4ef7514c Mon Sep 17 00:00:00 2001
From: Johannes Kirschbauer <hsjobeki@gmail.com>
Date: Thu, 21 Nov 2024 13:41:34 +0100
Subject: [PATCH 4/7] Docs/inventory: ad-hoc loading of user modules

---
 docs/site/clanmodules/index.md              | 40 ++++++++++++++++-----
 lib/eval-clan-modules/default.nix           |  2 +-
 lib/inventory/build-inventory/interface.nix | 32 +++++++++++++++--
 3 files changed, 62 insertions(+), 12 deletions(-)

diff --git a/docs/site/clanmodules/index.md b/docs/site/clanmodules/index.md
index 8434cbce7..0d650f979 100644
--- a/docs/site/clanmodules/index.md
+++ b/docs/site/clanmodules/index.md
@@ -6,10 +6,8 @@ This site will guide you through authoring your first module. Explaining which c
 Under construction
 :fontawesome-solid-road-barrier: :fontawesome-solid-road-barrier: :fontawesome-solid-road-barrier:
 
-!!! Note
-    Currently ClanModules should be contributed to the [clan-core repository](https://git.clan.lol/clan/clan-core) via a PR.
-
-    Ad-hoc loading of custom modules is not recommended / supported yet.
+!!! Tip
+    External ClanModules can be ad-hoc loaded via [`clan.inventory.modules`](../reference/nix-api/inventory.md#modules)
 
 ## Bootstrapping the `clanModule`
 
@@ -43,13 +41,37 @@ clanModules/borgbackup
 
     The `roles` folder is strictly required for `features = [ "inventory" ]`.
 
-The clanModule must be registered via the `clanModules` attribute in `clan-core`
+=== "User module"
 
-```nix title="clanModules/flake-module.nix"
---8<-- "clanModules/flake-module.nix:0:6"
-    # Register your new module here
+    If the module should be ad-hoc loaded.
+    It can be made avilable in any project via the [`clan.inventory.modules`](../reference/nix-api/inventory.md#modules) attribute.
+
+    ```nix title="flake.nix"
     # ...
-```
+    buildClan {
+        # 1. Add the module to the avilable inventory modules
+        inventory.modules = {
+            custom-module = ./modules/my_module;
+        };
+        # 2. Use the module in the inventory
+        inventory.services = {
+            custom-module.instance_1 = {
+                roles.default.machines = [ "machineA" ];
+            };
+        };
+    };
+    ```
+
+=== "Upstream module"
+
+    If the module will be contributed to [`clan-core`](https://git.clan.lol/clan-core)
+    The clanModule must be registered within the `clanModules` attribute in `clan-core`
+
+    ```nix title="clanModules/flake-module.nix"
+    --8<-- "clanModules/flake-module.nix:0:5"
+        # Register our new module here
+        # ...
+    ```
 
 ## Readme
 
diff --git a/lib/eval-clan-modules/default.nix b/lib/eval-clan-modules/default.nix
index 393d3cab8..b12e910de 100644
--- a/lib/eval-clan-modules/default.nix
+++ b/lib/eval-clan-modules/default.nix
@@ -58,7 +58,7 @@ let
       res = builtins.mapAttrs (
         moduleName: module:
         let
-          frontmatter = clan-core.lib.modules.getFrontmatter moduleName;
+          frontmatter = clan-core.lib.modules.getFrontmatter allModules.${moduleName} moduleName;
           roles =
             if builtins.elem "inventory" frontmatter.features or [ ] then
               assert lib.isPath module;
diff --git a/lib/inventory/build-inventory/interface.nix b/lib/inventory/build-inventory/interface.nix
index cad9ad055..3e152f424 100644
--- a/lib/inventory/build-inventory/interface.nix
+++ b/lib/inventory/build-inventory/interface.nix
@@ -94,10 +94,38 @@ in
   options = {
     modules = lib.mkOption {
       type = types.attrsOf types.path;
-      internal = true;
-      visible = false;
       default = { };
       defaultText = "clanModules of clan-core";
+      description = ''
+        A mapping of module names to their path.
+
+        Each module can be referenced by its `attributeName` in the `inventory.services` attribute set.
+
+        !!! Important
+            Each module MUST fullfill the following requirements to be usable with the inventory:
+
+            - The module MUST have a `README.md` file with a `description`.
+            - The module MUST have at least `features = [ "inventory" ]` in the frontmatter section.
+            - The module MUST have a subfolder `roles` with at least one `{roleName}.nix` file.
+
+            For further information see: [Module Authoring Guide](../../clanmodules/index.md).
+
+        ???+ example
+            ```nix
+            buildClan {
+                # 1. Add the module to the avilable inventory modules
+                inventory.modules = {
+                  custom-module = ./modules/my_module;
+                };
+                # 2. Use the module in the inventory
+                inventory.services = {
+                  custom-module.instance_1 = {
+                      roles.default.machines = [ "machineA" ];
+                  };
+                };
+            };
+            ```
+      '';
     };
 
     assertions = lib.mkOption {

From 28f907cc858f50fcafc1a15e51d88c62dba4c329 Mon Sep 17 00:00:00 2001
From: Johannes Kirschbauer <hsjobeki@gmail.com>
Date: Thu, 21 Nov 2024 13:46:20 +0100
Subject: [PATCH 5/7] Clan-cli: update inventory classes.py

---
 pkgs/clan-cli/clan_cli/inventory/classes.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkgs/clan-cli/clan_cli/inventory/classes.py b/pkgs/clan-cli/clan_cli/inventory/classes.py
index 296230db8..754cec604 100644
--- a/pkgs/clan-cli/clan_cli/inventory/classes.py
+++ b/pkgs/clan-cli/clan_cli/inventory/classes.py
@@ -35,5 +35,6 @@ Service = dict[str, Any]
 class Inventory:
     meta: Meta
     machines: dict[str, Machine] = field(default_factory = dict)
+    modules: dict[str, str] = field(default_factory = dict)
     services: dict[str, Service] = field(default_factory = dict)
     tags: dict[str, list[str]] = field(default_factory = dict)

From 6808f0a59fa015ab30463d22a89f18ec7314b9a2 Mon Sep 17 00:00:00 2001
From: Johannes Kirschbauer <hsjobeki@gmail.com>
Date: Thu, 21 Nov 2024 14:11:29 +0100
Subject: [PATCH 6/7] jsonschema: dont export defaultText as default

---
 lib/jsonschema/default.nix | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/lib/jsonschema/default.nix b/lib/jsonschema/default.nix
index 16a8e46b2..8dfddde94 100644
--- a/lib/jsonschema/default.nix
+++ b/lib/jsonschema/default.nix
@@ -72,7 +72,8 @@ rec {
       { }
     else if opt ? defaultText then
       {
-        default = "<thunk>";
+        # dont add default to jsonschema. It seems to alter the type
+        # default = "<thunk>";
       }
     else
       lib.optionalAttrs (opt ? default) {
@@ -93,11 +94,18 @@ rec {
     };
 
   makeModuleInfo =
-    { path }:
     {
-      "$exportedModuleInfo" = {
-        inherit path;
-      };
+      path,
+      defaultText ? null,
+    }:
+    {
+      "$exportedModuleInfo" =
+        {
+          inherit path;
+        }
+        // lib.optionalAttrs (defaultText != null) {
+          inherit defaultText;
+        };
     };
 
   # parses a set of evaluated nixos options to a jsonschema
@@ -172,6 +180,7 @@ rec {
       };
       exposedModuleInfo = makeModuleInfo {
         path = option.loc;
+        defaultText = option.defaultText or null;
       };
     in
     # either type

From aaaabafdf1b12ba7baf7dfdd892be3d97329aba0 Mon Sep 17 00:00:00 2001
From: Johannes Kirschbauer <hsjobeki@gmail.com>
Date: Thu, 21 Nov 2024 16:14:16 +0100
Subject: [PATCH 7/7] inventory/eval.-tests: add explicit modules

---
 lib/inventory/tests/default.nix | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib/inventory/tests/default.nix b/lib/inventory/tests/default.nix
index 7db165963..1a7b51be2 100644
--- a/lib/inventory/tests/default.nix
+++ b/lib/inventory/tests/default.nix
@@ -6,7 +6,6 @@ let
       inherit lib clan-core;
     }
   );
-
   inherit (inventory) buildInventory;
 in
 {
@@ -23,6 +22,7 @@ in
       configs = buildInventory {
         directory = ./.;
         inventory = {
+          modules = clan-core.clanModules;
           services = {
             borgbackup.instance_1 = {
               roles.server.machines = [ "backup_server" ];
@@ -64,6 +64,7 @@ in
       configs = buildInventory {
         directory = ./.;
         inventory = {
+          modules = clan-core.clanModules;
           services = {
             borgbackup.instance_1 = {
               roles.client.tags = [ "backup" ];
@@ -103,6 +104,7 @@ in
       configs = buildInventory {
         directory = ./.;
         inventory = {
+          modules = clan-core.clanModules;
           services = {
             borgbackup.instance_1 = {
               roles.client.machines = [ "machine_1" ];
@@ -132,6 +134,7 @@ in
       configs = buildInventory {
         directory = ./.;
         inventory = {
+          modules = clan-core.clanModules;
           services = {
             fanatasy.instance_1 = {
               roles.default.machines = [ "machine_1" ];
@@ -156,6 +159,7 @@ in
       configs = buildInventory {
         directory = ./.;
         inventory = {
+          modules = clan-core.clanModules;
           services = {
             borgbackup.instance_1 = {
               roles.roleXYZ.machines = [ "machine_1" ];
@@ -179,6 +183,7 @@ in
       configs = buildInventory {
         directory = ./.;
         inventory = {
+          modules = clan-core.clanModules;
           services = {
             borgbackup.instance_1 = {
               roles.client.machines = [ "machine_1" ];
@@ -205,6 +210,7 @@ in
       configs = buildInventory {
         directory = ./.;
         inventory = {
+          modules = clan-core.clanModules;
           services = {
             borgbackup.instance_1 = {
               enabled = false;