Merge pull request 'Doc: write inventory documentation' (#1741) from hsjobeki/clan-core:hsjobeki-main into main

docs/mkdocs.yml

@@ -52,6 +52,7 @@ nav:
       - Flake-parts: getting-started/flake-parts.md
   - Concepts:
       - Configuration: concepts/configuration.md
+      - Inventory: concepts/inventory.md
   - Reference:
       - Clan Modules:
           - reference/clanModules/borgbackup-static.md

docs/site/concepts/configuration.md

@@ -45,7 +45,7 @@ The core function that produces a clan. It returns a set of consistent configura
 `inventory`
 : Service set for easily configuring distributed services, such as backups
 
-: For more details see [Inventory](#inventory)
+: For more details see [Inventory](./inventory.md)
 
 `specialArgs`
 : Extra arguments to pass to nixosSystem i.e. useful to make self available
@@ -54,61 +54,3 @@ The core function that produces a clan. It returns a set of consistent configura
 : A function that maps from architecture to pkgs, if specified this nixpkgs will be only imported once for each system.
   This improves performance, but all nipxkgs.* options will be ignored.
   `(string -> pkgs )`
-
-## Inventory
-
-`Inventory` is an abstract service layer for consistently configuring distributed services across machine boundaries.
-
-The following is the specification of the inventory in `cuelang`
-
-```cue
-{
-    meta: {
-        // A name of the clan (primarily shown by the UI)
-        name: string
-        // A description of the clan
-        description?: string
-        // The icon path
-        icon?: string
-    }
-
-    // A map of services
-    services: [string]: [string]: {
-        // Required meta fields
-        meta: {
-            name: string,
-            icon?: string
-            description?: string,
-        },
-        // Machines are added via the avilable roles
-        // Membership depends only on this field
-        roles: [string]: {
-            machines: [...string],
-            tags: [...string],
-        }
-        machines?: {
-            [string]: {
-                config?: {
-                    ...
-                }
-            }
-        },
-
-        // Global Configuration for the service
-        // Applied to all machines.
-        config?: {
-            // Schema depends on the module.
-            // It declares the interface how the service can be configured.
-            ...
-        }
-    }
-    // A map of machines, extends the machines of `buildClan`
-    machines: [string]: {
-        name: string,
-        description?: string,
-        icon?: string
-        tags: [...string]
-        system: string
-    }
-}
-```

docs/site/concepts/inventory.md

@@ -0,0 +1,206 @@
+# Inventory
+
+`Inventory` is an abstract service layer for consistently configuring distributed services across machine boundaries.
+
+## Meta
+
+Metadata about the clan, will be displayed upfront in the upcomming clan-app, make sure to choose a unique name.
+
+```{.nix hl_lines="3-8"}
+buildClan {
+    inventory = {
+        meta = {
+            # The following options are available
+            # name:        string # Required, name of the clan.
+            # description: null | string
+            # icon:        null | string
+        };
+    };
+}
+```
+
+## Machines
+
+Machines and a small pieve of their configuration can be added via `inventory.machines`.
+
+!!! Note
+    It doesn't matter where the machine gets introduced to buildClan - All delarations are valid, duplications are merged.
+
+    However the clan-app (UI) will create machines in the inventory, because it cannot create arbitrary nixos configs.
+
+In the following example `backup_server` is one machine - it may specify parts of its configuration in different places.
+
+```{.nix hl_lines="3-5 12-20"}
+buildClan {
+    machines = {
+        "backup_server" = {
+            # Any valid nixos config
+        };
+        "jon" = {
+            # Any valid nixos config
+        };
+    };
+    inventory = {
+        machines = {
+            "backup_server" = {
+                # Don't include any nixos config here
+                # The following fields are avilable
+                # description: null | string
+                # icon:        null | string
+                # name:        string
+                # system:      null | string
+                # tags:        [...string]
+            };
+            "jon" = {
+                # Same as above
+            };
+        };
+    };
+}
+```
+
+## Services
+
+### Available clanModules
+
+Currently the inventory interface is implemented by the following clanModules
+
+- [borgbackup](../reference/clanModules/borgbackup.md)
+- [packages](../reference/clanModules/packages.md)
+- [single-disk](../reference/clanModules/single-disk.md)
+
+See the respective module documentation for available roles.
+
+### Adding services to machines
+
+A module can be added to one or multiple machines via `Roles`. clan's `Role` interface provide sane defaults for a module this allows the module author to reduce the configuration overhead to a minimum.
+
+Each service can still be customized and configured according to the modules options.
+
+- Per instance configuration via `services.<serviceName>.<instanceName>.config`
+- Per machine configuration via `services.<serviceName>.<instanceName>.machines.<machineName>.config`
+
+### Configuration Examples
+
+!!! Example "Borgbackup Example"
+
+    To configure a service it needs to be added to the machine.
+    It is required to assign the service (`borgbackup`) an arbitrary instance name. (`instance_1`)
+
+    See also: [Multiple Service Instances](#multiple-service-instances)
+
+    ```{.nix hl_lines="14-17"}
+    buildClan {
+        inventory = {
+            machines = {
+                "backup_server" = {
+                    # Don't include any nixos config here
+                    # See inventory.Machines for available options
+                };
+                "jon" = {
+                    # Don't include any nixos config here
+                    # See inventory.Machines for available options
+                };
+            };
+            services = {
+                borgbackup.instance_1 = {
+                    roles.client.machines = [ "jon" ];
+                    roles.server.machines = [ "backup_server" ];
+                };
+            };
+        };
+    }
+    ```
+
+!!! Example "Packages Example"
+
+    This example shows how to add `pkgs.firefox` via the inventory interface.
+
+    ```{.nix hl_lines="8-11"}
+    buildClan {
+        inventory = {
+            machines = {
+                "sara" = {};
+                "jon" = {};
+            };
+            services = {
+                packages.set_1 = {
+                    roles.default.machines = [ "jon" "sara" ];
+                    # Packages is a configuration option of the "packages" clanModule
+                    config.packages = ["firefox"];
+                };
+            };
+        };
+    }
+    ```
+
+### Tags
+
+It is possible to add services to multiple machines via tags. The service instance gets added in the specified role. In this case `role = "default"`
+
+!!! Example "Tags Example"
+
+    ```{.nix hl_lines="5 8 13"}
+    buildClan {
+        inventory = {
+            machines = {
+                "sara" = {
+                    tags = ["browsing"];
+                };
+                "jon" = {
+                    tags = ["browsing"];
+                };
+            };
+            services = {
+                packages.set_1 = {
+                    roles.default.tags = [ "browsing" ];
+                    config.packages = ["firefox"];
+                };
+            };
+        };
+    }
+    ```
+
+### Multiple Service Instances
+
+!!! danger "Important"
+    Not all modules support multiple instances yet.
+
+Some modules have support for adding multiple instances of the same service in different roles or configurations.
+
+!!! Example
+
+    In this example `backup_server` has role `client` and `server` in different instances.
+
+    ```{.nix hl_lines="11 14"}
+    buildClan {
+        inventory = {
+            machines = {
+                "jon" = {};
+                "backup_server" = {};
+                "backup_backup_server" = {}
+            };
+            services = {
+                borgbackup.instance_1 = {
+                    roles.client.machines = [ "jon" ];
+                    roles.server.machines = [ "backup_server" ];
+                };
+                borgbackup.instance_1 = {
+                    roles.client.machines = [ "backup_server" ];
+                    roles.server.machines = [ "backup_backup_server" ];
+                };
+            };
+        };
+    }
+    ```
+
+### Schema specification
+
+The complete schema specification can be retrieved via:
+
+```sh
+nix build git+https://git.clan.lol/clan/clan-core#inventory-schema
+> result
+> ├── schema.cue
+> └── schema.json
+```

lib/inventory/flake-module.nix

@@ -22,87 +22,11 @@ in
         inherit lib;
       };
 
-      optionsFromModule =
+      getSchema = import ./interface-to-schema.nix { inherit lib self; };
-        mName:
-        let
-          eval = self.lib.evalClanModules [ mName ];
-        in
-        if (eval.options.clan ? "${mName}") then eval.options.clan.${mName} else { };
-
-      modulesSchema = lib.mapAttrs (
-        moduleName: _: jsonLib'.parseOptions (optionsFromModule moduleName) { }
-      ) self.clanModules;
-
-      jsonLib = self.lib.jsonschema {
-        # includeDefaults = false;
-      };
-      jsonLib' = self.lib.jsonschema {
-        # includeDefaults = false;
-        header = { };
-      };
-      inventorySchema = jsonLib.parseModule (import ./build-inventory/interface.nix);
-
-      getRoles =
-        modulePath:
-        let
-          rolesDir = "${modulePath}/roles";
-        in
-        if builtins.pathExists rolesDir then
-          lib.pipe rolesDir [
-            builtins.readDir
-            (lib.filterAttrs (_n: v: v == "regular"))
-            lib.attrNames
-            (map (fileName: lib.removeSuffix ".nix" fileName))
-          ]
-        else
-          null;
-
-      schema = inventorySchema // {
-        properties = inventorySchema.properties // {
-          services = {
-            type = "object";
-            additionalProperties = false;
-            properties = lib.mapAttrs (moduleName: moduleSchema: {
-              type = "object";
-              additionalProperties = {
-                type = "object";
-                additionalProperties = false;
-                properties = {
-                  meta =
-                    inventorySchema.properties.services.additionalProperties.additionalProperties.properties.meta;
-                  config = moduleSchema;
-                  roles = {
-                    type = "object";
-                    additionalProperties = false;
-                    required = [ ];
-                    properties = lib.listToAttrs (
-                      map
-                        (role: {
-                          name = role;
-                          value =
-                            inventorySchema.properties.services.additionalProperties.additionalProperties.properties.roles.additionalProperties;
-                        })
-                        (
-                          let
-                            roles = getRoles self.clanModules.${moduleName};
-                          in
-                          if roles == null then [ ] else roles
-                        )
-                    );
-                  };
-                  machines =
-                    lib.recursiveUpdate
-                      inventorySchema.properties.services.additionalProperties.additionalProperties.properties.machines
-                      { additionalProperties.properties.config = moduleSchema; };
-                };
-              };
-            }) modulesSchema;
-          };
-        };
-      };
     in
     {
-      legacyPackages.inventorySchema = schema;
+      legacyPackages.inventorySchema = getSchema { };
+      legacyPackages.inventorySchemaPretty = getSchema { includeDefaults = false; };
 
       devShells.inventory-schema = pkgs.mkShell {
         inputsFrom = with config.checks; [
@@ -126,6 +50,19 @@ in
           cp schema.json $out
         '';
       };
+      packages.inventory-schema-pretty = pkgs.stdenv.mkDerivation {
+        name = "inventory-schema-pretty";
+        buildInputs = [ pkgs.cue ];
+        src = ./.;
+        buildPhase = ''
+          export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON self'.legacyPackages.inventorySchemaPretty)}
+          cp $SCHEMA schema.json
+          cue import -f -p compose -l '#Root:' schema.json
+          mkdir $out
+          cp schema.cue $out
+          cp schema.json $out
+        '';
+      };
 
       # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests
       legacyPackages.evalTests-inventory = import ./tests {

lib/inventory/interface-to-schema.nix

@@ -0,0 +1,98 @@
+{ lib, self, ... }:
+{
+  includeDefaults ? true,
+}:
+let
+  optionsFromModule =
+    mName:
+    let
+      eval = self.lib.evalClanModules [ mName ];
+    in
+    if (eval.options.clan ? "${mName}") then eval.options.clan.${mName} else { };
+
+  modulesSchema = lib.mapAttrs (
+    moduleName: _: jsonLib'.parseOptions (optionsFromModule moduleName) { }
+  ) self.clanModules;
+
+  jsonLib = self.lib.jsonschema { inherit includeDefaults; };
+  jsonLib' = self.lib.jsonschema {
+    inherit includeDefaults;
+    header = { };
+  };
+  inventorySchema = jsonLib.parseModule (import ./build-inventory/interface.nix);
+
+  getRoles =
+    modulePath:
+    let
+      rolesDir = "${modulePath}/roles";
+    in
+    if builtins.pathExists rolesDir then
+      lib.pipe rolesDir [
+        builtins.readDir
+        (lib.filterAttrs (_n: v: v == "regular"))
+        lib.attrNames
+        (map (fileName: lib.removeSuffix ".nix" fileName))
+      ]
+    else
+      null;
+
+  # The actual schema for the inventory
+  # !!! We cannot import the module into the interface.nix, because it would cause evaluation overhead.
+  # Modifies:
+  # - service.<serviceName>.<instanceName>.config = moduleSchema
+  # - service.<serviceName>.<instanceName>.machine.<machineName>.config = moduleSchema
+  # - service.<serviceName>.<instanceName>.roles = acutalRoles
+
+  schema =
+    let
+      moduleToService = moduleName: moduleSchema: {
+        type = "object";
+        additionalProperties = {
+          type = "object";
+          additionalProperties = false;
+          properties = {
+            meta =
+              inventorySchema.properties.services.additionalProperties.additionalProperties.properties.meta;
+            config = moduleSchema;
+            roles = {
+              type = "object";
+              additionalProperties = false;
+              required = [ ];
+              properties = lib.listToAttrs (
+                map (role: {
+                  name = role;
+                  value =
+                    inventorySchema.properties.services.additionalProperties.additionalProperties.properties.roles.additionalProperties;
+                }) (rolesOf moduleName)
+              );
+            };
+            machines =
+              lib.recursiveUpdate
+                inventorySchema.properties.services.additionalProperties.additionalProperties.properties.machines
+                { additionalProperties.properties.config = moduleSchema; };
+          };
+        };
+      };
+
+      rolesOf =
+        moduleName:
+        let
+          roles = getRoles self.clanModules.${moduleName};
+        in
+        if roles == null then [ ] else roles;
+      moduleServices = lib.mapAttrs moduleToService (
+        lib.filterAttrs (n: _v: rolesOf n != [ ]) modulesSchema
+      );
+    in
+    inventorySchema
+    // {
+      properties = inventorySchema.properties // {
+        services = {
+          type = "object";
+          additionalProperties = false;
+          properties = moduleServices;
+        };
+      };
+    };
+in
+schema