Inventory: move to lib.inventory

lib/inventory/.envrc

@@ -0,0 +1,5 @@
+source_up
+
+watch_file flake-module.nix
+
+use flake .#inventory-schema --builders ''

lib/inventory/README.md

@@ -0,0 +1,90 @@
+# Inventory
+
+The inventory is our concept for distributed services. Users can configure multiple machines with minimal effort.
+
+- The inventory acts as a declarative source of truth for all machine configurations.
+- Users can easily add or remove machines to/from services.
+- Ensures that all machines and services are configured consistently, across multiple nixosConfigs.
+- Defaults and predefined roles in our modules minimizes the need for manual configuration.
+
+Open questions:
+
+- [ ] How do we set default role, description and other metadata?
+  - It must be accessible from Python.
+  - It must set the value in the module system.
+
+- [ ] Inventory might use assertions. Should each machine inherit the inventory assertions ?
+
+- [ ] Is the service config interface the same as the module config interface ?
+
+- [ ] As a user do I want to see borgbackup as the high level category?
+
+
+Architecture
+
+```
+nixosConfig < machine_module        < inventory
+---------------------------------------------
+nixos   < borgbackup            <- inventory <-> UI
+
+        creates the config      Maps from high level services to the borgbackup clan module
+        for ONE machine         Inventory is completely serializable.
+                                UI can interact with the inventory to define machines, and services
+                                Defining Users is out of scope for the first prototype.
+```
+
+## Provides a specification for the inventory
+
+It is used for design phase and as validation helper.
+
+> Cue is less verbose and easier to understand and maintain than json-schema.
+> Json-schema, if needed can be easily generated on-the fly.
+
+## Checking validity
+
+Directly check a json against the schema
+
+`cue vet inventory.json root.cue -d '#Root'`
+
+## Json schema
+
+Export the json-schema i.e. for usage in python / javascript / nix
+
+`cue export --out openapi root.cue`
+
+## Usage
+
+Comments are rendered as descriptions in the json schema.
+
+```cue
+// A name of the clan (primarily shown by the UI)
+name: string
+```
+
+Cue open sets. In the following `foo = {...}` means that the key `foo` can contain any arbitrary json object.
+
+```cue
+foo: { ... }
+```
+
+Cue dynamic keys.
+
+```cue
+[string]: {
+    attr: string
+}
+```
+
+This is the schema of
+
+```json
+{
+    "a": {
+        "attr": "foo"
+    },
+    "b": {
+        "attr": "bar"
+    }
+    // ... Indefinitely more dynamic keys of type "string"
+}
+```

lib/inventory/_old_default.nix

@@ -0,0 +1,32 @@
+{ self, lib, ... }:
+let
+  clan-core = self;
+in
+{
+  clan = clan-core.lib.buildClan {
+    meta.name = "kenjis clan";
+    # Should usually point to the directory of flake.nix
+    directory = self;
+
+    inventory = {
+      services = {
+        borgbackup.instance_1 = {
+          roles.server.machines = [ "vyr_machine" ];
+          roles.client.tags = [ "laptop" ];
+        };
+      };
+    };
+
+    # merged with
+    machines = {
+      "vyr_machine" = { };
+      "vi_machine" = {
+        clan.tags = [ "laptop" ];
+      };
+      "camina_machine" = {
+        clan.tags = [ "laptop" ];
+        clan.meta.name = "camina";
+      };
+    };
+  };
+}

lib/inventory/build-inventory/default.nix

@@ -0,0 +1,106 @@
+# Generate partial NixOS configurations for every machine in the inventory
+# This function is responsible for generating the module configuration for every machine in the inventory.
+{ lib, clan-core }:
+inventory:
+let
+  machines = machinesFromInventory inventory;
+
+  resolveTags =
+    # Inventory, { machines :: [string], tags :: [string] }
+    inventory: members: {
+      machines =
+        members.machines or [ ]
+        ++ (builtins.foldl' (
+          acc: tag:
+          let
+            tagMembers = builtins.attrNames (
+              lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines
+            );
+          in
+          # throw "Machine tag ${tag} not found. Not machine with: tag ${tagName} not in inventory.";
+          if tagMembers == [ ] then
+            throw "Machine tag ${tag} not found. Not machine with: tag ${tag} not in inventory."
+          else
+            acc ++ tagMembers
+        ) [ ] members.tags or [ ]);
+    };
+
+  /*
+    Returns a NixOS configuration for every machine in the inventory.
+
+    machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration }
+  */
+  machinesFromInventory =
+    inventory:
+    # For every machine in the inventory, build a NixOS configuration
+    # For each machine generate config, forEach service, if the machine is used.
+    builtins.mapAttrs (
+      machineName: _:
+      lib.foldlAttrs (
+        # [ Modules ], String, { ${instance_name} :: ServiceConfig }
+        acc: moduleName: serviceConfigs:
+        acc
+        # Collect service config
+        ++ (lib.foldlAttrs (
+          # [ Modules ], String, ServiceConfig
+          acc2: instanceName: serviceConfig:
+          let
+            resolvedRoles = builtins.mapAttrs (
+              _roleName: members: resolveTags inventory members
+            ) serviceConfig.roles;
+
+            isInService = builtins.any (members: builtins.elem machineName members.machines) (
+              builtins.attrValues resolvedRoles
+            );
+
+            # Inverse map of roles. Allows for easy lookup of roles for a given machine.
+            # { ${machine_name} :: [roles]
+            inverseRoles = lib.foldlAttrs (
+              acc: roleName:
+              { machines }:
+              acc
+              // builtins.foldl' (
+                acc2: machineName: acc2 // { ${machineName} = (acc.${machineName} or [ ]) ++ [ roleName ]; }
+              ) { } machines
+            ) { } resolvedRoles;
+
+            machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { };
+            globalConfig = serviceConfig.config or { };
+
+            # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy
+            roleModules = builtins.map (
+              role:
+              let
+                path = "${clan-core.clanModules.${moduleName}}/roles/${role}.nix";
+              in
+              if builtins.pathExists path then
+                path
+              else
+                throw "Role doesnt have a module: ${role}. Path: ${path} not found."
+            ) inverseRoles.${machineName} or [ ];
+          in
+          if isInService then
+            acc2
+            ++ [
+              {
+                imports = [ clan-core.clanModules.${moduleName} ] ++ roleModules;
+                config.clan.${moduleName} = lib.mkMerge [
+                  globalConfig
+                  machineServiceConfig
+                ];
+              }
+              {
+                config.clan.inventory.services.${moduleName}.${instanceName} = {
+                  roles = resolvedRoles;
+                  # TODO: Add inverseRoles to the service config if needed
+                  # inherit inverseRoles;
+                };
+              }
+            ]
+          else
+            acc2
+        ) [ ] serviceConfigs)
+      ) [ ] inventory.services
+    ) inventory.machines;
+in
+machines

lib/inventory/build-inventory/interface.nix

@@ -0,0 +1,121 @@
+{ config, lib, ... }:
+let
+  t = lib.types;
+
+  metaOptions = {
+    name = lib.mkOption { type = t.str; };
+    description = lib.mkOption {
+      default = null;
+      type = t.nullOr t.str;
+    };
+    icon = lib.mkOption {
+      default = null;
+      type = t.nullOr t.str;
+    };
+  };
+
+  machineRef = lib.mkOptionType {
+    name = "machineRef";
+    description = "Machine :: [${builtins.concatStringsSep " | " (builtins.attrNames config.machines)}]";
+    check = v: lib.isString v && builtins.elem v (builtins.attrNames config.machines);
+    merge = lib.mergeEqualOption;
+  };
+
+  allTags = lib.unique (
+    lib.foldlAttrs (
+      tags: _: m:
+      tags ++ m.tags or [ ]
+    ) [ ] config.machines
+  );
+
+  tagRef = lib.mkOptionType {
+    name = "tagRef";
+    description = "Tags :: [${builtins.concatStringsSep " | " allTags}]";
+    check = v: lib.isString v && builtins.elem v allTags;
+    merge = lib.mergeEqualOption;
+  };
+in
+{
+  options.assertions = lib.mkOption {
+    type = t.listOf t.unspecified;
+    internal = true;
+    default = [ ];
+  };
+  config.assertions = lib.foldlAttrs (
+    ass1: serviceName: c:
+    ass1
+    ++ lib.foldlAttrs (
+      ass2: instanceName: instanceConfig:
+      let
+        serviceMachineNames = lib.attrNames instanceConfig.machines;
+        topLevelMachines = lib.attrNames config.machines;
+        # All machines must be defined in the top-level machines
+        assertions = builtins.map (m: {
+          assertion = builtins.elem m topLevelMachines;
+          message = "${serviceName}.${instanceName}.machines.${m}. Should be one of [ ${builtins.concatStringsSep " | " topLevelMachines} ]";
+        }) serviceMachineNames;
+      in
+      ass2 ++ assertions
+    ) [ ] c
+  ) [ ] config.services;
+
+  options.meta = metaOptions;
+
+  options.machines = lib.mkOption {
+    default = { };
+    type = t.attrsOf (
+      t.submodule {
+        options = {
+          inherit (metaOptions) name description icon;
+          tags = lib.mkOption {
+            default = [ ];
+            apply = lib.unique;
+            type = t.listOf t.str;
+          };
+        };
+      }
+    );
+  };
+
+  options.services = lib.mkOption {
+    default = { };
+    type = t.attrsOf (
+      t.attrsOf (
+        t.submodule {
+          options.meta = metaOptions;
+          options.config = lib.mkOption {
+            default = { };
+            type = t.anything;
+          };
+          options.machines = lib.mkOption {
+            default = { };
+            type = t.attrsOf (
+              t.submodule {
+                options.config = lib.mkOption {
+                  default = { };
+                  type = t.anything;
+                };
+              }
+            );
+          };
+          options.roles = lib.mkOption {
+            default = { };
+            type = t.attrsOf (
+              t.submodule {
+                options.machines = lib.mkOption {
+                  default = [ ];
+                  type = t.listOf machineRef;
+                };
+                options.tags = lib.mkOption {
+                  default = [ ];
+                  apply = lib.unique;
+                  type = t.listOf tagRef;
+                };
+              }
+            );
+          };
+        }
+      )
+    );
+  };
+}

lib/inventory/default.nix

@@ -0,0 +1,5 @@
+{ lib, clan-core }:
+{
+  buildInventory = import ./build-inventory { inherit lib clan-core; };
+  interface = ./build-inventory/interface.nix;
+}

lib/inventory/flake-module.nix

@@ -0,0 +1,48 @@
+{ ... }:
+{
+  # flake.inventory = import ./default.nix { inherit inputs self lib; };
+  perSystem =
+    { pkgs, config, ... }:
+    {
+      packages.inventory-schema = pkgs.stdenv.mkDerivation {
+        name = "inventory-schema";
+        src = ./src;
+
+        buildInputs = [ pkgs.cue ];
+
+        installPhase = ''
+          mkdir -p $out
+        '';
+      };
+
+      devShells.inventory-schema = pkgs.mkShell { inputsFrom = [ config.packages.inventory-schema ]; };
+
+      checks.inventory-schema-checks = pkgs.stdenv.mkDerivation {
+        name = "inventory-schema-checks";
+        src = ./src;
+        buildInputs = [ pkgs.cue ];
+        buildPhase = ''
+          echo "Running inventory tests..."
+
+          echo "Export cue as json-schema..."
+          cue export --out openapi root.cue
+
+          echo "Validate test/*.json against inventory-schema..."
+
+          test_dir="test"
+          for file in "$test_dir"/*; do
+            # Check if the item is a file
+            if [ -f "$file" ]; then
+              # Print the filename
+              echo "Running test on: $file"
+
+              # Run the cue vet command
+              cue vet "$file" root.cue -d "#Root"
+            fi
+          done
+
+          touch $out
+        '';
+      };
+    };
+}

lib/inventory/src/cue.mod/module.cue

@@ -0,0 +1,2 @@
+module: "clan.lol/inventory"
+language: version: "v0.8.2"

lib/inventory/src/root.cue

@@ -0,0 +1,23 @@
+package inventory
+
+import (
+	"clan.lol/inventory/schema"
+)
+
+@jsonschema(schema="http://json-schema.org/schema#")
+#Root: {
+	meta: {
+		// A name of the clan (primarily shown by the UI)
+		name: string
+		// A description of the clan
+		description?: string
+		// The icon path
+		icon?: string
+	}
+
+	// // A map of services
+	schema.#service
+
+	// // A map of machines
+	schema.#machine
+}

lib/inventory/src/schema/schema.cue

@@ -0,0 +1,39 @@
+package schema
+
+#machine: machines: [string]: {
+    name: string,
+    description?: string,
+    icon?: string
+    tags: [...string]
+}
+
+#role: string
+
+#service: services: [string]: [string]: {
+    // Required meta fields
+    meta: {
+        name: string,
+        icon?: string
+        description?: string,
+    },
+    // We moved the machine sepcific config to "machines".
+    // It may be moved back depending on what makes more sense in the future.
+    roles: [#role]: {
+        machines: [...string],
+        tags: [...string],
+    }
+    machines: {
+        [string]: {
+            config?: {
+                ...
+            }
+        }
+    },
+
+    // Configuration for the service
+    config: {
+        // Schema depends on the module.
+        // It declares the interface how the service can be configured.
+        ...
+    }
+}

lib/inventory/src/tests/borgbackup.json

@@ -0,0 +1,53 @@
+{
+  "machines": {
+    "camina_machine": {
+      "name": "camina",
+      "tags": ["laptop"]
+    },
+    "vyr_machine": {
+      "name": "vyr"
+    },
+    "vi_machine": {
+      "name": "vi",
+      "tags": ["laptop"]
+    }
+  },
+  "meta": {
+    "name": "kenjis clan"
+  },
+  "services": {
+    "borgbackup": {
+      "instance_1": {
+        "meta": {
+          "name": "My backup"
+        },
+        "roles": {
+          "server": {
+            "machines": ["vyr_machine"]
+          },
+          "client": {
+            "machines": ["vyr_machine"],
+            "tags": ["laptop"]
+          }
+        },
+        "machines": {},
+        "config": {}
+      },
+      "instance_2": {
+        "meta": {
+          "name": "My backup"
+        },
+        "roles": {
+          "server": {
+            "machines": ["vi_machine"]
+          },
+          "client": {
+            "machines": ["camina_machine"]
+          }
+        },
+        "machines": {},
+        "config": {}
+      }
+    }
+  }
+}

lib/inventory/src/tests/syncthing.json

@@ -0,0 +1,47 @@
+{
+  "machines": {
+    "camina_machine": {
+      "name": "camina"
+    },
+    "vyr_machine": {
+      "name": "vyr"
+    },
+    "vi_machine": {
+      "name": "vi"
+    }
+  },
+  "meta": {
+    "name": "kenjis clan"
+  },
+  "services": {
+    "syncthing": {
+      "instance_1": {
+        "meta": {
+          "name": "My sync"
+        },
+        "roles": {
+          "peer": {
+            "machines": ["vyr_machine", "vi_machine", "camina_machine"]
+          }
+        },
+        "machines": {},
+        "config": {
+          "folders": {
+            "test": {
+              "path": "~/data/docs",
+              "devices": ["camina_machine", "vyr_machine", "vi_machine"]
+            },
+            "videos": {
+              "path": "~/data/videos",
+              "devices": ["camina_machine", "vyr_machine"]
+            },
+            "playlist": {
+              "path": "~/data/playlist",
+              "devices": ["camina_machine", "vi_machine"]
+            }
+          }
+        }
+      }
+    }
+  }
+}

lib/inventory/src/tests/zerotier.json

@@ -0,0 +1,36 @@
+{
+  "machines": {
+    "camina_machine": {
+      "name": "camina"
+    },
+    "vyr_machine": {
+      "name": "vyr"
+    },
+    "vi_machine": {
+      "name": "vi"
+    }
+  },
+  "meta": {
+    "name": "kenjis clan"
+  },
+  "services": {
+    "zerotier": {
+      "instance_1": {
+        "meta": {
+          "name": "My Network"
+        },
+        "roles": {
+          "controller": { "machines": ["vyr_machine"] },
+          "moon": { "machines": ["vyr_machine"] },
+          "peer": { "machines": ["vi_machine", "camina_machine"] }
+        },
+        "machines": {
+          "vyr_machine": {
+            "config": {}
+          }
+        },
+        "config": {}
+      }
+    }
+  }
+}