Merge pull request 'Inventory: add global imports' (#1749) from inventory-config into main

2024-07-16 08:45:00 +00:00
parent 645bd98c3a a2c3c09564
commit 08cd44ea03
6 changed files with 143 additions and 100 deletions
--- a/flakeModules/clan.nix
+++ b/flakeModules/clan.nix
@@ -60,7 +60,8 @@ in
      description = "Allows to include machine-specific modules i.e. machines.\${name} = { ... }";
    };
    inventory = mkOption {
-      type = types.submodule { imports = [ ../lib/inventory/build-inventory/interface.nix ]; };
+      #type = types.submodule { imports = [ ../lib/inventory/build-inventory/interface.nix ]; };
+      type = types.attrsOf types.raw;
      default = { };
      description = ''
        An abstract service layer for consistently configuring distributed services across machine boundaries.
@@ -117,10 +118,10 @@ in
        directory
        specialArgs
        machines
-        inventory
        pkgsForSystem
        meta
        ;
+      inventory = (lib.traceValSeq cfg.inventory);
    };
  };
  _file = __curPos.file;
--- a/lib/build-clan/default.nix
+++ b/lib/build-clan/default.nix
@@ -106,7 +106,10 @@ let

  # map from machine name to service configuration
  # { ${machineName} :: Config }
-  serviceConfigs = buildInventory mergedInventory;
+  serviceConfigs = buildInventory {
+    inventory = mergedInventory;
+    inherit directory;
+  };

  machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (
    builtins.readDir (directory + /machines)
--- a/lib/inventory/build-inventory/default.nix
+++ b/lib/inventory/build-inventory/default.nix
@@ -1,13 +1,20 @@
 # Generate partial NixOS configurations for every machine in the inventory
 # This function is responsible for generating the module configuration for every machine in the inventory.
 { lib, clan-core }:
-inventory:
+{ inventory, directory }:
 let
  machines = machinesFromInventory inventory;

  resolveTags =
    # Inventory, { machines :: [string], tags :: [string] }
-    inventory: members: {
+    {
+      serviceName,
+      instanceName,
+      roleName,
+      inventory,
+      members,
+    }:
+    {
      machines =
        members.machines or [ ]
        ++ (builtins.foldl' (
@@ -17,14 +24,17 @@ let
            availableTags = lib.foldlAttrs (
              acc: _: v:
              v.tags or [ ] ++ acc
-            ) [ ] inventory.machines;
+            ) [ ] (lib.traceValSeq inventory.machines);

            tagMembers = builtins.attrNames (
              lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines
            );
          in
          if tagMembers == [ ] then
-            throw "Tag: '${tag}' not found. Available tags: ${builtins.toJSON (lib.unique availableTags)}"
+            throw ''
+              inventory.services.${serviceName}.${instanceName}: - ${roleName} tags: no machine with tag '${tag}' found.
+              Available tags: ${builtins.toJSON (lib.unique availableTags)}
+            ''
          else
            acc ++ tagMembers
        ) [ ] members.tags or [ ]);
@@ -43,7 +53,7 @@ let
      machineName: machineConfig:
      lib.foldlAttrs (
        # [ Modules ], String, { ${instance_name} :: ServiceConfig }
-        acc: moduleName: serviceConfigs:
+        acc: serviceName: serviceConfigs:
        acc
        # Collect service config
        ++ (lib.foldlAttrs (
@@ -51,7 +61,16 @@ let
          acc2: instanceName: serviceConfig:
          let
            resolvedRoles = builtins.mapAttrs (
-              _roleName: members: resolveTags inventory members
+              roleName: members:
+              resolveTags {
+                inherit
+                  serviceName
+                  instanceName
+                  roleName
+                  inventory
+                  members
+                  ;
+              }
            ) serviceConfig.roles;

            isInService = builtins.any (members: builtins.elem machineName members.machines) (
@@ -72,11 +91,17 @@ let
            machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { };
            globalConfig = serviceConfig.config or { };

+            globalImports = serviceConfig.imports or [ ];
+            machineImports = serviceConfig.machines.${machineName}.imports or [ ];
+            roleServiceImports = builtins.foldl' (
+              acc: role: acc ++ serviceConfig.roles.${role}.imports or [ ]
+            ) [ ] inverseRoles.${machineName} or [ ];
+
            # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy
            roleModules = builtins.map (
              role:
              let
-                path = "${clan-core.clanModules.${moduleName}}/roles/${role}.nix";
+                path = "${clan-core.clanModules.${serviceName}}/roles/${role}.nix";
              in
              if builtins.pathExists path then
                path
@@ -87,13 +112,18 @@ let
            roleServiceConfigs = builtins.map (
              role: serviceConfig.roles.${role}.config or { }
            ) inverseRoles.${machineName} or [ ];
+
+            customImports = map (s: "${directory}/${s}") (
+              globalImports ++ machineImports ++ roleServiceImports
+            );
          in
+
          if isInService then
            acc2
            ++ [
              {
-                imports = [ clan-core.clanModules.${moduleName} ] ++ roleModules;
-                config.clan.${moduleName} = lib.mkMerge (
+                imports = [ clan-core.clanModules.${serviceName} ] ++ roleModules ++ customImports;
+                config.clan.${serviceName} = lib.mkMerge (
                  [
                    globalConfig
                    machineServiceConfig
@@ -102,7 +132,7 @@ let
                );
              }
              {
-                config.clan.inventory.services.${moduleName}.${instanceName} = {
+                config.clan.inventory.services.${serviceName}.${instanceName} = {
                  roles = resolvedRoles;
                  # TODO: Add inverseRoles to the service config if needed
                  # inherit inverseRoles;
--- a/lib/inventory/build-inventory/interface.nix
+++ b/lib/inventory/build-inventory/interface.nix
@@ -1,49 +1,33 @@
 { config, lib, ... }:
 let
-  t = lib.types;
+  types = lib.types;

  metaOptions = {
-    name = lib.mkOption { type = t.str; };
+    name = lib.mkOption { type = types.str; };
    description = lib.mkOption {
      default = null;
-      type = t.nullOr t.str;
+      type = types.nullOr types.str;
    };
    icon = lib.mkOption {
      default = null;
-      type = t.nullOr t.str;
+      type = types.nullOr types.str;
    };
  };

-  machineRef = lib.mkOptionType {
-    name = "str";
-    description = "Machine :: [${builtins.concatStringsSep " | " (builtins.attrNames config.machines)}]";
-    check = v: lib.isString v && builtins.elem v (builtins.attrNames config.machines);
-    merge = lib.mergeEqualOption;
-  };
-
-  allTags = lib.unique (
-    lib.foldlAttrs (
-      tags: _: m:
-      tags ++ m.tags or [ ]
-    ) [ ] config.machines
-  );
-
-  tagRef = lib.mkOptionType {
-    name = "str";
-    description = "Tags :: [${builtins.concatStringsSep " | " allTags}]";
-    check = v: lib.isString v && builtins.elem v allTags;
-    merge = lib.mergeEqualOption;
-  };
-
  moduleConfig = lib.mkOption {
    default = { };
-    type = t.attrsOf t.anything;
+    type = types.attrsOf types.anything;
+  };
+
+  importsOption = lib.mkOption {
+    default = [ ];
+    type = types.listOf types.str;
  };
 in
 {
  options = {
    assertions = lib.mkOption {
-      type = t.listOf t.unspecified;
+      type = types.listOf types.unspecified;
      internal = true;
      visible = false;
      default = [ ];
@@ -52,18 +36,18 @@ in

    machines = lib.mkOption {
      default = { };
-      type = t.attrsOf (
-        t.submodule {
+      type = types.attrsOf (
+        types.submodule {
          options = {
            inherit (metaOptions) name description icon;
            tags = lib.mkOption {
              default = [ ];
              apply = lib.unique;
-              type = t.listOf t.str;
+              type = types.listOf types.str;
            };
            system = lib.mkOption {
              default = null;
-              type = t.nullOr t.str;
+              type = types.nullOr types.str;
            };
          };
        }
@@ -72,29 +56,36 @@ in

    services = lib.mkOption {
      default = { };
-      type = t.attrsOf (
-        t.attrsOf (
-          t.submodule {
+      type = types.attrsOf (
+        types.attrsOf (
+          types.submodule {
            options.meta = metaOptions;
+            options.imports = importsOption;
            options.config = moduleConfig;
            options.machines = lib.mkOption {
              default = { };
-              type = t.attrsOf (t.submodule { options.config = moduleConfig; });
+              type = types.attrsOf (
+                types.submodule {
+                  options.imports = importsOption;
+                  options.config = moduleConfig;
+                }
+              );
            };
            options.roles = lib.mkOption {
              default = { };
-              type = t.attrsOf (
-                t.submodule {
+              type = types.attrsOf (
+                types.submodule {
                  options.machines = lib.mkOption {
                    default = [ ];
-                    type = t.listOf machineRef;
+                    type = types.listOf types.str;
                  };
                  options.tags = lib.mkOption {
                    default = [ ];
                    apply = lib.unique;
-                    type = t.listOf tagRef;
+                    type = types.listOf types.str;
                  };
                  options.config = moduleConfig;
+                  options.imports = importsOption;
                }
              );
            };
--- a/lib/inventory/tests/default.nix
+++ b/lib/inventory/tests/default.nix
@@ -2,25 +2,31 @@
 {
  test_inventory_empty = {
    # Empty inventory should return an empty module
-    expr = buildInventory { };
+    expr = buildInventory {
+      inventory = { };
+      directory = ./.;
+    };
    expected = { };
  };
  test_inventory_role_imports =
    let
      configs = buildInventory {
-        services = {
-          borgbackup.instance_1 = {
-            roles.server.machines = [ "backup_server" ];
-            roles.client.machines = [
-              "client_1_machine"
-              "client_2_machine"
-            ];
+        directory = ./.;
+        inventory = {
+          services = {
+            borgbackup.instance_1 = {
+              roles.server.machines = [ "backup_server" ];
+              roles.client.machines = [
+                "client_1_machine"
+                "client_2_machine"
+              ];
+            };
+          };
+          machines = {
+            "backup_server" = { };
+            "client_1_machine" = { };
+            "client_2_machine" = { };
          };
-        };
-        machines = {
-          "backup_server" = { };
-          "client_1_machine" = { };
-          "client_2_machine" = { };
        };
      };
    in
@@ -49,18 +55,21 @@
  test_inventory_tag_resolve =
    let
      configs = buildInventory {
-        services = {
-          borgbackup.instance_1 = {
-            roles.client.tags = [ "backup" ];
+        directory = ./.;
+        inventory = {
+          services = {
+            borgbackup.instance_1 = {
+              roles.client.tags = [ "backup" ];
+            };
          };
-        };
-        machines = {
-          "not_used_machine" = { };
-          "client_1_machine" = {
-            tags = [ "backup" ];
-          };
-          "client_2_machine" = {
-            tags = [ "backup" ];
+          machines = {
+            "not_used_machine" = { };
+            "client_1_machine" = {
+              tags = [ "backup" ];
+            };
+            "client_2_machine" = {
+              tags = [ "backup" ];
+            };
          };
        };
      };
@@ -85,14 +94,17 @@
  test_inventory_multiple_roles =
    let
      configs = buildInventory {
-        services = {
-          borgbackup.instance_1 = {
-            roles.client.machines = [ "machine_1" ];
-            roles.server.machines = [ "machine_1" ];
+        directory = ./.;
+        inventory = {
+          services = {
+            borgbackup.instance_1 = {
+              roles.client.machines = [ "machine_1" ];
+              roles.server.machines = [ "machine_1" ];
+            };
+          };
+          machines = {
+            "machine_1" = { };
          };
-        };
-        machines = {
-          "machine_1" = { };
        };
      };
    in
@@ -112,13 +124,16 @@
  test_inventory_role_doesnt_exist =
    let
      configs = buildInventory {
-        services = {
-          borgbackup.instance_1 = {
-            roles.roleXYZ.machines = [ "machine_1" ];
+        directory = ./.;
+        inventory = {
+          services = {
+            borgbackup.instance_1 = {
+              roles.roleXYZ.machines = [ "machine_1" ];
+            };
+          };
+          machines = {
+            "machine_1" = { };
          };
-        };
-        machines = {
-          "machine_1" = { };
        };
      };
    in
@@ -132,15 +147,18 @@
  test_inventory_tag_doesnt_exist =
    let
      configs = buildInventory {
-        services = {
-          borgbackup.instance_1 = {
-            roles.client.machines = [ "machine_1" ];
-            roles.client.tags = [ "tagXYZ" ];
+        directory = ./.;
+        inventory = {
+          services = {
+            borgbackup.instance_1 = {
+              roles.client.machines = [ "machine_1" ];
+              roles.client.tags = [ "tagXYZ" ];
+            };
          };
-        };
-        machines = {
-          "machine_1" = {
-            tags = [ "tagABC" ];
+          machines = {
+            "machine_1" = {
+              tags = [ "tagABC" ];
+            };
          };
        };
      };
@@ -149,7 +167,7 @@
      expr = configs;
      expectedError = {
        type = "ThrownError";
-        msg = "Tag: '\\w+' not found";
+        msg = "no machine with tag '\\w+' found";
      };
    };
 }
--- a/nixosModules/clanCore/vars/default.nix
+++ b/nixosModules/clanCore/vars/default.nix
@@ -45,7 +45,7 @@ in
      );
      inherit (config.clan.core.vars.settings) secretUploadDirectory secretModule publicModule;
    };
-    inherit (config.clan.networking) targetHost buildHost;
-    inherit (config.clan.deployment) requireExplicitUpdate;
+    inherit (config.clan.core.networking) targetHost buildHost;
+    inherit (config.clan.core.deployment) requireExplicitUpdate;
  };
 }