From 6755aa2c7098deabb03e216f84441f9f4bf599a6 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 4 Feb 2025 12:13:29 +0700 Subject: [PATCH] inventory: migrate tests --- lib/build-clan/module.nix | 2 +- lib/inventory/build-inventory/default.nix | 258 +++++++++++++--------- lib/inventory/tests/default.nix | 115 +++++----- 3 files changed, 217 insertions(+), 158 deletions(-) diff --git a/lib/build-clan/module.nix b/lib/build-clan/module.nix index c431be07c..40eeff8b5 100644 --- a/lib/build-clan/module.nix +++ b/lib/build-clan/module.nix @@ -76,7 +76,7 @@ let (machines.${name} or { }) # Inherit the inventory assertions ? # { inherit (mergedInventory) assertions; } - { imports = serviceConfigs.machines.${name}.machineImports or []; } + { imports = serviceConfigs.machines.${name}.machineImports or [ ]; } ( { # Settings diff --git a/lib/inventory/build-inventory/default.nix b/lib/inventory/build-inventory/default.nix index 1cb54001a..21a3a4ae7 100644 --- a/lib/inventory/build-inventory/default.nix +++ b/lib/inventory/build-inventory/default.nix @@ -62,131 +62,177 @@ let directory, }: let - compileServiceModules = serviceName: serviceConfigs: { - # TODO: Add other attributes - machineImports = ( - lib.foldlAttrs ( - # [ Modules ], String, ServiceConfig - acc2: instanceName: serviceConfig: - + compileServiceModules = + serviceName: serviceConfigs: + let + supportedRoles = clan-core.lib.modules.getRoles inventory.modules serviceName; + resolvedRolesPerInstance = lib.mapAttrs ( + instanceName: instanceConfig: let - roles = clan-core.lib.modules.getRoles inventory.modules serviceName; - - resolvedRoles = lib.genAttrs roles ( + resolvedRoles = lib.genAttrs supportedRoles ( roleName: resolveTags { - members = serviceConfig.roles.${roleName} or { }; + members = instanceConfig.roles.${roleName} or { }; inherit - serviceName instanceName + serviceName roleName inventory ; } ); - - isInService = builtins.any (members: builtins.elem machineName members.machines) ( - builtins.attrValues resolvedRoles - ); - - # all roles where the machine is present - machineRoles = builtins.attrNames ( - lib.filterAttrs (_role: roleConfig: builtins.elem machineName roleConfig.machines) resolvedRoles - ); - machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { }; - globalConfig = serviceConfig.config or { }; - - globalExtraModules = serviceConfig.extraModules or [ ]; - machineExtraModules = serviceConfig.machines.${machineName}.extraModules or [ ]; - roleServiceExtraModules = builtins.foldl' ( - acc: role: acc ++ serviceConfig.roles.${role}.extraModules or [ ] - ) [ ] machineRoles; - - # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy - roleModules = builtins.map ( - role: - if builtins.elem role roles && inventory.modules ? ${serviceName} then - inventory.modules.${serviceName} + "/roles/${role}.nix" - else - throw "Module ${serviceName} doesn't have role: '${role}'. Role: ${ - inventory.modules.${serviceName} - }/roles/${role}.nix not found." - ) machineRoles; - - roleServiceConfigs = builtins.filter (m: m != { }) ( - builtins.map (role: serviceConfig.roles.${role}.config or { }) machineRoles - ); - - extraModules = map (s: if builtins.typeOf s == "string" then "${directory}/${s}" else s) ( - globalExtraModules ++ machineExtraModules ++ roleServiceExtraModules - ); - - nonExistingRoles = builtins.filter (role: !(builtins.elem role roles)) ( - builtins.attrNames (serviceConfig.roles or { }) - ); - - constraintAssertions = clan-core.lib.modules.checkConstraints { - moduleName = serviceName; - allModules = inventory.modules; - inherit resolvedRoles instanceName; - }; + usedRoles = builtins.attrNames instanceConfig.roles; + unmatchedRoles = builtins.filter (role: !builtins.elem role supportedRoles) usedRoles; in - if (nonExistingRoles != [ ]) then - throw "Roles ${builtins.toString nonExistingRoles} are not defined in the service ${serviceName}." - else if !(serviceConfig.enabled or true) then - acc2 - else if isInService then - acc2 - ++ [ - { - imports = roleModules ++ extraModules; - - clan.inventory.assertions = constraintAssertions; - clan.inventory.services.${serviceName}.${instanceName} = { - roles = resolvedRoles; - # TODO: Add inverseRoles to the service config if needed - # inherit inverseRoles; - }; - } - (lib.optionalAttrs (globalConfig != { } || machineServiceConfig != { } || roleServiceConfigs != [ ]) - { - clan.${serviceName} = lib.mkMerge ( - [ - globalConfig - machineServiceConfig - ] - ++ roleServiceConfigs - ); - } - ) - ] + if unmatchedRoles != [ ] then + throw '' + Service: '${serviceName}' Instance: '${instanceName}' + The following roles do not exist: ${builtins.toJSON unmatchedRoles} + Please use one of available roles: ${builtins.toJSON supportedRoles} + '' else - acc2 - ) [ ] (serviceConfigs) - ); + resolvedRoles + ) serviceConfigs; - assertions = lib.mapAttrs' (name: value: { - name = "checkservice.${serviceName}.${name}"; - value = { - assertion = checkService inventory.modules.${serviceName} serviceName; - message = '' - Service ${serviceName} cannot be used in inventory. It does not declare the 'inventory' feature. + machinesRoles = builtins.zipAttrsWith ( + _n: vs: + let + flat = builtins.foldl' (acc: s: acc ++ s.machines) [ ] vs; + in + lib.unique flat + ) (builtins.attrValues resolvedRolesPerInstance); + + matchedRoles = builtins.attrNames ( + lib.filterAttrs (_: ms: builtins.elem machineName ms) machinesRoles + ); + in + # roleImports = lib.mapAttrsToList ( + # roleName: _: inventory.modules.${serviceName} + "/roles/${roleName}.nix" + # ) (lib.filterAttrs (_: ms: builtins.elem machineName ms) machinesRoles); + # CompiledService :: { machineImports :: []; machineRoles :: [ String ] } + { + inherit machinesRoles matchedRoles resolvedRolesPerInstance; + # TODO: Add other attributes + machineImports = ( + lib.foldlAttrs ( + # [ Modules ], String, ServiceConfig + acc2: instanceName: serviceConfig: + let + resolvedRoles = lib.genAttrs supportedRoles ( + roleName: + resolveTags { + members = serviceConfig.roles.${roleName} or { }; + inherit + serviceName + instanceName + roleName + inventory + ; + } + ); + + isInService = builtins.any (members: builtins.elem machineName members.machines) ( + builtins.attrValues resolvedRoles + ); + + # all roles where the machine is present + machineRoles = builtins.attrNames ( + lib.filterAttrs (_role: roleConfig: builtins.elem machineName roleConfig.machines) resolvedRoles + ); + machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { }; + globalConfig = serviceConfig.config or { }; + + globalExtraModules = serviceConfig.extraModules or [ ]; + machineExtraModules = serviceConfig.machines.${machineName}.extraModules or [ ]; + roleServiceExtraModules = builtins.foldl' ( + acc: role: acc ++ serviceConfig.roles.${role}.extraModules or [ ] + ) [ ] machineRoles; + + # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy + roleModules = builtins.map ( + role: + if builtins.elem role supportedRoles && inventory.modules ? ${serviceName} then + inventory.modules.${serviceName} + "/roles/${role}.nix" + else + throw "Module ${serviceName} doesn't have role: '${role}'. Role: ${ + inventory.modules.${serviceName} + }/roles/${role}.nix not found." + ) machineRoles; + + roleServiceConfigs = builtins.filter (m: m != { }) ( + builtins.map (role: serviceConfig.roles.${role}.config or { }) machineRoles + ); + + extraModules = map (s: if builtins.typeOf s == "string" then "${directory}/${s}" else s) ( + globalExtraModules ++ machineExtraModules ++ roleServiceExtraModules + ); + + nonExistingRoles = builtins.filter (role: !(builtins.elem role supportedRoles)) ( + builtins.attrNames (serviceConfig.roles or { }) + ); + + constraintAssertions = clan-core.lib.modules.checkConstraints { + moduleName = serviceName; + allModules = inventory.modules; + inherit resolvedRoles instanceName; + }; + in + if (nonExistingRoles != [ ]) then + throw "Roles ${builtins.toString nonExistingRoles} are not defined in the service ${serviceName}." + else if !(serviceConfig.enabled or true) then + acc2 + else if isInService then + acc2 + ++ [ + { + imports = roleModules ++ extraModules; + + clan.inventory.assertions = constraintAssertions; + clan.inventory.services.${serviceName}.${instanceName} = { + roles = resolvedRoles; + # TODO: Add inverseRoles to the service config if needed + # inherit inverseRoles; + }; + } + (lib.optionalAttrs (globalConfig != { } || machineServiceConfig != { } || roleServiceConfigs != [ ]) + { + clan.${serviceName} = lib.mkMerge ( + [ + globalConfig + machineServiceConfig + ] + ++ roleServiceConfigs + ); + } + ) + ] + else + acc2 + ) [ ] (serviceConfigs) + ); + + assertions = lib.mapAttrs' (name: value: { + name = "checkservice.${serviceName}.${name}"; + value = { + assertion = checkService inventory.modules.${serviceName} serviceName; + message = '' + Service ${serviceName} cannot be used in inventory. It does not declare the 'inventory' feature. - To allow it add the following to the beginning of the README.md of the module: + To allow it add the following to the beginning of the README.md of the module: - --- - ... + --- + ... - features = [ "inventory" ] - --- + features = [ "inventory" ] + --- - Also make sure to test the module with the 'inventory' feature enabled. + Also make sure to test the module with the 'inventory' feature enabled. - ''; - }; - }) inventory.services; - }; + ''; + }; + }) inventory.services; + }; in lib.mapAttrs compileServiceModules inventory.services; diff --git a/lib/inventory/tests/default.nix b/lib/inventory/tests/default.nix index 182069e4a..d29c4cc0a 100644 --- a/lib/inventory/tests/default.nix +++ b/lib/inventory/tests/default.nix @@ -9,17 +9,21 @@ let inherit (inventory) buildInventory; in { - test_inventory_empty = { - # Empty inventory should return an empty module - expr = buildInventory { - inventory = { }; - directory = ./.; - }; - expected = { }; - }; - test_inventory_role_imports = + test_inventory_empty = let - configs = buildInventory { + compiled = buildInventory { + inventory = { }; + directory = ./.; + }; + in + { + # Empty inventory should return an empty module + expr = compiled.machines; + expected = { }; + }; + test_inventory_role_resolve = + let + compiled = buildInventory { directory = ./.; inventory = { modules = clan-core.clanModules; @@ -42,21 +46,37 @@ in in { expr = { - server_imports = (builtins.head configs."backup_server").imports; - client_1_imports = (builtins.head configs."client_1_machine").imports; - client_2_imports = (builtins.head configs."client_2_machine").imports; + m1 = (compiled.machines."backup_server").compiledServices.borgbackup.matchedRoles; + m2 = (compiled.machines."client_1_machine").compiledServices.borgbackup.matchedRoles; + m3 = (compiled.machines."client_2_machine").compiledServices.borgbackup.matchedRoles; + inherit ((compiled.machines."client_2_machine").compiledServices.borgbackup) + resolvedRolesPerInstance + ; }; expected = { - server_imports = [ - (clan-core.clanModules.borgbackup + "/roles/server.nix") + m1 = [ + "server" ]; - client_1_imports = [ - (clan-core.clanModules.borgbackup + "/roles/client.nix") + m2 = [ + "client" ]; - client_2_imports = [ - (clan-core.clanModules.borgbackup + "/roles/client.nix") + m3 = [ + "client" ]; + resolvedRolesPerInstance = { + instance_1 = { + client = { + machines = [ + "client_1_machine" + "client_2_machine" + ]; + }; + server = { + machines = [ "backup_server" ]; + }; + }; + }; }; }; test_inventory_tag_resolve = @@ -83,19 +103,19 @@ in }; in { - expr = { - # A machine that includes the backup service should have 3 imports - # - one for some service agnostic properties of the machine itself - # - One for the service itself (default.nix) - # - one for the role (roles/client.nix) - client_1_machine = builtins.length configs.client_1_machine; - client_2_machine = builtins.length configs.client_2_machine; - not_used_machine = builtins.length configs.not_used_machine; - }; + expr = configs.machines.client_1_machine.compiledServices.borgbackup.resolvedRolesPerInstance; expected = { - client_1_machine = 4; - client_2_machine = 4; - not_used_machine = 2; + instance_1 = { + client = { + machines = [ + "client_1_machine" + "client_2_machine" + ]; + }; + server = { + machines = [ ]; + }; + }; }; }; @@ -118,15 +138,11 @@ in }; in { - expr = { - machine_1_imports = (builtins.head configs."machine_1").imports; - }; - expected = { - machine_1_imports = [ - (clan-core.clanModules.borgbackup + "/roles/client.nix") - (clan-core.clanModules.borgbackup + "/roles/server.nix") - ]; - }; + expr = configs.machines.machine_1.compiledServices.borgbackup.matchedRoles; + expected = [ + "client" + "server" + ]; }; test_inventory_module_doesnt_exist = @@ -147,7 +163,8 @@ in }; in { - expr = configs; + inherit configs; + expr = configs.machines.machine_1.machineImports; expectedError = { type = "ThrownError"; msg = "ClanModule not found*"; @@ -172,7 +189,8 @@ in }; in { - expr = configs; + inherit configs; + expr = configs.machines.machine_1.machineImports; expectedError = { type = "ThrownError"; msg = "Roles roleXYZ are not defined in the service borgbackup."; @@ -201,7 +219,7 @@ in }; in { - expr = configs; + expr = configs.machines.machine_1.machineImports; expectedError = { type = "Error"; # TODO: Add warning matching in nix-unit @@ -229,13 +247,8 @@ in }; in { - expr = { - machine_1_config = (builtins.head configs."machine_1"); - }; - expected = { - # Empty config - machine_1_config = { }; - }; - + inherit configs; + expr = builtins.filter (v: v != { }) configs.machines.machine_1.machineImports; + expected = [ ]; }; }