yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'Inventory: add global imports' (#1749) from inventory-config into main

Browse Source
This commit is contained in:
clan-bot
2024-07-16 08:45:00 +00:00
parent dad94af825 5ec952d29d
commit 2d5a5dc203
6 changed files with 143 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
flakeModules/clan.nix
View File

@@ -60,7 +60,8 @@ in
description = "Allows to include machine-specific modules i.e. machines.\${name} = { ... }"; description = "Allows to include machine-specific modules i.e. machines.\${name} = { ... }";
}; };
inventory = mkOption { inventory = mkOption {
type = types.submodule { imports = [ ../lib/inventory/build-inventory/interface.nix ]; }; #type = types.submodule { imports = [ ../lib/inventory/build-inventory/interface.nix ]; };
type = types.attrsOf types.raw;
default = { }; default = { };
description = '' description = ''
An abstract service layer for consistently configuring distributed services across machine boundaries. An abstract service layer for consistently configuring distributed services across machine boundaries.
@@ -117,10 +118,10 @@ in
directory directory
specialArgs specialArgs
machines machines
inventory
pkgsForSystem pkgsForSystem
meta meta
; ;
inventory = (lib.traceValSeq cfg.inventory);
}; };
}; };
_file = __curPos.file; _file = __curPos.file;

5
lib/build-clan/default.nix
View File

@@ -106,7 +106,10 @@ let
# map from machine name to service configuration # map from machine name to service configuration
# { ${machineName} :: Config } # { ${machineName} :: Config }
serviceConfigs = buildInventory mergedInventory; serviceConfigs = buildInventory {
inventory = mergedInventory;
inherit directory;
};
machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (
builtins.readDir (directory + /machines) builtins.readDir (directory + /machines)

50
lib/inventory/build-inventory/default.nix
View File

@@ -1,13 +1,20 @@
# Generate partial NixOS configurations for every machine in the inventory # Generate partial NixOS configurations for every machine in the inventory
# This function is responsible for generating the module configuration for every machine in the inventory. # This function is responsible for generating the module configuration for every machine in the inventory.
{ lib, clan-core }: { lib, clan-core }:
inventory: { inventory, directory }:
let let
machines = machinesFromInventory inventory; machines = machinesFromInventory inventory;
resolveTags = resolveTags =
# Inventory, { machines :: [string], tags :: [string] } # Inventory, { machines :: [string], tags :: [string] }
inventory: members: { {
serviceName,
instanceName,
roleName,
inventory,
members,
}:
{
machines = machines =
members.machines or [ ] members.machines or [ ]
++ (builtins.foldl' ( ++ (builtins.foldl' (
@@ -17,14 +24,17 @@ let
availableTags = lib.foldlAttrs ( availableTags = lib.foldlAttrs (
acc: _: v: acc: _: v:
v.tags or [ ] ++ acc v.tags or [ ] ++ acc
) [ ] inventory.machines; ) [ ] (lib.traceValSeq inventory.machines);
tagMembers = builtins.attrNames ( tagMembers = builtins.attrNames (
lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines
); );
in in
if tagMembers == [ ] then if tagMembers == [ ] then
throw "Tag: '${tag}' not found. Available tags: ${builtins.toJSON (lib.unique availableTags)}" throw ''
inventory.services.${serviceName}.${instanceName}: - ${roleName} tags: no machine with tag '${tag}' found.
Available tags: ${builtins.toJSON (lib.unique availableTags)}
''
else else
acc ++ tagMembers acc ++ tagMembers
) [ ] members.tags or [ ]); ) [ ] members.tags or [ ]);
@@ -43,7 +53,7 @@ let
machineName: machineConfig: machineName: machineConfig:
lib.foldlAttrs ( lib.foldlAttrs (
# [ Modules ], String, { ${instance_name} :: ServiceConfig } # [ Modules ], String, { ${instance_name} :: ServiceConfig }
acc: moduleName: serviceConfigs: acc: serviceName: serviceConfigs:
acc acc
# Collect service config # Collect service config
++ (lib.foldlAttrs ( ++ (lib.foldlAttrs (
@@ -51,7 +61,16 @@ let
acc2: instanceName: serviceConfig: acc2: instanceName: serviceConfig:
let let
resolvedRoles = builtins.mapAttrs ( resolvedRoles = builtins.mapAttrs (
_roleName: members: resolveTags inventory members roleName: members:
resolveTags {
inherit
serviceName
instanceName
roleName
inventory
members
;
}
) serviceConfig.roles; ) serviceConfig.roles;
isInService = builtins.any (members: builtins.elem machineName members.machines) ( isInService = builtins.any (members: builtins.elem machineName members.machines) (
@@ -72,11 +91,17 @@ let
machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { }; machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { };
globalConfig = serviceConfig.config or { }; globalConfig = serviceConfig.config or { };
globalImports = serviceConfig.imports or [ ];
machineImports = serviceConfig.machines.${machineName}.imports or [ ];
roleServiceImports = builtins.foldl' (
acc: role: acc ++ serviceConfig.roles.${role}.imports or [ ]
) [ ] inverseRoles.${machineName} or [ ];
# TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy
roleModules = builtins.map ( roleModules = builtins.map (
role: role:
let let
path = "${clan-core.clanModules.${moduleName}}/roles/${role}.nix"; path = "${clan-core.clanModules.${serviceName}}/roles/${role}.nix";
in in
if builtins.pathExists path then if builtins.pathExists path then
path path
@@ -87,13 +112,18 @@ let
roleServiceConfigs = builtins.map ( roleServiceConfigs = builtins.map (
role: serviceConfig.roles.${role}.config or { } role: serviceConfig.roles.${role}.config or { }
) inverseRoles.${machineName} or [ ]; ) inverseRoles.${machineName} or [ ];
customImports = map (s: "${directory}/${s}") (
globalImports ++ machineImports ++ roleServiceImports
);
in in
if isInService then if isInService then
acc2 acc2
++ [ ++ [
{ {
imports = [ clan-core.clanModules.${moduleName} ] ++ roleModules; imports = [ clan-core.clanModules.${serviceName} ] ++ roleModules ++ customImports;
config.clan.${moduleName} = lib.mkMerge ( config.clan.${serviceName} = lib.mkMerge (
[ [
globalConfig globalConfig
machineServiceConfig machineServiceConfig
@@ -102,7 +132,7 @@ let
); );
} }
{ {
config.clan.inventory.services.${moduleName}.${instanceName} = { config.clan.inventory.services.${serviceName}.${instanceName} = {
roles = resolvedRoles; roles = resolvedRoles;
# TODO: Add inverseRoles to the service config if needed # TODO: Add inverseRoles to the service config if needed
# inherit inverseRoles; # inherit inverseRoles;

69
lib/inventory/build-inventory/interface.nix
View File

@@ -1,49 +1,33 @@
{ config, lib, ... }: { config, lib, ... }:
let let
t = lib.types; types = lib.types;
metaOptions = { metaOptions = {
name = lib.mkOption { type = t.str; }; name = lib.mkOption { type = types.str; };
description = lib.mkOption { description = lib.mkOption {
default = null; default = null;
type = t.nullOr t.str; type = types.nullOr types.str;
}; };
icon = lib.mkOption { icon = lib.mkOption {
default = null; default = null;
type = t.nullOr t.str; type = types.nullOr types.str;
}; };
}; };
machineRef = lib.mkOptionType {
name = "str";
description = "Machine :: [${builtins.concatStringsSep " | " (builtins.attrNames config.machines)}]";
check = v: lib.isString v && builtins.elem v (builtins.attrNames config.machines);
merge = lib.mergeEqualOption;
};
allTags = lib.unique (
lib.foldlAttrs (
tags: _: m:
tags ++ m.tags or [ ]
) [ ] config.machines
);
tagRef = lib.mkOptionType {
name = "str";
description = "Tags :: [${builtins.concatStringsSep " | " allTags}]";
check = v: lib.isString v && builtins.elem v allTags;
merge = lib.mergeEqualOption;
};
moduleConfig = lib.mkOption { moduleConfig = lib.mkOption {
default = { }; default = { };
type = t.attrsOf t.anything; type = types.attrsOf types.anything;
};
importsOption = lib.mkOption {
default = [ ];
type = types.listOf types.str;
}; };
in in
{ {
options = { options = {
assertions = lib.mkOption { assertions = lib.mkOption {
type = t.listOf t.unspecified; type = types.listOf types.unspecified;
internal = true; internal = true;
visible = false; visible = false;
default = [ ]; default = [ ];
@@ -52,18 +36,18 @@ in
machines = lib.mkOption { machines = lib.mkOption {
default = { }; default = { };
type = t.attrsOf ( type = types.attrsOf (
t.submodule { types.submodule {
options = { options = {
inherit (metaOptions) name description icon; inherit (metaOptions) name description icon;
tags = lib.mkOption { tags = lib.mkOption {
default = [ ]; default = [ ];
apply = lib.unique; apply = lib.unique;
type = t.listOf t.str; type = types.listOf types.str;
}; };
system = lib.mkOption { system = lib.mkOption {
default = null; default = null;
type = t.nullOr t.str; type = types.nullOr types.str;
}; };
}; };
} }
@@ -72,29 +56,36 @@ in
services = lib.mkOption { services = lib.mkOption {
default = { }; default = { };
type = t.attrsOf ( type = types.attrsOf (
t.attrsOf ( types.attrsOf (
t.submodule { types.submodule {
options.meta = metaOptions; options.meta = metaOptions;
options.imports = importsOption;
options.config = moduleConfig; options.config = moduleConfig;
options.machines = lib.mkOption { options.machines = lib.mkOption {
default = { }; default = { };
type = t.attrsOf (t.submodule { options.config = moduleConfig; }); type = types.attrsOf (
types.submodule {
options.imports = importsOption;
options.config = moduleConfig;
}
);
}; };
options.roles = lib.mkOption { options.roles = lib.mkOption {
default = { }; default = { };
type = t.attrsOf ( type = types.attrsOf (
t.submodule { types.submodule {
options.machines = lib.mkOption { options.machines = lib.mkOption {
default = [ ]; default = [ ];
type = t.listOf machineRef; type = types.listOf types.str;
}; };
options.tags = lib.mkOption { options.tags = lib.mkOption {
default = [ ]; default = [ ];
apply = lib.unique; apply = lib.unique;
type = t.listOf tagRef; type = types.listOf types.str;
}; };
options.config = moduleConfig; options.config = moduleConfig;
options.imports = importsOption;
} }
); );
}; };

110
lib/inventory/tests/default.nix
View File

@@ -2,25 +2,31 @@
{ {
test_inventory_empty = { test_inventory_empty = {
# Empty inventory should return an empty module # Empty inventory should return an empty module
expr = buildInventory { }; expr = buildInventory {
inventory = { };
directory = ./.;
};
expected = { }; expected = { };
}; };
test_inventory_role_imports = test_inventory_role_imports =
let let
configs = buildInventory { configs = buildInventory {
services = { directory = ./.;
borgbackup.instance_1 = { inventory = {
roles.server.machines = [ "backup_server" ]; services = {
roles.client.machines = [ borgbackup.instance_1 = {
"client_1_machine" roles.server.machines = [ "backup_server" ];
"client_2_machine" roles.client.machines = [
]; "client_1_machine"
"client_2_machine"
];
};
};
machines = {
"backup_server" = { };
"client_1_machine" = { };
"client_2_machine" = { };
}; };
};
machines = {
"backup_server" = { };
"client_1_machine" = { };
"client_2_machine" = { };
}; };
}; };
in in
@@ -49,18 +55,21 @@
test_inventory_tag_resolve = test_inventory_tag_resolve =
let let
configs = buildInventory { configs = buildInventory {
services = { directory = ./.;
borgbackup.instance_1 = { inventory = {
roles.client.tags = [ "backup" ]; services = {
borgbackup.instance_1 = {
roles.client.tags = [ "backup" ];
};
}; };
}; machines = {
machines = { "not_used_machine" = { };
"not_used_machine" = { }; "client_1_machine" = {
"client_1_machine" = { tags = [ "backup" ];
tags = [ "backup" ]; };
}; "client_2_machine" = {
"client_2_machine" = { tags = [ "backup" ];
tags = [ "backup" ]; };
}; };
}; };
}; };
@@ -85,14 +94,17 @@
test_inventory_multiple_roles = test_inventory_multiple_roles =
let let
configs = buildInventory { configs = buildInventory {
services = { directory = ./.;
borgbackup.instance_1 = { inventory = {
roles.client.machines = [ "machine_1" ]; services = {
roles.server.machines = [ "machine_1" ]; borgbackup.instance_1 = {
roles.client.machines = [ "machine_1" ];
roles.server.machines = [ "machine_1" ];
};
};
machines = {
"machine_1" = { };
}; };
};
machines = {
"machine_1" = { };
}; };
}; };
in in
@@ -112,13 +124,16 @@
test_inventory_role_doesnt_exist = test_inventory_role_doesnt_exist =
let let
configs = buildInventory { configs = buildInventory {
services = { directory = ./.;
borgbackup.instance_1 = { inventory = {
roles.roleXYZ.machines = [ "machine_1" ]; services = {
borgbackup.instance_1 = {
roles.roleXYZ.machines = [ "machine_1" ];
};
};
machines = {
"machine_1" = { };
}; };
};
machines = {
"machine_1" = { };
}; };
}; };
in in
@@ -132,15 +147,18 @@
test_inventory_tag_doesnt_exist = test_inventory_tag_doesnt_exist =
let let
configs = buildInventory { configs = buildInventory {
services = { directory = ./.;
borgbackup.instance_1 = { inventory = {
roles.client.machines = [ "machine_1" ]; services = {
roles.client.tags = [ "tagXYZ" ]; borgbackup.instance_1 = {
roles.client.machines = [ "machine_1" ];
roles.client.tags = [ "tagXYZ" ];
};
}; };
}; machines = {
machines = { "machine_1" = {
"machine_1" = { tags = [ "tagABC" ];
tags = [ "tagABC" ]; };
}; };
}; };
}; };
@@ -149,7 +167,7 @@
expr = configs; expr = configs;
expectedError = { expectedError = {
type = "ThrownError"; type = "ThrownError";
msg = "Tag: '\\w+' not found"; msg = "no machine with tag '\\w+' found";
}; };
}; };
} }

4
nixosModules/clanCore/vars/default.nix
View File

@@ -45,7 +45,7 @@ in
); );
inherit (config.clan.core.vars.settings) secretUploadDirectory secretModule publicModule; inherit (config.clan.core.vars.settings) secretUploadDirectory secretModule publicModule;
}; };
inherit (config.clan.networking) targetHost buildHost; inherit (config.clan.core.networking) targetHost buildHost;
inherit (config.clan.deployment) requireExplicitUpdate; inherit (config.clan.core.deployment) requireExplicitUpdate;
}; };
} }