Exports POC

This commit is contained in:
Johannes Kirschbauer
2025-10-30 10:05:59 +01:00
parent d1b2d43e5b
commit f9fc47093b
12 changed files with 680 additions and 208 deletions

View File

@@ -40,6 +40,9 @@ lib.fix (
# TODO: Flatten our lib functions like this: # TODO: Flatten our lib functions like this:
resolveModule = clanLib.callLib ./resolve-module { }; resolveModule = clanLib.callLib ./resolve-module { };
# Functions to help define exports
exports = clanLib.callLib ./exports.nix { };
fs = { fs = {
inherit (builtins) pathExists readDir; inherit (builtins) pathExists readDir;
}; };

88
lib/exports.nix Normal file
View File

@@ -0,0 +1,88 @@
{ lib }:
let
/**
Creates a scope string for global exports
At least one of serviceName or machineName must be set.
The scope string has the format:
"/SERVICE/INSTANCE/ROLE/MACHINE"
If the parameter is not set, the corresponding part is left empty.
Semantically this means "all".
Examples:
mkScope { serviceName = "A"; }
-> "/A///"
mkScope { machineName = "jon"; }
-> "///jon"
mkScope { serviceName = "A"; instanceName = "i1"; roleName = "peer"; machineName = "jon"; }
-> "/A/i1/peer/jon"
*/
mkScope =
{
serviceName ? "",
instanceName ? "",
roleName ? "",
machineName ? "",
}:
let
parts = [
serviceName
instanceName
roleName
machineName
];
checkedParts = lib.map (
part:
lib.throwIf (builtins.match ".?/.?" part != null) ''
clanLib.exports.mkScope: ${part} cannot contain the "/" character
''
) parts;
in
lib.throwIf ((serviceName == "" && machineName == "")) ''
clanLib.exports.mkScope requires at least 'serviceName' or 'machineName' to be set
In case your use case requires neither
'' (lib.join "/" checkedParts);
/**
Parses a scope string into its components
Returns an attribute set with the keys:
- serviceName
- instanceName
- roleName
- machineName
Example:
parseScope "A/i1/peer/jon"
->
{
serviceName = "A";
instanceName = "i1";
roleName = "peer";
machineName = "jon";
}
*/
parseScope =
scopeStr:
let
parts = lib.splitString "/" scopeStr;
checkedParts = lib.throwIf (lib.length parts != 4) ''
clanLib.exports.parseScope: invalid scope string format, expected 4 parts separated by 3 "/"
'' (parts);
in
{
serviceName = lib.elemAt 0 checkedParts;
instanceName = lib.elemAt 1 checkedParts;
roleName = lib.elemAt 2 checkedParts;
machineName = lib.elemAt 3 checkedParts;
};
in
{
inherit mkScope parseScope;
}

View File

@@ -103,6 +103,11 @@ rec {
inherit lib; inherit lib;
clan-core = self; clan-core = self;
}; };
# Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests-build-clan
legacyPackages.eval-exports = import ./new_exports.nix {
inherit lib;
clan-core = self;
};
checks = { checks = {
eval-lib-build-clan = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } '' eval-lib-build-clan = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
export HOME="$(realpath .)" export HOME="$(realpath .)"

View File

@@ -2,6 +2,7 @@
{ {
# TODO: consume directly from clan.config # TODO: consume directly from clan.config
directory, directory,
exports,
}: }:
{ {
lib, lib,
@@ -17,10 +18,10 @@ in
{ {
# TODO: merge these options into clan options # TODO: merge these options into clan options
options = { options = {
exportsModule = mkOption { # exportsModule = mkOption {
type = types.deferredModule; # type = types.deferredModule;
readOnly = true; # readOnly = true;
}; # };
mappedServices = mkOption { mappedServices = mkOption {
visible = false; visible = false;
type = attrsWith { type = attrsWith {
@@ -28,9 +29,11 @@ in
elemType = submoduleWith { elemType = submoduleWith {
class = "clan.service"; class = "clan.service";
specialArgs = { specialArgs = {
directory = directory;
clanLib = specialArgs.clanLib; clanLib = specialArgs.clanLib;
exports = config.exports; inherit
exports
directory
;
}; };
modules = [ modules = [
( (
@@ -51,34 +54,13 @@ in
default = { }; default = { };
}; };
exports = mkOption { exports = mkOption {
type = submoduleWith { type = types.lazyAttrsOf types.deferredModule;
modules = [
{ # collect exports from all services
options = { # zipAttrs is needed until we use the record type.
instances = lib.mkOption { default = lib.zipAttrsWith (_name: values: { imports = values; }) (
default = { }; lib.mapAttrsToList (_name: service: service.exports) config.mappedServices
# instances.<instanceName>... );
type = types.attrsOf (submoduleWith {
modules = [
config.exportsModule
];
});
};
# instances.<machineName>...
machines = lib.mkOption {
default = { };
type = types.attrsOf (submoduleWith {
modules = [
config.exportsModule
];
});
};
};
}
]
++ lib.mapAttrsToList (_: service: service.exports) config.mappedServices;
};
default = { };
}; };
}; };
} }

View File

@@ -504,7 +504,7 @@ in
staticModules = [ staticModules = [
({ ({
options.exports = mkOption { options.exports = mkOption {
type = types.deferredModule; type = types.lazyAttrsOf types.deferredModule;
default = { }; default = { };
description = '' description = ''
!!! Danger "Experimental Feature" !!! Danger "Experimental Feature"
@@ -634,8 +634,16 @@ in
type = types.deferredModuleWith { type = types.deferredModuleWith {
staticModules = [ staticModules = [
({ ({
# exports."///".generator.name = { _file ... import = []; _type = }
# exports."///".networking = { _file ... import = []; }
# generators."///".name = { name, ...}: { _file ... import = [];}
# networks."///" = { _file ... import = []; }
# { _file ... import = []; }
# { _file ... import = []; }
options.exports = mkOption { options.exports = mkOption {
type = types.deferredModule; type = types.lazyAttrsOf types.deferredModule;
default = { }; default = { };
description = '' description = ''
!!! Danger "Experimental Feature" !!! Danger "Experimental Feature"
@@ -767,79 +775,38 @@ in
``` ```
''; '';
default = { }; default = { };
type = types.submoduleWith { type = types.lazyAttrsOf (
# Static modules types.deferredModuleWith {
modules = [ # staticModules = [];
{ # lib.concatLists (
options.instances = mkOption { # lib.concatLists (
type = types.attrsOf types.deferredModule; # lib.mapAttrsToList (
description = '' # _roleName: role:
export modules defined in 'perInstance' # lib.mapAttrsToList (
mapped to their instance name # _instanceName: instance: lib.mapAttrsToList (_machineName: v: v.exports) instance.allMachines
# ) role.allInstances
Example # ) config.result.allRoles
# )
with instances: # )
# ++
```nix
instances.A = { ... };
instances.B= { ... };
roles.peer.perInstance = { instanceName, machine, ... }:
{
exports.foo = 1;
} }
);
This yields all other services can access these exports # # Lazy default via imports
=> # # should probably be moved to deferredModuleWith { staticModules = [ ]; }
exports.instances.A.foo = 1; # imports =
exports.instances.B.foo = 1; # if config._docs_rendering then
``` # [ ]
''; # else
}; # lib.mapAttrsToList (_roleName: role: {
options.machines = mkOption { # instances = lib.mapAttrs (_instanceName: instance: {
type = types.attrsOf types.deferredModule; # imports = lib.mapAttrsToList (_machineName: v: v.exports) instance.allMachines;
description = '' # }) role.allInstances;
export modules defined in 'perMachine' # }) config.result.allRoles
mapped to their machine name # ++ lib.mapAttrsToList (machineName: machine: {
# machines.${machineName} = machine.exports;
Example # }) config.result.allMachines;
# }
with machines: # ];
```nix
instances.A = { roles.peer.machines.jon = ... };
instances.B = { roles.peer.machines.jon = ... };
perMachine = { machine, ... }:
{
exports.foo = 1;
}
This yields all other services can access these exports
=>
exports.machines.jon.foo = 1;
exports.machines.sara.foo = 1;
```
'';
};
# Lazy default via imports
# should probably be moved to deferredModuleWith { staticModules = [ ]; }
imports =
if config._docs_rendering then
[ ]
else
lib.mapAttrsToList (_roleName: role: {
instances = lib.mapAttrs (_instanceName: instance: {
imports = lib.mapAttrsToList (_machineName: v: v.exports) instance.allMachines;
}) role.allInstances;
}) config.result.allRoles
++ lib.mapAttrsToList (machineName: machine: {
machines.${machineName} = machine.exports;
}) config.result.allMachines;
}
];
};
}; };
# --- # ---
# Place the result in _module.result to mark them as "internal" and discourage usage/overrides # Place the result in _module.result to mark them as "internal" and discourage usage/overrides
@@ -1024,5 +991,39 @@ in
} }
) config.result.allMachines; ) config.result.allMachines;
}; };
debug = mkOption {
default = lib.zipAttrsWith (_name: values: { imports = values; }) (
lib.mapAttrsToList (_machineName: machine: machine.exports) config.result.allMachines
);
}; };
};
imports = [
{
# collect exports from all machines
# zipAttrs is needed until we use the record type.
exports = lib.zipAttrsWith (_name: values: { imports = values; }) (
lib.mapAttrsToList (_machineName: machine: machine.exports) config.result.allMachines
);
}
{
# collect exports from all instances, roles and machines
# zipAttrs is needed until we use the record type.
exports = lib.zipAttrsWith (_name: values: { imports = values; }) (
lib.concatLists (
lib.concatLists (
lib.mapAttrsToList (
_roleName: role:
lib.mapAttrsToList (
_instanceName: instance: lib.mapAttrsToList (_machineName: v: v.exports) instance.allMachines
) role.allInstances
) config.result.allRoles
)
)
);
}
];
} }

221
lib/new_exports.nix Normal file
View File

@@ -0,0 +1,221 @@
{
clan-core,
lib,
}:
# TODO: TEST: define a clan without machines
{
test_simple =
let
eval = clan-core.clanLib.clan {
exports."///".foo = lib.mkForce eval.config.exports."///".bar;
directory = ./.;
self = {
clan = eval.config;
inputs = { };
};
machines.jon = { };
machines.sara = { };
exportsModule =
{ lib, ... }:
{
options.foo = lib.mkOption {
type = lib.types.number;
default = 0;
};
options.bar = lib.mkOption {
type = lib.types.number;
default = 0;
};
};
####### Service module "A"
modules.service-A =
{ ... }:
{
# config.exports
manifest.name = "A";
roles.default = {
# TODO: Remove automapping
# Currently exports are automapped
# scopes "/service=A/instance=hello/role=default/machine=jon"
# perInstance.exports.foo = 7;
# New style:
# Explizit scope
# perInstance.exports."service=A/instance=hello/role=default/machine=jon".foo = 7;
perInstance =
{ instanceName, machine, exports, ... }:
{
exports."A/${instanceName}/default/${machine.name}" = {
foo = 7;
# define export depending on B
bar = exports."B/B/default/${machine.name}".foo + 35;
};
# exports."A/${instanceName}/default/${machine.name}".
# default behavior
# exports = scope.mkExports { foo = 7; };
# We want to export things for different scopes from this scope;
# If this scope is used.
#
# Explicit scope; different from the function scope above
# exports = clanLib.scopedExport {
# # Different role export
# role = "peer";
# serviceName = config.manifest.name;
# inherit instanceName machineName;
# } { foo = 7; };
};
};
perMachine =
{ ... }:
{
#
# exports = scope.mkExports { foo = 7; };
# exports."A///${machine.name}".foo = 42;
# exports."B///".foo = 42;
};
# scope "/service=A/instance=??/role=??/machine=jon"
# perMachine.exports.foo = 42;
# scope "/service=A/instance=??/role=??/machine=??"
# exports."///".foo = 10;
};
####### Service module "A"
modules.service-B =
{ exports, ... }:
{
# config.exports
manifest.name = "B";
roles.default = {
# TODO: Remove automapping
# Currently exports are automapped
# scopes "/service=A/instance=hello/role=default/machine=jon"
# perInstance.exports.foo = 7;
# New style:
# Explizit scope
# perInstance.exports."service=A/instance=hello/role=default/machine=jon".foo = 7;
perInstance =
{ instanceName, machine, ... }:
{
# TODO: Test non-existing scope
# define export depending on A
exports."B/${instanceName}/default/${machine.name}".foo = exports."///".foo + exports."A/A/default/${machine.name}".foo;
# exports."B/B/default/jon".foo = exports."A/A/default/jon".foo;
# default behavior
# exports = scope.mkExports { foo = 7; };
# We want to export things for different scopes from this scope;
# If this scope is used.
#
# Explicit scope; different from the function scope above
# exports = clanLib.scopedExport {
# # Different role export
# role = "peer";
# serviceName = config.manifest.name;
# inherit instanceName machineName;
# } { foo = 7; };
};
};
perMachine =
{ ... }:
{
# exports = scope.mkExports { foo = 7; };
# exports."A///${machine.name}".foo = 42;
# exports."B///".foo = 42;
};
# scope "/service=A/instance=??/role=??/machine=jon"
# perMachine.exports.foo = 42;
# scope "/service=A/instance=??/role=??/machine=??"
exports."///".foo = 10;
};
#######
inventory = {
instances.A = {
module.name = "service-A";
module.input = "self";
roles.default.tags = [ "all" ];
};
instances.B = {
module.name = "service-B";
module.input = "self";
roles.default.tags = [ "all" ];
};
};
# <- inventory
#
# -> exports
/**
Current state
{
instances = {
hello = { networking = null; };
};
machines = {
jon = { networking = null; };
};
}
*/
/**
Target state: (Flat attribute set)
tdlr;
# roles / instance level definitions may not exist on their own
# role and instance names are completely arbitrary.
# For example what does it mean: this is a export for all "peer" roles of all service-instances? That would be magic on the roleName.
# Or exports for all instances with name "ifoo" ? That would be magic on the instanceName.
# Practical combinations
# always include either the service name or the machine name
exports = {
# Clan level (1)
"///" networks generators
# Service anchored (8) : min 1 instance is needed ; machines may not exist
"A///" <- service specific
"A/instance//" <- instance of a service
"A//peer/" <- role of a service
"A/instance/peer/" <- instance+role of a service
"A///machine" <- machine of a service
"A/instance//machine" <- machine + instance of a service
"A//role/machine" <- machine + role of a service
"A/instance/role/machine" <- machine + role + instance of a service
# Machine anchored (1 or 2)
"///jon" <- this machine
"A///jon" <- role on a machine (dupped with service anchored)
# Unpractical; probably not needed (5)
"//peer/jon" <- role on a machine
"/instance//jon" <- role on a machine
"/instance//" <- instance: All "foo" instances everywhere?
"//role/" <- role: All "peer" roles everywhere?
"/instance/role/" <- instance role: Applies to all services, whose instance name has "ifoo" and role is "peer" (double magic)
# TODO: lazyattrs poc
}
*/
};
in
{
inherit eval;
expr = eval;
expected = 42;
};
}

View File

@@ -1,4 +1,21 @@
{ lib, ... }: { lib, ... }:
let
inherit (lib)
mapAttrs
attrNames
showOption
setDefaultModuleLocation
mkOptionType
isAttrs
filterAttrs
intersectAttrs
mapAttrsToList
mkOptionDefault
zipAttrsWith
seq
fix
;
in
{ {
/** /**
A custom type for deferred modules that guarantee to be JSON serializable. A custom type for deferred modules that guarantee to be JSON serializable.
@@ -12,7 +29,7 @@
- Enforces that the definition is JSON serializable - Enforces that the definition is JSON serializable
- Disallows nested imports - Disallows nested imports
*/ */
uniqueDeferredSerializableModule = lib.fix ( uniqueDeferredSerializableModule = fix (
self: self:
let let
checkDef = checkDef =
@@ -23,19 +40,18 @@
def; def;
in in
# Essentially the "raw" type, but with a custom name and check # Essentially the "raw" type, but with a custom name and check
lib.mkOptionType { mkOptionType {
name = "deferredModule"; name = "deferredModule";
description = "deferred custom module. Must be JSON serializable."; description = "deferred custom module. Must be JSON serializable.";
descriptionClass = "noun"; descriptionClass = "noun";
# Unfortunately, tryEval doesn't catch JSON errors # Unfortunately, tryEval doesn't catch JSON errors
check = value: lib.seq (builtins.toJSON value) (lib.isAttrs value); check = value: seq (builtins.toJSON value) (isAttrs value);
merge = lib.options.mergeUniqueOption { merge = lib.options.mergeUniqueOption {
message = "------"; message = "------";
merge = loc: defs: { merge = loc: defs: {
imports = map ( imports = map (
def: def:
lib.seq (checkDef loc def) lib.setDefaultModuleLocation seq (checkDef loc def) setDefaultModuleLocation "${def.file}, via option ${showOption loc}"
"${def.file}, via option ${lib.showOption loc}"
def.value def.value
) defs; ) defs;
}; };
@@ -48,4 +64,113 @@
}; };
} }
); );
/**
New submodule type that allows merging at the attribute level.
:::note
'record' type adopted from https://github.com/NixOS/nixpkgs/pull/334680
:::
It applies additional constraints to immediate child options:
- No support for 'readOnly'
- No support for 'apply'
- No support for type-merging: That means the modules options must be pre-declared directly.
*/
record =
{
optional ? { },
required ? { },
wildcardType ? null,
}:
mkOptionType {
name = "record";
description =
if wildcardType == null then "record" else "open record of ${wildcardType.description}";
descriptionClass = if wildcardType == null then "noun" else "composite";
check = isAttrs;
merge.v2 =
{ loc, defs }:
let
pushPositions = map (
def:
mapAttrs (_n: v: {
inherit (def) file;
value = v;
}) def.value
);
# Checks
intersection = intersectAttrs optional required;
optionalDefault = filterAttrs (_: opt: opt ? default) optional;
# Definitions + option defaults
allDefs =
defs
++ (mapAttrsToList (name: opt: {
file = (builtins.unsafeGetAttrPos name required).file or "<unknown-file>";
value = {
${name} = mkOptionDefault opt.default;
};
}) (filterAttrs (_n: opt: opt ? default) required));
merged = zipAttrsWith (
name: defs:
let
elemType = optional.${name}.type or required.${name}.type or wildcardType;
in
lib.modules.mergeDefinitions (loc ++ [ name ]) elemType defs
) (pushPositions allDefs);
in
{
headError =
if intersection != { } then
{
message = "The following attributes of '${showOption loc}' are both declared in 'optional' and in 'required': ${lib.concatStringsSep ", " (attrNames intersection)}";
}
else if optionalDefault != { } then
{
message = "The following attributes of '${showOption loc}' are declared in 'optional' cannot have a default value: ${lib.concatStringsSep ", " (attrNames optionalDefault)}";
}
else
null;
# TODO: expose fields, fieldValues and extraValues
valueMeta = {
attrs = mapAttrs (_n: v: v.checkedAndMerged.valueMeta) merged;
};
value = mapAttrs (
name: v:
let
elemType = optional.${name}.type or required.${name}.type or wildcardType;
in
if required ? ${name} then
# Non-optional, lazy ?
v.mergedValue
else
# Optional, lazy
v.optionalValue.value or elemType.emptyValue.value or v.mergedValue
) merged;
};
nestedTypes = lib.optionalAttrs (wildcardType != null) {
inherit wildcardType;
};
getSubOptions =
prefix:
# Since this type doesn't support type merging, we can safely use the original attrs to display documentation.
mapAttrs (
name: opt:
(
opt
// {
loc = prefix ++ [ name ];
inherit name;
declarations = [
(builtins.unsafeGetAttrPos name optional).file or (builtins.unsafeGetAttrPos name required).file
or "<unknown-file>"
];
}
)
) (optional // required);
};
} }

View File

@@ -0,0 +1,44 @@
{ lib, clanLib, ... }:
let
inherit (lib) evalModules mkOption;
inherit (clanLib.types) record;
in
{
test_simple =
let
eval = evalModules {
modules = [
{
options.foo = mkOption {
type = record { };
default = { };
};
}
];
};
in
{
inherit eval;
expr = eval.config.foo;
expected = { };
};
test_wildcard =
let
eval = evalModules {
modules = [
{
options.foo = mkOption {
type = record { };
default = { };
};
}
];
};
in
{
inherit eval;
expr = eval.config.foo;
expected = { };
};
}

View File

@@ -1,92 +1,5 @@
{ lib, clanLib, ... }: { lib, clanLib, ... }:
let
evalSettingsModule =
m:
lib.evalModules {
modules = [
{
options.foo = lib.mkOption {
type = clanLib.types.uniqueDeferredSerializableModule;
};
}
m
];
};
in
{ {
test_simple = unique = import ./unique_tests.nix { inherit lib clanLib; };
let record = import ./record_tests.nix { inherit lib clanLib; };
eval = evalSettingsModule {
foo = { };
};
in
{
inherit eval;
expr = eval.config.foo;
expected = {
# Foo has imports
# This can only ever be one module due to the type of foo
imports = [
{
# This is the result of 'setDefaultModuleLocation'
# Which also returns exactly one module
_file = "<unknown-file>, via option foo";
imports = [
{ }
];
}
];
};
};
test_no_nested_imports =
let
eval = evalSettingsModule {
foo = {
imports = [ ];
};
};
in
{
inherit eval;
expr = eval.config.foo;
expectedError = {
type = "ThrownError";
message = "*nested imports";
};
};
test_no_function_modules =
let
eval = evalSettingsModule {
foo =
{ ... }:
{
};
};
in
{
inherit eval;
expr = eval.config.foo;
expectedError = {
type = "TypeError";
message = "cannot convert a function to JSON";
};
};
test_non_attrs_module =
let
eval = evalSettingsModule {
foo = "foo.nix";
};
in
{
inherit eval;
expr = eval.config.foo;
expectedError = {
type = "ThrownError";
message = ".*foo.* is not of type";
};
};
} }

View File

@@ -0,0 +1,92 @@
{ lib, clanLib, ... }:
let
evalSettingsModule =
m:
lib.evalModules {
modules = [
{
options.foo = lib.mkOption {
type = clanLib.types.uniqueDeferredSerializableModule;
};
}
m
];
};
in
{
test_not_defined =
let
eval = evalSettingsModule {
foo = { };
};
in
{
inherit eval;
expr = eval.config.foo;
expected = {
# Foo has imports
# This can only ever be one module due to the type of foo
imports = [
{
# This is the result of 'setDefaultModuleLocation'
# Which also returns exactly one module
_file = "<unknown-file>, via option foo";
imports = [
{ }
];
}
];
};
};
test_no_nested_imports =
let
eval = evalSettingsModule {
foo = {
imports = [ ];
};
};
in
{
inherit eval;
expr = eval.config.foo;
expectedError = {
type = "ThrownError";
message = "*nested imports";
};
};
test_no_function_modules =
let
eval = evalSettingsModule {
foo =
{ ... }:
{
};
};
in
{
inherit eval;
expr = eval.config.foo;
expectedError = {
type = "TypeError";
message = "cannot convert a function to JSON";
};
};
test_non_attrs_module =
let
eval = evalSettingsModule {
foo = "foo.nix";
};
in
{
inherit eval;
expr = eval.config.foo;
expectedError = {
type = "ThrownError";
message = ".*foo.* is not of type";
};
};
}

View File

@@ -111,11 +111,11 @@ in
}; };
modules = [ modules = [
(import ../../lib/inventory/distributed-service/all-services-wrapper.nix { (import ../../lib/inventory/distributed-service/all-services-wrapper.nix {
inherit (clanConfig) directory; inherit (clanConfig) directory exports;
}) })
# Dependencies # Dependencies
{ {
exportsModule = clanConfig.exportsModule; # exportsModule = clanConfig.exportsModule;
} }
{ {
# TODO: Rename to "allServices" # TODO: Rename to "allServices"

View File

@@ -110,9 +110,7 @@ in
# TODO: make this writable by moving the options from inventoryClass into clan. # TODO: make this writable by moving the options from inventoryClass into clan.
exports = lib.mkOption { exports = lib.mkOption {
readOnly = true; type = types.lazyAttrsOf (types.submoduleWith { modules = [ config.exportsModule ]; });
visible = false;
internal = true;
}; };
exportsModule = lib.mkOption { exportsModule = lib.mkOption {