yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Inventory: generate exact schema for validation & documentation

Browse Source
This commit is contained in:
Johannes Kirschbauer
2024-07-14 13:30:02 +02:00
parent 77577a9f27
commit d49a23b890
8 changed files with 209 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lib/build-clan/default.nix
View File

@@ -32,6 +32,9 @@ let
# - Each service role must reference a valid machine after all machines are merged # - Each service role must reference a valid machine after all machines are merged
mergedInventory = mergedInventory =
(lib.evalModules { (lib.evalModules {
specialArgs = {
inherit clan-core;
};
modules = [ modules = [
clan-core.lib.inventory.interface clan-core.lib.inventory.interface
{ inherit meta; } { inherit meta; }

146
lib/inventory/build-inventory/interface.nix
View File

@@ -15,7 +15,7 @@ let
}; };
machineRef = lib.mkOptionType { machineRef = lib.mkOptionType {
name = "machineRef"; name = "str";
description = "Machine :: [${builtins.concatStringsSep " | " (builtins.attrNames config.machines)}]"; description = "Machine :: [${builtins.concatStringsSep " | " (builtins.attrNames config.machines)}]";
check = v: lib.isString v && builtins.elem v (builtins.attrNames config.machines); check = v: lib.isString v && builtins.elem v (builtins.attrNames config.machines);
merge = lib.mergeEqualOption; merge = lib.mergeEqualOption;
@@ -29,20 +29,85 @@ let
); );
tagRef = lib.mkOptionType { tagRef = lib.mkOptionType {
name = "tagRef"; name = "str";
description = "Tags :: [${builtins.concatStringsSep " | " allTags}]"; description = "Tags :: [${builtins.concatStringsSep " | " allTags}]";
check = v: lib.isString v && builtins.elem v allTags; check = v: lib.isString v && builtins.elem v allTags;
merge = lib.mergeEqualOption; merge = lib.mergeEqualOption;
}; };
moduleConfig = lib.mkOption {
default = { };
type = t.attrsOf t.anything;
};
in in
{ {
options.assertions = lib.mkOption { options = {
type = t.listOf t.unspecified; assertions = lib.mkOption {
internal = true; type = t.listOf t.unspecified;
default = [ ]; internal = true;
visible = false;
default = [ ];
};
meta = metaOptions;
machines = lib.mkOption {
default = { };
type = t.attrsOf (
t.submodule {
options = {
inherit (metaOptions) name description icon;
tags = lib.mkOption {
default = [ ];
apply = lib.unique;
type = t.listOf t.str;
};
system = lib.mkOption {
default = null;
type = t.nullOr t.str;
};
};
}
);
};
services = lib.mkOption {
default = { };
type = t.attrsOf (
t.attrsOf (
t.submodule {
options.meta = metaOptions;
options.config = moduleConfig;
options.machines = lib.mkOption {
default = { };
type = t.attrsOf (t.submodule { options.config = moduleConfig; });
};
options.roles = lib.mkOption {
default = { };
type = t.attrsOf (
t.submodule {
options.machines = lib.mkOption {
default = [ ];
type = t.listOf machineRef;
};
options.tags = lib.mkOption {
default = [ ];
apply = lib.unique;
type = t.listOf tagRef;
};
}
);
};
}
)
);
};
}; };
# Smoke validation of the inventory
config.assertions = config.assertions =
let let
# Inventory assertions
# - All referenced machines must exist in the top-level machines
serviceAssertions = lib.foldlAttrs ( serviceAssertions = lib.foldlAttrs (
ass1: serviceName: c: ass1: serviceName: c:
ass1 ass1
@@ -60,8 +125,11 @@ in
ass2 ++ assertions ass2 ++ assertions
) [ ] c ) [ ] c
) [ ] config.services; ) [ ] config.services;
# Machine assertions
# - A machine must define their host system
machineAssertions = map ( machineAssertions = map (
{ name, value }: { name }:
{ {
assertion = true; assertion = true;
message = "Machine ${name} should define its host system in the inventory. ()"; message = "Machine ${name} should define its host system in the inventory. ()";
@@ -69,68 +137,4 @@ in
) (lib.attrsToList (lib.filterAttrs (_n: v: v.system or null == null) config.machines)); ) (lib.attrsToList (lib.filterAttrs (_n: v: v.system or null == null) config.machines));
in in
machineAssertions ++ serviceAssertions; machineAssertions ++ serviceAssertions;
options.meta = metaOptions;
options.machines = lib.mkOption {
default = { };
type = t.attrsOf (
t.submodule {
options = {
inherit (metaOptions) name description icon;
tags = lib.mkOption {
default = [ ];
apply = lib.unique;
type = t.listOf t.str;
};
system = lib.mkOption {
default = null;
type = t.nullOr t.str;
};
};
}
);
};
options.services = lib.mkOption {
default = { };
type = t.attrsOf (
t.attrsOf (
t.submodule {
options.meta = metaOptions;
options.config = lib.mkOption {
default = { };
type = t.anything;
};
options.machines = lib.mkOption {
default = { };
type = t.attrsOf (
t.submodule {
options.config = lib.mkOption {
default = { };
type = t.anything;
};
}
);
};
options.roles = lib.mkOption {
default = { };
type = t.attrsOf (
t.submodule {
options.machines = lib.mkOption {
default = [ ];
type = t.listOf machineRef;
};
options.tags = lib.mkOption {
default = [ ];
apply = lib.unique;
type = t.listOf tagRef;
};
}
);
};
}
)
);
};
} }

47
lib/inventory/examples/syncthing.json
View File

@@ -1,47 +0,0 @@
{
"machines": {
"camina_machine": {
"name": "camina"
},
"vyr_machine": {
"name": "vyr"
},
"vi_machine": {
"name": "vi"
}
},
"meta": {
"name": "kenjis clan"
},
"services": {
"syncthing": {
"instance_1": {
"meta": {
"name": "My sync"
},
"roles": {
"peer": {
"machines": ["vyr_machine", "vi_machine", "camina_machine"]
}
},
"machines": {},
"config": {
"folders": {
"test": {
"path": "~/data/docs",
"devices": ["camina_machine", "vyr_machine", "vi_machine"]
},
"videos": {
"path": "~/data/videos",
"devices": ["camina_machine", "vyr_machine"]
},
"playlist": {
"path": "~/data/playlist",
"devices": ["camina_machine", "vi_machine"]
}
}
}
}
}
}
}

36
lib/inventory/examples/zerotier.json
View File

@@ -1,36 +0,0 @@
{
"machines": {
"camina_machine": {
"name": "camina"
},
"vyr_machine": {
"name": "vyr"
},
"vi_machine": {
"name": "vi"
}
},
"meta": {
"name": "kenjis clan"
},
"services": {
"zerotier": {
"instance_1": {
"meta": {
"name": "My Network"
},
"roles": {
"controller": { "machines": ["vyr_machine"] },
"moon": { "machines": ["vyr_machine"] },
"peer": { "machines": ["vi_machine", "camina_machine"] }
},
"machines": {
"vyr_machine": {
"config": {}
}
},
"config": {}
}
}
}
}

128
lib/inventory/flake-module.nix
View File

@@ -21,16 +21,113 @@ in
clan-core = self; clan-core = self;
inherit lib; inherit lib;
}; };
optionsFromModule =
mName:
let
eval = self.lib.evalClanModules [ mName ];
in
if (eval.options.clan ? "${mName}") then eval.options.clan.${mName} else { };
modulesSchema = lib.mapAttrs (
moduleName: _: jsonLib'.parseOptions (optionsFromModule moduleName) { }
) self.clanModules;
jsonLib = self.lib.jsonschema {
# includeDefaults = false;
};
jsonLib' = self.lib.jsonschema {
# includeDefaults = false;
header = { };
};
inventorySchema = jsonLib.parseModule (import ./build-inventory/interface.nix);
getRoles =
modulePath:
let
rolesDir = "${modulePath}/roles";
in
if builtins.pathExists rolesDir then
lib.pipe rolesDir [
builtins.readDir
(lib.filterAttrs (_n: v: v == "regular"))
lib.attrNames
(map (fileName: lib.removeSuffix ".nix" fileName))
]
else
null;
schema = inventorySchema // {
properties = inventorySchema.properties // {
services = {
type = "object";
additionalProperties = false;
properties = lib.mapAttrs (moduleName: moduleSchema: {
type = "object";
additionalProperties = {
type = "object";
additionalProperties = false;
properties = {
meta =
inventorySchema.properties.services.additionalProperties.additionalProperties.properties.meta;
config = moduleSchema;
roles = {
type = "object";
additionalProperties = false;
required = [ ];
properties = lib.listToAttrs (
map
(role: {
name = role;
value =
inventorySchema.properties.services.additionalProperties.additionalProperties.properties.roles.additionalProperties;
})
(
let
roles = getRoles self.clanModules.${moduleName};
in
if roles == null then [ ] else roles
)
);
};
machines =
lib.recursiveUpdate
inventorySchema.properties.services.additionalProperties.additionalProperties.properties.machines
{ additionalProperties.properties.config = moduleSchema; };
};
};
}) modulesSchema;
};
};
};
in in
{ {
legacyPackages.inventorySchema = schema;
devShells.inventory-schema = pkgs.mkShell { devShells.inventory-schema = pkgs.mkShell {
inputsFrom = with config.checks; [ inputsFrom = with config.checks; [
lib-inventory-examples-cue
lib-inventory-schema lib-inventory-schema
lib-inventory-eval lib-inventory-eval
self'.devShells.default self'.devShells.default
]; ];
}; };
# Inventory schema with concrete module implementations
packages.inventory-schema = pkgs.stdenv.mkDerivation {
name = "inventory-schema";
buildInputs = [ pkgs.cue ];
src = ./.;
buildPhase = ''
export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON self'.legacyPackages.inventorySchema)}
cp $SCHEMA schema.json
cue import -f -p compose -l '#Root:' schema.json
mkdir $out
cp schema.cue $out
cp schema.json $out
'';
};
# Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests
legacyPackages.evalTests-inventory = import ./tests { legacyPackages.evalTests-inventory = import ./tests {
inherit buildInventory; inherit buildInventory;
@@ -38,32 +135,21 @@ in
}; };
checks = { checks = {
lib-inventory-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } '' lib-inventory-examples-cue = pkgs.stdenv.mkDerivation {
export HOME="$(realpath .)"
nix-unit --eval-store "$HOME" \
--extra-experimental-features flakes \
${inputOverrides} \
--flake ${self}#legacyPackages.${system}.evalTests-inventory
touch $out
'';
lib-inventory-schema = pkgs.stdenv.mkDerivation {
name = "inventory-schema-checks"; name = "inventory-schema-checks";
src = ./.; src = ./.;
buildInputs = [ pkgs.cue ]; buildInputs = [ pkgs.cue ];
buildPhase = '' buildPhase = ''
echo "Running inventory tests..." echo "Running inventory tests..."
# Cue is easier to run in the same directory as the schema # Cue is easier to run in the same directory as the schema
cd spec cp ${self'.packages.inventory-schema}/schema.cue root.cue
echo "Export cue as json-schema..." ls -la .
cue export --out openapi root.cue
echo "Validate test/*.json against inventory-schema..." echo "Validate test/*.json against inventory-schema..."
cat root.cue
test_dir="../examples" test_dir="./examples"
for file in "$test_dir"/*; do for file in "$test_dir"/*; do
# Check if the item is a file # Check if the item is a file
if [ -f "$file" ]; then if [ -f "$file" ]; then
@@ -78,6 +164,16 @@ in
touch $out touch $out
''; '';
}; };
lib-inventory-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
export HOME="$(realpath .)"
nix-unit --eval-store "$HOME" \
--extra-experimental-features flakes \
${inputOverrides} \
--flake ${self}#legacyPackages.${system}.evalTests-inventory
touch $out
'';
}; };
}; };
} }

46
lib/jsonschema/default.nix
View File

@@ -1,9 +1,16 @@
{ {
lib ? import <nixpkgs/lib>, lib ? import <nixpkgs/lib>,
}:
{
excludedTypes ? [ excludedTypes ? [
"functionTo" "functionTo"
"package" "package"
], ],
includeDefaults ? true,
header ? {
"$schema" = "http://json-schema.org/draft-07/schema#";
},
specialArgs ? { },
}: }:
let let
# remove _module attribute from options # remove _module attribute from options
@@ -40,12 +47,14 @@ let
]; ];
in in
rec { rec {
# parses a nixos module to a jsonschema # parses a nixos module to a jsonschema
parseModule = parseModule =
module: module:
let let
evaled = lib.evalModules { modules = [ module ]; }; evaled = lib.evalModules {
modules = [ module ];
inherit specialArgs;
};
in in
parseOptions evaled.options { }; parseOptions evaled.options { };
@@ -57,9 +66,6 @@ rec {
{ {
# The top-level header object should specify at least the schema version # The top-level header object should specify at least the schema version
# Can be customized if needed # Can be customized if needed
header ? {
"$schema" = "http://json-schema.org/draft-07/schema#";
},
# By default the header is not added to the schema # By default the header is not added to the schema
addHeader ? true, addHeader ? true,
}: }:
@@ -86,7 +92,7 @@ rec {
parseOption = parseOption =
option: option:
let let
default = lib.optionalAttrs (option ? default) { inherit (option) default; }; default = lib.optionalAttrs (option ? default && includeDefaults) { inherit (option) default; };
example = lib.optionalAttrs (option ? example) { example = lib.optionalAttrs (option ? example) {
examples = examples =
if (builtins.typeOf option.example) == "list" then option.example else [ option.example ]; if (builtins.typeOf option.example) == "list" then option.example else [ option.example ];
@@ -95,7 +101,6 @@ rec {
description = option.description.text or option.description; description = option.description.text or option.description;
}; };
in in
# either type # either type
# TODO: if all nested options are excluded, the parent should be excluded too # TODO: if all nested options are excluded, the parent should be excluded too
if if
@@ -117,16 +122,13 @@ rec {
]; ];
optionsList = filterExcluded optionsList'; optionsList = filterExcluded optionsList';
in in
default // example // description // { anyOf = map parseOption optionsList; } default // example // description // { oneOf = map parseOption optionsList; }
# handle nested options (not a submodule) # handle nested options (not a submodule)
else if !option ? _type then else if !option ? _type then
parseOptions' option parseOptions' option
# throw if not an option # throw if not an option
else if option._type != "option" && option._type != "option-type" then else if option._type != "option" && option._type != "option-type" then
throw "parseOption: not an option" throw "parseOption: not an option"
# parse nullOr # parse nullOr
else if else if
option.type.name == "nullOr" option.type.name == "nullOr"
@@ -143,32 +145,28 @@ rec {
// example // example
// description // description
// { // {
anyOf = [ oneOf = [
{ type = "null"; } { type = "null"; }
] ++ (lib.optional (!isExcludedOption nestedOption) (parseOption nestedOption)); ] ++ (lib.optional (!isExcludedOption nestedOption) (parseOption nestedOption));
} }
# parse bool # parse bool
else if else if
option.type.name == "bool" option.type.name == "bool"
# return jsonschema property definition for bool # return jsonschema property definition for bool
then then
default // example // description // { type = "boolean"; } default // example // description // { type = "boolean"; }
# parse float # parse float
else if else if
option.type.name == "float" option.type.name == "float"
# return jsonschema property definition for float # return jsonschema property definition for float
then then
default // example // description // { type = "number"; } default // example // description // { type = "number"; }
# parse int # parse int
else if else if
(option.type.name == "int" || option.type.name == "positiveInt") (option.type.name == "int" || option.type.name == "positiveInt")
# return jsonschema property definition for int # return jsonschema property definition for int
then then
default // example // description // { type = "integer"; } default // example // description // { type = "integer"; }
# TODO: Add support for intMatching in jsonschema # TODO: Add support for intMatching in jsonschema
# parse port type aka. "unsignedInt16" # parse port type aka. "unsignedInt16"
else if else if
@@ -178,7 +176,6 @@ rec {
|| option.type.name == "intBetween" || option.type.name == "intBetween"
then then
default // example // description // { type = "integer"; } default // example // description // { type = "integer"; }
# parse string # parse string
# TODO: parse more precise string types # TODO: parse more precise string types
else if else if
@@ -189,51 +186,43 @@ rec {
# return jsonschema property definition for string # return jsonschema property definition for string
then then
default // example // description // { type = "string"; } default // example // description // { type = "string"; }
# TODO: Add support for stringMatching in jsonschema # TODO: Add support for stringMatching in jsonschema
# parse stringMatching # parse stringMatching
else if lib.strings.hasPrefix "strMatching" option.type.name then else if lib.strings.hasPrefix "strMatching" option.type.name then
default // example // description // { type = "string"; } default // example // description // { type = "string"; }
# TODO: Add support for separatedString in jsonschema # TODO: Add support for separatedString in jsonschema
else if lib.strings.hasPrefix "separatedString" option.type.name then else if lib.strings.hasPrefix "separatedString" option.type.name then
default // example // description // { type = "string"; } default // example // description // { type = "string"; }
# parse string # parse string
else if else if
option.type.name == "path" option.type.name == "path"
# return jsonschema property definition for path # return jsonschema property definition for path
then then
default // example // description // { type = "string"; } default // example // description // { type = "string"; }
# parse anything # parse anything
else if else if
option.type.name == "anything" option.type.name == "anything"
# return jsonschema property definition for anything # return jsonschema property definition for anything
then then
default // example // description // { type = allBasicTypes; } default // example // description // { type = allBasicTypes; }
# parse unspecified # parse unspecified
else if else if
option.type.name == "unspecified" option.type.name == "unspecified"
# return jsonschema property definition for unspecified # return jsonschema property definition for unspecified
then then
default // example // description // { type = allBasicTypes; } default // example // description // { type = allBasicTypes; }
# parse raw # parse raw
else if else if
option.type.name == "raw" option.type.name == "raw"
# return jsonschema property definition for raw # return jsonschema property definition for raw
then then
default // example // description // { type = allBasicTypes; } default // example // description // { type = allBasicTypes; }
# parse enum # parse enum
else if else if
option.type.name == "enum" option.type.name == "enum"
# return jsonschema property definition for enum # return jsonschema property definition for enum
then then
default // example // description // { enum = option.type.functor.payload; } default // example // description // { enum = option.type.functor.payload; }
# parse listOf submodule # parse listOf submodule
else if else if
option.type.name == "listOf" && option.type.functor.wrapped.name == "submodule" option.type.name == "listOf" && option.type.functor.wrapped.name == "submodule"
@@ -246,7 +235,6 @@ rec {
type = "array"; type = "array";
items = parseOptions' (option.type.functor.wrapped.getSubOptions option.loc); items = parseOptions' (option.type.functor.wrapped.getSubOptions option.loc);
} }
# parse list # parse list
else if else if
(option.type.name == "listOf") (option.type.name == "listOf")
@@ -266,14 +254,12 @@ rec {
type = "array"; type = "array";
} }
// (lib.optionalAttrs (!isExcludedOption nestedOption) { items = parseOption nestedOption; }) // (lib.optionalAttrs (!isExcludedOption nestedOption) { items = parseOption nestedOption; })
# parse list of unspecified # parse list of unspecified
else if else if
(option.type.name == "listOf") && (option.type.functor.wrapped.name == "unspecified") (option.type.name == "listOf") && (option.type.functor.wrapped.name == "unspecified")
# return jsonschema property definition for list # return jsonschema property definition for list
then then
default // example // description // { type = "array"; } default // example // description // { type = "array"; }
# parse attrsOf submodule # parse attrsOf submodule
else if else if
option.type.name == "attrsOf" && option.type.nestedTypes.elemType.name == "submodule" option.type.name == "attrsOf" && option.type.nestedTypes.elemType.name == "submodule"
@@ -286,7 +272,6 @@ rec {
type = "object"; type = "object";
additionalProperties = parseOptions' (option.type.nestedTypes.elemType.getSubOptions option.loc); additionalProperties = parseOptions' (option.type.nestedTypes.elemType.getSubOptions option.loc);
} }
# parse attrs # parse attrs
else if else if
option.type.name == "attrs" option.type.name == "attrs"
@@ -299,7 +284,6 @@ rec {
type = "object"; type = "object";
additionalProperties = true; additionalProperties = true;
} }
# parse attrsOf # parse attrsOf
# TODO: if nested option is excluded, the parent sould be excluded too # TODO: if nested option is excluded, the parent sould be excluded too
else if else if
@@ -328,7 +312,6 @@ rec {
else else
false; false;
} }
# parse submodule # parse submodule
else if else if
option.type.name == "submodule" option.type.name == "submodule"
@@ -336,7 +319,6 @@ rec {
# then (lib.attrNames (option.type.getSubOptions option.loc).opt) # then (lib.attrNames (option.type.getSubOptions option.loc).opt)
then then
parseOptions' (option.type.getSubOptions option.loc) parseOptions' (option.type.getSubOptions option.loc)
# throw error if option type is not supported # throw error if option type is not supported
else else
notSupported option; notSupported option;

2
nixosModules/clanCore/schema.nix
View File

@@ -1,6 +1,6 @@
{ options, lib, ... }: { options, lib, ... }:
let let
jsonschema = import ../../lib/jsonschema { inherit lib; }; jsonschema = import ../../lib/jsonschema { inherit lib; } { };
in in
{ {
options.clanSchema = lib.mkOption { options.clanSchema = lib.mkOption {

6
pkgs/schemas/flake-module.nix
View File

@@ -5,6 +5,8 @@
let let
clanModules = self.clanModules; clanModules = self.clanModules;
jsonLib = self.lib.jsonschema { };
# Uncomment if you only want one module to be available # Uncomment if you only want one module to be available
# clanModules = { # clanModules = {
# borgbackup = self.clanModules.borgbackup; # borgbackup = self.clanModules.borgbackup;
@@ -18,13 +20,13 @@
if (eval.options.clan ? "${mName}") then eval.options.clan.${mName} else { }; if (eval.options.clan ? "${mName}") then eval.options.clan.${mName} else { };
clanModuleSchemas = lib.mapAttrs ( clanModuleSchemas = lib.mapAttrs (
modulename: _: self.lib.jsonschema.parseOptions (optionsFromModule modulename) { } modulename: _: jsonLib.parseOptions (optionsFromModule modulename) { }
) clanModules; ) clanModules;
clanModuleFunctionSchemas = lib.mapAttrsFlatten (modulename: _: { clanModuleFunctionSchemas = lib.mapAttrsFlatten (modulename: _: {
name = modulename; name = modulename;
description = self.lib.modules.getShortDescription modulename; description = self.lib.modules.getShortDescription modulename;
parameters = self.lib.jsonschema.parseOptions (optionsFromModule modulename) { }; parameters = jsonLib.parseOptions (optionsFromModule modulename) { };
}) clanModules; }) clanModules;
in in
rec { rec {