yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Inventory/schemas: use less schema versions

Browse Source
This commit is contained in:
Johannes Kirschbauer
2024-10-09 11:30:22 +02:00
parent 4ef6a37540
commit 666f87e7e4
7 changed files with 24 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
lib/inventory/.envrc
View File

@@ -1,6 +0,0 @@
# shellcheck shell=bash
source_up
watch_file flake-module.nix
use flake .#inventory-schema --builders ''

53
lib/inventory/examples/borgbackup.json
View File

@@ -1,53 +0,0 @@
{
"machines": {
"camina_machine": {
"name": "camina",
"tags": ["laptop"]
},
"vyr_machine": {
"name": "vyr"
},
"vi_machine": {
"name": "vi",
"tags": ["laptop"]
}
},
"meta": {
"name": "kenjis clan"
},
"services": {
"borgbackup": {
"instance_1": {
"meta": {
"name": "My backup"
},
"roles": {
"server": {
"machines": ["vyr_machine"]
},
"client": {
"machines": ["vyr_machine"],
"tags": ["laptop"]
}
},
"machines": {},
"config": {}
},
"instance_2": {
"meta": {
"name": "My backup"
},
"roles": {
"server": {
"machines": ["vi_machine"]
},
"client": {
"machines": ["camina_machine"]
}
},
"machines": {},
"config": {}
}
}
}
}

97
lib/inventory/flake-module.nix
View File

@@ -16,22 +16,7 @@ in
self',
...
}:
let
getSchema = import ./interface-to-schema.nix { inherit lib self; };
# The schema for the inventory, without default values, from the module system.
# This is better suited for human reading and for generating code.
bareSchema = getSchema { includeDefaults = false; };
# The schema for the inventory with default values, from the module system.
# This is better suited for validation, since default values are included.
fullSchema = getSchema { };
in
{
legacyPackages.inventory = {
inherit fullSchema;
inherit bareSchema;
};
devShells.inventory-schema = pkgs.mkShell {
inputsFrom = with config.checks; [
lib-inventory-examples-cue
@@ -40,49 +25,12 @@ in
];
};
# Inventory schema with concrete module implementations
packages.inventory-api-docs = pkgs.stdenv.mkDerivation {
name = "inventory-schema";
buildInputs = [ ];
src = ./.;
buildPhase = ''
cat <<EOF > "$out"
# Inventory API
*Inventory* is an abstract service layer for consistently configuring distributed services across machine boundaries.
The following is a specification of the inventory in [cuelang](https://cuelang.org/) format.
\`\`\`cue
EOF
cat ${self'.packages.inventory-schema-pretty}/schema.cue >> $out
cat <<EOF >> $out
\`\`\`
EOF
'';
};
packages.inventory-schema = pkgs.stdenv.mkDerivation {
name = "inventory-schema";
buildInputs = [ pkgs.cue ];
src = ./.;
buildPhase = ''
export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON fullSchema.schemaWithModules)}
cp $SCHEMA schema.json
cue import -f -p compose -l '#Root:' schema.json
mkdir $out
cp schema.cue $out
cp schema.json $out
'';
};
packages.inventory-schema-abstract = pkgs.stdenv.mkDerivation {
name = "inventory-schema";
buildInputs = [ pkgs.cue ];
src = ./.;
buildPhase = ''
export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON bareSchema.abstractSchema)}
export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON self'.legacyPackages.schemas.inventory)}
cp $SCHEMA schema.json
cue import -f -p compose -l '#Root:' schema.json
mkdir $out
@@ -102,20 +50,6 @@ in
}
);
packages.inventory-schema-pretty = pkgs.stdenv.mkDerivation {
name = "inventory-schema-pretty";
buildInputs = [ pkgs.cue ];
src = ./.;
buildPhase = ''
export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON bareSchema.schemaWithModules)}
cp $SCHEMA schema.json
cue import -f -p compose -l '#Root:' schema.json
mkdir $out
cp schema.cue $out
cp schema.json $out
'';
};
# Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests
legacyPackages.evalTests-inventory = import ./tests {
inherit lib;
@@ -123,35 +57,6 @@ in
};
checks = {
lib-inventory-examples-cue = pkgs.stdenv.mkDerivation {
name = "inventory-schema-checks";
src = ./.;
buildInputs = [ pkgs.cue ];
buildPhase = ''
echo "Running inventory tests..."
# Cue is easier to run in the same directory as the schema
cp ${self'.packages.inventory-schema}/schema.cue root.cue
ls -la .
echo "Validate test/*.json against inventory-schema..."
cat root.cue
test_dir="./examples"
for file in "$test_dir"/*; do
# Check if the item is a file
if [ -f "$file" ]; then
# Print the filename
echo "Running test on: $file"
# Run the cue vet command
cue vet "$file" root.cue -d "#Root"
fi
done
touch $out
'';
};
lib-inventory-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
export HOME="$(realpath .)"

135
lib/inventory/interface-to-schema.nix
View File

@@ -1,135 +0,0 @@
{ lib, self, ... }:
{
includeDefaults ? true,
}:
let
optionsFromModule =
mName:
let
eval = self.lib.evalClanModules [ mName ];
in
if (eval.options.clan ? "${mName}") then eval.options.clan.${mName} else { };
modulesSchema = lib.mapAttrs (
moduleName: _: jsonLib'.parseOptions (optionsFromModule moduleName) { }
) self.clanModules;
jsonLib = self.lib.jsonschema { inherit includeDefaults; };
jsonLib' = self.lib.jsonschema {
inherit includeDefaults;
header = { };
};
inventorySchema = jsonLib.parseModule (import ./build-inventory/interface.nix);
getRoles =
modulePath:
let
rolesDir = "${modulePath}/roles";
in
if builtins.pathExists rolesDir then
lib.pipe rolesDir [
builtins.readDir
(lib.filterAttrs (_n: v: v == "regular"))
lib.attrNames
(map (fileName: lib.removeSuffix ".nix" fileName))
]
else
null;
# The actual schema for the inventory
# !!! We cannot import the module into the interface.nix, because it would cause evaluation overhead.
# Modifies:
# - service.<serviceName>.<instanceName>.config = moduleSchema
# - service.<serviceName>.<instanceName>.machine.<machineName>.config = moduleSchema
# - service.<serviceName>.<instanceName>.roles = acutalRoles
schema =
let
moduleToService = moduleName: moduleSchema: {
type = "object";
additionalProperties = {
type = "object";
additionalProperties = false;
properties = {
meta = {
title = "service-meta";
} // inventorySchema.properties.services.additionalProperties.additionalProperties.properties.meta;
config = {
title = "${moduleName}-config";
default = { };
} // moduleSchema;
roles = {
type = "object";
additionalProperties = false;
required = [ ];
properties = lib.listToAttrs (
map (role: {
name = role;
value =
lib.recursiveUpdate
inventorySchema.properties.services.additionalProperties.additionalProperties.properties.roles.additionalProperties
{
properties.config = {
title = "${moduleName}-config";
default = { };
} // moduleSchema;
};
}) (rolesOf moduleName)
);
};
machines =
lib.recursiveUpdate
inventorySchema.properties.services.additionalProperties.additionalProperties.properties.machines
{
additionalProperties.properties.config = {
title = "${moduleName}-config";
default = { };
} // moduleSchema;
};
};
};
};
rolesOf =
moduleName:
let
# null | [ string ]
roles = getRoles self.clanModules.${moduleName};
in
if roles == null then [ ] else roles;
moduleServices = lib.mapAttrs moduleToService (
lib.filterAttrs (n: _v: rolesOf n != [ ]) modulesSchema
);
in
inventorySchema
// {
properties = inventorySchema.properties // {
services = {
type = "object";
additionalProperties = false;
properties = moduleServices;
};
};
};
in
{
/*
The abstract inventory without the exact schema for each module filled
InventorySchema<T extends Any> :: {
serviceConfig :: dict[str, T];
}
*/
abstractSchema = inventorySchema;
/*
The inventory with each module schema filled.
InventorySchema<T extends ModuleSchema> :: {
${serviceConfig} :: T; # <- each concrete module name is filled
}
*/
schemaWithModules = schema;
inherit modulesSchema;
}

1
lib/inventory/schemas/default.nix
View File

@@ -23,6 +23,7 @@ let
};
inventorySchema = jsonLib.parseModule (import ../build-inventory/interface.nix);
renderSchema = pkgs.writers.writePython3Bin "render-schema" {
flakeIgnore = [
"F401"

27
lib/inventory/schemas/render_schema.py
View File

@@ -36,13 +36,19 @@ def service_roles_to_schema(
schema: dict[str, Any],
service_name: str,
roles: list[str],
roles_schemas: list[dict[str, Any]],
roles_schemas: dict[str, dict[str, Any]],
# Original service properties: {'config': Schema, 'machines': Schema, 'meta': Schema, 'extraModules': Schema, ...?}
orig: dict[str, Any],
) -> dict[str, Any]:
"""
Add roles to the service schema
"""
# collect all the roles for the service, to form a type union
all_roles_schema: list[dict[str, Any]] = []
for role_name, role_schema in roles_schemas.items():
role_schema["title"] = f"{module_name}-config-role-{role_name}"
all_roles_schema.append(role_schema)
role_schema = {}
for role in roles:
role_schema[role] = {
@@ -51,8 +57,8 @@ def service_roles_to_schema(
"properties": {
**orig["roles"]["additionalProperties"]["properties"],
"config": {
**roles_schemas.get(role, {}),
"title": f"{service_name}-config-role-{role}",
"oneOf": roles_schemas,
"type": "object",
"default": {},
"additionalProperties": False,
@@ -68,7 +74,7 @@ def service_roles_to_schema(
**orig["machines"]["additionalProperties"]["properties"],
"config": {
"title": f"{service_name}-config",
"oneOf": roles_schemas,
"oneOf": all_roles_schema,
"type": "object",
"default": {},
"additionalProperties": False,
@@ -95,7 +101,7 @@ def service_roles_to_schema(
"machines": machines_schema,
"config": {
"title": f"{service_name}-config",
"oneOf": roles_schemas,
"oneOf": all_roles_schema,
"type": "object",
"default": {},
"additionalProperties": False,
@@ -134,17 +140,10 @@ if __name__ == "__main__":
"additionalProperties": False,
}
for module_name, roles_schema in modules_schema.items():
# collect all the roles for the service
roles_schemas = []
for role_name, role_schema in roles_schema.items():
role_schema["title"] = f"{module_name}-config-role-{role_name}"
roles_schemas.append(role_schema)
for module_name, roles_schemas in modules_schema.items():
# Add the roles schemas to the service schema
if roles_schemas:
roles = list(roles_schema.keys())
roles = list(roles_schemas.keys())
if roles:
services = service_roles_to_schema(
services,
module_name,