yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'Inventory/schemas: move all schemas to legacyPackages.schemas' (#2234) from hsjobeki/clan-core:hsjobeki-main into main

Browse Source
This commit is contained in:
clan-bot
2024-10-09 11:44:05 +00:00
parent 86fb0bee26 2de5b83ed0
commit e6a1bcadf1
11 changed files with 57 additions and 344 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/site/manual/inventory.md
View File

@@ -128,7 +128,7 @@ It is possible to add services to multiple machines via tags as shown
Or it can build anytime via:
```sh
nix build git+https://git.clan.lol/clan/clan-core#inventory-schema
nix build git+https://git.clan.lol/clan/clan-core#schemas.inventory
> result
> ├── schema.cue
> └── schema.json

5
lib/eval-clan-modules/default.nix
View File

@@ -26,7 +26,7 @@ let
modules = [
baseModule
{
clan.core.clanDir = ./.;
clan.core.clanDir = clan-core;
}
clan-core.nixosModules.clanCore
] ++ (map (name: clanModules.${name}) modulenames);
@@ -90,6 +90,9 @@ let
modules = [
baseModule
clan-core.nixosModules.clanCore
{
clan.core.clanDir = clan-core;
}
# Role interface
(module + "/roles/${role}.nix")
];

6
lib/inventory/.envrc
View File

@@ -1,6 +0,0 @@
# shellcheck shell=bash
source_up
watch_file flake-module.nix
use flake .#inventory-schema --builders ''

53
lib/inventory/examples/borgbackup.json
View File

@@ -1,53 +0,0 @@
{
"machines": {
"camina_machine": {
"name": "camina",
"tags": ["laptop"]
},
"vyr_machine": {
"name": "vyr"
},
"vi_machine": {
"name": "vi",
"tags": ["laptop"]
}
},
"meta": {
"name": "kenjis clan"
},
"services": {
"borgbackup": {
"instance_1": {
"meta": {
"name": "My backup"
},
"roles": {
"server": {
"machines": ["vyr_machine"]
},
"client": {
"machines": ["vyr_machine"],
"tags": ["laptop"]
}
},
"machines": {},
"config": {}
},
"instance_2": {
"meta": {
"name": "My backup"
},
"roles": {
"server": {
"machines": ["vi_machine"]
},
"client": {
"machines": ["camina_machine"]
}
},
"machines": {},
"config": {}
}
}
}
}

110
lib/inventory/flake-module.nix
View File

@@ -16,81 +16,14 @@ in
self',
...
}:
let
getSchema = import ./interface-to-schema.nix { inherit lib self; };
# The schema for the inventory, without default values, from the module system.
# This is better suited for human reading and for generating code.
bareSchema = getSchema { includeDefaults = false; };
# The schema for the inventory with default values, from the module system.
# This is better suited for validation, since default values are included.
fullSchema = getSchema { };
in
{
legacyPackages.inventory = {
inherit fullSchema;
inherit bareSchema;
};
devShells.inventory-schema = pkgs.mkShell {
inputsFrom = with config.checks; [
lib-inventory-examples-cue
lib-inventory-eval
self'.devShells.default
];
};
# Inventory schema with concrete module implementations
packages.inventory-api-docs = pkgs.stdenv.mkDerivation {
name = "inventory-schema";
buildInputs = [ ];
src = ./.;
buildPhase = ''
cat <<EOF > "$out"
# Inventory API
*Inventory* is an abstract service layer for consistently configuring distributed services across machine boundaries.
The following is a specification of the inventory in [cuelang](https://cuelang.org/) format.
\`\`\`cue
EOF
cat ${self'.packages.inventory-schema-pretty}/schema.cue >> $out
cat <<EOF >> $out
\`\`\`
EOF
'';
};
packages.inventory-schema = pkgs.stdenv.mkDerivation {
name = "inventory-schema";
buildInputs = [ pkgs.cue ];
src = ./.;
buildPhase = ''
export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON fullSchema.schemaWithModules)}
cp $SCHEMA schema.json
cue import -f -p compose -l '#Root:' schema.json
mkdir $out
cp schema.cue $out
cp schema.json $out
'';
};
packages.inventory-schema-abstract = pkgs.stdenv.mkDerivation {
name = "inventory-schema";
buildInputs = [ pkgs.cue ];
src = ./.;
buildPhase = ''
export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON bareSchema.abstractSchema)}
cp $SCHEMA schema.json
cue import -f -p compose -l '#Root:' schema.json
mkdir $out
cp schema.cue $out
cp schema.json $out
'';
};
legacyPackages.schemas = (
import ./schemas {
inherit
@@ -102,20 +35,6 @@ in
}
);
packages.inventory-schema-pretty = pkgs.stdenv.mkDerivation {
name = "inventory-schema-pretty";
buildInputs = [ pkgs.cue ];
src = ./.;
buildPhase = ''
export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON bareSchema.schemaWithModules)}
cp $SCHEMA schema.json
cue import -f -p compose -l '#Root:' schema.json
mkdir $out
cp schema.cue $out
cp schema.json $out
'';
};
# Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests
legacyPackages.evalTests-inventory = import ./tests {
inherit lib;
@@ -123,35 +42,6 @@ in
};
checks = {
lib-inventory-examples-cue = pkgs.stdenv.mkDerivation {
name = "inventory-schema-checks";
src = ./.;
buildInputs = [ pkgs.cue ];
buildPhase = ''
echo "Running inventory tests..."
# Cue is easier to run in the same directory as the schema
cp ${self'.packages.inventory-schema}/schema.cue root.cue
ls -la .
echo "Validate test/*.json against inventory-schema..."
cat root.cue
test_dir="./examples"
for file in "$test_dir"/*; do
# Check if the item is a file
if [ -f "$file" ]; then
# Print the filename
echo "Running test on: $file"
# Run the cue vet command
cue vet "$file" root.cue -d "#Root"
fi
done
touch $out
'';
};
lib-inventory-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
export HOME="$(realpath .)"

135
lib/inventory/interface-to-schema.nix
View File

@@ -1,135 +0,0 @@
{ lib, self, ... }:
{
includeDefaults ? true,
}:
let
optionsFromModule =
mName:
let
eval = self.lib.evalClanModules [ mName ];
in
if (eval.options.clan ? "${mName}") then eval.options.clan.${mName} else { };
modulesSchema = lib.mapAttrs (
moduleName: _: jsonLib'.parseOptions (optionsFromModule moduleName) { }
) self.clanModules;
jsonLib = self.lib.jsonschema { inherit includeDefaults; };
jsonLib' = self.lib.jsonschema {
inherit includeDefaults;
header = { };
};
inventorySchema = jsonLib.parseModule (import ./build-inventory/interface.nix);
getRoles =
modulePath:
let
rolesDir = "${modulePath}/roles";
in
if builtins.pathExists rolesDir then
lib.pipe rolesDir [
builtins.readDir
(lib.filterAttrs (_n: v: v == "regular"))
lib.attrNames
(map (fileName: lib.removeSuffix ".nix" fileName))
]
else
null;
# The actual schema for the inventory
# !!! We cannot import the module into the interface.nix, because it would cause evaluation overhead.
# Modifies:
# - service.<serviceName>.<instanceName>.config = moduleSchema
# - service.<serviceName>.<instanceName>.machine.<machineName>.config = moduleSchema
# - service.<serviceName>.<instanceName>.roles = acutalRoles
schema =
let
moduleToService = moduleName: moduleSchema: {
type = "object";
additionalProperties = {
type = "object";
additionalProperties = false;
properties = {
meta = {
title = "service-meta";
} // inventorySchema.properties.services.additionalProperties.additionalProperties.properties.meta;
config = {
title = "${moduleName}-config";
default = { };
} // moduleSchema;
roles = {
type = "object";
additionalProperties = false;
required = [ ];
properties = lib.listToAttrs (
map (role: {
name = role;
value =
lib.recursiveUpdate
inventorySchema.properties.services.additionalProperties.additionalProperties.properties.roles.additionalProperties
{
properties.config = {
title = "${moduleName}-config";
default = { };
} // moduleSchema;
};
}) (rolesOf moduleName)
);
};
machines =
lib.recursiveUpdate
inventorySchema.properties.services.additionalProperties.additionalProperties.properties.machines
{
additionalProperties.properties.config = {
title = "${moduleName}-config";
default = { };
} // moduleSchema;
};
};
};
};
rolesOf =
moduleName:
let
# null | [ string ]
roles = getRoles self.clanModules.${moduleName};
in
if roles == null then [ ] else roles;
moduleServices = lib.mapAttrs moduleToService (
lib.filterAttrs (n: _v: rolesOf n != [ ]) modulesSchema
);
in
inventorySchema
// {
properties = inventorySchema.properties // {
services = {
type = "object";
additionalProperties = false;
properties = moduleServices;
};
};
};
in
{
/*
The abstract inventory without the exact schema for each module filled
InventorySchema<T extends Any> :: {
serviceConfig :: dict[str, T];
}
*/
abstractSchema = inventorySchema;
/*
The inventory with each module schema filled.
InventorySchema<T extends ModuleSchema> :: {
${serviceConfig} :: T; # <- each concrete module name is filled
}
*/
schemaWithModules = schema;
inherit modulesSchema;
}

24
lib/inventory/schemas/default.nix
View File

@@ -23,16 +23,38 @@ let
};
inventorySchema = jsonLib.parseModule (import ../build-inventory/interface.nix);
renderSchema = pkgs.writers.writePython3Bin "render-schema" {
flakeIgnore = [
"F401"
"E501"
];
} ./render_schema.py;
inventory-schema-abstract = pkgs.stdenv.mkDerivation {
name = "inventory-schema-files";
buildInputs = [ pkgs.cue ];
src = ./.;
buildPhase = ''
export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON inventorySchema)}
cp $SCHEMA schema.json
# Also generate a CUE schema version that is derived from the JSON schema
cue import -f -p compose -l '#Root:' schema.json
mkdir $out
cp schema.cue $out
cp schema.json $out
'';
};
in
{
inherit inventorySchema modulesSchema renderSchema;
inherit
inventorySchema
modulesSchema
renderSchema
inventory-schema-abstract
;
# Inventory schema, with the modules schema added per role
inventory =
pkgs.runCommand "rendered"
{

27
lib/inventory/schemas/render_schema.py
View File

@@ -36,13 +36,19 @@ def service_roles_to_schema(
schema: dict[str, Any],
service_name: str,
roles: list[str],
roles_schemas: list[dict[str, Any]],
roles_schemas: dict[str, dict[str, Any]],
# Original service properties: {'config': Schema, 'machines': Schema, 'meta': Schema, 'extraModules': Schema, ...?}
orig: dict[str, Any],
) -> dict[str, Any]:
"""
Add roles to the service schema
"""
# collect all the roles for the service, to form a type union
all_roles_schema: list[dict[str, Any]] = []
for role_name, role_schema in roles_schemas.items():
role_schema["title"] = f"{module_name}-config-role-{role_name}"
all_roles_schema.append(role_schema)
role_schema = {}
for role in roles:
role_schema[role] = {
@@ -51,8 +57,8 @@ def service_roles_to_schema(
"properties": {
**orig["roles"]["additionalProperties"]["properties"],
"config": {
**roles_schemas.get(role, {}),
"title": f"{service_name}-config-role-{role}",
"oneOf": roles_schemas,
"type": "object",
"default": {},
"additionalProperties": False,
@@ -68,7 +74,7 @@ def service_roles_to_schema(
**orig["machines"]["additionalProperties"]["properties"],
"config": {
"title": f"{service_name}-config",
"oneOf": roles_schemas,
"oneOf": all_roles_schema,
"type": "object",
"default": {},
"additionalProperties": False,
@@ -95,7 +101,7 @@ def service_roles_to_schema(
"machines": machines_schema,
"config": {
"title": f"{service_name}-config",
"oneOf": roles_schemas,
"oneOf": all_roles_schema,
"type": "object",
"default": {},
"additionalProperties": False,
@@ -134,17 +140,10 @@ if __name__ == "__main__":
"additionalProperties": False,
}
for module_name, roles_schema in modules_schema.items():
# collect all the roles for the service
roles_schemas = []
for role_name, role_schema in roles_schema.items():
role_schema["title"] = f"{module_name}-config-role-{role_name}"
roles_schemas.append(role_schema)
for module_name, roles_schemas in modules_schema.items():
# Add the roles schemas to the service schema
if roles_schemas:
roles = list(roles_schema.keys())
roles = list(roles_schemas.keys())
if roles:
services = service_roles_to_schema(
services,
module_name,

2
pkgs/clan-cli/clan_cli/inventory/update.sh
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env bash
jsonSchema=$(nix build .#inventory-schema-abstract --print-out-paths)/schema.json
jsonSchema=$(nix build .#schemas.inventory-schema-abstract --print-out-paths)/schema.json
nix run .#classgen "$jsonSchema" "$PKG_ROOT/clan_cli/inventory/classes.py"

35
pkgs/clan-cli/flake-module.nix
View File

@@ -47,7 +47,8 @@
packages = {
clan-cli = pkgs.python3.pkgs.callPackage ./default.nix {
inherit (inputs) nixpkgs;
inherit (self'.packages) inventory-schema-abstract classgen;
inherit (self'.packages) classgen;
inherit (self'.legacyPackages.schemas) inventory-schema-abstract;
clan-core-path = clanCoreWithVendoredDeps;
includedRuntimeDeps = [
"age"
@@ -56,7 +57,8 @@
};
clan-cli-full = pkgs.python3.pkgs.callPackage ./default.nix {
inherit (inputs) nixpkgs;
inherit (self'.packages) inventory-schema-abstract classgen;
inherit (self'.packages) classgen;
inherit (self'.legacyPackages.schemas) inventory-schema-abstract;
clan-core-path = clanCoreWithVendoredDeps;
includedRuntimeDeps = lib.importJSON ./clan_cli/nix/allowed-programs.json;
};
@@ -71,7 +73,7 @@
];
installPhase = ''
${self'.packages.classgen}/bin/classgen ${self'.packages.inventory-schema}/schema.json ./clan_cli/inventory/classes.py --stop-at "Service"
${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json ./clan_cli/inventory/classes.py --stop-at "Service"
python docs.py reference
mkdir -p $out
@@ -85,30 +87,21 @@
buildInputs = [
pkgs.python3
self'.packages.json2ts
# TODO: see postFixup clan-cli/default.nix:L188
self'.packages.clan-cli.propagatedBuildInputs
pkgs.json2ts
];
installPhase = ''
${self'.packages.classgen}/bin/classgen ${self'.packages.inventory-schema}/schema.json ./clan_cli/inventory/classes.py --stop-at "Service"
${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json ./clan_cli/inventory/classes.py --stop-at "Service"
mkdir -p $out
# Retrieve python API Typescript types
python api.py > $out/API.json
${self'.packages.json2ts}/bin/json2ts --input $out/API.json > $out/API.ts
${self'.packages.json2ts}/bin/json2ts --input ${self'.packages.inventory-schema}/schema.json > $out/Inventory.ts
cp ${self'.packages.inventory-schema}/schema.json $out/inventory-schema.json
json2ts --input $out/API.json > $out/API.ts
# Retrieve python API Typescript types
json2ts --input ${self'.legacyPackages.schemas.inventory}/schema.json > $out/Inventory.ts
cp ${self'.legacyPackages.schemas.inventory}/* $out
'';
};
json2ts = pkgs.buildNpmPackage {
name = "json2ts";
src = pkgs.fetchFromGitHub {
owner = "bcherny";
repo = "json-schema-to-typescript";
rev = "118d6a8e7a5a9397d1d390ce297f127ae674a623";
hash = "sha256-ldAFfw3E0A0lIJyDSsshgPRPR7OmV/FncPsDhC3waT8=";
};
npmDepsHash = "sha256-kLKau4SBxI9bMAd7X8/FQfCza2sYl/+0bg2LQcOQIJo=";
};
default = self'.packages.clan-cli;
};
@@ -122,7 +115,7 @@
classFile = "classes.py";
};
installPhase = ''
${self'.packages.classgen}/bin/classgen ${self'.packages.inventory-schema-abstract}/schema.json b_classes.py --stop-at "Service"
${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json b_classes.py --stop-at "Service"
file1=$classFile
file2=b_classes.py

2
pkgs/clan-cli/shell.nix
View File

@@ -46,6 +46,6 @@ mkShell {
# Generate classes.py from inventory schema
# This file is in .gitignore
${self'.packages.classgen}/bin/classgen ${self'.packages.inventory-schema-abstract}/schema.json $PKG_ROOT/clan_cli/inventory/classes.py --stop-at "Service"
${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json $PKG_ROOT/clan_cli/inventory/classes.py --stop-at "Service"
'';
}