diff --git a/lib/inventory/.envrc b/lib/inventory/.envrc deleted file mode 100644 index 22103077a..000000000 --- a/lib/inventory/.envrc +++ /dev/null @@ -1,6 +0,0 @@ -# shellcheck shell=bash -source_up - -watch_file flake-module.nix - -use flake .#inventory-schema --builders '' diff --git a/lib/inventory/examples/borgbackup.json b/lib/inventory/examples/borgbackup.json deleted file mode 100644 index 7e02bde58..000000000 --- a/lib/inventory/examples/borgbackup.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "machines": { - "camina_machine": { - "name": "camina", - "tags": ["laptop"] - }, - "vyr_machine": { - "name": "vyr" - }, - "vi_machine": { - "name": "vi", - "tags": ["laptop"] - } - }, - "meta": { - "name": "kenjis clan" - }, - "services": { - "borgbackup": { - "instance_1": { - "meta": { - "name": "My backup" - }, - "roles": { - "server": { - "machines": ["vyr_machine"] - }, - "client": { - "machines": ["vyr_machine"], - "tags": ["laptop"] - } - }, - "machines": {}, - "config": {} - }, - "instance_2": { - "meta": { - "name": "My backup" - }, - "roles": { - "server": { - "machines": ["vi_machine"] - }, - "client": { - "machines": ["camina_machine"] - } - }, - "machines": {}, - "config": {} - } - } - } -} diff --git a/lib/inventory/flake-module.nix b/lib/inventory/flake-module.nix index 38d14258f..81ac138c6 100644 --- a/lib/inventory/flake-module.nix +++ b/lib/inventory/flake-module.nix @@ -16,22 +16,7 @@ in self', ... }: - let - getSchema = import ./interface-to-schema.nix { inherit lib self; }; - - # The schema for the inventory, without default values, from the module system. - # This is better suited for human reading and for generating code. - bareSchema = getSchema { includeDefaults = false; }; - # The schema for the inventory with default values, from the module system. - # This is better suited for validation, since default values are included. - fullSchema = getSchema { }; - in { - legacyPackages.inventory = { - inherit fullSchema; - inherit bareSchema; - }; - devShells.inventory-schema = pkgs.mkShell { inputsFrom = with config.checks; [ lib-inventory-examples-cue @@ -40,49 +25,12 @@ in ]; }; - # Inventory schema with concrete module implementations - packages.inventory-api-docs = pkgs.stdenv.mkDerivation { - name = "inventory-schema"; - buildInputs = [ ]; - src = ./.; - buildPhase = '' - cat < "$out" - # Inventory API - - *Inventory* is an abstract service layer for consistently configuring distributed services across machine boundaries. - - The following is a specification of the inventory in [cuelang](https://cuelang.org/) format. - - \`\`\`cue - EOF - - cat ${self'.packages.inventory-schema-pretty}/schema.cue >> $out - - cat <> $out - \`\`\` - EOF - ''; - }; - - packages.inventory-schema = pkgs.stdenv.mkDerivation { - name = "inventory-schema"; - buildInputs = [ pkgs.cue ]; - src = ./.; - buildPhase = '' - export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON fullSchema.schemaWithModules)} - cp $SCHEMA schema.json - cue import -f -p compose -l '#Root:' schema.json - mkdir $out - cp schema.cue $out - cp schema.json $out - ''; - }; packages.inventory-schema-abstract = pkgs.stdenv.mkDerivation { name = "inventory-schema"; buildInputs = [ pkgs.cue ]; src = ./.; buildPhase = '' - export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON bareSchema.abstractSchema)} + export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON self'.legacyPackages.schemas.inventory)} cp $SCHEMA schema.json cue import -f -p compose -l '#Root:' schema.json mkdir $out @@ -102,20 +50,6 @@ in } ); - packages.inventory-schema-pretty = pkgs.stdenv.mkDerivation { - name = "inventory-schema-pretty"; - buildInputs = [ pkgs.cue ]; - src = ./.; - buildPhase = '' - export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON bareSchema.schemaWithModules)} - cp $SCHEMA schema.json - cue import -f -p compose -l '#Root:' schema.json - mkdir $out - cp schema.cue $out - cp schema.json $out - ''; - }; - # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests legacyPackages.evalTests-inventory = import ./tests { inherit lib; @@ -123,35 +57,6 @@ in }; checks = { - lib-inventory-examples-cue = pkgs.stdenv.mkDerivation { - name = "inventory-schema-checks"; - src = ./.; - buildInputs = [ pkgs.cue ]; - buildPhase = '' - echo "Running inventory tests..." - # Cue is easier to run in the same directory as the schema - cp ${self'.packages.inventory-schema}/schema.cue root.cue - - ls -la . - - echo "Validate test/*.json against inventory-schema..." - cat root.cue - - test_dir="./examples" - for file in "$test_dir"/*; do - # Check if the item is a file - if [ -f "$file" ]; then - # Print the filename - echo "Running test on: $file" - - # Run the cue vet command - cue vet "$file" root.cue -d "#Root" - fi - done - - touch $out - ''; - }; lib-inventory-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } '' export HOME="$(realpath .)" diff --git a/lib/inventory/interface-to-schema.nix b/lib/inventory/interface-to-schema.nix deleted file mode 100644 index 0c2a7f215..000000000 --- a/lib/inventory/interface-to-schema.nix +++ /dev/null @@ -1,135 +0,0 @@ -{ lib, self, ... }: -{ - includeDefaults ? true, -}: -let - optionsFromModule = - mName: - let - eval = self.lib.evalClanModules [ mName ]; - in - if (eval.options.clan ? "${mName}") then eval.options.clan.${mName} else { }; - - modulesSchema = lib.mapAttrs ( - moduleName: _: jsonLib'.parseOptions (optionsFromModule moduleName) { } - ) self.clanModules; - - jsonLib = self.lib.jsonschema { inherit includeDefaults; }; - jsonLib' = self.lib.jsonschema { - inherit includeDefaults; - header = { }; - }; - inventorySchema = jsonLib.parseModule (import ./build-inventory/interface.nix); - - getRoles = - modulePath: - let - rolesDir = "${modulePath}/roles"; - in - if builtins.pathExists rolesDir then - lib.pipe rolesDir [ - builtins.readDir - (lib.filterAttrs (_n: v: v == "regular")) - lib.attrNames - (map (fileName: lib.removeSuffix ".nix" fileName)) - ] - else - null; - - # The actual schema for the inventory - # !!! We cannot import the module into the interface.nix, because it would cause evaluation overhead. - # Modifies: - # - service...config = moduleSchema - # - service...machine..config = moduleSchema - # - service...roles = acutalRoles - - schema = - let - moduleToService = moduleName: moduleSchema: { - type = "object"; - additionalProperties = { - type = "object"; - additionalProperties = false; - properties = { - meta = { - title = "service-meta"; - } // inventorySchema.properties.services.additionalProperties.additionalProperties.properties.meta; - - config = { - title = "${moduleName}-config"; - default = { }; - } // moduleSchema; - roles = { - type = "object"; - additionalProperties = false; - required = [ ]; - properties = lib.listToAttrs ( - map (role: { - name = role; - value = - lib.recursiveUpdate - inventorySchema.properties.services.additionalProperties.additionalProperties.properties.roles.additionalProperties - { - properties.config = { - title = "${moduleName}-config"; - default = { }; - } // moduleSchema; - }; - }) (rolesOf moduleName) - ); - }; - machines = - lib.recursiveUpdate - inventorySchema.properties.services.additionalProperties.additionalProperties.properties.machines - { - additionalProperties.properties.config = { - title = "${moduleName}-config"; - default = { }; - } // moduleSchema; - }; - }; - }; - }; - - rolesOf = - moduleName: - let - # null | [ string ] - roles = getRoles self.clanModules.${moduleName}; - in - if roles == null then [ ] else roles; - moduleServices = lib.mapAttrs moduleToService ( - lib.filterAttrs (n: _v: rolesOf n != [ ]) modulesSchema - ); - in - inventorySchema - // { - properties = inventorySchema.properties // { - services = { - type = "object"; - additionalProperties = false; - properties = moduleServices; - }; - }; - }; -in -{ - /* - The abstract inventory without the exact schema for each module filled - - InventorySchema :: { - serviceConfig :: dict[str, T]; - } - */ - abstractSchema = inventorySchema; - /* - The inventory with each module schema filled. - - InventorySchema :: { - ${serviceConfig} :: T; # <- each concrete module name is filled - } - */ - schemaWithModules = schema; - - inherit modulesSchema; -} diff --git a/lib/inventory/schemas/default.nix b/lib/inventory/schemas/default.nix index 5ad5291ec..01bc36aaf 100644 --- a/lib/inventory/schemas/default.nix +++ b/lib/inventory/schemas/default.nix @@ -23,6 +23,7 @@ let }; inventorySchema = jsonLib.parseModule (import ../build-inventory/interface.nix); + renderSchema = pkgs.writers.writePython3Bin "render-schema" { flakeIgnore = [ "F401" diff --git a/lib/inventory/schemas/render_schema.py b/lib/inventory/schemas/render_schema.py index f18bd1e61..8f743fc9f 100644 --- a/lib/inventory/schemas/render_schema.py +++ b/lib/inventory/schemas/render_schema.py @@ -36,13 +36,19 @@ def service_roles_to_schema( schema: dict[str, Any], service_name: str, roles: list[str], - roles_schemas: list[dict[str, Any]], + roles_schemas: dict[str, dict[str, Any]], # Original service properties: {'config': Schema, 'machines': Schema, 'meta': Schema, 'extraModules': Schema, ...?} orig: dict[str, Any], ) -> dict[str, Any]: """ Add roles to the service schema """ + # collect all the roles for the service, to form a type union + all_roles_schema: list[dict[str, Any]] = [] + for role_name, role_schema in roles_schemas.items(): + role_schema["title"] = f"{module_name}-config-role-{role_name}" + all_roles_schema.append(role_schema) + role_schema = {} for role in roles: role_schema[role] = { @@ -51,8 +57,8 @@ def service_roles_to_schema( "properties": { **orig["roles"]["additionalProperties"]["properties"], "config": { + **roles_schemas.get(role, {}), "title": f"{service_name}-config-role-{role}", - "oneOf": roles_schemas, "type": "object", "default": {}, "additionalProperties": False, @@ -68,7 +74,7 @@ def service_roles_to_schema( **orig["machines"]["additionalProperties"]["properties"], "config": { "title": f"{service_name}-config", - "oneOf": roles_schemas, + "oneOf": all_roles_schema, "type": "object", "default": {}, "additionalProperties": False, @@ -95,7 +101,7 @@ def service_roles_to_schema( "machines": machines_schema, "config": { "title": f"{service_name}-config", - "oneOf": roles_schemas, + "oneOf": all_roles_schema, "type": "object", "default": {}, "additionalProperties": False, @@ -134,17 +140,10 @@ if __name__ == "__main__": "additionalProperties": False, } - for module_name, roles_schema in modules_schema.items(): - # collect all the roles for the service - roles_schemas = [] - for role_name, role_schema in roles_schema.items(): - role_schema["title"] = f"{module_name}-config-role-{role_name}" - roles_schemas.append(role_schema) - + for module_name, roles_schemas in modules_schema.items(): # Add the roles schemas to the service schema - if roles_schemas: - roles = list(roles_schema.keys()) - + roles = list(roles_schemas.keys()) + if roles: services = service_roles_to_schema( services, module_name, diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index 28601e146..d0747c94d 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -71,7 +71,7 @@ ]; installPhase = '' - ${self'.packages.classgen}/bin/classgen ${self'.packages.inventory-schema}/schema.json ./clan_cli/inventory/classes.py --stop-at "Service" + ${self'.packages.classgen}/bin/classgen ${self'.packages.inventory-schema-abstract}/schema.json ./clan_cli/inventory/classes.py --stop-at "Service" python docs.py reference mkdir -p $out @@ -85,30 +85,21 @@ buildInputs = [ pkgs.python3 - self'.packages.json2ts - # TODO: see postFixup clan-cli/default.nix:L188 - self'.packages.clan-cli.propagatedBuildInputs + pkgs.json2ts ]; installPhase = '' - ${self'.packages.classgen}/bin/classgen ${self'.packages.inventory-schema}/schema.json ./clan_cli/inventory/classes.py --stop-at "Service" + ${self'.packages.classgen}/bin/classgen ${self'.packages.inventory-schema-abstract}/schema.json ./clan_cli/inventory/classes.py --stop-at "Service" mkdir -p $out + # Retrieve python API Typescript types python api.py > $out/API.json - ${self'.packages.json2ts}/bin/json2ts --input $out/API.json > $out/API.ts - ${self'.packages.json2ts}/bin/json2ts --input ${self'.packages.inventory-schema}/schema.json > $out/Inventory.ts - cp ${self'.packages.inventory-schema}/schema.json $out/inventory-schema.json + json2ts --input $out/API.json > $out/API.ts + + # Retrieve python API Typescript types + json2ts --input ${self'.legacyPackages.schemas.inventory}/schema.json > $out/Inventory.ts + cp ${self'.legacyPackages.schemas.inventory}/* $out ''; }; - json2ts = pkgs.buildNpmPackage { - name = "json2ts"; - src = pkgs.fetchFromGitHub { - owner = "bcherny"; - repo = "json-schema-to-typescript"; - rev = "118d6a8e7a5a9397d1d390ce297f127ae674a623"; - hash = "sha256-ldAFfw3E0A0lIJyDSsshgPRPR7OmV/FncPsDhC3waT8="; - }; - npmDepsHash = "sha256-kLKau4SBxI9bMAd7X8/FQfCza2sYl/+0bg2LQcOQIJo="; - }; default = self'.packages.clan-cli; };