Inventory/schemas: use less schema versions

lib/inventory/.envrc

@@ -1,6 +0,0 @@
-# shellcheck shell=bash
-source_up
-
-watch_file flake-module.nix
-
-use flake .#inventory-schema --builders ''

lib/inventory/examples/borgbackup.json

@@ -1,53 +0,0 @@
-{
-  "machines": {
-    "camina_machine": {
-      "name": "camina",
-      "tags": ["laptop"]
-    },
-    "vyr_machine": {
-      "name": "vyr"
-    },
-    "vi_machine": {
-      "name": "vi",
-      "tags": ["laptop"]
-    }
-  },
-  "meta": {
-    "name": "kenjis clan"
-  },
-  "services": {
-    "borgbackup": {
-      "instance_1": {
-        "meta": {
-          "name": "My backup"
-        },
-        "roles": {
-          "server": {
-            "machines": ["vyr_machine"]
-          },
-          "client": {
-            "machines": ["vyr_machine"],
-            "tags": ["laptop"]
-          }
-        },
-        "machines": {},
-        "config": {}
-      },
-      "instance_2": {
-        "meta": {
-          "name": "My backup"
-        },
-        "roles": {
-          "server": {
-            "machines": ["vi_machine"]
-          },
-          "client": {
-            "machines": ["camina_machine"]
-          }
-        },
-        "machines": {},
-        "config": {}
-      }
-    }
-  }
-}

lib/inventory/flake-module.nix

@@ -16,22 +16,7 @@ in
       self',
       ...
     }:
-    let
-      getSchema = import ./interface-to-schema.nix { inherit lib self; };
-
-      # The schema for the inventory, without default values, from the module system.
-      # This is better suited for human reading and for generating code.
-      bareSchema = getSchema { includeDefaults = false; };
-      # The schema for the inventory with default values, from the module system.
-      # This is better suited for validation, since default values are included.
-      fullSchema = getSchema { };
-    in
     {
-      legacyPackages.inventory = {
-        inherit fullSchema;
-        inherit bareSchema;
-      };
-
       devShells.inventory-schema = pkgs.mkShell {
         inputsFrom = with config.checks; [
           lib-inventory-examples-cue
@@ -40,49 +25,12 @@ in
         ];
       };
 
-      # Inventory schema with concrete module implementations
-      packages.inventory-api-docs = pkgs.stdenv.mkDerivation {
-        name = "inventory-schema";
-        buildInputs = [ ];
-        src = ./.;
-        buildPhase = ''
-          cat <<EOF > "$out"
-          # Inventory API
-
-          *Inventory* is an abstract service layer for consistently configuring distributed services across machine boundaries.
-
-          The following is a specification of the inventory in [cuelang](https://cuelang.org/) format.
-
-          \`\`\`cue
-          EOF
-
-          cat ${self'.packages.inventory-schema-pretty}/schema.cue >> $out
-
-          cat <<EOF >> $out
-          \`\`\`
-          EOF
-        '';
-      };
-
-      packages.inventory-schema = pkgs.stdenv.mkDerivation {
-        name = "inventory-schema";
-        buildInputs = [ pkgs.cue ];
-        src = ./.;
-        buildPhase = ''
-          export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON fullSchema.schemaWithModules)}
-          cp $SCHEMA schema.json
-          cue import -f -p compose -l '#Root:' schema.json
-          mkdir $out
-          cp schema.cue $out
-          cp schema.json $out
-        '';
-      };
       packages.inventory-schema-abstract = pkgs.stdenv.mkDerivation {
         name = "inventory-schema";
         buildInputs = [ pkgs.cue ];
         src = ./.;
         buildPhase = ''
-          export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON bareSchema.abstractSchema)}
+          export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON self'.legacyPackages.schemas.inventory)}
           cp $SCHEMA schema.json
           cue import -f -p compose -l '#Root:' schema.json
           mkdir $out
@@ -102,20 +50,6 @@ in
         }
       );
 
-      packages.inventory-schema-pretty = pkgs.stdenv.mkDerivation {
-        name = "inventory-schema-pretty";
-        buildInputs = [ pkgs.cue ];
-        src = ./.;
-        buildPhase = ''
-          export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON bareSchema.schemaWithModules)}
-          cp $SCHEMA schema.json
-          cue import -f -p compose -l '#Root:' schema.json
-          mkdir $out
-          cp schema.cue $out
-          cp schema.json $out
-        '';
-      };
-
       # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests
       legacyPackages.evalTests-inventory = import ./tests {
         inherit lib;
@@ -123,35 +57,6 @@ in
       };
 
       checks = {
-        lib-inventory-examples-cue = pkgs.stdenv.mkDerivation {
-          name = "inventory-schema-checks";
-          src = ./.;
-          buildInputs = [ pkgs.cue ];
-          buildPhase = ''
-            echo "Running inventory tests..."
-            # Cue is easier to run in the same directory as the schema
-            cp ${self'.packages.inventory-schema}/schema.cue root.cue
-
-            ls -la .
-
-            echo "Validate test/*.json against inventory-schema..."
-            cat root.cue
-
-            test_dir="./examples"
-            for file in "$test_dir"/*; do
-              # Check if the item is a file
-              if [ -f "$file" ]; then
-                # Print the filename
-                echo "Running test on: $file"
-
-                # Run the cue vet command
-                cue vet "$file" root.cue -d "#Root"
-              fi
-            done
-
-            touch $out
-          '';
-        };
         lib-inventory-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
           export HOME="$(realpath .)"
 

lib/inventory/interface-to-schema.nix

@@ -1,135 +0,0 @@
-{ lib, self, ... }:
-{
-  includeDefaults ? true,
-}:
-let
-  optionsFromModule =
-    mName:
-    let
-      eval = self.lib.evalClanModules [ mName ];
-    in
-    if (eval.options.clan ? "${mName}") then eval.options.clan.${mName} else { };
-
-  modulesSchema = lib.mapAttrs (
-    moduleName: _: jsonLib'.parseOptions (optionsFromModule moduleName) { }
-  ) self.clanModules;
-
-  jsonLib = self.lib.jsonschema { inherit includeDefaults; };
-  jsonLib' = self.lib.jsonschema {
-    inherit includeDefaults;
-    header = { };
-  };
-  inventorySchema = jsonLib.parseModule (import ./build-inventory/interface.nix);
-
-  getRoles =
-    modulePath:
-    let
-      rolesDir = "${modulePath}/roles";
-    in
-    if builtins.pathExists rolesDir then
-      lib.pipe rolesDir [
-        builtins.readDir
-        (lib.filterAttrs (_n: v: v == "regular"))
-        lib.attrNames
-        (map (fileName: lib.removeSuffix ".nix" fileName))
-      ]
-    else
-      null;
-
-  # The actual schema for the inventory
-  # !!! We cannot import the module into the interface.nix, because it would cause evaluation overhead.
-  # Modifies:
-  # - service.<serviceName>.<instanceName>.config = moduleSchema
-  # - service.<serviceName>.<instanceName>.machine.<machineName>.config = moduleSchema
-  # - service.<serviceName>.<instanceName>.roles = acutalRoles
-
-  schema =
-    let
-      moduleToService = moduleName: moduleSchema: {
-        type = "object";
-        additionalProperties = {
-          type = "object";
-          additionalProperties = false;
-          properties = {
-            meta = {
-              title = "service-meta";
-            } // inventorySchema.properties.services.additionalProperties.additionalProperties.properties.meta;
-
-            config = {
-              title = "${moduleName}-config";
-              default = { };
-            } // moduleSchema;
-            roles = {
-              type = "object";
-              additionalProperties = false;
-              required = [ ];
-              properties = lib.listToAttrs (
-                map (role: {
-                  name = role;
-                  value =
-                    lib.recursiveUpdate
-                      inventorySchema.properties.services.additionalProperties.additionalProperties.properties.roles.additionalProperties
-                      {
-                        properties.config = {
-                          title = "${moduleName}-config";
-                          default = { };
-                        } // moduleSchema;
-                      };
-                }) (rolesOf moduleName)
-              );
-            };
-            machines =
-              lib.recursiveUpdate
-                inventorySchema.properties.services.additionalProperties.additionalProperties.properties.machines
-                {
-                  additionalProperties.properties.config = {
-                    title = "${moduleName}-config";
-                    default = { };
-                  } // moduleSchema;
-                };
-          };
-        };
-      };
-
-      rolesOf =
-        moduleName:
-        let
-          # null | [ string ]
-          roles = getRoles self.clanModules.${moduleName};
-        in
-        if roles == null then [ ] else roles;
-      moduleServices = lib.mapAttrs moduleToService (
-        lib.filterAttrs (n: _v: rolesOf n != [ ]) modulesSchema
-      );
-    in
-    inventorySchema
-    // {
-      properties = inventorySchema.properties // {
-        services = {
-          type = "object";
-          additionalProperties = false;
-          properties = moduleServices;
-        };
-      };
-    };
-in
-{
-  /*
-    The abstract inventory without the exact schema for each module filled
-
-    InventorySchema<T extends Any> :: {
-      serviceConfig :: dict[str, T];
-    }
-  */
-  abstractSchema = inventorySchema;
-  /*
-    The inventory with each module schema filled.
-
-    InventorySchema<T extends ModuleSchema> :: {
-      ${serviceConfig} :: T; # <- each concrete module name is filled
-    }
-  */
-  schemaWithModules = schema;
-
-  inherit modulesSchema;
-}

lib/inventory/schemas/default.nix

@@ -23,6 +23,7 @@ let
   };
 
   inventorySchema = jsonLib.parseModule (import ../build-inventory/interface.nix);
+
   renderSchema = pkgs.writers.writePython3Bin "render-schema" {
     flakeIgnore = [
       "F401"

lib/inventory/schemas/render_schema.py

@@ -36,13 +36,19 @@ def service_roles_to_schema(
     schema: dict[str, Any],
     service_name: str,
     roles: list[str],
-    roles_schemas: list[dict[str, Any]],
+    roles_schemas: dict[str, dict[str, Any]],
     # Original service properties: {'config': Schema, 'machines': Schema, 'meta': Schema, 'extraModules': Schema, ...?}
     orig: dict[str, Any],
 ) -> dict[str, Any]:
     """
     Add roles to the service schema
     """
+    # collect all the roles for the service, to form a type union
+    all_roles_schema: list[dict[str, Any]] = []
+    for role_name, role_schema in roles_schemas.items():
+        role_schema["title"] = f"{module_name}-config-role-{role_name}"
+        all_roles_schema.append(role_schema)
+
     role_schema = {}
     for role in roles:
         role_schema[role] = {
@@ -51,8 +57,8 @@ def service_roles_to_schema(
             "properties": {
                 **orig["roles"]["additionalProperties"]["properties"],
                 "config": {
+                    **roles_schemas.get(role, {}),
                     "title": f"{service_name}-config-role-{role}",
-                    "oneOf": roles_schemas,
                     "type": "object",
                     "default": {},
                     "additionalProperties": False,
@@ -68,7 +74,7 @@ def service_roles_to_schema(
                 **orig["machines"]["additionalProperties"]["properties"],
                 "config": {
                     "title": f"{service_name}-config",
-                    "oneOf": roles_schemas,
+                    "oneOf": all_roles_schema,
                     "type": "object",
                     "default": {},
                     "additionalProperties": False,
@@ -95,7 +101,7 @@ def service_roles_to_schema(
                 "machines": machines_schema,
                 "config": {
                     "title": f"{service_name}-config",
-                    "oneOf": roles_schemas,
+                    "oneOf": all_roles_schema,
                     "type": "object",
                     "default": {},
                     "additionalProperties": False,
@@ -134,17 +140,10 @@ if __name__ == "__main__":
         "additionalProperties": False,
     }
 
-    for module_name, roles_schema in modules_schema.items():
+    for module_name, roles_schemas in modules_schema.items():
-        # collect all the roles for the service
-        roles_schemas = []
-        for role_name, role_schema in roles_schema.items():
-            role_schema["title"] = f"{module_name}-config-role-{role_name}"
-            roles_schemas.append(role_schema)
-
         # Add the roles schemas to the service schema
-        if roles_schemas:
+        roles = list(roles_schemas.keys())
-            roles = list(roles_schema.keys())
+        if roles:
-
             services = service_roles_to_schema(
                 services,
                 module_name,

pkgs/clan-cli/flake-module.nix

@@ -71,7 +71,7 @@
           ];
 
           installPhase = ''
-            ${self'.packages.classgen}/bin/classgen ${self'.packages.inventory-schema}/schema.json ./clan_cli/inventory/classes.py --stop-at "Service"
+            ${self'.packages.classgen}/bin/classgen ${self'.packages.inventory-schema-abstract}/schema.json ./clan_cli/inventory/classes.py --stop-at "Service"
 
             python docs.py reference
             mkdir -p $out
@@ -85,30 +85,21 @@
 
           buildInputs = [
             pkgs.python3
-            self'.packages.json2ts
+            pkgs.json2ts
-            # TODO: see postFixup clan-cli/default.nix:L188
-            self'.packages.clan-cli.propagatedBuildInputs
           ];
 
           installPhase = ''
-            ${self'.packages.classgen}/bin/classgen ${self'.packages.inventory-schema}/schema.json ./clan_cli/inventory/classes.py --stop-at "Service"
+            ${self'.packages.classgen}/bin/classgen ${self'.packages.inventory-schema-abstract}/schema.json ./clan_cli/inventory/classes.py --stop-at "Service"
             mkdir -p $out
+            # Retrieve python API Typescript types
             python api.py > $out/API.json
-            ${self'.packages.json2ts}/bin/json2ts --input $out/API.json > $out/API.ts
+            json2ts --input $out/API.json > $out/API.ts
-            ${self'.packages.json2ts}/bin/json2ts --input ${self'.packages.inventory-schema}/schema.json > $out/Inventory.ts
+
-            cp ${self'.packages.inventory-schema}/schema.json $out/inventory-schema.json
+            # Retrieve python API Typescript types
+            json2ts --input ${self'.legacyPackages.schemas.inventory}/schema.json > $out/Inventory.ts
+            cp ${self'.legacyPackages.schemas.inventory}/* $out
           '';
         };
-        json2ts = pkgs.buildNpmPackage {
-          name = "json2ts";
-          src = pkgs.fetchFromGitHub {
-            owner = "bcherny";
-            repo = "json-schema-to-typescript";
-            rev = "118d6a8e7a5a9397d1d390ce297f127ae674a623";
-            hash = "sha256-ldAFfw3E0A0lIJyDSsshgPRPR7OmV/FncPsDhC3waT8=";
-          };
-          npmDepsHash = "sha256-kLKau4SBxI9bMAd7X8/FQfCza2sYl/+0bg2LQcOQIJo=";
-        };
 
         default = self'.packages.clan-cli;
       };