Merge pull request 'feat(docs,api): expose inventory.instances interface' (#3721) from hsjobeki/clan-core:inventory-services-1 into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3721

lib/inventory/build-inventory/interface.nix

@@ -359,9 +359,6 @@ in
     };
 
     instances = lib.mkOption {
-      # Keep as internal until all de-/serialization issues are resolved
-      visible = false;
-      internal = true;
       description = "Multi host service module instances";
       type = types.attrsOf (
         types.submodule {
@@ -402,14 +399,7 @@ in
                       default = { };
                     };
                     tags = lib.mkOption {
-                      type = types.attrsOf (
-                        types.submodule {
-                          options.settings = lib.mkOption {
-                            default = { };
-                            type = types.deferredModule;
-                          };
-                        }
-                      );
+                      type = types.attrsOf (types.submodule { });
                       default = { };
                     };
                     settings = lib.mkOption {

lib/jsonschema/default.nix

@@ -304,6 +304,17 @@ rec {
     # return jsonschema property definition for raw
     then
       exposedModuleInfo // default // example // description // { type = allBasicTypes; }
+    else if
+      # This is a special case for the deferred clan.service 'settings', we assume it is JSON serializable
+      # To get the type of a Deferred modules we need to know the interface of the place where it is evaluated.
+      # i.e. in case of a clan.service this is the interface of the service which dynamically changes depending on the service
+      # We assign "type" = []
+      # This means any value is valid — or like TypeScript’s unknown.
+      # We can assign the type later, when we know the exact interface.
+      # tsType = "unknown" is a type that we preload for json2ts, such that it gets the correct type in typescript
+      (option.type.name == "deferredModule")
+    then
+      exposedModuleInfo // default // example // description // { tsType = "unknown"; }
     # parse enum
     else if
       option.type.name == "enum"

pkgs/clan-cli/clan_cli/tests/test_deserializers.py

@@ -294,3 +294,31 @@ def test_enum_roundtrip() -> None:
     assert from_dict(Person, data2) == expected2
 
     assert dataclass_to_dict(expected2) == data2
+
+
+# for the test below
+# we would import this from the nix_models
+class Unknown:
+    pass
+
+
+def test_unknown_deserialize() -> None:
+    @dataclass
+    class Person:
+        name: Unknown
+
+    data = {"name": ["a", "b"]}
+
+    person = from_dict(Person, data)
+    person.name = ["a", "b"]
+
+
+def test_unknown_serialize() -> None:
+    @dataclass
+    class Person:
+        name: Unknown
+
+    data = Person(["a", "b"])  # type: ignore
+
+    person = dataclass_to_dict(data)
+    assert person == {"name": ["a", "b"]}

pkgs/clan-cli/clan_lib/api/serde.py

@@ -30,6 +30,7 @@ Note: This module assumes the presence of other modules and classes such as `Cla
 """
 
 import dataclasses
+import inspect
 from dataclasses import dataclass, fields, is_dataclass
 from enum import Enum
 from pathlib import Path
@@ -261,6 +262,10 @@ def construct_value(
 
         return t(field_value)  # type: ignore
 
+    if inspect.isclass(t) and t.__name__ == "Unknown":
+        # Return the field value as is
+        return field_value
+
     msg = f"Unhandled field type {t} with value {field_value}"
     raise ClanError(msg)
 

pkgs/clan-cli/clan_lib/api/util.py

@@ -1,5 +1,6 @@
 import copy
 import dataclasses
+import inspect
 import pathlib
 from dataclasses import MISSING
 from enum import EnumType
@@ -110,6 +111,12 @@ def type_to_dict(
     if t is None:
         return {"type": "null"}
 
+    if inspect.isclass(t) and t.__name__ == "Unknown":
+        # Empty should represent unknown
+        # We don't know anything about this type
+        # Nor about the nested fields, if there are any
+        return {}
+
     if dataclasses.is_dataclass(t):
         fields = dataclasses.fields(t)
         properties = {}

pkgs/clan-cli/clan_lib/nix_models/inventory.py

@@ -8,53 +8,104 @@
 from typing import Any, Literal, NotRequired, TypedDict
 
 
+# Mimic "unknown".
+# 'Any' is unsafe because it allows any operations
+# This forces the user to use type-narrowing or casting in the code
+class Unknown:
+    pass
+
+
+InstanceModuleNameType = str
+InstanceModuleInputType = str
+
+class InstanceModule(TypedDict):
+    name: str
+    input: NotRequired[InstanceModuleInputType]
+
+
+
+InstanceRoleMachineSettingsType = Unknown
+
+class InstanceRoleMachine(TypedDict):
+    settings: NotRequired[InstanceRoleMachineSettingsType]
+
+
+
+
+
+class InstanceRoleTag(TypedDict):
+    pass
+
+
+
+InstanceRoleMachinesType = dict[str, InstanceRoleMachine]
+InstanceRoleSettingsType = Unknown
+InstanceRoleTagsType = dict[str, InstanceRoleTag]
+
+class InstanceRole(TypedDict):
+    machines: NotRequired[InstanceRoleMachinesType]
+    settings: NotRequired[InstanceRoleSettingsType]
+    tags: NotRequired[InstanceRoleTagsType]
+
+
+
+InstanceModuleType = InstanceModule
+InstanceRolesType = dict[str, InstanceRole]
+
+class Instance(TypedDict):
+    module: NotRequired[InstanceModuleType]
+    roles: NotRequired[InstanceRolesType]
+
+
+
+MachineDeployTargethostType = str
+
 class MachineDeploy(TypedDict):
-    targetHost: NotRequired[str]
-
-MachineDeployTargethostType = NotRequired[str]
+    targetHost: NotRequired[MachineDeployTargethostType]
 
 
 
+MachineDeployType = MachineDeploy
+MachineDescriptionType = str
+MachineIconType = str
+MachineMachineclassType = Literal["nixos", "darwin"]
+MachineNameType = str
+MachineTagsType = list[str]
+
 class Machine(TypedDict):
-    deploy: NotRequired[MachineDeploy]
-    description: NotRequired[str]
-    icon: NotRequired[str]
-    machineClass: NotRequired[Literal["nixos", "darwin"]]
-    name: NotRequired[str]
-    tags: NotRequired[list[str]]
-
-MachineDeployType = NotRequired[MachineDeploy]
-MachineDescriptionType = NotRequired[str]
-MachineIconType = NotRequired[str]
-MachineMachineclassType = NotRequired[Literal["nixos", "darwin"]]
-MachineNameType = NotRequired[str]
-MachineTagsType = NotRequired[list[str]]
+    deploy: NotRequired[MachineDeployType]
+    description: NotRequired[MachineDescriptionType]
+    icon: NotRequired[MachineIconType]
+    machineClass: NotRequired[MachineMachineclassType]
+    name: NotRequired[MachineNameType]
+    tags: NotRequired[MachineTagsType]
 
 
 
+MetaNameType = str
+MetaDescriptionType = str
+MetaIconType = str
+
 class Meta(TypedDict):
     name: str
-    description: NotRequired[str]
-    icon: NotRequired[str]
-
-MetaNameType = str
-MetaDescriptionType = NotRequired[str]
-MetaIconType = NotRequired[str]
-
+    description: NotRequired[MetaDescriptionType]
+    icon: NotRequired[MetaIconType]
 
 Service = dict[str, Any]
 
 
+
+InventoryInstancesType = dict[str, Instance]
+InventoryMachinesType = dict[str, Machine]
+InventoryMetaType = Meta
+InventoryModulesType = dict[str, Any]
+InventoryServicesType = dict[str, Service]
+InventoryTagsType = dict[str, Any]
+
 class Inventory(TypedDict):
-    machines: NotRequired[dict[str, Machine]]
-    meta: NotRequired[Meta]
-    modules: NotRequired[dict[str, Any]]
-    services: NotRequired[dict[str, Service]]
-    tags: NotRequired[dict[str, Any]]
-
-InventoryMachinesType = NotRequired[dict[str, Machine]]
-InventoryMetaType = NotRequired[Meta]
-InventoryModulesType = NotRequired[dict[str, Any]]
-InventoryServicesType = NotRequired[dict[str, Service]]
-InventoryTagsType = NotRequired[dict[str, Any]]
-
+    instances: NotRequired[InventoryInstancesType]
+    machines: NotRequired[InventoryMachinesType]
+    meta: NotRequired[InventoryMetaType]
+    modules: NotRequired[InventoryModulesType]
+    services: NotRequired[InventoryServicesType]
+    tags: NotRequired[InventoryTagsType]

pkgs/classgen/main.py

@@ -1,12 +1,15 @@
 # ruff: noqa: RUF001
 import argparse
 import json
+import logging
 import sys
 from collections.abc import Callable
 from functools import partial
 from pathlib import Path
 from typing import Any
 
+logger = logging.getLogger(__name__)
+
 
 class Error(Exception):
     pass
@@ -56,7 +59,7 @@ known_classes = set()
 root_class = "Inventory"
 # TODO: make this configurable
 # For now this only includes static top-level attributes of the inventory.
-attrs = ["machines", "meta", "services"]
+attrs = ["machines", "meta", "services", "instances"]
 
 static: dict[str, str] = {"Service": "dict[str, Any]"}
 
@@ -130,6 +133,12 @@ def field_def_from_default_value(
     finalize_field: Callable[..., tuple[str, str]],
 ) -> tuple[str, str] | None:
     # default_value = prop_info.get("default")
+    if "Unknown" in field_types:
+        # Unknown type, doesnt matter what the default value is
+        # type Unknown | a -> Unknown
+        return finalize_field(
+            field_types=field_types,
+        )
     if default_value is None:
         return finalize_field(
             field_types=field_types | {"None"},
@@ -189,7 +198,7 @@ def get_field_def(
         if "None" in field_types:
             field_types.remove("None")
         serialised_types = " | ".join(field_types) + type_appendix
-        serialised_types = f"NotRequired[{serialised_types}]"
+        serialised_types = f"{serialised_types}"
     else:
         serialised_types = " | ".join(field_types) + type_appendix
 
@@ -218,7 +227,7 @@ def generate_dataclass(
         field_name = prop.replace("-", "_")
 
         if len(attr_path) == 0 and prop not in attrs:
-            field_def = field_name, "NotRequired[dict[str, Any]]"
+            field_def = field_name, "dict[str, Any]"
             fields_with_default.append(field_def)
             # breakpoint()
             continue
@@ -235,8 +244,9 @@ def generate_dataclass(
         nested_class_name = f"""{class_name if class_name != root_class and not prop_info.get("title") else ""}{title_sanitized}"""
 
         if not prop_type and not union_variants and not enum_variants:
-            msg = f"Type not found for property {prop} {prop_info}"
-            raise Error(msg)
+            msg = f"Type not found for property {prop} {prop_info}.\nConverting to unknown type.\n"
+            logger.warning(msg)
+            prop_type = "Unknown"
 
         if union_variants:
             field_types = map_json_type(union_variants)
@@ -283,6 +293,8 @@ def generate_dataclass(
                         )
                     )
                     known_classes.add(nested_class_name)
+        elif prop_type == "Unknown":
+            field_types = {"Unknown"}
         else:
             field_types = map_json_type(
                 prop_type,
@@ -347,8 +359,9 @@ def generate_dataclass(
 
     # Join field name with type to form a complete field declaration
     # e.g. "name: str"
-    all_field_declarations = [
-        f"{n}: {t}" for n, t in (required_fields + fields_with_default)
+    all_field_declarations = [f"{n}: {t}" for n, t in (required_fields)] + [
+        f"{n}: NotRequired[{class_name}{n.capitalize()}Type]"
+        for n, t in (fields_with_default)
     ]
     hoisted_types: str = "\n".join(
         [
@@ -359,14 +372,13 @@ def generate_dataclass(
     fields_str = "\n    ".join(all_field_declarations)
     nested_classes_str = "\n\n".join(nested_classes)
 
-    class_def = f"\nclass {class_name}(TypedDict):\n"
+    class_def = f"\n\n{hoisted_types}\n"
+    class_def += f"\nclass {class_name}(TypedDict):\n"
     if not required_fields + fields_with_default:
         class_def += "    pass"
     else:
         class_def += f"    {fields_str}"
 
-    class_def += f"\n\n{hoisted_types}\n"
-
     return f"{nested_classes_str}\n\n{class_def}" if nested_classes_str else class_def
 
 
@@ -388,6 +400,12 @@ def run_gen(args: argparse.Namespace) -> None:
 # ruff: noqa: F401
 # fmt: off
 from typing import Any, Literal, NotRequired, TypedDict\n
+
+# Mimic "unknown".
+# 'Any' is unsafe because it allows any operations
+# This forces the user to use type-narrowing or casting in the code
+class Unknown:
+    pass
 """
         )
         f.write(dataclass_code)