Merge pull request 'Api/schema: improve types top schema conversion' (#4799) from api-types into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/4799

pkgs/clan-app/ui/src/workflows/Install/Install.stories.tsx

@@ -64,6 +64,7 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
             description: "Name of the gritty",
             prompt_type: "line",
             display: {
+              helperText: null,
               label: "(1) Name",
               group: "User",
               required: true,
@@ -74,6 +75,7 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
             description: "Name of the gritty",
             prompt_type: "line",
             display: {
+              helperText: null,
               label: "(2) Password",
               group: "Root",
               required: true,
@@ -84,6 +86,7 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
             description: "Name of the gritty",
             prompt_type: "line",
             display: {
+              helperText: null,
               label: "(3) Gritty",
               group: "Root",
               required: true,
@@ -99,6 +102,7 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
             description: "Name of the gritty",
             prompt_type: "line",
             display: {
+              helperText: null,
               label: "(4) Name",
               group: "User",
               required: true,
@@ -109,6 +113,7 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
             description: "Name of the gritty",
             prompt_type: "line",
             display: {
+              helperText: null,
               label: "(5) Password",
               group: "Lonely",
               required: true,
@@ -119,6 +124,7 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
             description: "Name of the gritty",
             prompt_type: "line",
             display: {
+              helperText: null,
               label: "(6) Batty",
               group: "Root",
               required: true,
@@ -130,6 +136,7 @@ const mockFetcher: Fetcher = <K extends OperationNames>(
     run_generators: null,
     get_machine_hardware_summary: {
       hardware_config: "nixos-facter",
+      platform: "x86_64-linux",
     },
   };
 

pkgs/clan-app/ui/src/workflows/Install/steps/createInstaller.tsx

@@ -324,9 +324,9 @@ const FlashProgress = () => {
       <img
         src="/logos/usb-stick-min.png"
         alt="usb logo"
-        class="absolute z-0 top-2"
+        class="absolute top-2 z-0"
       />
-      <div class="mb-6 flex w-full max-w-md flex-col items-center gap-3 fg-inv-1 z-10">
+      <div class="z-10 mb-6 flex w-full max-w-md flex-col items-center gap-3 fg-inv-1">
         <Typography
           hierarchy="title"
           size="default"
@@ -338,7 +338,7 @@ const FlashProgress = () => {
         <LoadingBar />
         <Button
           hierarchy="primary"
-          class="w-fit mt-3"
+          class="mt-3 w-fit"
           size="s"
           onClick={handleCancel}
         >

pkgs/clan-app/ui/src/workflows/Install/steps/installSteps.tsx

@@ -653,9 +653,9 @@ const InstallProgress = () => {
       <img
         src="/logos/usb-stick-min.png"
         alt="usb logo"
-        class="absolute z-0 top-2"
+        class="absolute top-2 z-0"
       />
-      <div class="mb-6 flex w-full max-w-md flex-col items-center gap-3 fg-inv-1 z-10">
+      <div class="z-10 mb-6 flex w-full max-w-md flex-col items-center gap-3 fg-inv-1">
         <Typography
           hierarchy="title"
           size="default"
@@ -706,7 +706,7 @@ const InstallProgress = () => {
         </Typography>
         <Button
           hierarchy="primary"
-          class="w-fit mt-3"
+          class="mt-3 w-fit"
           size="s"
           onClick={handleCancel}
         >

pkgs/clan-cli/clan_cli/tests/test_api_dataclass_compat.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from typing import cast
 
 from clan_lib.api import API
-from clan_lib.api.util import JSchemaTypeError, type_to_dict
+from clan_lib.api.type_to_jsonschema import JSchemaTypeError, type_to_dict
 from clan_lib.errors import ClanError
 
 

pkgs/clan-cli/clan_lib/api/__init__.py

@@ -16,7 +16,6 @@ from typing import (
     get_type_hints,
 )
 
-from clan_lib.api.util import JSchemaTypeError
 from clan_lib.async_run import get_current_thread_opkey
 from clan_lib.errors import ClanError
 
@@ -204,7 +203,7 @@ API.register(get_system_file)
     def to_json_schema(self) -> dict[str, Any]:
         from typing import get_type_hints
 
-        from .util import type_to_dict
+        from .type_to_jsonschema import JSchemaTypeError, type_to_dict
 
         api_schema: dict[str, Any] = {
             "$comment": "An object containing API methods. ",
@@ -221,7 +220,9 @@ API.register(get_system_file)
             try:
                 serialized_hints = {
                     key: type_to_dict(
-                        value, scope=name + " argument" if key != "return" else "return"
+                        value,
+                        scope=name + " argument" if key != "return" else "return",
+                        narrow_unsupported_union_types=True,
                     )
                     for key, value in hints.items()
                 }

pkgs/clan-cli/clan_lib/api/type_to_jsonschema.py

@@ -104,7 +104,10 @@ def is_total(typed_dict_class: type) -> bool:
 
 
 def type_to_dict(
-    t: Any, scope: str = "", type_map: dict[TypeVar, type] | None = None
+    t: Any,
+    scope: str = "",
+    type_map: dict[TypeVar, type] | None = None,
+    narrow_unsupported_union_types: bool = False,
 ) -> dict:
     if type_map is None:
         type_map = {}
@@ -148,13 +151,13 @@ def type_to_dict(
             if f.default is MISSING and f.default_factory is MISSING
         }
 
-        # Find intersection
+        # TODO: figure out why we needed to do this
-        intersection = required & required_fields
+        # intersection = required_fields & required
 
         return {
             "type": "object",
             "properties": properties,
-            "required": list(intersection),
+            "required": sorted(required_fields),
             # Dataclasses can only have the specified properties
             "additionalProperties": False,
         }
@@ -162,28 +165,59 @@ def type_to_dict(
     if is_typed_dict(t):
         dict_fields = get_typed_dict_fields(t, scope)
         dict_properties: dict = {}
-        dict_required: list[str] = []
+        explicit_optional: set[str] = set()
+        explicit_required: set[str] = set()
         for field_name, field_type in dict_fields.items():
-            if (
+            # Unwrap special case for "NotRequired" and "Required"
-                not is_type_in_union(field_type, type(None))
+            # A field type that only exist for TypedDicts
-                and get_origin(field_type) is not NotRequired
+            if get_origin(field_type) is NotRequired:
-            ) or get_origin(field_type) is Required:
+                explicit_optional.add(field_name)
-                dict_required.append(field_name)
+
+            if get_origin(field_type) is Required:
+                explicit_required.add(field_name)
 
             dict_properties[field_name] = type_to_dict(
                 field_type, f"{scope} {t.__name__}.{field_name}", type_map
             )
 
+        optional = set(dict_fields) - explicit_optional
         return {
             "type": "object",
             "properties": dict_properties,
-            "required": dict_required if is_total(t) else [],
+            "required": sorted(optional) if is_total(t) else sorted(explicit_required),
             "additionalProperties": False,
         }
 
-    if type(t) is UnionType:
+    origin = get_origin(t)
+    # UnionTypes
+    if type(t) is UnionType or origin is Union:
+        supported = []
+        for arg in get_args(t):
+            try:
+                supported.append(
+                    type_to_dict(arg, scope, type_map, narrow_unsupported_union_types)
+                )
+            except JSchemaTypeError:
+                if narrow_unsupported_union_types:
+                    # If we are narrowing unsupported union types, we skip the error
+                    continue
+                raise
+
+        if len(supported) == 0:
+            msg = f"{scope} - No supported types in Union {t!s}, type_map: {type_map}"
+            raise JSchemaTypeError(msg)
+
+        if len(supported) == 1:
+            # If there's only one supported type, return it directly
+            return supported[0]
+
+        # TODO: it would maybe be better to return 'anyOf' this should work for typescript
+        # But is more correct for JSON Schema validation
+        # i.e. 42 would match all of "int | float" which would be an invalid value for that using "oneOf"
+
+        # If there are multiple supported types, return them as oneOf
         return {
-            "oneOf": [type_to_dict(arg, scope, type_map) for arg in t.__args__],
+            "oneOf": supported,
         }
 
     if isinstance(t, TypeVar):
@@ -221,12 +255,6 @@ def type_to_dict(
             schema = type_to_dict(base_type, scope)  # Generate schema for the base type
             return apply_annotations(schema, metadata)
 
-        if origin is Union:
-            union_types = [type_to_dict(arg, scope, type_map) for arg in t.__args__]
-            return {
-                "oneOf": union_types,
-            }
-
         if origin in {list, set, frozenset, tuple}:
             return {
                 "type": "array",

pkgs/clan-cli/clan_lib/api/type_to_jsonschema_test.py

@@ -0,0 +1,331 @@
+from dataclasses import dataclass, field
+from typing import Any, NotRequired, Required
+
+import pytest
+
+from .type_to_jsonschema import JSchemaTypeError, type_to_dict
+
+
+def test_simple_primitives() -> None:
+    assert type_to_dict(int) == {
+        "type": "integer",
+    }
+    assert type_to_dict(float) == {
+        "type": "number",
+    }
+
+    assert type_to_dict(str) == {
+        "type": "string",
+    }
+
+    assert type_to_dict(bool) == {
+        "type": "boolean",
+    }
+    assert type_to_dict(object) == {
+        "type": "object",
+    }
+
+
+def test_enum_type() -> None:
+    from enum import Enum
+
+    class Color(Enum):
+        RED = "red"
+        GREEN = "green"
+        BLUE = "blue"
+
+    assert type_to_dict(Color) == {
+        "type": "string",
+        "enum": ["red", "green", "blue"],
+    }
+
+
+def test_unsupported_any_types() -> None:
+    with pytest.raises(JSchemaTypeError) as exc_info:
+        type_to_dict(Any)
+    assert "Usage of the Any type is not supported" in str(exc_info.value)
+
+    with pytest.raises(JSchemaTypeError) as exc_info:
+        type_to_dict(list[Any])
+    assert "Usage of the Any type is not supported" in str(exc_info.value)
+
+    # TBD.
+    # with pytest.raises(JSchemaTypeError) as exc_info:
+    #     type_to_dict(dict[str, Any])
+    # assert "Usage of the Any type is not supported" in str(exc_info.value)
+
+    with pytest.raises(JSchemaTypeError) as exc_info:
+        type_to_dict(tuple[Any, ...])
+    assert "Usage of the Any type is not supported" in str(exc_info.value)
+
+    with pytest.raises(JSchemaTypeError) as exc_info:
+        type_to_dict(set[Any])
+    assert "Usage of the Any type is not supported" in str(exc_info.value)
+
+    with pytest.raises(JSchemaTypeError) as exc_info:
+        type_to_dict(str | Any)
+    assert "Usage of the Any type is not supported" in str(exc_info.value)
+
+
+def test_allowed_any_types() -> None:
+    # Object with arbitrary keys
+    assert type_to_dict(dict[str, Any]) == {
+        "type": "object",
+        "additionalProperties": True,
+    }
+
+    # Union where Any is discarded
+    assert type_to_dict(str | Any, narrow_unsupported_union_types=True) == {
+        "type": "string",
+    }
+
+
+def test_simple_union_types() -> None:
+    assert type_to_dict(int | str) == {
+        "oneOf": [
+            {"type": "integer"},
+            {"type": "string"},
+        ]
+    }
+
+    assert type_to_dict(int | str | float) == {
+        "oneOf": [
+            {"type": "integer"},
+            {"type": "string"},
+            {"type": "number"},
+        ]
+    }
+
+    assert type_to_dict(int | str | None) == {
+        "oneOf": [
+            {"type": "integer"},
+            {"type": "string"},
+            {"type": "null"},
+        ]
+    }
+
+
+def test_complex_union_types() -> None:
+    @dataclass
+    class Foo:
+        foo: str
+
+    @dataclass
+    class Bar:
+        bar: str
+
+    assert type_to_dict(Foo | Bar | None) == {
+        "oneOf": [
+            {
+                "type": "object",
+                "properties": {
+                    "foo": {"type": "string"},
+                },
+                "additionalProperties": False,
+                "required": ["foo"],
+            },
+            {
+                "type": "object",
+                "properties": {
+                    "bar": {"type": "string"},
+                },
+                "additionalProperties": False,
+                "required": ["bar"],
+            },
+            {"type": "null"},
+        ]
+    }
+
+
+def test_dataclasses() -> None:
+    # @dataclass
+    # class Example:
+    #     name: str
+    #     value: bool
+
+    # assert type_to_dict(Example) == {
+    #     "type": "object",
+    #     "properties": {
+    #         "name": {"type": "string"},
+    #         "value": {"type": "boolean"},
+    #     },
+    #     "additionalProperties": False,
+    #     "required": [
+    #         "name",
+    #         "value",
+    #     ],
+    # }
+
+    @dataclass
+    class ExampleWithNullable:
+        name: str
+        value: int | None
+
+    assert type_to_dict(ExampleWithNullable) == {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string"},
+            "value": {"oneOf": [{"type": "integer"}, {"type": "null"}]},
+        },
+        "additionalProperties": False,
+        "required": [
+            "name",
+            "value",
+        ],  # value is required because it has no default value
+    }
+
+    @dataclass
+    class ExampleWithOptional:
+        name: str
+        value: int | None = None
+
+    assert type_to_dict(ExampleWithOptional) == {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string"},
+            "value": {"oneOf": [{"type": "integer"}, {"type": "null"}]},
+        },
+        "additionalProperties": False,
+        "required": [
+            "name"
+        ],  # value is optional because it has a default value of None
+    }
+
+
+def test_dataclass_with_optional_fields() -> None:
+    @dataclass
+    class Example:
+        value: dict[str, Any] = field(default_factory=dict)
+
+    assert type_to_dict(Example) == {
+        "type": "object",
+        "properties": {
+            "value": {
+                "type": "object",
+                "additionalProperties": True,
+            },
+        },
+        "additionalProperties": False,
+        "required": [],  # value is optional because it has default factory
+    }
+
+
+def test_nested_open_dicts() -> None:
+    assert type_to_dict(dict[str, dict[str, list[str]]]) == {
+        "type": "object",
+        "additionalProperties": {
+            "type": "object",
+            "additionalProperties": {
+                "type": "array",
+                "items": {"type": "string"},
+            },
+        },
+    }
+
+
+def test_type_variables() -> None:
+    from typing import Generic, TypeVar
+
+    T = TypeVar("T")
+
+    @dataclass
+    class Wrapper(Generic[T]):
+        value: T
+
+    assert type_to_dict(Wrapper[int]) == {
+        "type": "object",
+        "properties": {
+            "value": {"type": "integer"},
+        },
+        "additionalProperties": False,
+        "required": ["value"],
+    }
+
+    assert type_to_dict(Wrapper[str]) == {
+        "type": "object",
+        "properties": {
+            "value": {"type": "string"},
+        },
+        "additionalProperties": False,
+        "required": ["value"],
+    }
+
+
+def test_type_variable_nested_scopes() -> None:
+    # Define two type variables with the same name "T" but in different scopes
+
+    from typing import Generic, TypeVar
+
+    T = TypeVar("T")
+
+    @dataclass
+    class Outer(Generic[T]):
+        foo: T
+
+    @dataclass
+    class Inner(Generic[T]):
+        bar: T
+
+    assert type_to_dict(Outer[Inner[int]]) == {
+        "type": "object",
+        "properties": {
+            "foo": {
+                "type": "object",
+                "properties": {
+                    "bar": {"type": "integer"},
+                },
+                "additionalProperties": False,
+                "required": ["bar"],
+            },
+        },
+        "additionalProperties": False,
+        "required": ["foo"],
+    }
+
+
+def test_total_typed_dict() -> None:
+    from typing import TypedDict
+
+    class ExampleTypedDict(TypedDict):
+        name: str
+        value: NotRequired[int]
+        bar: int | None
+
+    assert type_to_dict(ExampleTypedDict) == {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string"},
+            "value": {"type": "integer"},
+            "bar": {
+                "oneOf": [
+                    {
+                        "type": "integer",
+                    },
+                    {
+                        "type": "null",
+                    },
+                ],
+            },
+        },
+        "additionalProperties": False,
+        # bar is required because it's not explicitly marked as 'NotRequired'
+        "required": ["bar", "name"],
+    }
+
+
+def test_open_typed_dict() -> None:
+    from typing import TypedDict
+
+    class ExampleTypedDict(TypedDict, total=False):
+        name: Required[str]
+        value: int
+
+    assert type_to_dict(ExampleTypedDict) == {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string"},
+            "value": {"type": "integer"},
+        },
+        "additionalProperties": False,
+        "required": ["name"],
+    }