From f6544d1cda0b51a166de845a5742d1f206d2b7e6 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 20 May 2025 15:48:28 +0200 Subject: [PATCH 1/7] Feat(jsonschema): convert deferredModule to unknown --- lib/jsonschema/default.nix | 11 ++++++++ .../clan_cli/tests/test_deserializers.py | 28 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/lib/jsonschema/default.nix b/lib/jsonschema/default.nix index b0bad30de..094407869 100644 --- a/lib/jsonschema/default.nix +++ b/lib/jsonschema/default.nix @@ -304,6 +304,17 @@ rec { # return jsonschema property definition for raw then exposedModuleInfo // default // example // description // { type = allBasicTypes; } + else if + # This is a special case for the deferred clan.service 'settings', we assume it is JSON serializable + # To get the type of a Deferred modules we need to know the interface of the place where it is evaluated. + # i.e. in case of a clan.service this is the interface of the service which dynamically changes depending on the service + # We assign "type" = [] + # This means any value is valid — or like TypeScript’s unknown. + # We can assign the type later, when we know the exact interface. + # tsType = "unknown" is a type that we preload for json2ts, such that it gets the correct type in typescript + (option.type.name == "deferredModule") + then + exposedModuleInfo // default // example // description // { tsType = "unknown"; } # parse enum else if option.type.name == "enum" diff --git a/pkgs/clan-cli/clan_cli/tests/test_deserializers.py b/pkgs/clan-cli/clan_cli/tests/test_deserializers.py index 0bc64d0fd..58b24c36b 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_deserializers.py +++ b/pkgs/clan-cli/clan_cli/tests/test_deserializers.py @@ -294,3 +294,31 @@ def test_enum_roundtrip() -> None: assert from_dict(Person, data2) == expected2 assert dataclass_to_dict(expected2) == data2 + + +# for the test below +# we would import this from the nix_models +class Unknown: + pass + + +def test_unknown_deserialize() -> None: + @dataclass + class Person: + name: Unknown + + data = {"name": ["a", "b"]} + + person = from_dict(Person, data) + person.name = ["a", "b"] + + +def test_unknown_serialize() -> None: + @dataclass + class Person: + name: Unknown + + data = Person(["a", "b"]) # type: ignore + + person = dataclass_to_dict(data) + assert person == {"name": ["a", "b"]} From 42de68966b22ea7cc3480f29a5603812adde2984 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 20 May 2025 15:49:56 +0200 Subject: [PATCH 2/7] feat(classgen): add support for unknown types --- pkgs/clan-cli/clan_lib/api/serde.py | 5 +++++ pkgs/clan-cli/clan_lib/api/util.py | 7 +++++++ pkgs/classgen/main.py | 20 ++++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/pkgs/clan-cli/clan_lib/api/serde.py b/pkgs/clan-cli/clan_lib/api/serde.py index 59a611cbe..aeefe9402 100644 --- a/pkgs/clan-cli/clan_lib/api/serde.py +++ b/pkgs/clan-cli/clan_lib/api/serde.py @@ -30,6 +30,7 @@ Note: This module assumes the presence of other modules and classes such as `Cla """ import dataclasses +import inspect from dataclasses import dataclass, fields, is_dataclass from enum import Enum from pathlib import Path @@ -261,6 +262,10 @@ def construct_value( return t(field_value) # type: ignore + if inspect.isclass(t) and t.__name__ == "Unknown": + # Return the field value as is + return field_value + msg = f"Unhandled field type {t} with value {field_value}" raise ClanError(msg) diff --git a/pkgs/clan-cli/clan_lib/api/util.py b/pkgs/clan-cli/clan_lib/api/util.py index d06de4f0d..d4ac6a206 100644 --- a/pkgs/clan-cli/clan_lib/api/util.py +++ b/pkgs/clan-cli/clan_lib/api/util.py @@ -1,5 +1,6 @@ import copy import dataclasses +import inspect import pathlib from dataclasses import MISSING from enum import EnumType @@ -110,6 +111,12 @@ def type_to_dict( if t is None: return {"type": "null"} + if inspect.isclass(t) and t.__name__ == "Unknown": + # Empty should represent unknown + # We don't know anything about this type + # Nor about the nested fields, if there are any + return {} + if dataclasses.is_dataclass(t): fields = dataclasses.fields(t) properties = {} diff --git a/pkgs/classgen/main.py b/pkgs/classgen/main.py index 856b1c545..4f01a3645 100644 --- a/pkgs/classgen/main.py +++ b/pkgs/classgen/main.py @@ -1,12 +1,15 @@ # ruff: noqa: RUF001 import argparse import json +import logging import sys from collections.abc import Callable from functools import partial from pathlib import Path from typing import Any +logger = logging.getLogger(__name__) + class Error(Exception): pass @@ -130,6 +133,12 @@ def field_def_from_default_value( finalize_field: Callable[..., tuple[str, str]], ) -> tuple[str, str] | None: # default_value = prop_info.get("default") + if "Unknown" in field_types: + # Unknown type, doesnt matter what the default value is + # type Unknown | a -> Unknown + return finalize_field( + field_types=field_types, + ) if default_value is None: return finalize_field( field_types=field_types | {"None"}, @@ -237,6 +246,9 @@ def generate_dataclass( if not prop_type and not union_variants and not enum_variants: msg = f"Type not found for property {prop} {prop_info}" raise Error(msg) + msg = f"Type not found for property {prop} {prop_info}.\nConverting to unknown type.\n" + logger.warning(msg) + prop_type = "Unknown" if union_variants: field_types = map_json_type(union_variants) @@ -283,6 +295,8 @@ def generate_dataclass( ) ) known_classes.add(nested_class_name) + elif prop_type == "Unknown": + field_types = {"Unknown"} else: field_types = map_json_type( prop_type, @@ -388,6 +402,12 @@ def run_gen(args: argparse.Namespace) -> None: # ruff: noqa: F401 # fmt: off from typing import Any, Literal, NotRequired, TypedDict\n + +# Mimic "unknown". +# 'Any' is unsafe because it allows any operations +# This forces the user to use type-narrowing or casting in the code +class Unknown: + pass """ ) f.write(dataclass_code) From d842a1378990efa0f7465ea225458251dec09ad3 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 20 May 2025 15:51:10 +0200 Subject: [PATCH 3/7] feat(classgen): defer NotRequired type wrapper --- pkgs/classgen/main.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pkgs/classgen/main.py b/pkgs/classgen/main.py index 4f01a3645..0a2e71776 100644 --- a/pkgs/classgen/main.py +++ b/pkgs/classgen/main.py @@ -198,7 +198,7 @@ def get_field_def( if "None" in field_types: field_types.remove("None") serialised_types = " | ".join(field_types) + type_appendix - serialised_types = f"NotRequired[{serialised_types}]" + serialised_types = f"{serialised_types}" else: serialised_types = " | ".join(field_types) + type_appendix @@ -227,7 +227,7 @@ def generate_dataclass( field_name = prop.replace("-", "_") if len(attr_path) == 0 and prop not in attrs: - field_def = field_name, "NotRequired[dict[str, Any]]" + field_def = field_name, "dict[str, Any]" fields_with_default.append(field_def) # breakpoint() continue @@ -244,8 +244,6 @@ def generate_dataclass( nested_class_name = f"""{class_name if class_name != root_class and not prop_info.get("title") else ""}{title_sanitized}""" if not prop_type and not union_variants and not enum_variants: - msg = f"Type not found for property {prop} {prop_info}" - raise Error(msg) msg = f"Type not found for property {prop} {prop_info}.\nConverting to unknown type.\n" logger.warning(msg) prop_type = "Unknown" @@ -361,8 +359,9 @@ def generate_dataclass( # Join field name with type to form a complete field declaration # e.g. "name: str" - all_field_declarations = [ - f"{n}: {t}" for n, t in (required_fields + fields_with_default) + all_field_declarations = [f"{n}: {t}" for n, t in (required_fields)] + [ + f"{n}: NotRequired[{class_name}{n.capitalize()}Type]" + for n, t in (fields_with_default) ] hoisted_types: str = "\n".join( [ @@ -373,14 +372,13 @@ def generate_dataclass( fields_str = "\n ".join(all_field_declarations) nested_classes_str = "\n\n".join(nested_classes) - class_def = f"\nclass {class_name}(TypedDict):\n" + class_def = f"\n\n{hoisted_types}\n" + class_def += f"\nclass {class_name}(TypedDict):\n" if not required_fields + fields_with_default: class_def += " pass" else: class_def += f" {fields_str}" - class_def += f"\n\n{hoisted_types}\n" - return f"{nested_classes_str}\n\n{class_def}" if nested_classes_str else class_def From bd92170bedd9333d2ad331ae5b27496fe4e73a41 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 20 May 2025 15:51:33 +0200 Subject: [PATCH 4/7] feat(classgen): add 'instances' as top level key --- pkgs/classgen/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/classgen/main.py b/pkgs/classgen/main.py index 0a2e71776..cb94fe7e1 100644 --- a/pkgs/classgen/main.py +++ b/pkgs/classgen/main.py @@ -59,7 +59,7 @@ known_classes = set() root_class = "Inventory" # TODO: make this configurable # For now this only includes static top-level attributes of the inventory. -attrs = ["machines", "meta", "services"] +attrs = ["machines", "meta", "services", "instances"] static: dict[str, str] = {"Service": "dict[str, Any]"} From 6863ce136f6e955914b005bad81346002a684dd0 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 20 May 2025 15:53:02 +0200 Subject: [PATCH 5/7] chore(inventory/tags): remove unsupported settings from tags interface --- lib/inventory/build-inventory/interface.nix | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/lib/inventory/build-inventory/interface.nix b/lib/inventory/build-inventory/interface.nix index 63bebb21c..faf0e3c09 100644 --- a/lib/inventory/build-inventory/interface.nix +++ b/lib/inventory/build-inventory/interface.nix @@ -402,14 +402,7 @@ in default = { }; }; tags = lib.mkOption { - type = types.attrsOf ( - types.submodule { - options.settings = lib.mkOption { - default = { }; - type = types.deferredModule; - }; - } - ); + type = types.attrsOf (types.submodule { }); default = { }; }; settings = lib.mkOption { From 40e34d9307c1b9c2b09d072245b649755b357618 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 20 May 2025 15:53:43 +0200 Subject: [PATCH 6/7] feat(docs,api): expose inventory.instances interface" --- lib/inventory/build-inventory/interface.nix | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/inventory/build-inventory/interface.nix b/lib/inventory/build-inventory/interface.nix index faf0e3c09..542ce8818 100644 --- a/lib/inventory/build-inventory/interface.nix +++ b/lib/inventory/build-inventory/interface.nix @@ -359,9 +359,6 @@ in }; instances = lib.mkOption { - # Keep as internal until all de-/serialization issues are resolved - visible = false; - internal = true; description = "Multi host service module instances"; type = types.attrsOf ( types.submodule { From 228cdfd0e9b97e397dda93572a5d93eadd914db5 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 20 May 2025 15:54:32 +0200 Subject: [PATCH 7/7] Chore(nix_models): update models after refactoring for inventory.instances --- .../clan-cli/clan_lib/nix_models/inventory.py | 121 +++++++++++++----- 1 file changed, 86 insertions(+), 35 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/nix_models/inventory.py b/pkgs/clan-cli/clan_lib/nix_models/inventory.py index 69814941c..628ce0c72 100644 --- a/pkgs/clan-cli/clan_lib/nix_models/inventory.py +++ b/pkgs/clan-cli/clan_lib/nix_models/inventory.py @@ -8,53 +8,104 @@ from typing import Any, Literal, NotRequired, TypedDict +# Mimic "unknown". +# 'Any' is unsafe because it allows any operations +# This forces the user to use type-narrowing or casting in the code +class Unknown: + pass + + +InstanceModuleNameType = str +InstanceModuleInputType = str + +class InstanceModule(TypedDict): + name: str + input: NotRequired[InstanceModuleInputType] + + + +InstanceRoleMachineSettingsType = Unknown + +class InstanceRoleMachine(TypedDict): + settings: NotRequired[InstanceRoleMachineSettingsType] + + + + + +class InstanceRoleTag(TypedDict): + pass + + + +InstanceRoleMachinesType = dict[str, InstanceRoleMachine] +InstanceRoleSettingsType = Unknown +InstanceRoleTagsType = dict[str, InstanceRoleTag] + +class InstanceRole(TypedDict): + machines: NotRequired[InstanceRoleMachinesType] + settings: NotRequired[InstanceRoleSettingsType] + tags: NotRequired[InstanceRoleTagsType] + + + +InstanceModuleType = InstanceModule +InstanceRolesType = dict[str, InstanceRole] + +class Instance(TypedDict): + module: NotRequired[InstanceModuleType] + roles: NotRequired[InstanceRolesType] + + + +MachineDeployTargethostType = str + class MachineDeploy(TypedDict): - targetHost: NotRequired[str] - -MachineDeployTargethostType = NotRequired[str] + targetHost: NotRequired[MachineDeployTargethostType] +MachineDeployType = MachineDeploy +MachineDescriptionType = str +MachineIconType = str +MachineMachineclassType = Literal["nixos", "darwin"] +MachineNameType = str +MachineTagsType = list[str] + class Machine(TypedDict): - deploy: NotRequired[MachineDeploy] - description: NotRequired[str] - icon: NotRequired[str] - machineClass: NotRequired[Literal["nixos", "darwin"]] - name: NotRequired[str] - tags: NotRequired[list[str]] - -MachineDeployType = NotRequired[MachineDeploy] -MachineDescriptionType = NotRequired[str] -MachineIconType = NotRequired[str] -MachineMachineclassType = NotRequired[Literal["nixos", "darwin"]] -MachineNameType = NotRequired[str] -MachineTagsType = NotRequired[list[str]] + deploy: NotRequired[MachineDeployType] + description: NotRequired[MachineDescriptionType] + icon: NotRequired[MachineIconType] + machineClass: NotRequired[MachineMachineclassType] + name: NotRequired[MachineNameType] + tags: NotRequired[MachineTagsType] +MetaNameType = str +MetaDescriptionType = str +MetaIconType = str + class Meta(TypedDict): name: str - description: NotRequired[str] - icon: NotRequired[str] - -MetaNameType = str -MetaDescriptionType = NotRequired[str] -MetaIconType = NotRequired[str] - + description: NotRequired[MetaDescriptionType] + icon: NotRequired[MetaIconType] Service = dict[str, Any] + +InventoryInstancesType = dict[str, Instance] +InventoryMachinesType = dict[str, Machine] +InventoryMetaType = Meta +InventoryModulesType = dict[str, Any] +InventoryServicesType = dict[str, Service] +InventoryTagsType = dict[str, Any] + class Inventory(TypedDict): - machines: NotRequired[dict[str, Machine]] - meta: NotRequired[Meta] - modules: NotRequired[dict[str, Any]] - services: NotRequired[dict[str, Service]] - tags: NotRequired[dict[str, Any]] - -InventoryMachinesType = NotRequired[dict[str, Machine]] -InventoryMetaType = NotRequired[Meta] -InventoryModulesType = NotRequired[dict[str, Any]] -InventoryServicesType = NotRequired[dict[str, Service]] -InventoryTagsType = NotRequired[dict[str, Any]] - + instances: NotRequired[InventoryInstancesType] + machines: NotRequired[InventoryMachinesType] + meta: NotRequired[InventoryMetaType] + modules: NotRequired[InventoryModulesType] + services: NotRequired[InventoryServicesType] + tags: NotRequired[InventoryTagsType]