From ad1f3bfa92dc2208390ef0535674682a98bc4a05 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 27 May 2025 10:25:00 +0200 Subject: [PATCH 1/7] fix(clan/interface): minor fixes --- lib/build-clan/interface.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/build-clan/interface.nix b/lib/build-clan/interface.nix index 3d652a72b..ca7a02c65 100644 --- a/lib/build-clan/interface.nix +++ b/lib/build-clan/interface.nix @@ -14,6 +14,7 @@ in _prefix = lib.mkOption { type = types.listOf types.str; internal = true; + visible = false; default = [ ]; }; self = lib.mkOption { @@ -79,6 +80,8 @@ in }; modules = lib.mkOption { + # Correct type would be `types.attrsOf types.deferredModule` but that allows for + # Merging and transforms the value, which add eval overhead. type = types.attrsOf types.raw; default = { }; description = '' From 7957fbaa4feb75fa92610489ba7f26c495119ac1 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 27 May 2025 10:26:19 +0200 Subject: [PATCH 2/7] feat(classgen): make type generation more predictable across --- pkgs/classgen/main.py | 108 +++++++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 49 deletions(-) diff --git a/pkgs/classgen/main.py b/pkgs/classgen/main.py index e8dac154e..bc15fb94c 100644 --- a/pkgs/classgen/main.py +++ b/pkgs/classgen/main.py @@ -3,7 +3,7 @@ import argparse import json import logging import sys -from collections.abc import Callable +from collections.abc import Callable, Iterable from functools import partial from pathlib import Path from typing import Any @@ -15,58 +15,70 @@ class Error(Exception): pass +def sort_types(items: Iterable[str]) -> list[str]: + def sort_key(item: str) -> tuple[int, str]: + # Priority order: lower number = higher priority + if item.startswith(("dict", "list")): + return (0, item) # Highest priority, dicts and lists should be first + if item == "None": + return (2, item) # Lowest priority, None should be last + return (1, item) # Middle priority, sorted alphabetically + + return sorted(items, key=sort_key) + + # Function to map JSON schemas and types to Python types def map_json_type( - json_type: Any, nested_types: set[str] | None = None, parent: Any = None -) -> set[str]: + json_type: Any, nested_types: list[str] | None = None, parent: Any = None +) -> list[str]: if nested_types is None: - nested_types = {"Any"} + nested_types = ["Any"] if isinstance(json_type, list): - res = set() + res: list[str] = [] for t in json_type: - res |= map_json_type(t) - return res + res.extend(map_json_type(t)) + return sort_types(set(res)) if isinstance(json_type, dict): items = json_type.get("items") if items: nested_types = map_json_type(items) - return map_json_type(json_type.get("type"), nested_types) + + if not json_type.get("type") and json_type.get("tsType") == "unknown": + return ["Unknown"] + + return sort_types(map_json_type(json_type.get("type"), nested_types)) if json_type == "string": - return {"str"} + return ["str"] if json_type == "integer": - return {"int"} + return ["int"] if json_type == "number": - return {"float"} + return ["float"] if json_type == "boolean": - return {"bool"} + return ["bool"] # In Python, "number" is analogous to the float type. # https://json-schema.org/understanding-json-schema/reference/numeric#number if json_type == "number": - return {"float"} + return ["float"] if json_type == "array": assert nested_types, f"Array type not found for {parent}" - return {f"""list[{" | ".join(nested_types)}]"""} + return [f"""list[{" | ".join(sort_types(nested_types))}]"""] if json_type == "object": assert nested_types, f"dict type not found for {parent}" - return {f"""dict[str, {" | ".join(nested_types)}]"""} + return [f"""dict[str, {" | ".join(sort_types(nested_types))}]"""] if json_type == "null": - return {"None"} + return ["None"] msg = f"Python type not found for {json_type}" raise Error(msg) known_classes = set() -root_class = "Inventory" # TODO: make this configurable -# For now this only includes static top-level attributes of the inventory. -attrs = ["machines", "meta", "services", "instances"] - -static: dict[str, str] = {"Service": "dict[str, Any]"} +root_class = "Clan" def field_def_from_default_type( field_name: str, - field_types: set[str], + field_types: list[str], class_name: str, finalize_field: Callable[..., tuple[str, str]], ) -> tuple[str, str] | None: @@ -128,7 +140,7 @@ def field_def_from_default_type( def field_def_from_default_value( default_value: Any, field_name: str, - field_types: set[str], + field_types: list[str], nested_class_name: str, finalize_field: Callable[..., tuple[str, str]], ) -> tuple[str, str] | None: @@ -141,7 +153,7 @@ def field_def_from_default_value( ) if default_value is None: return finalize_field( - field_types=field_types | {"None"}, + field_types=[*field_types, "None"], default="None", ) if isinstance(default_value, list): @@ -189,7 +201,7 @@ def field_def_from_default_value( def get_field_def( field_name: str, field_meta: str | None, - field_types: set[str], + field_types: list[str], default: str | None = None, default_factory: str | None = None, type_appendix: str = "", @@ -197,10 +209,10 @@ def get_field_def( if "None" in field_types or default or default_factory: if "None" in field_types: field_types.remove("None") - serialised_types = " | ".join(field_types) + type_appendix + serialised_types = " | ".join(sort_types(field_types)) + type_appendix serialised_types = f"{serialised_types}" else: - serialised_types = " | ".join(field_types) + type_appendix + serialised_types = " | ".join(sort_types(field_types)) + type_appendix return (field_name, serialised_types) @@ -217,26 +229,21 @@ def generate_dataclass( fields_with_default: list[tuple[str, str]] = [] nested_classes: list[str] = [] - # if We are at the top level, and the attribute name is in shallow - # return f"{class_name} = dict[str, Any]" - if class_name in static: - return f"{class_name} = {static[class_name]}" - for prop, prop_info in properties.items(): # If we are at the top level, and the attribute name is not explicitly included we only do shallow field_name = prop.replace("-", "_") - if len(attr_path) == 0 and prop not in attrs: - field_def = field_name, "dict[str, Any]" - fields_with_default.append(field_def) - continue + # if len(attr_path) == 0 and prop in shallow_attrs: + # field_def = field_name, "dict[str, Any]" + # fields_with_default.append(field_def) + # continue prop_type = prop_info.get("type", None) union_variants = prop_info.get("oneOf", []) enum_variants = prop_info.get("enum", []) # Collect all types - field_types = set() + field_types: list[str] = [] title = prop_info.get("title", prop.removesuffix("s")) title_sanitized = "".join([p.capitalize() for p in title.split("-")]) @@ -259,13 +266,13 @@ def generate_dataclass( ) elif enum := prop_info.get("enum"): literals = ", ".join([f'"{string}"' for string in enum]) - field_types = {f"""Literal[{literals}]"""} + field_types = [f"""Literal[{literals}]"""] elif prop_type == "object": inner_type = prop_info.get("additionalProperties") if inner_type and inner_type.get("type") == "object": # Inner type is a class - field_types = map_json_type(prop_type, {nested_class_name}, field_name) + field_types = map_json_type(prop_type, [nested_class_name], field_name) if nested_class_name not in known_classes: nested_classes.append( @@ -278,13 +285,13 @@ def generate_dataclass( elif inner_type and inner_type.get("type") != "object": # Trivial type: # dict[str, inner_type] - field_types = { + field_types = [ f"""dict[str, {" | ".join(map_json_type(inner_type))}]""" - } + ] elif not inner_type: # The type is a class - field_types = {nested_class_name} + field_types = [nested_class_name] if nested_class_name not in known_classes: nested_classes.append( generate_dataclass( @@ -293,11 +300,11 @@ def generate_dataclass( ) known_classes.add(nested_class_name) elif prop_type == "Unknown": - field_types = {"Unknown"} + field_types = ["Unknown"] else: field_types = map_json_type( prop_type, - nested_types=set(), + nested_types=[], parent=field_name, ) @@ -309,6 +316,9 @@ def generate_dataclass( finalize_field = partial(get_field_def, field_name, field_meta) + # Sort and remove potential duplicates + field_types_sorted = sort_types(set(field_types)) + if "default" in prop_info or field_name not in prop_info.get("required", []): if prop_info.get("type") == "object": prop_info.update({"default": {}}) @@ -318,7 +328,7 @@ def generate_dataclass( field_def = field_def_from_default_value( default_value=default_value, field_name=field_name, - field_types=field_types, + field_types=field_types_sorted, nested_class_name=nested_class_name, finalize_field=finalize_field, ) @@ -328,7 +338,7 @@ def generate_dataclass( if not field_def: # Finalize without the default value field_def = finalize_field( - field_types=field_types, + field_types=field_types_sorted, ) required_fields.append(field_def) @@ -337,7 +347,7 @@ def generate_dataclass( # Trying to infer default value from type field_def = field_def_from_default_type( field_name=field_name, - field_types=field_types, + field_types=field_types_sorted, class_name=class_name, finalize_field=finalize_field, ) @@ -346,13 +356,13 @@ def generate_dataclass( fields_with_default.append(field_def) if not field_def: field_def = finalize_field( - field_types=field_types, + field_types=field_types_sorted, ) required_fields.append(field_def) else: field_def = finalize_field( - field_types=field_types, + field_types=field_types_sorted, ) required_fields.append(field_def) From 1ef2e13c851d29ab2847fb1682fe64cae3023070 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 27 May 2025 10:27:17 +0200 Subject: [PATCH 3/7] feat(nix_models): replace inventory model by holistic clan model --- lib/inventory/flake-module.nix | 8 +++++++- lib/inventory/schemas/default.nix | 13 +++++++++---- pkgs/clan-cli/clan_lib/nix_models/update.sh | 4 ++-- pkgs/clan-cli/flake-module.nix | 4 ++-- pkgs/clan-cli/shell.nix | 5 ++--- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/lib/inventory/flake-module.nix b/lib/inventory/flake-module.nix index 9d4f48b08..4c36fb9ee 100644 --- a/lib/inventory/flake-module.nix +++ b/lib/inventory/flake-module.nix @@ -1,4 +1,9 @@ -{ self, inputs, ... }: +{ + self, + inputs, + options, + ... +}: let inputOverrides = builtins.concatStringsSep " " ( builtins.map (input: " --override-input ${input} ${inputs.${input}}") (builtins.attrNames inputs) @@ -28,6 +33,7 @@ in legacyPackages.schemas = ( import ./schemas { + flakeOptions = options; inherit pkgs self diff --git a/lib/inventory/schemas/default.nix b/lib/inventory/schemas/default.nix index 048a03ae9..65d16fe88 100644 --- a/lib/inventory/schemas/default.nix +++ b/lib/inventory/schemas/default.nix @@ -2,6 +2,7 @@ self, self', pkgs, + flakeOptions, ... }: let @@ -21,6 +22,8 @@ let import ../build-inventory/interface.nix { inherit (self) clanLib; } ); + clanSchema = jsonLib.parseOptions (flakeOptions.clan.type.getSubOptions [ "clan" ]) { }; + renderSchema = pkgs.writers.writePython3Bin "render-schema" { flakeIgnore = [ "F401" @@ -28,12 +31,12 @@ let ]; } ./render_schema.py; - inventory-schema-abstract = pkgs.stdenv.mkDerivation { - name = "inventory-schema-files"; + clan-schema-abstract = pkgs.stdenv.mkDerivation { + name = "clan-schema-files"; buildInputs = [ pkgs.cue ]; src = ./.; buildPhase = '' - export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON inventorySchema)} + export SCHEMA=${builtins.toFile "clan-schema.json" (builtins.toJSON clanSchema)} cp $SCHEMA schema.json # Also generate a CUE schema version that is derived from the JSON schema cue import -f -p compose -l '#Root:' schema.json @@ -45,11 +48,13 @@ let in { inherit + flakeOptions frontMatterSchema + clanSchema inventorySchema modulesSchema renderSchema - inventory-schema-abstract + clan-schema-abstract ; # Inventory schema, with the modules schema added per role diff --git a/pkgs/clan-cli/clan_lib/nix_models/update.sh b/pkgs/clan-cli/clan_lib/nix_models/update.sh index 10c44d2cf..123d5d19a 100755 --- a/pkgs/clan-cli/clan_lib/nix_models/update.sh +++ b/pkgs/clan-cli/clan_lib/nix_models/update.sh @@ -2,7 +2,7 @@ set -euo pipefail -jsonSchema=$(nix build .#schemas.inventory-schema-abstract --print-out-paths)/schema.json +clanSchema=$(nix build .#schemas.clan-schema-abstract --print-out-paths)/schema.json SCRIPT_DIR=$(dirname "$0") cd "$SCRIPT_DIR" -nix run .#classgen -- "$jsonSchema" "./inventory.py" +nix run .#classgen -- "$clanSchema" "./clan.py" diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index 95a42ab0f..c4b261606 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -106,10 +106,10 @@ src = ./clan_lib/nix_models; env = { - classFile = "inventory.py"; + classFile = "clan.py"; }; installPhase = '' - ${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json b_classes.py + ${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.clan-schema-abstract}/schema.json b_classes.py file1=$classFile file2=b_classes.py diff --git a/pkgs/clan-cli/shell.nix b/pkgs/clan-cli/shell.nix index 7057c4423..74cab1fce 100644 --- a/pkgs/clan-cli/shell.nix +++ b/pkgs/clan-cli/shell.nix @@ -46,8 +46,7 @@ mkShell { # Add clan command to PATH export PATH="$PKG_ROOT/bin":"$PATH" - # Generate classes.py from inventory schema - # This file is in .gitignore - ${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.inventory-schema-abstract}/schema.json $PKG_ROOT/clan_lib/nix_models/inventory.py + # Generate classes.py from schemas + ${self'.packages.classgen}/bin/classgen ${self'.legacyPackages.schemas.clan-schema-abstract}/schema.json $PKG_ROOT/clan_lib/nix_models/clan.py ''; } From d482e226102101acd1b44f44ba24eed80471dae6 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 27 May 2025 10:27:46 +0200 Subject: [PATCH 4/7] chore(nix_models): migrate import to use exported clan models --- pkgs/clan-cli/clan_cli/clan/show.py | 2 +- pkgs/clan-cli/clan_cli/clan/update.py | 3 +- pkgs/clan-cli/clan_cli/machines/create.py | 8 +- pkgs/clan-cli/clan_cli/machines/list.py | 2 +- pkgs/clan-cli/clan_cli/machines/morph.py | 2 +- .../clan-cli/clan_cli/tests/test_inventory.py | 5 +- pkgs/clan-cli/clan_cli/tests/test_modules.py | 8 +- pkgs/clan-cli/clan_lib/clan/create.py | 2 +- pkgs/clan-cli/clan_lib/inventory/__init__.py | 2 +- pkgs/clan-cli/clan_lib/machines/actions.py | 4 +- pkgs/clan-cli/clan_lib/nix_models/clan.py | 200 ++++++++++++++++++ .../clan-cli/clan_lib/nix_models/inventory.py | 111 ---------- .../clan_lib/persist/inventory_store.py | 2 +- pkgs/clan-cli/clan_lib/tests/test_create.py | 4 +- 14 files changed, 223 insertions(+), 132 deletions(-) create mode 100644 pkgs/clan-cli/clan_lib/nix_models/clan.py delete mode 100644 pkgs/clan-cli/clan_lib/nix_models/inventory.py diff --git a/pkgs/clan-cli/clan_cli/clan/show.py b/pkgs/clan-cli/clan_cli/clan/show.py index 93f24d2e5..f182d396b 100644 --- a/pkgs/clan-cli/clan_cli/clan/show.py +++ b/pkgs/clan-cli/clan_cli/clan/show.py @@ -9,7 +9,7 @@ from clan_lib.cmd import run from clan_lib.errors import ClanCmdError, ClanError from clan_lib.flake import Flake from clan_lib.nix import nix_eval -from clan_lib.nix_models.inventory import Meta +from clan_lib.nix_models.clan import InventoryMeta as Meta log = logging.getLogger(__name__) diff --git a/pkgs/clan-cli/clan_cli/clan/update.py b/pkgs/clan-cli/clan_cli/clan/update.py index e47b002cb..7e1ac3355 100644 --- a/pkgs/clan-cli/clan_cli/clan/update.py +++ b/pkgs/clan-cli/clan_cli/clan/update.py @@ -2,7 +2,8 @@ from dataclasses import dataclass from clan_lib.api import API from clan_lib.flake import Flake -from clan_lib.nix_models.inventory import Inventory, Meta +from clan_lib.nix_models.clan import Inventory +from clan_lib.nix_models.clan import InventoryMeta as Meta from clan_lib.persist.inventory_store import InventoryStore from clan_lib.persist.util import set_value_by_path diff --git a/pkgs/clan-cli/clan_cli/machines/create.py b/pkgs/clan-cli/clan_cli/machines/create.py index 76a358070..9aa8359f8 100644 --- a/pkgs/clan-cli/clan_cli/machines/create.py +++ b/pkgs/clan-cli/clan_cli/machines/create.py @@ -9,12 +9,8 @@ from clan_lib.dirs import get_clan_flake_toplevel_or_env from clan_lib.errors import ClanError from clan_lib.flake import Flake from clan_lib.git import commit_file -from clan_lib.nix_models.inventory import ( - Machine as InventoryMachine, -) -from clan_lib.nix_models.inventory import ( - MachineDeploy, -) +from clan_lib.nix_models.clan import InventoryMachine +from clan_lib.nix_models.clan import InventoryMachineDeploy as MachineDeploy from clan_lib.persist.inventory_store import InventoryStore from clan_lib.persist.util import set_value_by_path from clan_lib.templates import ( diff --git a/pkgs/clan-cli/clan_cli/machines/list.py b/pkgs/clan-cli/clan_cli/machines/list.py index cff1eb1e3..121bcdd56 100644 --- a/pkgs/clan-cli/clan_cli/machines/list.py +++ b/pkgs/clan-cli/clan_cli/machines/list.py @@ -11,7 +11,7 @@ from clan_lib.errors import ClanError from clan_lib.flake import Flake from clan_lib.machines.actions import get_machine from clan_lib.machines.machines import Machine -from clan_lib.nix_models.inventory import Machine as InventoryMachine +from clan_lib.nix_models.clan import InventoryMachine from clan_lib.persist.inventory_store import InventoryStore from clan_cli.completions import add_dynamic_completer, complete_tags diff --git a/pkgs/clan-cli/clan_cli/machines/morph.py b/pkgs/clan-cli/clan_cli/machines/morph.py index f9adb5739..6dbf7f669 100644 --- a/pkgs/clan-cli/clan_cli/machines/morph.py +++ b/pkgs/clan-cli/clan_cli/machines/morph.py @@ -13,7 +13,7 @@ from clan_lib.errors import ClanError from clan_lib.flake import Flake from clan_lib.machines.machines import Machine from clan_lib.nix import nix_build, nix_command -from clan_lib.nix_models.inventory import Machine as InventoryMachine +from clan_lib.nix_models.clan import InventoryMachine from clan_cli.machines.create import CreateOptions, create_machine from clan_cli.vars.generate import generate_vars diff --git a/pkgs/clan-cli/clan_cli/tests/test_inventory.py b/pkgs/clan-cli/clan_cli/tests/test_inventory.py index 773fd284f..d91939e6f 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_inventory.py +++ b/pkgs/clan-cli/clan_cli/tests/test_inventory.py @@ -1,4 +1,7 @@ -from clan_lib.nix_models.inventory import Inventory, Machine, Meta, Service +from clan_lib.nix_models.clan import Inventory +from clan_lib.nix_models.clan import InventoryMachine as Machine +from clan_lib.nix_models.clan import InventoryMeta as Meta +from clan_lib.nix_models.clan import InventoryService as Service def test_make_meta_minimal() -> None: diff --git a/pkgs/clan-cli/clan_cli/tests/test_modules.py b/pkgs/clan-cli/clan_cli/tests/test_modules.py index ca6d2df1b..78e9ab835 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_modules.py +++ b/pkgs/clan-cli/clan_cli/tests/test_modules.py @@ -8,9 +8,11 @@ from clan_cli.tests.fixtures_flakes import FlakeForTest from clan_lib.api.modules import list_modules from clan_lib.flake import Flake from clan_lib.nix import nix_eval, run -from clan_lib.nix_models.inventory import ( - Machine, - MachineDeploy, +from clan_lib.nix_models.clan import ( + InventoryMachine as Machine, +) +from clan_lib.nix_models.clan import ( + InventoryMachineDeploy as MachineDeploy, ) from clan_lib.persist.inventory_store import InventoryStore from clan_lib.persist.util import set_value_by_path diff --git a/pkgs/clan-cli/clan_lib/clan/create.py b/pkgs/clan-cli/clan_lib/clan/create.py index 9922e36e9..4fb7a1a61 100644 --- a/pkgs/clan-cli/clan_lib/clan/create.py +++ b/pkgs/clan-cli/clan_lib/clan/create.py @@ -7,7 +7,7 @@ from clan_lib.cmd import CmdOut, RunOpts, run from clan_lib.errors import ClanError from clan_lib.flake import Flake from clan_lib.nix import nix_command, nix_metadata, nix_shell -from clan_lib.nix_models.inventory import Inventory +from clan_lib.nix_models.clan import Inventory from clan_lib.persist.inventory_store import InventoryStore from clan_lib.templates import ( InputPrio, diff --git a/pkgs/clan-cli/clan_lib/inventory/__init__.py b/pkgs/clan-cli/clan_lib/inventory/__init__.py index 70a64cfe8..8e5114de2 100644 --- a/pkgs/clan-cli/clan_lib/inventory/__init__.py +++ b/pkgs/clan-cli/clan_lib/inventory/__init__.py @@ -13,7 +13,7 @@ Interacting with 'clan_lib.inventory' is NOT recommended and will be removed from clan_lib.api import API from clan_lib.flake import Flake -from clan_lib.nix_models.inventory import Inventory +from clan_lib.nix_models.clan import Inventory from clan_lib.persist.inventory_store import InventoryStore diff --git a/pkgs/clan-cli/clan_lib/machines/actions.py b/pkgs/clan-cli/clan_lib/machines/actions.py index e34a34a08..067b05142 100644 --- a/pkgs/clan-cli/clan_lib/machines/actions.py +++ b/pkgs/clan-cli/clan_lib/machines/actions.py @@ -1,8 +1,8 @@ from clan_lib.api import API from clan_lib.errors import ClanError from clan_lib.machines.machines import Machine -from clan_lib.nix_models.inventory import ( - Machine as InventoryMachine, +from clan_lib.nix_models.clan import ( + InventoryMachine, ) from clan_lib.persist.inventory_store import InventoryStore from clan_lib.persist.util import set_value_by_path diff --git a/pkgs/clan-cli/clan_lib/nix_models/clan.py b/pkgs/clan-cli/clan_lib/nix_models/clan.py new file mode 100644 index 000000000..066ef7a9e --- /dev/null +++ b/pkgs/clan-cli/clan_lib/nix_models/clan.py @@ -0,0 +1,200 @@ +# DO NOT EDIT THIS FILE MANUALLY. IT IS GENERATED. +# This file was generated by running `pkgs/clan-cli/clan_lib.inventory/update.sh` +# +# ruff: noqa: N815 +# ruff: noqa: N806 +# ruff: noqa: F401 +# fmt: off +from typing import Any, Literal, NotRequired, TypedDict + + +# Mimic "unknown". +# 'Any' is unsafe because it allows any operations +# This forces the user to use type-narrowing or casting in the code +class Unknown: + pass + + +InventoryInstanceModuleNameType = str +InventoryInstanceModuleInputType = str + +class InventoryInstanceModule(TypedDict): + name: str + input: NotRequired[InventoryInstanceModuleInputType] + + + +InventoryInstanceRoleMachineSettingsType = Unknown + +class InventoryInstanceRoleMachine(TypedDict): + settings: NotRequired[InventoryInstanceRoleMachineSettingsType] + + + + + +class InventoryInstanceRoleTag(TypedDict): + pass + + + +InventoryInstanceRoleMachinesType = dict[str, InventoryInstanceRoleMachine] +InventoryInstanceRoleSettingsType = Unknown +InventoryInstanceRoleTagsType = dict[str, InventoryInstanceRoleTag] + +class InventoryInstanceRole(TypedDict): + machines: NotRequired[InventoryInstanceRoleMachinesType] + settings: NotRequired[InventoryInstanceRoleSettingsType] + tags: NotRequired[InventoryInstanceRoleTagsType] + + + +InventoryInstanceModuleType = InventoryInstanceModule +InventoryInstanceRolesType = dict[str, InventoryInstanceRole] + +class InventoryInstance(TypedDict): + module: NotRequired[InventoryInstanceModuleType] + roles: NotRequired[InventoryInstanceRolesType] + + + +InventoryMachineDeployTargethostType = str | None + +class InventoryMachineDeploy(TypedDict): + targetHost: NotRequired[InventoryMachineDeployTargethostType] + + + +InventoryMachineDeployType = InventoryMachineDeploy +InventoryMachineDescriptionType = str | None +InventoryMachineIconType = str | None +InventoryMachineMachineclassType = Literal["nixos", "darwin"] +InventoryMachineNameType = str +InventoryMachineTagsType = list[str] + +class InventoryMachine(TypedDict): + deploy: NotRequired[InventoryMachineDeployType] + description: NotRequired[InventoryMachineDescriptionType] + icon: NotRequired[InventoryMachineIconType] + machineClass: NotRequired[InventoryMachineMachineclassType] + name: NotRequired[InventoryMachineNameType] + tags: NotRequired[InventoryMachineTagsType] + + + +InventoryMetaNameType = str +InventoryMetaDescriptionType = str | None +InventoryMetaIconType = str | None + +class InventoryMeta(TypedDict): + name: str + description: NotRequired[InventoryMetaDescriptionType] + icon: NotRequired[InventoryMetaIconType] + + + + + +class InventoryService(TypedDict): + pass + + + +InventoryInstancesType = dict[str, InventoryInstance] +InventoryMachinesType = dict[str, InventoryMachine] +InventoryMetaType = InventoryMeta +InventoryModulesType = dict[str, dict[str, Any] | list[Any] | bool | float | int | str | None] +InventoryServicesType = dict[str, InventoryService] +InventoryTagsType = dict[str, list[str]] + +class Inventory(TypedDict): + instances: NotRequired[InventoryInstancesType] + machines: NotRequired[InventoryMachinesType] + meta: NotRequired[InventoryMetaType] + modules: NotRequired[InventoryModulesType] + services: NotRequired[InventoryServicesType] + tags: NotRequired[InventoryTagsType] + + + +OutputModuleformachineType = dict[str, Unknown] + +class Output(TypedDict): + moduleForMachine: NotRequired[OutputModuleformachineType] + + + +SecretAgePluginsType = list[str] + +class SecretAge(TypedDict): + plugins: NotRequired[SecretAgePluginsType] + + + +SecretAgeType = SecretAge + +class Secret(TypedDict): + age: NotRequired[SecretAgeType] + + + +TemplateClanPathType = str +TemplateClanDescriptionType = str + +class TemplateClan(TypedDict): + path: str + description: NotRequired[TemplateClanDescriptionType] + + + +TemplateDiskoPathType = str +TemplateDiskoDescriptionType = str + +class TemplateDisko(TypedDict): + path: str + description: NotRequired[TemplateDiskoDescriptionType] + + + +TemplateMachinePathType = str +TemplateMachineDescriptionType = str + +class TemplateMachine(TypedDict): + path: str + description: NotRequired[TemplateMachineDescriptionType] + + + +TemplateClanType = dict[str, TemplateClan] +TemplateDiskoType = dict[str, TemplateDisko] +TemplateMachineType = dict[str, TemplateMachine] + +class Template(TypedDict): + clan: NotRequired[TemplateClanType] + disko: NotRequired[TemplateDiskoType] + machine: NotRequired[TemplateMachineType] + + + +ClanDirectoryType = dict[str, Any] | list[Any] | bool | float | int | str +ClanInventoryType = Inventory +ClanMachinesType = dict[str, Unknown] +ClanMetaType = Unknown +ClanModulesType = dict[str, Unknown] +ClanOutputsType = Output +ClanSecretsType = Secret +ClanSelfType = dict[str, Any] | list[Any] | bool | float | int | str +ClanSpecialargsType = dict[str, dict[str, Any] | list[Any] | bool | float | int | str | None] +ClanTemplatesType = Template + +class Clan(TypedDict): + directory: NotRequired[ClanDirectoryType] + inventory: NotRequired[ClanInventoryType] + machines: NotRequired[ClanMachinesType] + meta: NotRequired[ClanMetaType] + modules: NotRequired[ClanModulesType] + outputs: NotRequired[ClanOutputsType] + secrets: NotRequired[ClanSecretsType] + self: NotRequired[ClanSelfType] + specialArgs: NotRequired[ClanSpecialargsType] + templates: NotRequired[ClanTemplatesType] diff --git a/pkgs/clan-cli/clan_lib/nix_models/inventory.py b/pkgs/clan-cli/clan_lib/nix_models/inventory.py deleted file mode 100644 index 1d57a281b..000000000 --- a/pkgs/clan-cli/clan_lib/nix_models/inventory.py +++ /dev/null @@ -1,111 +0,0 @@ -# DO NOT EDIT THIS FILE MANUALLY. IT IS GENERATED. -# This file was generated by running `pkgs/clan-cli/clan_lib.inventory/update.sh` -# -# ruff: noqa: N815 -# ruff: noqa: N806 -# ruff: noqa: F401 -# fmt: off -from typing import Any, Literal, NotRequired, TypedDict - - -# Mimic "unknown". -# 'Any' is unsafe because it allows any operations -# This forces the user to use type-narrowing or casting in the code -class Unknown: - pass - - -InstanceModuleNameType = str -InstanceModuleInputType = str - -class InstanceModule(TypedDict): - name: str - input: NotRequired[InstanceModuleInputType] - - - -InstanceRoleMachineSettingsType = Unknown - -class InstanceRoleMachine(TypedDict): - settings: NotRequired[InstanceRoleMachineSettingsType] - - - - - -class InstanceRoleTag(TypedDict): - pass - - - -InstanceRoleMachinesType = dict[str, InstanceRoleMachine] -InstanceRoleSettingsType = Unknown -InstanceRoleTagsType = dict[str, InstanceRoleTag] - -class InstanceRole(TypedDict): - machines: NotRequired[InstanceRoleMachinesType] - settings: NotRequired[InstanceRoleSettingsType] - tags: NotRequired[InstanceRoleTagsType] - - - -InstanceModuleType = InstanceModule -InstanceRolesType = dict[str, InstanceRole] - -class Instance(TypedDict): - module: NotRequired[InstanceModuleType] - roles: NotRequired[InstanceRolesType] - - - -MachineDeployTargethostType = str - -class MachineDeploy(TypedDict): - targetHost: NotRequired[MachineDeployTargethostType] - - - -MachineDeployType = MachineDeploy -MachineDescriptionType = str -MachineIconType = str -MachineMachineclassType = Literal["nixos", "darwin"] -MachineNameType = str -MachineTagsType = list[str] - -class Machine(TypedDict): - deploy: NotRequired[MachineDeployType] - description: NotRequired[MachineDescriptionType] - icon: NotRequired[MachineIconType] - machineClass: NotRequired[MachineMachineclassType] - name: NotRequired[MachineNameType] - tags: NotRequired[MachineTagsType] - - - -MetaNameType = str -MetaDescriptionType = str -MetaIconType = str - -class Meta(TypedDict): - name: str - description: NotRequired[MetaDescriptionType] - icon: NotRequired[MetaIconType] - -Service = dict[str, Any] - - - -InventoryInstancesType = dict[str, Instance] -InventoryMachinesType = dict[str, Machine] -InventoryMetaType = Meta -InventoryModulesType = dict[str, Any] -InventoryServicesType = dict[str, Service] -InventoryTagsType = dict[str, Any] - -class Inventory(TypedDict): - instances: NotRequired[InventoryInstancesType] - machines: NotRequired[InventoryMachinesType] - meta: NotRequired[InventoryMetaType] - modules: NotRequired[InventoryModulesType] - services: NotRequired[InventoryServicesType] - tags: NotRequired[InventoryTagsType] diff --git a/pkgs/clan-cli/clan_lib/persist/inventory_store.py b/pkgs/clan-cli/clan_lib/persist/inventory_store.py index 898638c8a..92944dfde 100644 --- a/pkgs/clan-cli/clan_lib/persist/inventory_store.py +++ b/pkgs/clan-cli/clan_lib/persist/inventory_store.py @@ -5,7 +5,7 @@ from typing import Any, Protocol from clan_lib.errors import ClanError from clan_lib.git import commit_file -from clan_lib.nix_models.inventory import Inventory +from clan_lib.nix_models.clan import Inventory from .util import ( calc_patches, diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index ff889506e..eff98593a 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -24,8 +24,8 @@ from clan_lib.errors import ClanError from clan_lib.flake import Flake from clan_lib.machines.machines import Machine from clan_lib.nix import nix_command -from clan_lib.nix_models.inventory import Machine as InventoryMachine -from clan_lib.nix_models.inventory import MachineDeploy +from clan_lib.nix_models.clan import InventoryMachine +from clan_lib.nix_models.clan import InventoryMachineDeploy as MachineDeploy from clan_lib.ssh.remote import Remote log = logging.getLogger(__name__) From bac3e6c2b27b9cec54c3bae034723d959cb042bf Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 27 May 2025 10:46:30 +0200 Subject: [PATCH 5/7] Chore: update clan nix_models --- pkgs/clan-cli/clan_lib/nix_models/clan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_lib/nix_models/clan.py b/pkgs/clan-cli/clan_lib/nix_models/clan.py index 066ef7a9e..f68db5c8a 100644 --- a/pkgs/clan-cli/clan_lib/nix_models/clan.py +++ b/pkgs/clan-cli/clan_lib/nix_models/clan.py @@ -180,7 +180,7 @@ ClanDirectoryType = dict[str, Any] | list[Any] | bool | float | int | str ClanInventoryType = Inventory ClanMachinesType = dict[str, Unknown] ClanMetaType = Unknown -ClanModulesType = dict[str, Unknown] +ClanModulesType = dict[str, dict[str, Any] | list[Any] | bool | float | int | str | None] ClanOutputsType = Output ClanSecretsType = Secret ClanSelfType = dict[str, Any] | list[Any] | bool | float | int | str From 104343a334eb615bba0dc775a2c762a949e0d85b Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 27 May 2025 17:08:24 +0200 Subject: [PATCH 6/7] feat(InventoryStore): return a restricted view of the inventory --- pkgs/clan-cli/clan_cli/clan/update.py | 5 +- pkgs/clan-cli/clan_lib/inventory/__init__.py | 5 +- .../clan-cli/clan_lib/nix_models/inventory.py | 111 ++++++++++++++++++ .../clan_lib/persist/inventory_store.py | 44 +++++-- 4 files changed, 148 insertions(+), 17 deletions(-) create mode 100644 pkgs/clan-cli/clan_lib/nix_models/inventory.py diff --git a/pkgs/clan-cli/clan_cli/clan/update.py b/pkgs/clan-cli/clan_cli/clan/update.py index 7e1ac3355..8cf32f1bc 100644 --- a/pkgs/clan-cli/clan_cli/clan/update.py +++ b/pkgs/clan-cli/clan_cli/clan/update.py @@ -2,9 +2,8 @@ from dataclasses import dataclass from clan_lib.api import API from clan_lib.flake import Flake -from clan_lib.nix_models.clan import Inventory from clan_lib.nix_models.clan import InventoryMeta as Meta -from clan_lib.persist.inventory_store import InventoryStore +from clan_lib.persist.inventory_store import InventorySnapshot, InventoryStore from clan_lib.persist.util import set_value_by_path @@ -15,7 +14,7 @@ class UpdateOptions: @API.register -def update_clan_meta(options: UpdateOptions) -> Inventory: +def update_clan_meta(options: UpdateOptions) -> InventorySnapshot: inventory_store = InventoryStore(options.flake) inventory = inventory_store.read() set_value_by_path(inventory, "meta", options.meta) diff --git a/pkgs/clan-cli/clan_lib/inventory/__init__.py b/pkgs/clan-cli/clan_lib/inventory/__init__.py index 8e5114de2..7409ce8fe 100644 --- a/pkgs/clan-cli/clan_lib/inventory/__init__.py +++ b/pkgs/clan-cli/clan_lib/inventory/__init__.py @@ -13,12 +13,11 @@ Interacting with 'clan_lib.inventory' is NOT recommended and will be removed from clan_lib.api import API from clan_lib.flake import Flake -from clan_lib.nix_models.clan import Inventory -from clan_lib.persist.inventory_store import InventoryStore +from clan_lib.persist.inventory_store import InventorySnapshot, InventoryStore @API.register -def get_inventory(flake: Flake) -> Inventory: +def get_inventory(flake: Flake) -> InventorySnapshot: inventory_store = InventoryStore(flake) inventory = inventory_store.read() return inventory diff --git a/pkgs/clan-cli/clan_lib/nix_models/inventory.py b/pkgs/clan-cli/clan_lib/nix_models/inventory.py new file mode 100644 index 000000000..1d57a281b --- /dev/null +++ b/pkgs/clan-cli/clan_lib/nix_models/inventory.py @@ -0,0 +1,111 @@ +# DO NOT EDIT THIS FILE MANUALLY. IT IS GENERATED. +# This file was generated by running `pkgs/clan-cli/clan_lib.inventory/update.sh` +# +# ruff: noqa: N815 +# ruff: noqa: N806 +# ruff: noqa: F401 +# fmt: off +from typing import Any, Literal, NotRequired, TypedDict + + +# Mimic "unknown". +# 'Any' is unsafe because it allows any operations +# This forces the user to use type-narrowing or casting in the code +class Unknown: + pass + + +InstanceModuleNameType = str +InstanceModuleInputType = str + +class InstanceModule(TypedDict): + name: str + input: NotRequired[InstanceModuleInputType] + + + +InstanceRoleMachineSettingsType = Unknown + +class InstanceRoleMachine(TypedDict): + settings: NotRequired[InstanceRoleMachineSettingsType] + + + + + +class InstanceRoleTag(TypedDict): + pass + + + +InstanceRoleMachinesType = dict[str, InstanceRoleMachine] +InstanceRoleSettingsType = Unknown +InstanceRoleTagsType = dict[str, InstanceRoleTag] + +class InstanceRole(TypedDict): + machines: NotRequired[InstanceRoleMachinesType] + settings: NotRequired[InstanceRoleSettingsType] + tags: NotRequired[InstanceRoleTagsType] + + + +InstanceModuleType = InstanceModule +InstanceRolesType = dict[str, InstanceRole] + +class Instance(TypedDict): + module: NotRequired[InstanceModuleType] + roles: NotRequired[InstanceRolesType] + + + +MachineDeployTargethostType = str + +class MachineDeploy(TypedDict): + targetHost: NotRequired[MachineDeployTargethostType] + + + +MachineDeployType = MachineDeploy +MachineDescriptionType = str +MachineIconType = str +MachineMachineclassType = Literal["nixos", "darwin"] +MachineNameType = str +MachineTagsType = list[str] + +class Machine(TypedDict): + deploy: NotRequired[MachineDeployType] + description: NotRequired[MachineDescriptionType] + icon: NotRequired[MachineIconType] + machineClass: NotRequired[MachineMachineclassType] + name: NotRequired[MachineNameType] + tags: NotRequired[MachineTagsType] + + + +MetaNameType = str +MetaDescriptionType = str +MetaIconType = str + +class Meta(TypedDict): + name: str + description: NotRequired[MetaDescriptionType] + icon: NotRequired[MetaIconType] + +Service = dict[str, Any] + + + +InventoryInstancesType = dict[str, Instance] +InventoryMachinesType = dict[str, Machine] +InventoryMetaType = Meta +InventoryModulesType = dict[str, Any] +InventoryServicesType = dict[str, Service] +InventoryTagsType = dict[str, Any] + +class Inventory(TypedDict): + instances: NotRequired[InventoryInstancesType] + machines: NotRequired[InventoryMachinesType] + meta: NotRequired[InventoryMetaType] + modules: NotRequired[InventoryModulesType] + services: NotRequired[InventoryServicesType] + tags: NotRequired[InventoryTagsType] diff --git a/pkgs/clan-cli/clan_lib/persist/inventory_store.py b/pkgs/clan-cli/clan_lib/persist/inventory_store.py index 92944dfde..601b0b3e6 100644 --- a/pkgs/clan-cli/clan_lib/persist/inventory_store.py +++ b/pkgs/clan-cli/clan_lib/persist/inventory_store.py @@ -1,11 +1,17 @@ import json from dataclasses import dataclass from pathlib import Path -from typing import Any, Protocol +from typing import Any, NotRequired, Protocol, TypedDict from clan_lib.errors import ClanError from clan_lib.git import commit_file -from clan_lib.nix_models.clan import Inventory +from clan_lib.nix_models.clan import ( + Inventory, + InventoryInstancesType, + InventoryMachinesType, + InventoryMetaType, + InventoryServicesType, +) from .util import ( calc_patches, @@ -75,8 +81,8 @@ def sanitize(data: Any, whitelist_paths: list[str], current_path: list[str]) -> @dataclass class WriteInfo: writeables: dict[str, set[str]] - data_eval: Inventory - data_disk: Inventory + data_eval: "InventorySnapshot" + data_disk: "InventorySnapshot" class FlakeInterface(Protocol): @@ -92,6 +98,19 @@ class FlakeInterface(Protocol): def path(self) -> Path: ... +class InventorySnapshot(TypedDict): + """ + Restricted view of an Inventory. + + It contains only the keys that are convertible to python types and can be serialized to JSON. + """ + + machines: NotRequired[InventoryMachinesType] + instances: NotRequired[InventoryInstancesType] + meta: NotRequired[InventoryMetaType] + services: NotRequired[InventoryServicesType] + + class InventoryStore: def __init__( self, @@ -117,10 +136,11 @@ class InventoryStore: self._allowed_path_transforms = _allowed_path_transforms if _keys is None: - _keys = ["machines", "instances", "meta", "services"] + _keys = list(InventorySnapshot.__annotations__.keys()) + self._keys = _keys - def _load_merged_inventory(self) -> Inventory: + def _load_merged_inventory(self) -> InventorySnapshot: """ Loads the evaluated inventory. After all merge operations with eventual nix code in buildClan. @@ -140,7 +160,7 @@ class InventoryStore: return sanitized - def _get_persisted(self) -> Inventory: + def _get_persisted(self) -> InventorySnapshot: """ Load the inventory FILE from the flake directory If no file is found, returns an empty dictionary @@ -189,8 +209,8 @@ class InventoryStore: """ current_priority = self._get_inventory_current_priority() - data_eval: Inventory = self._load_merged_inventory() - data_disk: Inventory = self._get_persisted() + data_eval: InventorySnapshot = self._load_merged_inventory() + data_disk: InventorySnapshot = self._get_persisted() writeables = determine_writeability( current_priority, dict(data_eval), dict(data_disk) @@ -198,7 +218,7 @@ class InventoryStore: return WriteInfo(writeables, data_eval, data_disk) - def read(self) -> Inventory: + def read(self) -> InventorySnapshot: """ Accessor to the merged inventory @@ -226,7 +246,9 @@ class InventoryStore: commit_message=f"Delete inventory keys {delete_set}", ) - def write(self, update: Inventory, message: str, commit: bool = True) -> None: + def write( + self, update: InventorySnapshot, message: str, commit: bool = True + ) -> None: """ Write the inventory to the flake directory and commit it to git with the given message From 4b5880d1cbeceaac9097364db2a87de9e6a5d372 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 28 May 2025 14:09:30 +0200 Subject: [PATCH 7/7] Fix(machines/create): use 'InventorySnapshot' as init argument Inventory as a model is generally not json serializable InventorySnapshot is the return value of InventoryStore.{read, write} And should be used in API calls, because the other fields might not be compatible in general --- pkgs/clan-cli/clan_lib/clan/create.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/clan/create.py b/pkgs/clan-cli/clan_lib/clan/create.py index 4fb7a1a61..d9be66f06 100644 --- a/pkgs/clan-cli/clan_lib/clan/create.py +++ b/pkgs/clan-cli/clan_lib/clan/create.py @@ -7,8 +7,7 @@ from clan_lib.cmd import CmdOut, RunOpts, run from clan_lib.errors import ClanError from clan_lib.flake import Flake from clan_lib.nix import nix_command, nix_metadata, nix_shell -from clan_lib.nix_models.clan import Inventory -from clan_lib.persist.inventory_store import InventoryStore +from clan_lib.persist.inventory_store import InventorySnapshot, InventoryStore from clan_lib.templates import ( InputPrio, TemplateName, @@ -35,7 +34,7 @@ class CreateOptions: src_flake: Flake | None = None input_prio: InputPrio | None = None setup_git: bool = True - initial: Inventory | None = None + initial: InventorySnapshot | None = None update_clan: bool = True