yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'inventory.{cli,api}: use only dictionaries' (#2572) from hsjobeki/clan-core:hsjobeki-main into main

Browse Source
This commit is contained in:
clan-bot
2024-12-06 20:12:49 +00:00
parent 8fd4d82f1d b52c5beb9d
commit 3e05edbeb4
19 changed files with 263 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
lib/build-clan/module.nix
View File

@@ -183,7 +183,11 @@ in
inherit serviceConfigs;
inherit (clan-core) clanModules;
inherit inventoryFile;
inventoryValuesPrios = (clan-core.lib.values.getPrios { options = inventory.options; });
inventoryValuesPrios =
# Temporary workaround
builtins.removeAttrs (clan-core.lib.values.getPrios { options = inventory.options; })
# tags are freeformType which is not supported yet.
[ "tags" ];
inventory = config.inventory;
meta = config.inventory.meta;

41
lib/values/default.nix
View File

@@ -23,11 +23,9 @@ let
prio = {
__prio = opt.highestPrio;
};
subOptions = opt.type.getSubOptions opt.loc;
filteredSubOptions = filterOptions (opt.type.getSubOptions opt.loc);
attrDefinitions = (lib.modules.mergeAttrDefinitionsWithPrio opt);
zipDefs = builtins.zipAttrsWith (_ns: vs: vs);
defs = zipDefs opt.definitions;
zipDefs = builtins.zipAttrsWith (_: vs: vs);
prioPerValue =
{ type, defs }:
@@ -35,10 +33,13 @@ let
attrName: prioSet:
let
# Evaluate the submodule
options = filterOptions subOptions;
options = filteredSubOptions;
modules = (
[
{ inherit options; }
{
inherit options;
_file = "<artifical submodule>";
}
]
++ map (config: { inherit config; }) defs.${attrName}
);
@@ -48,7 +49,6 @@ let
in
(lib.optionalAttrs (prioSet ? highestPrio) {
__prio = prioSet.highestPrio;
# inherit defs options;
})
// (
if type.nestedTypes.elemType.name == "submodule" then
@@ -64,21 +64,24 @@ let
)
);
attributePrios = prioPerValue {
type = opt.type;
inherit defs;
} attrDefinitions;
submodulePrios =
let
modules = (opt.definitions ++ opt.type.getSubModules);
submoduleEval = lib.evalModules {
inherit modules;
};
in
getPrios { options = filterOptions submoduleEval.options; };
in
if opt ? type && opt.type.name == "submodule" then
prio // (getPrios { options = subOptions; })
(prio) // submodulePrios
else if opt ? type && opt.type.name == "attrsOf" then
# prio // attributePrios
# else if
# opt ? type && opt.type.name == "attrsOf" && opt.type.nestedTypes.elemType.name == "attrsOf"
# then
# prio // attributePrios
# else if opt ? type && opt.type.name == "attrsOf" then
prio // attributePrios
prio
// (prioPerValue {
type = opt.type;
defs = zipDefs opt.definitions;
} (lib.modules.mergeAttrDefinitionsWithPrio opt))
else if opt ? type && opt._type == "option" then
prio
else

86
lib/values/test.nix
View File

@@ -80,6 +80,92 @@ in
};
};
test_submodule_with_merging =
let
evaluated = (
eval [
{
options.foo = lib.mkOption {
type = lib.types.submodule {
options = {
normal = lib.mkOption {
type = lib.types.bool;
};
default = lib.mkOption {
type = lib.types.bool;
};
optionDefault = lib.mkOption {
type = lib.types.bool;
default = true;
};
unset = lib.mkOption {
type = lib.types.bool;
};
};
};
};
}
{
foo.default = lib.mkDefault true;
}
{
foo.normal = false;
}
]
);
in
{
inherit evaluated;
expr = slib.getPrios {
options = evaluated.options;
};
expected = {
foo = {
__prio = 100;
normal.__prio = 100; # Set via other module
default.__prio = 1000;
optionDefault.__prio = 1500;
unset.__prio = 9999;
};
};
};
test_submoduleWith =
let
evaluated = (
eval [
{
options.foo = lib.mkOption {
type = lib.types.submoduleWith {
modules = [
{
options.bar = lib.mkOption {
type = lib.types.bool;
};
}
];
};
};
}
{
foo.bar = false;
}
]
);
in
{
inherit evaluated;
expr = slib.getPrios {
options = evaluated.options;
};
expected = {
foo = {
__prio = 100;
bar.__prio = 100; # Set via other module
};
};
};
# TODO(@hsjobeki): Cover this edge case
# test_freeform =
# let

7
pkgs/clan-cli/clan_cli/api/modules.py
View File

@@ -272,11 +272,14 @@ def set_service_instance(
inventory = load_inventory_json(base_path)
target_type = get_args(get_type_hints(Service)[module_name])[1]
module_instance_map: dict[str, Any] = getattr(inventory.services, module_name, {})
module_instance_map: dict[str, Any] = inventory.get("services", {}).get(
module_name, {}
)
module_instance_map[instance_name] = from_dict(target_type, config)
setattr(inventory.services, module_name, module_instance_map)
inventory["services"] = inventory.get("services", {})
inventory["services"][module_name] = module_instance_map
set_inventory(
inventory, base_path, f"Update {module_name} instance {instance_name}"

9
pkgs/clan-cli/clan_cli/api/serde.py
View File

@@ -42,6 +42,7 @@ from typing import (
Union,
get_args,
get_origin,
is_typeddict,
)
from clan_cli.errors import ClanError
@@ -251,7 +252,13 @@ def construct_value(
if t is Any:
return field_value
# Unhandled
if is_typeddict(t):
if not isinstance(field_value, dict):
msg = f"Expected TypedDict {t}, got {field_value}"
raise ClanError(msg, location=f"{loc}")
return t(field_value) # type: ignore
msg = f"Unhandled field type {t} with value {field_value}"
raise ClanError(msg)

14
pkgs/clan-cli/clan_cli/clan/show.py
View File

@@ -63,9 +63,11 @@ def show_clan_meta(uri: str | Path) -> Meta:
)
return Meta(
name=clan_meta.get("name"),
description=clan_meta.get("description", None),
icon=icon_path,
{
"name": clan_meta.get("name"),
"description": clan_meta.get("description"),
"icon": icon_path if icon_path else "",
}
)
@@ -73,9 +75,9 @@ def show_command(args: argparse.Namespace) -> None:
flake_path = args.flake.path
meta = show_clan_meta(flake_path)
print(f"Name: {meta.name}")
print(f"Description: {meta.description or '-'}")
print(f"Icon: {meta.icon or '-'}")
print(f"Name: {meta.get("name")}")
print(f"Description: {meta.get("description", '-')}")
print(f"Icon: {meta.get("icon", '-')}")
def register_parser(parser: argparse.ArgumentParser) -> None:

8
pkgs/clan-cli/clan_cli/clan/update.py
View File

@@ -1,7 +1,7 @@
from dataclasses import dataclass
from clan_cli.api import API
from clan_cli.inventory import Meta, load_inventory_json, set_inventory
from clan_cli.inventory import Inventory, Meta, load_inventory_json, set_inventory
@dataclass
@@ -11,10 +11,10 @@ class UpdateOptions:
@API.register
def update_clan_meta(options: UpdateOptions) -> Meta:
def update_clan_meta(options: UpdateOptions) -> Inventory:
inventory = load_inventory_json(options.directory)
inventory.meta = options.meta
inventory["meta"] = options.meta
set_inventory(inventory, options.directory, "Update clan metadata")
return inventory.meta
return inventory

23
pkgs/clan-cli/clan_cli/inventory/__init__.py
View File

@@ -61,9 +61,7 @@ def get_inventory_path(flake_dir: str | Path, create: bool = True) -> Path:
# Default inventory
default_inventory = Inventory(
meta=Meta(name="New Clan"), machines={}, services=Service()
)
default_inventory: Inventory = {"meta": {"name": "New Clan"}}
@API.register
@@ -381,9 +379,7 @@ def patch_inventory_with(base_dir: Path, section: str, content: dict[str, Any])
@API.register
def set_inventory(
inventory: Inventory | dict[str, Any], flake_dir: str | Path, message: str
) -> None:
def set_inventory(inventory: Inventory, flake_dir: str | Path, message: str) -> None:
"""
Write the inventory to the flake directory
and commit it to git with the given message
@@ -396,18 +392,11 @@ def set_inventory(
filtered_modules = lambda m: {
key: value for key, value in m.items() if "/nix/store" not in value
}
if isinstance(inventory, dict):
modules = filtered_modules(inventory.get("modules", {})) # type: ignore
inventory["modules"] = modules
else:
modules = filtered_modules(inventory.modules) # type: ignore
inventory.modules = modules
modules = filtered_modules(inventory.get("modules", {})) # type: ignore
inventory["modules"] = modules
with inventory_file.open("w") as f:
if isinstance(inventory, Inventory):
json.dump(dataclass_to_dict(inventory), f, indent=2)
else:
json.dump(inventory, f, indent=2)
json.dump(inventory, f, indent=2)
commit_file(inventory_file, Path(flake_dir), commit_message=message)
@@ -436,7 +425,7 @@ def merge_template_inventory(
Merge the template inventory into the current inventory
The template inventory is expected to be a subset of the current inventory
"""
for service_name, instance in template_inventory.services.items():
for service_name, instance in template_inventory.get("services", {}).items():
if len(instance.keys()) > 0:
msg = f"Service {service_name} in template inventory has multiple instances"
description = (

47
pkgs/clan-cli/clan_cli/inventory/classes.py
View File

@@ -5,37 +5,32 @@
# ruff: noqa: N806
# ruff: noqa: F401
# fmt: off
from dataclasses import dataclass, field
from typing import Any, Literal
from typing import Any, Literal, NotRequired, TypedDict
@dataclass
class MachineDeploy:
targetHost: None | str = field(default = None)
class MachineDeploy(TypedDict):
targetHost: NotRequired[str]
@dataclass
class Machine:
deploy: MachineDeploy
class Machine(TypedDict):
deploy: NotRequired[MachineDeploy]
description: NotRequired[str]
icon: NotRequired[str]
name: NotRequired[str]
tags: NotRequired[list[str]]
class Meta(TypedDict):
name: str
description: None | str = field(default = None)
icon: None | str = field(default = None)
tags: list[str] = field(default_factory = list)
@dataclass
class Meta:
name: str
description: None | str = field(default = None)
icon: None | str = field(default = None)
description: NotRequired[str]
icon: NotRequired[str]
Service = dict[str, Any]
@dataclass
class Inventory:
meta: Meta
machines: dict[str, Machine] = field(default_factory = dict)
modules: dict[str, str] = field(default_factory = dict)
services: dict[str, Service] = field(default_factory = dict)
tags: dict[str, list[str]] = field(default_factory = dict)
class Inventory(TypedDict):
machines: NotRequired[dict[str, Machine]]
meta: NotRequired[Meta]
modules: NotRequired[dict[str, str]]
services: NotRequired[dict[str, Service]]
tags: NotRequired[dict[str, list[str]]]

24
pkgs/clan-cli/clan_cli/machines/create.py
View File

@@ -16,7 +16,6 @@ from clan_cli.git import commit_file
from clan_cli.inventory import Machine as InventoryMachine
from clan_cli.inventory import (
MachineDeploy,
dataclass_to_dict,
load_inventory_json,
merge_template_inventory,
set_inventory,
@@ -64,15 +63,17 @@ def create_machine(opts: CreateOptions) -> None:
clan_dir = opts.clan_dir.path
log.debug(f"Importing machine '{opts.template_name}' from {opts.template_src}")
if opts.template_name in list_nixos_machines(clan_dir) and not opts.machine.name:
machine_name = opts.machine.get("name")
if opts.template_name in list_nixos_machines(clan_dir) and not opts.machine.get(
"name"
):
msg = f"{opts.template_name} is already defined in {clan_dir}"
description = (
"Please add the --rename option to import the machine with a different name"
)
raise ClanError(msg, description=description)
machine_name = opts.template_name if not opts.machine.name else opts.machine.name
machine_name = machine_name if machine_name else opts.template_name
dst = clan_dir / "machines" / machine_name
# TODO: Move this into nix code
@@ -138,19 +139,24 @@ def create_machine(opts: CreateOptions) -> None:
merge_template_inventory(inventory, template_inventory, machine_name)
deploy = MachineDeploy()
deploy.targetHost = opts.target_host
target_host = opts.target_host
if target_host:
deploy["targetHost"] = target_host
# TODO: We should allow the template to specify machine metadata if not defined by user
new_machine = InventoryMachine(
name=machine_name, deploy=deploy, tags=opts.machine.tags
name=machine_name, deploy=deploy, tags=opts.machine.get("tags", [])
)
if (
not has_inventory
and len(opts.machine.tags) == 0
and new_machine.deploy.targetHost is None
and len(opts.machine.get("tags", [])) == 0
and new_machine.get("deploy", {}).get("targetHost") is None
):
# no need to update inventory if there are no tags or target host
return
inventory.machines.update({new_machine.name: dataclass_to_dict(new_machine)})
inventory["machines"] = inventory.get("machines", {})
inventory["machines"][machine_name] = new_machine
set_inventory(inventory, clan_dir, "Imported machine from template")

2
pkgs/clan-cli/clan_cli/machines/delete.py
View File

@@ -13,7 +13,7 @@ from clan_cli.inventory import load_inventory_json, set_inventory
def delete_machine(flake: FlakeId, name: str) -> None:
inventory = load_inventory_json(flake.path)
machine = inventory.machines.pop(name, None)
machine = inventory.get("machines", {}).pop(name, None)
if machine is None:
msg = f"Machine {name} does not exist"
raise ClanError(msg)

8
pkgs/clan-cli/clan_cli/machines/list.py
View File

@@ -34,7 +34,7 @@ def set_machine(flake_url: Path, machine_name: str, machine: Machine) -> None:
@API.register
def list_inventory_machines(flake_url: str | Path) -> dict[str, Machine]:
inventory = load_inventory_eval(flake_url)
return inventory.machines
return inventory.get("machines", {})
@dataclass
@@ -61,7 +61,7 @@ def extract_header(c: str) -> str:
@API.register
def get_inventory_machine_details(flake_url: Path, machine_name: str) -> MachineDetails:
inventory = load_inventory_eval(flake_url)
machine = inventory.machines.get(machine_name)
machine = inventory.get("machines", {}).get(machine_name)
if machine is None:
msg = f"Machine {machine_name} not found in inventory"
raise ClanError(msg)
@@ -113,12 +113,12 @@ class ConnectionOptions:
def check_machine_online(
flake_url: str | Path, machine_name: str, opts: ConnectionOptions | None
) -> Literal["Online", "Offline"]:
machine = load_inventory_eval(flake_url).machines.get(machine_name)
machine = load_inventory_eval(flake_url).get("machines", {}).get(machine_name)
if not machine:
msg = f"Machine {machine_name} not found in inventory"
raise ClanError(msg)
hostname = machine.deploy.targetHost
hostname = machine.get("deploy", {}).get("targetHost")
if not hostname:
msg = f"Machine {machine_name} does not specify a targetHost"

12
pkgs/clan-cli/clan_cli/machines/update.py
View File

@@ -96,15 +96,19 @@ def update_machines(base_path: str, machines: list[InventoryMachine]) -> None:
# Convert InventoryMachine to Machine
for machine in machines:
name = machine.get("name")
if not name:
msg = "Machine name is not set"
raise ClanError(msg)
m = Machine(
name=machine.name,
name,
flake=FlakeId(base_path),
)
if not machine.deploy.targetHost:
msg = f"'TargetHost' is not set for machine '{machine.name}'"
if not machine.get("deploy", {}).get("targetHost"):
msg = f"'TargetHost' is not set for machine '{name}'"
raise ClanError(msg)
# Copy targetHost to machine
m.override_target_host = machine.deploy.targetHost
m.override_target_host = machine.get("deploy", {}).get("targetHost")
# Would be nice to have?
# m.override_build_host = machine.deploy.buildHost
group_machines.append(m)

41
pkgs/clan-cli/tests/test_inventory.py Normal file
View File

@@ -0,0 +1,41 @@
from clan_cli.inventory.classes import Inventory, Machine, Meta, Service
def test_make_meta_minimal() -> None:
# Name is required
res = Meta(
{
"name": "foo",
}
)
assert res == {"name": "foo"}
def test_make_inventory_minimal() -> None:
# Meta is required
res = Inventory(
{
"meta": Meta(
{
"name": "foo",
}
),
}
)
assert res == {"meta": {"name": "foo"}}
def test_make_machine_minimal() -> None:
# Empty is valid
res = Machine({})
assert res == {}
def test_make_service_minimal() -> None:
# Empty is valid
res = Service({})
assert res == {}

2
pkgs/clan-cli/tests/test_modules.py
View File

@@ -72,7 +72,7 @@ def test_add_module_to_inventory(
inventory = load_inventory_json(base_path)
inventory.services = {
inventory["services"] = {
"borgbackup": {
"borg1": {
"meta": {"name": "borg1"},

38
pkgs/classgen/main.py
View File

@@ -148,8 +148,7 @@ def field_def_from_default_value(
default_factory="dict",
type_appendix=" | dict[str,Any]",
)
if default_value == "‹name›":
return None
# Primitive types
if isinstance(default_value, str):
return finalize_field(
@@ -176,23 +175,15 @@ def get_field_def(
default_factory: str | None = None,
type_appendix: str = "",
) -> str:
sorted_field_types = sorted(field_types)
serialised_types = " | ".join(sorted_field_types) + type_appendix
if not default and not default_factory and not field_meta:
return f"{field_name}: {serialised_types}"
field_init = "field("
if "None" in field_types or default or default_factory:
if "None" in field_types:
field_types.remove("None")
serialised_types = " | ".join(field_types) + type_appendix
serialised_types = f"NotRequired[{serialised_types}]"
else:
serialised_types = " | ".join(field_types) + type_appendix
init_args = []
if default:
init_args.append(f"default = {default}")
if default_factory:
init_args.append(f"default_factory = {default_factory}")
if field_meta:
init_args.append(f"metadata = {field_meta}")
field_init += ", ".join(init_args) + ")"
return f"{field_name}: {serialised_types} = {field_init}"
return f"{field_name}: {serialised_types}"
# Recursive function to generate dataclasses from JSON schema
@@ -281,6 +272,9 @@ def generate_dataclass(schema: dict[str, Any], class_name: str = root_class) ->
finalize_field = partial(get_field_def, field_name, field_meta)
if "default" in prop_info or field_name not in prop_info.get("required", []):
if prop_info.get("type") == "object":
prop_info.update({"default": {}})
if "default" in prop_info:
default_value = prop_info.get("default")
field_def = field_def_from_default_value(
@@ -327,11 +321,11 @@ def generate_dataclass(schema: dict[str, Any], class_name: str = root_class) ->
fields_str = "\n ".join(required_fields + fields_with_default)
nested_classes_str = "\n\n".join(nested_classes)
class_def = f"@dataclass\nclass {class_name}:\n"
class_def = f"\nclass {class_name}(TypedDict):\n"
if not required_fields + fields_with_default:
class_def += " pass"
else:
class_def += f" {fields_str}\n"
class_def += f" {fields_str}"
return f"{nested_classes_str}\n\n{class_def}" if nested_classes_str else class_def
@@ -356,11 +350,11 @@ def run_gen(args: argparse.Namespace) -> None:
# ruff: noqa: N806
# ruff: noqa: F401
# fmt: off
from dataclasses import dataclass, field
from typing import Any, Literal\n\n
from typing import Any, Literal, NotRequired, TypedDict\n
"""
)
f.write(dataclass_code)
f.write("\n")
def main() -> None:

16
pkgs/webview-ui/app/src/components/MachineListItem.tsx
View File

@@ -29,7 +29,7 @@ export const MachineListItem = (props: MachineListItemProps) => {
const navigate = useNavigate();
const handleInstall = async () => {
if (!info?.deploy.targetHost || installing()) {
if (!info?.deploy?.targetHost || installing()) {
return;
}
@@ -38,7 +38,7 @@ export const MachineListItem = (props: MachineListItemProps) => {
toast.error("No active clan selected");
return;
}
if (!info?.deploy.targetHost) {
if (!info?.deploy?.targetHost) {
toast.error(
"Machine does not have a target host. Specify where the machine should be deployed.",
);
@@ -69,7 +69,7 @@ export const MachineListItem = (props: MachineListItemProps) => {
};
const handleUpdate = async () => {
if (!info?.deploy.targetHost || installing()) {
if (!info?.deploy?.targetHost || installing()) {
return;
}
@@ -158,7 +158,7 @@ export const MachineListItem = (props: MachineListItemProps) => {
</span>
)}
</Show>
{d()?.deploy.targetHost}
{d()?.deploy?.targetHost}
</>
)}
</Show>
@@ -182,24 +182,24 @@ export const MachineListItem = (props: MachineListItemProps) => {
</li>
<li
classList={{
disabled: !info?.deploy.targetHost || installing(),
disabled: !info?.deploy?.targetHost || installing(),
}}
onClick={handleInstall}
>
<a>
<Show when={info?.deploy.targetHost} fallback={"Deploy"}>
<Show when={info?.deploy?.targetHost} fallback={"Deploy"}>
{(d) => `Install to ${d()}`}
</Show>
</a>
</li>
<li
classList={{
disabled: !info?.deploy.targetHost || updating(),
disabled: !info?.deploy?.targetHost || updating(),
}}
onClick={handleUpdate}
>
<a>
<Show when={info?.deploy.targetHost} fallback={"Deploy"}>
<Show when={info?.deploy?.targetHost} fallback={"Deploy"}>
{(d) => `Update (${d()})`}
</Show>
</a>

6
pkgs/webview-ui/app/src/routes/machines/details.tsx
View File

@@ -243,7 +243,7 @@ const MachineForm = (props: MachineDetailsProps) => {
const machine_response = await callApi("set_machine", {
flake_url: curr_uri,
machine_name: props.initialData.machine.name,
machine_name: props.initialData.machine.name || "My machine",
machine: {
...values.machine,
// TODO: Remove this workaround
@@ -316,7 +316,7 @@ const MachineForm = (props: MachineDetailsProps) => {
}
>
<div class="w-32 rounded-lg border p-2 bg-def-4 border-inv-3">
<RndThumbnail name={machineName()} />
<RndThumbnail name={machineName() || "M"} />
</div>
</div>
</figure>
@@ -342,7 +342,7 @@ const MachineForm = (props: MachineDetailsProps) => {
{(tag) => (
<label class="p-1">
Tags
<span class="mx-2 rounded-full px-3 py-1 bg-inv-4 fg-inv-1 w-fit">
<span class="mx-2 w-fit rounded-full px-3 py-1 bg-inv-4 fg-inv-1">
{tag}
</span>
</label>

2
pkgs/webview-ui/app/src/routes/machines/list.tsx
View File

@@ -84,7 +84,7 @@ export const MachineListView: Component = () => {
</div>
{/* <Show when={filter()}> */}
<div class="my-1 flex w-full gap-2 p-2">
<div class="h-6 w-6 p-1">
<div class="size-6 p-1">
<Icon icon="Filter" />
</div>
<For each={filter().tags.sort()}>