yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'clan-cli: Unify list_machines and use flake caching' (#3673) from Qubasa/clan-core:fix_ui_stuff into main

Browse Source 
Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3673
This commit is contained in:
hsjobeki
2025-05-16 09:08:59 +00:00
parent 507fba8ede 2f0114a236
commit 2544e8df8c
21 changed files with 198 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/build-clan/interface.nix
View File

@@ -180,8 +180,8 @@ in
meta = lib.mkOption { type = lib.types.raw; };
secrets = lib.mkOption { type = lib.types.raw; };
clanLib = lib.mkOption { type = lib.types.raw; };
all-machines-json = lib.mkOption { type = lib.types.raw; };
machines = lib.mkOption { type = lib.types.raw; };
all-machines-json = lib.mkOption { type = lib.types.raw; };
};
};
};

7
lib/build-clan/module.nix
View File

@@ -231,11 +231,14 @@ in
# machine specifics
machines = configsPerSystem;
all-machines-json = lib.mapAttrs (
all-machines-json =
lib.trace "Your clan-cli and the clan-core input have incompatible versions" lib.mapAttrs
(
system: configs:
nixpkgs.legacyPackages.${system}.writers.writeJSON "machines.json" (
lib.mapAttrs (_: m: m.config.system.clan.deployment.data) configs
)
) configsPerSystem;
)
configsPerSystem;
};
}

3
pkgs/clan-app/clan-app.code-workspace
View File

@@ -23,6 +23,9 @@
},
{
"path": "../clan-cli/clan_lib"
},
{
"path": "../webview-ui"
}
],
"settings": {

10
pkgs/clan-app/ui/src/api/manual_types.tsx Normal file
View File

@@ -0,0 +1,10 @@
export interface Machine {
machine: {
name: string;
flake: {
identifier: string;
};
override_target_host: string | null;
private_key: string | null;
};
}

2
pkgs/clan-app/ui/src/components/machine-list-item/index.tsx
View File

@@ -10,7 +10,7 @@ import { Filter } from "../../routes/machines";
import { Typography } from "../Typography";
import "./css/index.css";
type MachineDetails = SuccessQuery<"list_machines">["data"][string];
type MachineDetails = SuccessQuery<"list_inv_machines">["data"][string];
interface MachineListItemProps {
name: string;

2
pkgs/clan-app/ui/src/routes/machines/create.tsx
View File

@@ -61,7 +61,7 @@ export function CreateMachine() {
reset(formStore);
queryClient.invalidateQueries({
queryKey: [activeURI(), "list_machines"],
queryKey: [activeURI(), "list_inv_machines"],
});
navigate("/machines");
} else {

8
pkgs/clan-app/ui/src/routes/machines/details.tsx
View File

@@ -391,12 +391,14 @@ const MachineForm = (props: MachineDetailsProps) => {
return;
}
const machine_response = await callApi("set_machine", {
const machine_response = await callApi("set_inv_machine", {
machine: {
name: props.initialData.machine.name || "My machine",
flake: {
identifier: curr_uri,
},
machine_name: props.initialData.machine.name || "My machine",
machine: {
},
inventory_machine: {
...values.machine,
// TODO: Remove this workaround
tags: Array.from(

8
pkgs/clan-app/ui/src/routes/machines/list.tsx
View File

@@ -18,7 +18,7 @@ import { Header } from "@/src/layout/header";
import { makePersisted } from "@solid-primitives/storage";
type MachinesModel = Extract<
OperationResponse<"list_machines">,
OperationResponse<"list_inv_machines">,
{ status: "success" }
>["data"];
@@ -32,13 +32,13 @@ export const MachineListView: Component = () => {
const [filter, setFilter] = createSignal<Filter>({ tags: [] });
const inventoryQuery = createQuery<MachinesModel>(() => ({
queryKey: [activeURI(), "list_machines"],
queryKey: [activeURI(), "list_inv_machines"],
placeholderData: {},
enabled: !!activeURI(),
queryFn: async () => {
const uri = activeURI();
if (uri) {
const response = await callApi("list_machines", {
const response = await callApi("list_inv_machines", {
flake: {
identifier: uri,
},
@@ -56,7 +56,7 @@ export const MachineListView: Component = () => {
const refresh = async () => {
queryClient.invalidateQueries({
// Invalidates the cache for of all types of machine list at once
queryKey: [activeURI(), "list_machines"],
queryKey: [activeURI(), "list_inv_machines"],
});
};

4
pkgs/clan-cli/clan_cli/clan/inspect.py
View File

@@ -7,7 +7,7 @@ from clan_cli.cmd import run
from clan_cli.dirs import machine_gcroot
from clan_cli.errors import ClanError
from clan_cli.flake import Flake
from clan_cli.machines.list import list_nixos_machines
from clan_cli.machines.list import list_machines
from clan_cli.machines.machines import Machine
from clan_cli.nix import (
nix_add_to_gcroots,
@@ -57,7 +57,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig:
system = config["system"]
# Check if the machine exists
machines: list[str] = list_nixos_machines(flake_url)
machines: dict[str, Machine] = list_machines(Flake(str(flake_url)))
if machine_name not in machines:
msg = f"Machine {machine_name} not found in {flake_url}. Available machines: {', '.join(machines)}"
raise ClanError(msg)

15
pkgs/clan-cli/clan_cli/facts/generate.py
View File

@@ -15,7 +15,7 @@ from clan_cli.completions import (
)
from clan_cli.errors import ClanError
from clan_cli.git import commit_files
from clan_cli.machines.inventory import get_all_machines, get_selected_machines
from clan_cli.machines.list import list_machines
from clan_cli.machines.machines import Machine
from clan_cli.nix import nix_shell
@@ -225,10 +225,15 @@ def generate_command(args: argparse.Namespace) -> None:
if args.flake is None:
msg = "Could not find clan flake toplevel directory"
raise ClanError(msg)
if len(args.machines) == 0:
machines = get_all_machines(args.flake, args.option)
else:
machines = get_selected_machines(args.flake, args.option, args.machines)
machines: list[Machine] = list(list_machines(args.flake).values())
if len(args.machines) > 0:
machines = list(
filter(
lambda m: m.name in args.machines,
machines,
)
)
generate_facts(machines, args.service, args.regenerate)

26
pkgs/clan-cli/clan_cli/inventory/__init__.py
View File

@@ -25,11 +25,9 @@ from clan_lib.persist.util import (
determine_writeability,
)
from clan_cli.cmd import run
from clan_cli.errors import ClanError
from clan_cli.flake import Flake
from clan_cli.git import commit_file
from clan_cli.nix import nix_eval
def get_inventory_path(flake: Flake) -> Path:
@@ -40,7 +38,7 @@ def get_inventory_path(flake: Flake) -> Path:
return inventory_file
def load_inventory_eval(flake_dir: Flake) -> Inventory:
def load_inventory_eval(flake: Flake) -> Inventory:
"""
Loads the evaluated inventory.
After all merge operations with eventual nix code in buildClan.
@@ -51,18 +49,9 @@ def load_inventory_eval(flake_dir: Flake) -> Inventory:
- Contains all machines
- and more
"""
cmd = nix_eval(
[
f"{flake_dir}#clanInternals.inventory",
"--json",
]
)
proc = run(cmd)
data = flake.select("clanInternals.inventory")
try:
res = proc.stdout.strip()
data: dict = json.loads(res)
inventory = Inventory(data) # type: ignore
except json.JSONDecodeError as e:
msg = f"Error decoding inventory from flake: {e}"
@@ -89,18 +78,9 @@ def get_inventory_current_priority(flake: Flake) -> dict:
};
}
"""
cmd = nix_eval(
[
f"{flake}#clanInternals.inventoryClass.introspection",
"--json",
]
)
proc = run(cmd)
try:
res = proc.stdout.strip()
data = json.loads(res)
data = flake.select("clanInternals.inventoryClass.introspection")
except json.JSONDecodeError as e:
msg = f"Error decoding inventory from flake: {e}"
raise ClanError(msg) from e

8
pkgs/clan-cli/clan_cli/machines/create.py
View File

@@ -21,7 +21,7 @@ from clan_cli.git import commit_file
from clan_cli.inventory import (
patch_inventory_with,
)
from clan_cli.machines.list import list_nixos_machines
from clan_cli.machines.list import list_machines
from clan_cli.templates import (
InputPrio,
TemplateName,
@@ -62,9 +62,9 @@ def create_machine(opts: CreateOptions, commit: bool = True) -> None:
log.info(f"Found template '{template.name}' in '{template.input_variant}'")
machine_name = opts.machine.get("name")
if opts.template_name in list_nixos_machines(clan_dir) and not opts.machine.get(
"name"
):
if opts.template_name in list_machines(
Flake(str(clan_dir))
) and not opts.machine.get("name"):
msg = f"{opts.template_name} is already defined in {clan_dir}"
description = (
"Please add the --rename option to import the machine with a different name"

71
pkgs/clan-cli/clan_cli/machines/inventory.py
View File

@@ -1,45 +1,34 @@
import json
from pathlib import Path
from clan_lib.api import API
from clan_lib.nix_models.inventory import (
Machine as InventoryMachine,
)
from clan_lib.persist.inventory_store import InventoryStore
from clan_lib.persist.util import apply_patch
from clan_cli.cmd import run
from clan_cli.flake import Flake
from clan_cli.nix import nix_build, nix_config, nix_test_store
from .machines import Machine
from clan_cli.errors import ClanError
from clan_cli.machines.machines import Machine
# function to speedup eval if we want to evaluate all machines
def get_all_machines(flake: Flake, nix_options: list[str]) -> list[Machine]:
config = nix_config()
system = config["system"]
json_path = Path(
run(
nix_build([f'{flake}#clanInternals.all-machines-json."{system}"'])
).stdout.rstrip()
@API.register
def get_inv_machine(machine: Machine) -> InventoryMachine:
inventory_store = InventoryStore(flake=machine.flake)
inventory = inventory_store.read()
machine_inv = inventory.get("machines", {}).get(machine.name)
if machine_inv is None:
msg = f"Machine {machine.name} not found in inventory"
raise ClanError(msg)
return InventoryMachine(**machine_inv)
@API.register
def set_inv_machine(machine: Machine, inventory_machine: InventoryMachine) -> None:
assert machine.name == inventory_machine["name"], "Machine name mismatch"
inventory_store = InventoryStore(flake=machine.flake)
inventory = inventory_store.read()
apply_patch(inventory, f"machines.{machine.name}", inventory_machine)
inventory_store.write(
inventory, message=f"Update information about machine {machine.name}"
)
if test_store := nix_test_store():
json_path = test_store.joinpath(*json_path.parts[1:])
machines_json = json.loads(json_path.read_text())
machines = []
for name, machine_data in machines_json.items():
machines.append(
Machine(
name=name,
flake=flake,
cached_deployment=machine_data,
nix_options=nix_options,
)
)
return machines
def get_selected_machines(
flake: Flake, nix_options: list[str], machine_names: list[str]
) -> list[Machine]:
machines = []
for name in machine_names:
machines.append(Machine(name=name, flake=flake, nix_options=nix_options))
return machines

138
pkgs/clan-cli/clan_cli/machines/list.py
View File

@@ -1,47 +1,71 @@
import argparse
import json
import logging
import re
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Literal
from clan_lib.api import API
from clan_lib.api.disk import MachineDiskMatter
from clan_lib.api.modules import parse_frontmatter
from clan_lib.nix_models.inventory import Machine as InventoryMachine
from clan_lib.persist.inventory_store import InventoryStore
from clan_lib.persist.util import apply_patch
from clan_cli.cmd import RunOpts, run
from clan_cli.completions import add_dynamic_completer, complete_tags
from clan_cli.dirs import specific_machine_dir
from clan_cli.errors import ClanError
from clan_cli.flake import Flake
from clan_cli.machines.hardware import HardwareConfig
from clan_cli.machines.inventory import get_inv_machine
from clan_cli.machines.machines import Machine
from clan_cli.nix import nix_eval
from clan_cli.tags import list_nixos_machines_by_tags
log = logging.getLogger(__name__)
@API.register
def set_machine(flake: Flake, machine_name: str, machine: InventoryMachine) -> None:
def list_inv_machines(flake: Flake) -> dict[str, InventoryMachine]:
"""
List machines in the inventory for the UI.
"""
inventory_store = InventoryStore(flake=flake)
inventory = inventory_store.read()
apply_patch(inventory, f"machines.{machine_name}", machine)
inventory_store.write(
inventory, message=f"Update information about machine {machine_name}"
res = inventory.get("machines", {})
return res
def list_machines(
flake: Flake, nix_options: list[str] | None = None
) -> dict[str, Machine]:
inventory_store = InventoryStore(flake=flake)
inventory = inventory_store.read()
res = {}
if nix_options is None:
nix_options = []
for inv_machine in inventory.get("machines", {}).values():
machine = Machine(
name=inv_machine["name"],
flake=flake,
nix_options=nix_options,
)
res[machine.name] = machine
return res
@API.register
def list_machines(flake: Flake) -> dict[str, InventoryMachine]:
inventory_store = InventoryStore(flake=flake)
inventory = inventory_store.read()
return inventory.get("machines", {})
def query_machines_by_tags(flake: Flake, tags: list[str]) -> dict[str, Machine]:
"""
Query machines by their respective tags, if multiple tags are specified
then only machines that have those respective tags specified will be listed.
It is an intersection of the tags and machines.
"""
machines = list_machines(flake)
filtered_machines = {}
for machine in machines.values():
inv_machine = get_inv_machine(machine)
if all(tag in inv_machine["tags"] for tag in tags):
filtered_machines[machine.name] = machine
return filtered_machines
@dataclass
@@ -64,13 +88,7 @@ def extract_header(c: str) -> str:
@API.register
def get_machine_details(machine: Machine) -> MachineDetails:
inventory_store = InventoryStore(flake=machine.flake)
inventory = inventory_store.read()
machine_inv = inventory.get("machines", {}).get(machine.name)
if machine_inv is None:
msg = f"Machine {machine.name} not found in inventory"
raise ClanError(msg)
machine_inv = get_inv_machine(machine)
hw_config = HardwareConfig.detect_type(machine)
machine_dir = specific_machine_dir(machine)
@@ -85,72 +103,20 @@ def get_machine_details(machine: Machine) -> MachineDetails:
disk_schema = data # type: ignore
return MachineDetails(
machine=machine_inv, hw_config=hw_config, disk_schema=disk_schema
machine=machine_inv,
hw_config=hw_config,
disk_schema=disk_schema,
)
def list_nixos_machines(flake_url: str | Path) -> list[str]:
cmd = nix_eval(
[
f"{flake_url}#clanInternals.machines.x86_64-linux",
"--apply",
"builtins.attrNames",
"--json",
]
)
proc = run(cmd)
try:
res = proc.stdout.strip()
data = json.loads(res)
except json.JSONDecodeError as e:
msg = f"Error decoding machines from flake: {e}"
raise ClanError(msg) from e
else:
return data
@dataclass
class ConnectionOptions:
timeout: int = 2
retries: int = 10
from clan_cli.machines.machines import Machine
@API.register
def check_machine_online(
machine: Machine, opts: ConnectionOptions | None = None
) -> Literal["Online", "Offline"]:
hostname = machine.target_host_address
if not hostname:
msg = f"Machine {machine.name} does not specify a targetHost"
raise ClanError(msg)
timeout = opts.timeout if opts and opts.timeout else 2
for _ in range(opts.retries if opts and opts.retries else 10):
with machine.target_host() as target:
res = target.run(
["true"],
RunOpts(timeout=timeout, check=False, needs_user_terminal=True),
)
if res.returncode == 0:
return "Online"
time.sleep(timeout)
return "Offline"
def list_command(args: argparse.Namespace) -> None:
flake_path = args.flake.path
flake: Flake = args.flake
if args.tags:
list_nixos_machines_by_tags(flake_path, args.tags)
return
for name in list_nixos_machines(flake_path):
for name in query_machines_by_tags(flake, args.tags):
print(name)
else:
for name in list_machines(flake):
print(name)

2
pkgs/clan-cli/clan_cli/machines/machines.py
View File

@@ -30,8 +30,8 @@ if TYPE_CHECKING:
class Machine:
name: str
flake: Flake
nix_options: list[str] = field(default_factory=list)
cached_deployment: None | dict[str, Any] = None
override_target_host: None | str = None
override_build_host: None | str = None
private_key: Path | None = None

58
pkgs/clan-cli/clan_cli/tags.py
View File

@@ -1,58 +0,0 @@
import json
from pathlib import Path
from typing import Any
from clan_cli.cmd import run
from clan_cli.errors import ClanError
from clan_cli.nix import nix_eval
def list_tagged_machines(flake_url: str | Path) -> dict[str, Any]:
"""
Query machines from the inventory with their meta information intact.
The meta information includes tags.
"""
cmd = nix_eval(
[
f"{flake_url}#clanInternals.inventory.machines",
"--json",
]
)
proc = run(cmd)
try:
res = proc.stdout.strip()
data = json.loads(res)
except json.JSONDecodeError as e:
msg = f"Error decoding tagged inventory machines from flake: {e}"
raise ClanError(msg) from e
else:
return data
def query_machines_by_tags(
flake_path: str | Path, tags: list[str] | None = None
) -> list[str]:
"""
Query machines by their respective tags, if multiple tags are specified
then only machines that have those respective tags specified will be listed.
It is an intersection of the tags and machines.
"""
machines = list_tagged_machines(flake_path)
if not tags:
return list(machines.keys())
filtered_machines = []
for machine_id, machine_values in machines.items():
if all(tag in machine_values["tags"] for tag in tags):
filtered_machines.append(machine_id)
return filtered_machines
def list_nixos_machines_by_tags(
flake_path: str | Path, tags: list[str] | None = None
) -> None:
for name in query_machines_by_tags(flake_path, tags):
print(name)

3
pkgs/clan-cli/clan_cli/tests/test_inventory_serde.py
View File

@@ -1,6 +1,7 @@
from typing import Any
import pytest
from clan_cli.flake import Flake
# Functions to test
from clan_cli.inventory import load_inventory_eval
@@ -43,7 +44,7 @@ def test_inventory_deserialize_variants(
Testing different inventory deserializations
Inventory should always be deserializable to a dict
"""
inventory: dict[str, Any] = load_inventory_eval(test_flake_with_core.path) # type: ignore
inventory: dict[str, Any] = load_inventory_eval(Flake(test_flake_with_core.path)) # type: ignore
# Check that the inventory is a dict
assert isinstance(inventory, dict)

16
pkgs/clan-cli/clan_cli/vars/generate.py
View File

@@ -18,7 +18,7 @@ from clan_cli.completions import (
from clan_cli.errors import ClanError
from clan_cli.flake import Flake
from clan_cli.git import commit_files
from clan_cli.machines.inventory import get_all_machines, get_selected_machines
from clan_cli.machines.list import list_machines
from clan_cli.nix import nix_config, nix_shell, nix_test_store
from clan_cli.vars._types import StoreBase
from clan_cli.vars.migration import check_can_migrate, migrate_files
@@ -481,10 +481,16 @@ def generate_command(args: argparse.Namespace) -> None:
if args.flake is None:
msg = "Could not find clan flake toplevel directory"
raise ClanError(msg)
if len(args.machines) == 0:
machines = get_all_machines(args.flake, args.option)
else:
machines = get_selected_machines(args.flake, args.option, args.machines)
machines: list[Machine] = list(list_machines(args.flake, args.option).values())
if len(args.machines) > 0:
machines = list(
filter(
lambda m: m.name in args.machines,
machines,
)
)
# prefetch all vars
config = nix_config()

43
pkgs/clan-cli/clan_lib/api/network.py Normal file
View File

@@ -0,0 +1,43 @@
import logging
import time
from dataclasses import dataclass
from typing import Literal
from clan_cli.cmd import RunOpts
from clan_cli.errors import ClanError
from clan_cli.machines.machines import Machine
from clan_lib.api import API
log = logging.getLogger(__name__)
@dataclass
class ConnectionOptions:
timeout: int = 2
retries: int = 10
@API.register
def check_machine_online(
machine: Machine, opts: ConnectionOptions | None = None
) -> Literal["Online", "Offline"]:
hostname = machine.target_host_address
if not hostname:
msg = f"Machine {machine.name} does not specify a targetHost"
raise ClanError(msg)
timeout = opts.timeout if opts and opts.timeout else 2
for _ in range(opts.retries if opts and opts.retries else 10):
with machine.target_host() as target:
res = target.run(
["true"],
RunOpts(timeout=timeout, check=False, needs_user_terminal=True),
)
if res.returncode == 0:
return "Online"
time.sleep(timeout)
return "Offline"

2
pkgs/clan-cli/clan_lib/tests/test_create.py
View File

@@ -15,7 +15,6 @@ from clan_cli.flake import Flake
from clan_cli.inventory import patch_inventory_with
from clan_cli.machines.create import CreateOptions as ClanCreateOptions
from clan_cli.machines.create import create_machine
from clan_cli.machines.list import check_machine_online
from clan_cli.machines.machines import Machine
from clan_cli.nix import nix_command
from clan_cli.secrets.key import generate_key
@@ -26,6 +25,7 @@ from clan_cli.ssh.host_key import HostKeyCheck
from clan_cli.vars.generate import generate_vars_for_machine, get_generators_closure
from clan_lib.api.disk import hw_main_disk_options, set_machine_disk_schema
from clan_lib.api.network import check_machine_online
from clan_lib.nix_models.inventory import Machine as InventoryMachine
from clan_lib.nix_models.inventory import MachineDeploy

5
pkgs/clan-vm-manager/clan_vm_manager/history.py
View File

@@ -8,8 +8,9 @@ from typing import Any
from clan_cli.clan.inspect import FlakeConfig, inspect_flake
from clan_cli.dirs import user_history_file
from clan_cli.errors import ClanError
from clan_cli.flake import Flake
from clan_cli.locked_open import read_history_file, write_history_file
from clan_cli.machines.list import list_nixos_machines
from clan_cli.machines.list import list_machines
from clan_vm_manager.clan_uri import ClanURI
@@ -75,7 +76,7 @@ def new_history_entry(url: str, machine: str) -> HistoryEntry:
def add_all_to_history(uri: ClanURI) -> list[HistoryEntry]:
history = list_history()
new_entries: list[HistoryEntry] = []
for machine in list_nixos_machines(uri.get_url()):
for machine in list_machines(Flake(uri.get_url())):
new_entry = _add_maschine_to_history_list(uri.get_url(), machine, history)
new_entries.append(new_entry)
write_history_file(history)