diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 467971f77..01808d6db 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -58,11 +58,11 @@ let { # { ${name} :: meta // { name, tags } } machines = lib.mapAttrs ( - name: config: + name: machineConfig: (lib.attrByPath [ "clan" "meta" - ] { } config) + ] { } machineConfig) // { # meta.name default is the attribute name of the machine name = lib.mkDefault ( @@ -70,11 +70,11 @@ let "clan" "meta" "name" - ] name config + ] name machineConfig ); } # tags - // (clanToInventory config { + // (clanToInventory machineConfig { clanPath = [ "clan" "tags" @@ -82,15 +82,15 @@ let inventoryPath = [ "tags" ]; }) # system - // (clanToInventory config { + // (clanToInventory machineConfig { clanPath = [ "nixpkgs" - "hostSystem" + "hostPlatform" ]; inventoryPath = [ "system" ]; }) # deploy.targetHost - // (clanToInventory config { + // (clanToInventory machineConfig { clanPath = [ "clan" "core" diff --git a/pkgs/clan-cli/clan_cli/api/modules.py b/pkgs/clan-cli/clan_cli/api/modules.py index 6c02e8ac2..290689bf4 100644 --- a/pkgs/clan-cli/clan_cli/api/modules.py +++ b/pkgs/clan-cli/clan_cli/api/modules.py @@ -6,7 +6,7 @@ from pathlib import Path from clan_cli.cmd import run_no_stdout from clan_cli.errors import ClanCmdError, ClanError -from clan_cli.inventory import Inventory, load_inventory +from clan_cli.inventory import Inventory, load_inventory_json from clan_cli.nix import nix_eval from . import API @@ -152,4 +152,4 @@ def get_module_info( @API.register def get_inventory(base_path: str) -> Inventory: - return load_inventory(base_path) + return load_inventory_json(base_path) diff --git a/pkgs/clan-cli/clan_cli/clan/create.py b/pkgs/clan-cli/clan_cli/clan/create.py index 75921131d..a9c70b613 100644 --- a/pkgs/clan-cli/clan_cli/clan/create.py +++ b/pkgs/clan-cli/clan_cli/clan/create.py @@ -1,12 +1,11 @@ # !/usr/bin/env python3 import argparse import os -from dataclasses import dataclass, fields +from dataclasses import dataclass from pathlib import Path from clan_cli.api import API -from clan_cli.arg_actions import AppendOptionAction -from clan_cli.inventory import Meta, load_inventory, save_inventory +from clan_cli.inventory import Inventory, init_inventory from ..cmd import CmdOut, run from ..errors import ClanError @@ -29,11 +28,9 @@ class CreateClanResponse: @dataclass class CreateOptions: directory: Path | str - # Metadata for the clan - # Metadata can be shown with `clan show` - meta: Meta | None = None # URL to the template to use. Defaults to the "minimal" template template_url: str = minimal_template_url + initial: Inventory | None = None def git_command(directory: Path, *args: str) -> list[str]: @@ -88,17 +85,13 @@ def create_clan(options: CreateOptions) -> CreateClanResponse: git_command(directory, "config", "user.email", "clan@example.com") ) - # Write inventory.json file - inventory = load_inventory(directory) - if options.meta is not None: - inventory.meta = options.meta - # Persist creates a commit message for each change - save_inventory(inventory, directory, "Init inventory") - flake_update = run( nix_shell(["nixpkgs#nix"], ["nix", "flake", "update"]), cwd=directory ) + if options.initial: + init_inventory(options.directory, init=options.initial) + response = CreateClanResponse( flake_init=flake_init, git_init=git_init, @@ -118,15 +111,6 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None: default=default_template_url, ) - parser.add_argument( - "--meta", - help=f"""Metadata to set for the clan. Available options are: {", ".join([f.name for f in fields(Meta)]) }""", - nargs=2, - metavar=("name", "value"), - action=AppendOptionAction, - default=[], - ) - parser.add_argument( "path", type=Path, help="Path to the clan directory", default=Path(".") ) diff --git a/pkgs/clan-cli/clan_cli/clan/update.py b/pkgs/clan-cli/clan_cli/clan/update.py index 595b7f125..65ff096f6 100644 --- a/pkgs/clan-cli/clan_cli/clan/update.py +++ b/pkgs/clan-cli/clan_cli/clan/update.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from clan_cli.api import API -from clan_cli.inventory import Meta, load_inventory, save_inventory +from clan_cli.inventory import Meta, load_inventory_json, save_inventory @dataclass @@ -12,7 +12,7 @@ class UpdateOptions: @API.register def update_clan_meta(options: UpdateOptions) -> Meta: - inventory = load_inventory(options.directory) + inventory = load_inventory_json(options.directory) inventory.meta = options.meta save_inventory(inventory, options.directory, "Update clan metadata") diff --git a/pkgs/clan-cli/clan_cli/clan_uri.py b/pkgs/clan-cli/clan_cli/clan_uri.py index 5f5b7eb85..3a255dbd9 100644 --- a/pkgs/clan-cli/clan_cli/clan_uri.py +++ b/pkgs/clan-cli/clan_cli/clan_uri.py @@ -9,32 +9,49 @@ from .errors import ClanError @dataclass class FlakeId: - # FIXME: this is such a footgun if you accidnetally pass a string - _value: str | Path + loc: str | Path def __post_init__(self) -> None: assert isinstance( - self._value, str | Path - ), f"Flake {self._value} has an invalid type: {type(self._value)}" + self.loc, str | Path + ), f"Flake {self.loc} has an invalid format: {type(self.loc)}" def __str__(self) -> str: - return str(self._value) + return str(self.loc) @property def path(self) -> Path: - assert isinstance(self._value, Path), f"Flake {self._value} is not a local path" - return self._value + assert self.is_local(), f"Flake {self.loc} is not a local path" + return Path(self.loc) @property def url(self) -> str: - assert isinstance(self._value, str), f"Flake {self._value} is not a remote url" - return self._value + assert self.is_remote(), f"Flake {self.loc} is not a remote url" + return str(self.loc) def is_local(self) -> bool: - return isinstance(self._value, Path) + """ + https://nix.dev/manual/nix/2.22/language/builtins.html?highlight=urlS#source-types + + Examples: + + - file:///home/eelco/nix/README.md file LOCAL + - git+file://git:github.com:NixOS/nixpkgs git+file LOCAL + - https://example.com/index.html https REMOTE + - github:nixos/nixpkgs github REMOTE + - ftp://serv.file ftp REMOTE + - ./. '' LOCAL + + """ + x = urllib.parse.urlparse(str(self.loc)) + if x.scheme == "" or "file" in x.scheme: + # See above *file* or empty are the only local schemas + return True + + return False def is_remote(self) -> bool: - return isinstance(self._value, str) + return not self.is_local() # Define the ClanURI class diff --git a/pkgs/clan-cli/clan_cli/git.py b/pkgs/clan-cli/clan_cli/git.py index ec9e693f7..b2abbcf19 100644 --- a/pkgs/clan-cli/clan_cli/git.py +++ b/pkgs/clan-cli/clan_cli/git.py @@ -93,6 +93,7 @@ def _commit_file_to_git( "commit", "-m", commit_message, + "--no-verify", # dont run pre-commit hooks ] + [str(file_path) for file_path in file_paths], ) diff --git a/pkgs/clan-cli/clan_cli/inventory/__init__.py b/pkgs/clan-cli/clan_cli/inventory/__init__.py index 31ffecc00..68a8e8bf9 100644 --- a/pkgs/clan-cli/clan_cli/inventory/__init__.py +++ b/pkgs/clan-cli/clan_cli/inventory/__init__.py @@ -1,3 +1,17 @@ +""" +All read/write operations MUST use the inventory. + +Machine data, clan data or service data can be accessed in a performant way. + +This file exports stable classnames for static & dynamic type safety. + +Utilize: + +- load_inventory_eval: To load the actual inventory with nix declarations merged. +Operate on the returned inventory to make changes +- save_inventory: To persist changes. +""" + import dataclasses import json from dataclasses import fields, is_dataclass @@ -5,9 +19,12 @@ from pathlib import Path from types import UnionType from typing import Any, get_args, get_origin -from clan_cli.errors import ClanError +from clan_cli.api import API +from clan_cli.errors import ClanCmdError, ClanError from clan_cli.git import commit_file +from ..cmd import run_no_stdout +from ..nix import nix_eval from .classes import ( Inventory, Machine, @@ -165,14 +182,42 @@ default_inventory = Inventory( ) -def load_inventory( +def load_inventory_eval(flake_dir: str | Path) -> Inventory: + """ + Loads the actual inventory. + After all merge operations with eventual nix code in buildClan. + + Evaluates clanInternals.inventory with nix. Which is performant. + + - Contains all clan metadata + - Contains all machines + - and more + """ + cmd = nix_eval( + [ + f"{flake_dir}#clanInternals.inventory", + "--json", + ] + ) + proc = run_no_stdout(cmd) + + try: + res = proc.stdout.strip() + data = json.loads(res) + inventory = from_dict(Inventory, data) + return inventory + except json.JSONDecodeError as e: + raise ClanError(f"Error decoding inventory from flake: {e}") + + +def load_inventory_json( flake_dir: str | Path, default: Inventory = default_inventory ) -> Inventory: """ Load the inventory file from the flake directory If not file is found, returns the default inventory """ - inventory = default_inventory + inventory = default inventory_file = get_path(flake_dir) if inventory_file.exists(): @@ -184,6 +229,10 @@ def load_inventory( # Error decoding the inventory file raise ClanError(f"Error decoding inventory file: {e}") + if not inventory_file.exists(): + # Copy over the meta from the flake if the inventory is not initialized + inventory.meta = load_inventory_eval(flake_dir).meta + return inventory @@ -198,3 +247,22 @@ def save_inventory(inventory: Inventory, flake_dir: str | Path, message: str) -> json.dump(dataclass_to_dict(inventory), f, indent=2) commit_file(inventory_file, Path(flake_dir), commit_message=message) + + +@API.register +def init_inventory(directory: str, init: Inventory | None = None) -> None: + inventory = None + # Try reading the current flake + if init is None: + try: + inventory = load_inventory_eval(directory) + except ClanCmdError: + pass + + if init is not None: + inventory = init + + # Write inventory.json file + if inventory is not None: + # Persist creates a commit message for each change + save_inventory(inventory, directory, "Init inventory") diff --git a/pkgs/clan-cli/clan_cli/machines/create.py b/pkgs/clan-cli/clan_cli/machines/create.py index 0f5df27ee..cea2412a7 100644 --- a/pkgs/clan-cli/clan_cli/machines/create.py +++ b/pkgs/clan-cli/clan_cli/machines/create.py @@ -1,13 +1,17 @@ import argparse import logging import re -from pathlib import Path from ..api import API from ..clan_uri import FlakeId from ..errors import ClanError -from ..git import commit_file -from ..inventory import Machine, MachineDeploy, get_path, load_inventory, save_inventory +from ..inventory import ( + Machine, + MachineDeploy, + load_inventory_eval, + load_inventory_json, + save_inventory, +) log = logging.getLogger(__name__) @@ -20,12 +24,16 @@ def create_machine(flake: FlakeId, machine: Machine) -> None: "Machine name must be a valid hostname", location="Create Machine" ) - inventory = load_inventory(flake.path) + inventory = load_inventory_json(flake.path) + + full_inventory = load_inventory_eval(flake.path) + + if machine.name in full_inventory.machines.keys(): + raise ClanError(f"Machine with the name {machine.name} already exists") + inventory.machines.update({machine.name: machine}) save_inventory(inventory, flake.path, f"Create machine {machine.name}") - commit_file(get_path(flake.path), Path(flake.path)) - def create_command(args: argparse.Namespace) -> None: create_machine( diff --git a/pkgs/clan-cli/clan_cli/machines/delete.py b/pkgs/clan-cli/clan_cli/machines/delete.py index e9d64e785..228dba9f1 100644 --- a/pkgs/clan-cli/clan_cli/machines/delete.py +++ b/pkgs/clan-cli/clan_cli/machines/delete.py @@ -6,12 +6,12 @@ from ..clan_uri import FlakeId from ..completions import add_dynamic_completer, complete_machines from ..dirs import specific_machine_dir from ..errors import ClanError -from ..inventory import load_inventory, save_inventory +from ..inventory import load_inventory_json, save_inventory @API.register def delete_machine(flake: FlakeId, name: str) -> None: - inventory = load_inventory(flake.path) + inventory = load_inventory_json(flake.path) machine = inventory.machines.pop(name, None) if machine is None: diff --git a/pkgs/clan-cli/clan_cli/machines/list.py b/pkgs/clan-cli/clan_cli/machines/list.py index e2943cf57..f8df311d5 100644 --- a/pkgs/clan-cli/clan_cli/machines/list.py +++ b/pkgs/clan-cli/clan_cli/machines/list.py @@ -1,31 +1,17 @@ import argparse -import json import logging from pathlib import Path from clan_cli.api import API -from clan_cli.inventory import Machine, from_dict - -from ..cmd import run_no_stdout -from ..nix import nix_eval +from clan_cli.inventory import Machine, load_inventory_eval log = logging.getLogger(__name__) @API.register def list_machines(flake_url: str | Path, debug: bool = False) -> dict[str, Machine]: - cmd = nix_eval( - [ - f"{flake_url}#clanInternals.inventory.machines", - "--json", - ] - ) - - proc = run_no_stdout(cmd) - - res = proc.stdout.strip() - data = {name: from_dict(Machine, v) for name, v in json.loads(res).items()} - return data + inventory = load_inventory_eval(flake_url) + return inventory.machines def list_command(args: argparse.Namespace) -> None: diff --git a/pkgs/clan-cli/tests/test_create_flake.py b/pkgs/clan-cli/tests/test_create_flake.py index db4c7c8ef..d0e920c80 100644 --- a/pkgs/clan-cli/tests/test_create_flake.py +++ b/pkgs/clan-cli/tests/test_create_flake.py @@ -20,12 +20,12 @@ def test_create_flake( cli.run(["flakes", "create", str(flake_dir), f"--url={url}"]) assert (flake_dir / ".clan-flake").exists() - # Replace the inputs.clan.url in the template flake.nix substitute( flake_dir / "flake.nix", clan_core, ) + # Dont evaluate the inventory before the substitute call monkeypatch.chdir(flake_dir) cli.run(["machines", "create", "machine1"]) diff --git a/pkgs/clan-cli/tests/test_modules.py b/pkgs/clan-cli/tests/test_modules.py index b637ba147..4b45f7a6f 100644 --- a/pkgs/clan-cli/tests/test_modules.py +++ b/pkgs/clan-cli/tests/test_modules.py @@ -14,7 +14,7 @@ from clan_cli.inventory import ( ServiceBorgbackupRoleClient, ServiceBorgbackupRoleServer, ServiceMeta, - load_inventory, + load_inventory_json, save_inventory, ) from clan_cli.machines.create import create_machine @@ -67,7 +67,7 @@ def test_add_module_to_inventory( ), ) - inventory = load_inventory(base_path) + inventory = load_inventory_json(base_path) inventory.services.borgbackup = { "borg1": ServiceBorgbackup( diff --git a/pkgs/installer/flake-module.nix b/pkgs/installer/flake-module.nix index bfd4dc551..b00c22a79 100644 --- a/pkgs/installer/flake-module.nix +++ b/pkgs/installer/flake-module.nix @@ -32,7 +32,7 @@ let } // flashDiskoConfig; - # Important: The partition names need to be different to the clan install + # Important: The partition names need to be different to the clan install flashDiskoConfig = { boot.loader.grub.efiSupport = lib.mkDefault true; boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true; diff --git a/pkgs/webview-ui/.gitignore b/pkgs/webview-ui/.gitignore index 9e5bfb42d..590601772 100644 --- a/pkgs/webview-ui/.gitignore +++ b/pkgs/webview-ui/.gitignore @@ -1 +1,3 @@ -api \ No newline at end of file +api + +.vite \ No newline at end of file diff --git a/pkgs/webview-ui/app/src/Routes.tsx b/pkgs/webview-ui/app/src/Routes.tsx index 51f0c01d6..aceef3271 100644 --- a/pkgs/webview-ui/app/src/Routes.tsx +++ b/pkgs/webview-ui/app/src/Routes.tsx @@ -8,6 +8,7 @@ import { Flash } from "./routes/flash/view"; import { Settings } from "./routes/settings"; import { Welcome } from "./routes/welcome"; import { Deploy } from "./routes/deploy"; +import { CreateMachine } from "./routes/machines/create"; export type Route = keyof typeof routes; @@ -22,6 +23,11 @@ export const routes = { label: "Machines", icon: "devices_other", }, + "machines/add": { + child: CreateMachine, + label: "create Machine", + icon: "add", + }, hosts: { child: HostList, label: "hosts", diff --git a/pkgs/webview-ui/app/src/api.ts b/pkgs/webview-ui/app/src/api.ts index 602ef26c8..bcb1689cf 100644 --- a/pkgs/webview-ui/app/src/api.ts +++ b/pkgs/webview-ui/app/src/api.ts @@ -67,6 +67,17 @@ function createFunctions( dispatch: (args: OperationArgs) => void; receive: (fn: (response: OperationResponse) => void, id: string) => void; } { + window.clan[operationName] = (s: string) => { + const f = (response: OperationResponse) => { + // Get the correct receiver function for the op_key + const receiver = registry[operationName][response.op_key]; + if (receiver) { + receiver(response); + } + }; + deserialize(f)(s); + }; + return { dispatch: (args: OperationArgs) => { // Send the data to the gtk app @@ -78,15 +89,6 @@ function createFunctions( receive: (fn: (response: OperationResponse) => void, id: string) => { // @ts-expect-error: This should work although typescript doesn't let us write registry[operationName][id] = fn; - - window.clan[operationName] = (s: string) => { - const f = (response: OperationResponse) => { - if (response.op_key === id) { - registry[operationName][id](response); - } - }; - deserialize(f)(s); - }; }, }; } diff --git a/pkgs/webview-ui/app/src/index.css b/pkgs/webview-ui/app/src/index.css index 169082015..bbde75111 100644 --- a/pkgs/webview-ui/app/src/index.css +++ b/pkgs/webview-ui/app/src/index.css @@ -4,7 +4,6 @@ @tailwind components; @tailwind utilities; - html { overflow-x: hidden; overflow-y: scroll; diff --git a/pkgs/webview-ui/app/src/routes/clan/clanDetails.tsx b/pkgs/webview-ui/app/src/routes/clan/clanDetails.tsx index eb0356b44..fa6e872fe 100644 --- a/pkgs/webview-ui/app/src/routes/clan/clanDetails.tsx +++ b/pkgs/webview-ui/app/src/routes/clan/clanDetails.tsx @@ -42,7 +42,15 @@ export const ClanForm = () => { await toast.promise( (async () => { await callApi("create_clan", { - options: { directory: target_dir, meta, template_url }, + options: { + directory: target_dir, + template_url, + initial: { + meta, + services: {}, + machines: {}, + }, + }, }); setActiveURI(target_dir); setRoute("machines"); diff --git a/pkgs/webview-ui/app/src/routes/machines/create.tsx b/pkgs/webview-ui/app/src/routes/machines/create.tsx new file mode 100644 index 000000000..0a379c0b1 --- /dev/null +++ b/pkgs/webview-ui/app/src/routes/machines/create.tsx @@ -0,0 +1,124 @@ +import { callApi, OperationArgs, pyApi } from "@/src/api"; +import { activeURI } from "@/src/App"; +import { createForm, required } from "@modular-forms/solid"; +import toast from "solid-toast"; + +type CreateMachineForm = OperationArgs<"create_machine">; + +export function CreateMachine() { + const [formStore, { Form, Field }] = createForm({}); + + const handleSubmit = async (values: CreateMachineForm) => { + const active_dir = activeURI(); + if (!active_dir) { + toast.error("Open a clan to create the machine in"); + return; + } + + callApi("create_machine", { + flake: { + loc: active_dir, + }, + machine: { + name: "jon", + deploy: { + targetHost: null, + }, + }, + }); + console.log("submit", values); + }; + return ( +
+ Create new Machine +
+ + {(field, props) => ( + <> + +
+ {field.error && ( + + {field.error} + + )} +
+ + )} +
+ + {(field, props) => ( + <> + +
+ {field.error && ( + + {field.error} + + )} +
+ + )} +
+ + {(field, props) => ( + <> + +
+ + Must be set before deployment for the following tasks: +
    +
  • + Detect hardware config +
  • +
  • + Detect disk layout +
  • +
  • + Remote installation +
  • +
+
+ {field.error && ( + + {field.error} + + )} +
+ + )} +
+ +
+
+ ); +} diff --git a/pkgs/webview-ui/app/src/routes/machines/view.tsx b/pkgs/webview-ui/app/src/routes/machines/view.tsx index 969033ddc..441fe755e 100644 --- a/pkgs/webview-ui/app/src/routes/machines/view.tsx +++ b/pkgs/webview-ui/app/src/routes/machines/view.tsx @@ -7,7 +7,7 @@ import { createSignal, type Component, } from "solid-js"; -import { activeURI, route, setActiveURI } from "@/src/App"; +import { activeURI, route, setActiveURI, setRoute } from "@/src/App"; import { OperationResponse, callApi, pyApi } from "@/src/api"; import toast from "solid-toast"; import { MachineListItem } from "@/src/components/MachineListItem"; @@ -86,6 +86,11 @@ export const MachineListView: Component = () => { refresh +
+ +
{/* {(services) => ( diff --git a/templates/minimal/inventory.json b/templates/minimal/inventory.json new file mode 100644 index 000000000..40109ebd5 --- /dev/null +++ b/templates/minimal/inventory.json @@ -0,0 +1,5 @@ +{ + "meta": { "name": "__CHANGE_ME__" }, + "machines": {}, + "services": {} +}