From 334fe45adcb31d22be36cc73d950e568da067acd Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 23 Jul 2025 16:41:24 +0200 Subject: [PATCH 01/15] ui/cubes: add labels --- .../ui/src/components/Sidebar/Sidebar.css | 2 +- .../ui/src/components/Sidebar/SidebarPane.css | 2 +- pkgs/clan-app/ui/src/scene/cubes.css | 6 +++++ pkgs/clan-app/ui/src/scene/cubes.tsx | 26 +++++++++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-app/ui/src/components/Sidebar/Sidebar.css b/pkgs/clan-app/ui/src/components/Sidebar/Sidebar.css index 7884eae7d..2d19e39b9 100644 --- a/pkgs/clan-app/ui/src/components/Sidebar/Sidebar.css +++ b/pkgs/clan-app/ui/src/components/Sidebar/Sidebar.css @@ -1,5 +1,5 @@ div.sidebar { - @apply w-60 border-none; + @apply w-60 border-none z-10; & > div.header { } diff --git a/pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.css b/pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.css index 13dacc11c..793cc44a9 100644 --- a/pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.css +++ b/pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.css @@ -1,5 +1,5 @@ div.sidebar-pane { - @apply w-full max-w-60 border-none; + @apply w-full max-w-60 border-none z-10; & > div.header { @apply flex items-center justify-between px-3 py-2 rounded-t-[0.5rem]; diff --git a/pkgs/clan-app/ui/src/scene/cubes.css b/pkgs/clan-app/ui/src/scene/cubes.css index dd5175df0..563e6e0c3 100644 --- a/pkgs/clan-app/ui/src/scene/cubes.css +++ b/pkgs/clan-app/ui/src/scene/cubes.css @@ -13,3 +13,9 @@ justify-content: center; align-items: center; } + + +.machine-label { + @apply text-white bg-fg-def-2 py-1 px-3 rounded-sm; + font-size: 0.8rem; +} \ No newline at end of file diff --git a/pkgs/clan-app/ui/src/scene/cubes.tsx b/pkgs/clan-app/ui/src/scene/cubes.tsx index 228123ba9..0626f9650 100644 --- a/pkgs/clan-app/ui/src/scene/cubes.tsx +++ b/pkgs/clan-app/ui/src/scene/cubes.tsx @@ -10,6 +10,10 @@ import "./cubes.css"; import * as THREE from "three"; import { MapControls } from "three/examples/jsm/controls/MapControls.js"; +import { + CSS2DRenderer, + CSS2DObject, +} from "three/examples/jsm/renderers/CSS2DRenderer.js"; import { Toolbar } from "../components/Toolbar/Toolbar"; import { ToolbarButton } from "../components/Toolbar/ToolbarButton"; @@ -77,6 +81,7 @@ export function CubeScene(props: { let scene: THREE.Scene; let camera: THREE.OrthographicCamera; let renderer: THREE.WebGLRenderer; + let labelRenderer: CSS2DRenderer; let floor: THREE.Mesh; let controls: MapControls; // Raycaster for clicking @@ -195,8 +200,11 @@ export function CubeScene(props: { renderer.autoClear = false; renderer.render(bgScene, bgCamera); + controls.update(); // optional; see note below + renderer.render(scene, camera); + labelRenderer.render(scene, camera); if (frameCount % 30 === 0) logMemoryUsage(); } @@ -523,6 +531,15 @@ export function CubeScene(props: { renderer.shadowMap.type = THREE.PCFSoftShadowMap; container.appendChild(renderer.domElement); + // Label renderer + labelRenderer = new CSS2DRenderer(); + labelRenderer.setSize(container.clientWidth, container.clientHeight); + labelRenderer.domElement.style.position = "absolute"; + labelRenderer.domElement.style.top = "0px"; + labelRenderer.domElement.style.pointerEvents = "none"; + labelRenderer.domElement.style.zIndex = "0"; + container.appendChild(labelRenderer.domElement); + controls = new MapControls(camera, renderer.domElement); controls.enableDamping = true; // an animation loop is required when either damping or auto-rotation are enabled // Enable the context menu, @@ -716,6 +733,7 @@ export function CubeScene(props: { camera.updateProjectionMatrix(); renderer.setSize(container.clientWidth, container.clientHeight); + labelRenderer.setSize(container.clientWidth, container.clientHeight); // Update background shader resolution uniforms.resolution.value.set( @@ -791,6 +809,14 @@ export function CubeScene(props: { const baseMesh = createCubeBase([0, BASE_HEIGHT / 2, 0]); baseMesh.name = "base"; // Name for easy identification + const nameDiv = document.createElement("div"); + nameDiv.className = "machine-label"; + nameDiv.textContent = `${userData.id}`; + + const nameLabel = new CSS2DObject(nameDiv); + nameLabel.position.set(0, CUBE_Y + CUBE_SIZE / 2 + 0.2, 0); + cubeMesh.add(nameLabel); + // TODO: Destroy Group in onCleanup const group = new THREE.Group(); group.add(cubeMesh); From 94662b722d0cd00b7b5bdb688109b5bfed24eb37 Mon Sep 17 00:00:00 2001 From: Qubasa Date: Thu, 24 Jul 2025 14:25:05 +0700 Subject: [PATCH 02/15] clan-lib: Remove injected "op_key" argument from all functions and do it over the threadcontext instead. Remove double threading in http server --- pkgs/clan-app/clan_app/api/api_bridge.py | 5 ++- pkgs/clan-app/clan_app/api/file_gtk.py | 15 +++++-- .../api/middleware/argument_parsing.py | 6 --- .../clan_app/deps/http/http_bridge.py | 40 ++++++++++++++--- pkgs/clan-app/ui/src/hooks/api.ts | 13 +++--- pkgs/clan-cli/clan_lib/api/__init__.py | 44 +++---------------- pkgs/clan-cli/clan_lib/async_run/__init__.py | 17 +++++++ 7 files changed, 80 insertions(+), 60 deletions(-) diff --git a/pkgs/clan-app/clan_app/api/api_bridge.py b/pkgs/clan-app/clan_app/api/api_bridge.py index cdbdee676..b1e38a697 100644 --- a/pkgs/clan-app/clan_app/api/api_bridge.py +++ b/pkgs/clan-app/clan_app/api/api_bridge.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any from clan_lib.api import ApiResponse from clan_lib.api.tasks import WebThread -from clan_lib.async_run import set_should_cancel +from clan_lib.async_run import set_current_thread_opkey, set_should_cancel if TYPE_CHECKING: from .middleware import Middleware @@ -98,7 +98,7 @@ class ApiBridge(ABC): *, thread_name: str = "ApiBridgeThread", wait_for_completion: bool = False, - timeout: float = 60.0, + timeout: float = 60.0 * 60, # 1 hour default timeout ) -> None: """Process an API request in a separate thread with cancellation support. @@ -112,6 +112,7 @@ class ApiBridge(ABC): def thread_task(stop_event: threading.Event) -> None: set_should_cancel(lambda: stop_event.is_set()) + set_current_thread_opkey(op_key) try: log.debug( f"Processing {request.method_name} with args {request.args} " diff --git a/pkgs/clan-app/clan_app/api/file_gtk.py b/pkgs/clan-app/clan_app/api/file_gtk.py index a5e87e51d..9f2b0e097 100644 --- a/pkgs/clan-app/clan_app/api/file_gtk.py +++ b/pkgs/clan-app/clan_app/api/file_gtk.py @@ -9,6 +9,7 @@ gi.require_version("Gtk", "4.0") from clan_lib.api import ApiError, ErrorDataClass, SuccessDataClass from clan_lib.api.directory import FileRequest +from clan_lib.async_run import get_current_thread_opkey from clan_lib.clan.check import check_clan_valid from clan_lib.flake import Flake from gi.repository import Gio, GLib, Gtk @@ -24,7 +25,7 @@ def remove_none(_list: list) -> list: RESULT: dict[str, SuccessDataClass[list[str] | None] | ErrorDataClass] = {} -def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass: +def get_clan_folder() -> SuccessDataClass[Flake] | ErrorDataClass: """ Opens the clan folder using the GTK file dialog. Returns the path to the clan folder or an error if it fails. @@ -34,7 +35,10 @@ def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass: title="Select Clan Folder", initial_folder=str(Path.home()), ) - response = get_system_file(file_request, op_key=op_key) + + response = get_system_file(file_request) + + op_key = response.op_key if isinstance(response, ErrorDataClass): return response @@ -70,8 +74,13 @@ def get_clan_folder(*, op_key: str) -> SuccessDataClass[Flake] | ErrorDataClass: def get_system_file( - file_request: FileRequest, *, op_key: str + file_request: FileRequest, ) -> SuccessDataClass[list[str] | None] | ErrorDataClass: + op_key = get_current_thread_opkey() + + if not op_key: + msg = "No operation key found in the current thread context." + raise RuntimeError(msg) GLib.idle_add(gtk_open_file, file_request, op_key) while RESULT.get(op_key) is None: diff --git a/pkgs/clan-app/clan_app/api/middleware/argument_parsing.py b/pkgs/clan-app/clan_app/api/middleware/argument_parsing.py index 107271f91..6a1becbb0 100644 --- a/pkgs/clan-app/clan_app/api/middleware/argument_parsing.py +++ b/pkgs/clan-app/clan_app/api/middleware/argument_parsing.py @@ -21,18 +21,12 @@ class ArgumentParsingMiddleware(Middleware): # Convert dictionary arguments to dataclass instances reconciled_arguments = {} for k, v in context.request.args.items(): - if k == "op_key": - continue - # Get the expected argument type from the API arg_class = self.api.get_method_argtype(context.request.method_name, k) # Convert dictionary to dataclass instance reconciled_arguments[k] = from_dict(arg_class, v) - # Add op_key to arguments - reconciled_arguments["op_key"] = context.request.op_key - # Create a new request with reconciled arguments updated_request = BackendRequest( diff --git a/pkgs/clan-app/clan_app/deps/http/http_bridge.py b/pkgs/clan-app/clan_app/deps/http/http_bridge.py index 1a9cfda4b..883ce31cd 100644 --- a/pkgs/clan-app/clan_app/deps/http/http_bridge.py +++ b/pkgs/clan-app/clan_app/deps/http/http_bridge.py @@ -1,13 +1,22 @@ import json import logging +import threading import uuid from http.server import BaseHTTPRequestHandler from pathlib import Path from typing import TYPE_CHECKING, Any from urllib.parse import urlparse -from clan_lib.api import MethodRegistry, SuccessDataClass, dataclass_to_dict +from clan_lib.api import ( + MethodRegistry, + SuccessDataClass, + dataclass_to_dict, +) from clan_lib.api.tasks import WebThread +from clan_lib.async_run import ( + set_current_thread_opkey, + set_should_cancel, +) from clan_app.api.api_bridge import ApiBridge, BackendRequest, BackendResponse @@ -324,17 +333,34 @@ class HttpBridge(ApiBridge, BaseHTTPRequestHandler): msg = f"Operation key '{op_key}' is already in use. Please try again." raise ValueError(msg) + def process_request_in_thread( + self, + request: BackendRequest, + *, + thread_name: str = "ApiBridgeThread", + wait_for_completion: bool = False, + timeout: float = 60.0 * 60, # 1 hour default timeout + ) -> None: + pass + def _process_api_request_in_thread( self, api_request: BackendRequest, method_name: str ) -> None: """Process the API request in a separate thread.""" - # Use the inherited thread processing method - self.process_request_in_thread( - api_request, - thread_name="HttpThread", - wait_for_completion=True, - timeout=60.0, + stop_event = threading.Event() + request = api_request + op_key = request.op_key or "unknown" + set_should_cancel(lambda: stop_event.is_set()) + set_current_thread_opkey(op_key) + + curr_thread = threading.current_thread() + self.threads[op_key] = WebThread(thread=curr_thread, stop_event=stop_event) + + log.debug( + f"Processing {request.method_name} with args {request.args} " + f"and header {request.header}" ) + self.process_request(request) def log_message(self, format: str, *args: Any) -> None: # noqa: A002 """Override default logging to use our logger.""" diff --git a/pkgs/clan-app/ui/src/hooks/api.ts b/pkgs/clan-app/ui/src/hooks/api.ts index d84c24367..c164c9f75 100644 --- a/pkgs/clan-app/ui/src/hooks/api.ts +++ b/pkgs/clan-app/ui/src/hooks/api.ts @@ -20,6 +20,7 @@ export type SuccessData = SuccessQuery["data"]; interface SendHeaderType { logging?: { group_path: string[] }; + op_key?: string; } interface BackendSendType { body: OperationArgs; @@ -64,9 +65,14 @@ export const callApi = ( }; } - const req: BackendSendType = { + const op_key = backendOpts?.op_key ?? crypto.randomUUID(); + + let req: BackendSendType = { body: args, - header: backendOpts, + header: { + ...backendOpts, + op_key, + }, }; const result = ( @@ -78,9 +84,6 @@ export const callApi = ( > )[method](req) as Promise>; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const op_key = (result as any)._webviewMessageId as string; - return { uuid: op_key, result: result.then(({ body }) => body), diff --git a/pkgs/clan-cli/clan_lib/api/__init__.py b/pkgs/clan-cli/clan_lib/api/__init__.py index c06996977..62df22996 100644 --- a/pkgs/clan-cli/clan_lib/api/__init__.py +++ b/pkgs/clan-cli/clan_lib/api/__init__.py @@ -15,6 +15,7 @@ from typing import ( ) from clan_lib.api.util import JSchemaTypeError +from clan_lib.async_run import get_current_thread_opkey from clan_lib.errors import ClanError from .serde import dataclass_to_dict, from_dict, sanitize_string @@ -54,26 +55,6 @@ class ErrorDataClass: ApiResponse = SuccessDataClass[ResponseDataType] | ErrorDataClass -def update_wrapper_signature(wrapper: Callable, wrapped: Callable) -> None: - sig = signature(wrapped) - params = list(sig.parameters.values()) - - # Add 'op_key' parameter - op_key_param = Parameter( - "op_key", - Parameter.KEYWORD_ONLY, - # we add a None default value so that typescript code gen drops the parameter - # FIXME: this is a hack, we should filter out op_key in the typescript code gen - default=None, - annotation=str, - ) - params.append(op_key_param) - - # Create a new signature - new_sig = sig.replace(parameters=params) - wrapper.__signature__ = new_sig # type: ignore - - class MethodRegistry: def __init__(self) -> None: self._orig_signature: dict[str, Signature] = {} @@ -130,18 +111,8 @@ API.register(get_system_file) fn_signature = signature(fn) abstract_signature = signature(self._registry[fn_name]) - # Remove the default argument of op_key from abstract_signature - # FIXME: This is a hack to make the signature comparison work - # because the other hack above where default value of op_key is None in the wrapper - abstract_params = list(abstract_signature.parameters.values()) - for i, param in enumerate(abstract_params): - if param.name == "op_key": - abstract_params[i] = param.replace(default=Parameter.empty) - break - abstract_signature = abstract_signature.replace(parameters=abstract_params) - if fn_signature != abstract_signature: - msg = f"Expected signature: {abstract_signature}\nActual signature: {fn_signature}" + msg = f"For function: {fn_name}. Expected signature: {abstract_signature}\nActual signature: {fn_signature}" raise ClanError(msg) self._registry[fn_name] = fn @@ -159,7 +130,11 @@ API.register(get_system_file) self._orig_signature[fn.__name__] = signature(fn) @wraps(fn) - def wrapper(*args: Any, op_key: str, **kwargs: Any) -> ApiResponse[T]: + def wrapper(*args: Any, **kwargs: Any) -> ApiResponse[T]: + op_key = get_current_thread_opkey() + if op_key is None: + msg = f"While executing {fn.__name__}. Middleware forgot to set_current_thread_opkey()" + raise RuntimeError(msg) try: data: T = fn(*args, **kwargs) return SuccessDataClass(status="success", data=data, op_key=op_key) @@ -196,11 +171,6 @@ API.register(get_system_file) orig_return_type = get_type_hints(fn).get("return") wrapper.__annotations__["return"] = ApiResponse[orig_return_type] # type: ignore - # Add additional argument for the operation key - wrapper.__annotations__["op_key"] = str # type: ignore - - update_wrapper_signature(wrapper, fn) - self._registry[fn.__name__] = wrapper return fn diff --git a/pkgs/clan-cli/clan_lib/async_run/__init__.py b/pkgs/clan-cli/clan_lib/async_run/__init__.py index 8d7f8d728..d6a03b681 100644 --- a/pkgs/clan-cli/clan_lib/async_run/__init__.py +++ b/pkgs/clan-cli/clan_lib/async_run/__init__.py @@ -74,6 +74,7 @@ class AsyncContext: should_cancel: Callable[[], bool] = ( lambda: False ) # Used to signal cancellation of task + op_key: str | None = None @dataclass @@ -90,6 +91,22 @@ class AsyncOpts: ASYNC_CTX_THREAD_LOCAL = threading.local() +def set_current_thread_opkey(op_key: str) -> None: + """ + Set the current thread's operation key. + """ + ctx = get_async_ctx() + ctx.op_key = op_key + + +def get_current_thread_opkey() -> str | None: + """ + Get the current thread's operation key. + """ + ctx = get_async_ctx() + return ctx.op_key + + def is_async_cancelled() -> bool: """ Check if the current task has been cancelled. From 59105bd1dab1a03bc54d19ad2b65edc43e6fd7a8 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 24 Jul 2025 09:41:34 +0200 Subject: [PATCH 03/15] =?UTF-8?q?docs/options:=20expose=20all=20clan=20opt?= =?UTF-8?q?ions=20in=20N=C3=BCschtOS=20search?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/nix/options/flake-module.nix | 44 ++++++++++++++++++++----------- lib/modules/clan/interface.nix | 4 +-- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/docs/nix/options/flake-module.nix b/docs/nix/options/flake-module.nix index 32fcf8eae..80b8bb653 100644 --- a/docs/nix/options/flake-module.nix +++ b/docs/nix/options/flake-module.nix @@ -114,9 +114,6 @@ in { options = { - _ = mkOption { - type = types.raw; - }; instances.${name} = lib.mkOption { inherit description; type = types.submodule { @@ -149,20 +146,29 @@ }; }; - mkScope = name: modules: { - inherit name; - modules = [ - { - _module.args = { inherit clanLib; }; - _file = "docs mkScope"; - } - { noInstanceOptions = true; } - ../../../lib/modules/inventoryClass/interface.nix - ] ++ mapAttrsToList fakeInstanceOptions modules; - urlPrefix = "https://github.com/nix-community/dream2nix/blob/main/"; - }; + docModules = [ + { + inherit self; + } + self.modules.clan.default + { + options.inventory = lib.mkOption { + type = types.submoduleWith { + modules = [ + { noInstanceOptions = true; } + ] ++ mapAttrsToList fakeInstanceOptions serviceModules; + }; + }; + } + ]; + in { + # Uncomment for debugging + # legacyPackages.docModules = lib.evalModules { + # modules = docModules; + # }; + packages = lib.optionalAttrs ((privateInputs ? nuschtos) || (inputs ? nuschtos)) { docs-options = (privateInputs.nuschtos or inputs.nuschtos) @@ -171,7 +177,13 @@ inherit baseHref; title = "Clan Options"; # scopes = mapAttrsToList mkScope serviceModules; - scopes = [ (mkScope "Clan Inventory" serviceModules) ]; + scopes = [ + { + name = "Clan"; + modules = docModules; + urlPrefix = "https://git.clan.lol/clan/clan-core/src/branch/main/"; + } + ]; }; }; }; diff --git a/lib/modules/clan/interface.nix b/lib/modules/clan/interface.nix index a9d52066e..a097f7344 100644 --- a/lib/modules/clan/interface.nix +++ b/lib/modules/clan/interface.nix @@ -229,8 +229,8 @@ in }; inventory = lib.mkOption { - type = types.submodule { - imports = [ + type = types.submoduleWith { + modules = [ { _module.args = { inherit clanLib; }; _file = "clan interface"; From 694059d3ce3e7ae42e29219a0f873b52a1d41e5f Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Thu, 24 Jul 2025 09:48:57 +0100 Subject: [PATCH 04/15] feat(ui): waiting for necessary queries before dropping clan loader --- .../ui/src/components/Sidebar/SidebarBody.tsx | 30 ++++++----- .../src/components/Sidebar/SidebarHeader.tsx | 4 +- pkgs/clan-app/ui/src/hooks/api.ts | 2 +- pkgs/clan-app/ui/src/queries/queries.ts | 6 ++- pkgs/clan-app/ui/src/routes/Clan/Clan.tsx | 50 +++++++++++++++---- 5 files changed, 63 insertions(+), 29 deletions(-) diff --git a/pkgs/clan-app/ui/src/components/Sidebar/SidebarBody.tsx b/pkgs/clan-app/ui/src/components/Sidebar/SidebarBody.tsx index ccd762208..a5757bacb 100644 --- a/pkgs/clan-app/ui/src/components/Sidebar/SidebarBody.tsx +++ b/pkgs/clan-app/ui/src/components/Sidebar/SidebarBody.tsx @@ -3,7 +3,7 @@ import { A } from "@solidjs/router"; import { Accordion } from "@kobalte/core/accordion"; import Icon from "../Icon/Icon"; import { Typography } from "@/src/components/Typography/Typography"; -import { For, Suspense } from "solid-js"; +import { For } from "solid-js"; import { MachineStatus } from "@/src/components/MachineStatus/MachineStatus"; import { buildMachinePath, useClanURI } from "@/src/hooks/clan"; import { useMachinesQuery } from "@/src/queries/queries"; @@ -89,21 +89,19 @@ export const SidebarBody = (props: SidebarProps) => { - - - + diff --git a/pkgs/clan-app/ui/src/components/Sidebar/SidebarHeader.tsx b/pkgs/clan-app/ui/src/components/Sidebar/SidebarHeader.tsx index d22c75fc3..c0acc59db 100644 --- a/pkgs/clan-app/ui/src/components/Sidebar/SidebarHeader.tsx +++ b/pkgs/clan-app/ui/src/components/Sidebar/SidebarHeader.tsx @@ -4,7 +4,7 @@ import { DropdownMenu } from "@kobalte/core/dropdown-menu"; import { useNavigate } from "@solidjs/router"; import { Typography } from "../Typography/Typography"; import { createSignal, For, Suspense } from "solid-js"; -import { useAllClanDetailsQuery } from "@/src/queries/queries"; +import { useClanListQuery } from "@/src/queries/queries"; import { navigateToClan, useClanURI } from "@/src/hooks/clan"; import { clanURIs } from "@/src/stores/clan"; @@ -15,7 +15,7 @@ export const SidebarHeader = () => { // get information about the current active clan const clanURI = useClanURI(); - const allClans = useAllClanDetailsQuery(clanURIs()); + const allClans = useClanListQuery(clanURIs()); const activeClan = () => allClans.find(({ data }) => data?.uri === clanURI); diff --git a/pkgs/clan-app/ui/src/hooks/api.ts b/pkgs/clan-app/ui/src/hooks/api.ts index c164c9f75..282edd77d 100644 --- a/pkgs/clan-app/ui/src/hooks/api.ts +++ b/pkgs/clan-app/ui/src/hooks/api.ts @@ -67,7 +67,7 @@ export const callApi = ( const op_key = backendOpts?.op_key ?? crypto.randomUUID(); - let req: BackendSendType = { + const req: BackendSendType = { body: args, header: { ...backendOpts, diff --git a/pkgs/clan-app/ui/src/queries/queries.ts b/pkgs/clan-app/ui/src/queries/queries.ts index 38e880400..387b2d764 100644 --- a/pkgs/clan-app/ui/src/queries/queries.ts +++ b/pkgs/clan-app/ui/src/queries/queries.ts @@ -3,8 +3,12 @@ import { callApi, SuccessData } from "../hooks/api"; import { encodeBase64 } from "@/src/hooks/clan"; export type ClanDetails = SuccessData<"get_clan_details">; +export type ClanDetailsWithURI = ClanDetails & { uri: string }; + export type ListMachines = SuccessData<"list_machines">; + export type MachinesQueryResult = UseQueryResult; +export type ClanListQueryResult = UseQueryResult[]; export const useMachinesQuery = (clanURI: string) => useQuery(() => ({ @@ -48,7 +52,7 @@ export const useClanDetailsQuery = (clanURI: string) => }, })); -export const useAllClanDetailsQuery = (clanURIs: string[]) => +export const useClanListQuery = (clanURIs: string[]): ClanListQueryResult => useQueries(() => ({ queries: clanURIs.map((clanURI) => ({ queryKey: ["clans", encodeBase64(clanURI), "details"], diff --git a/pkgs/clan-app/ui/src/routes/Clan/Clan.tsx b/pkgs/clan-app/ui/src/routes/Clan/Clan.tsx index c26a35e97..d6211fc05 100644 --- a/pkgs/clan-app/ui/src/routes/Clan/Clan.tsx +++ b/pkgs/clan-app/ui/src/routes/Clan/Clan.tsx @@ -15,9 +15,14 @@ import { useClanURI, } from "@/src/hooks/clan"; import { CubeScene } from "@/src/scene/cubes"; -import { MachinesQueryResult, useMachinesQuery } from "@/src/queries/queries"; +import { + ClanListQueryResult, + MachinesQueryResult, + useClanListQuery, + useMachinesQuery, +} from "@/src/queries/queries"; import { callApi } from "@/src/hooks/api"; -import { store, setStore } from "@/src/stores/clan"; +import { store, setStore, clanURIs } from "@/src/stores/clan"; import { produce } from "solid-js/store"; import { Button } from "@/src/components/Button/Button"; import { Splash } from "@/src/scene/splash"; @@ -42,10 +47,12 @@ export const Clan: Component = (props) => { interface CreateFormValues extends FieldValues { name: string; } + interface MockProps { onClose: () => void; onSubmit: (formValues: CreateFormValues) => void; } + const MockCreateMachine = (props: MockProps) => { let container: Node; @@ -173,7 +180,26 @@ const ClanSceneController = (props: RouteSectionProps) => { return ( - {({ query }) => { + {({ clansQuery, machinesQuery }) => { + // a combination of the individual clan details query status and the machines query status + // the cube scene needs the machines query, the sidebar needs the clans query and machines query results + // so we wait on both before removing the loader to avoid any loading artefacts + const isLoading = (): boolean => { + // check the machines query first + if (machinesQuery.isLoading) { + return true; + } + + // otherwise iterate the clans query and return early if we find a queries that is still loading + for (const query of clansQuery) { + if (query.isLoading) { + return true; + } + } + + return false; + }; + return ( <> @@ -217,7 +243,7 @@ const ClanSceneController = (props: RouteSectionProps) => { ghost onClick={() => { console.log("Refetching API"); - query.refetch(); + machinesQuery.refetch(); }} > Refetch API @@ -225,7 +251,9 @@ const ClanSceneController = (props: RouteSectionProps) => { {/* TODO: Add minimal display time */}
@@ -233,8 +261,8 @@ const ClanSceneController = (props: RouteSectionProps) => { { const clanURI = useClanURI(); @@ -268,10 +296,14 @@ const ClanSceneController = (props: RouteSectionProps) => { const SceneDataProvider = (props: { clanURI: string; - children: (sceneData: { query: MachinesQueryResult }) => JSX.Element; + children: (sceneData: { + clansQuery: ClanListQueryResult; + machinesQuery: MachinesQueryResult; + }) => JSX.Element; }) => { + const clansQuery = useClanListQuery(clanURIs()); const machinesQuery = useMachinesQuery(props.clanURI); // This component can be used to provide scene data or context if needed - return props.children({ query: machinesQuery }); + return props.children({ clansQuery, machinesQuery }); }; From fb5229a5f353b74ba5af0321f46456f1e80b6f6b Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 23 Jul 2025 17:43:05 +0200 Subject: [PATCH 05/15] ui/cubes: adjust label style --- pkgs/clan-app/ui/src/scene/cubes.css | 18 ++++++++++++++---- pkgs/clan-app/ui/src/scene/cubes.tsx | 6 +++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pkgs/clan-app/ui/src/scene/cubes.css b/pkgs/clan-app/ui/src/scene/cubes.css index 563e6e0c3..d9d7a3f63 100644 --- a/pkgs/clan-app/ui/src/scene/cubes.css +++ b/pkgs/clan-app/ui/src/scene/cubes.css @@ -14,8 +14,18 @@ align-items: center; } - .machine-label { - @apply text-white bg-fg-def-2 py-1 px-3 rounded-sm; - font-size: 0.8rem; -} \ No newline at end of file + @apply text-white bg-inv-4 py-1 px-2 rounded-sm; + font-size: 0.75rem; +} + +.machine-label::after { + content: ""; + position: absolute; + top: 100%; + left: 50%; + margin-left: -5px; + border-width: 5px; + border-style: solid; + border-color: #203637 transparent transparent transparent; +} diff --git a/pkgs/clan-app/ui/src/scene/cubes.tsx b/pkgs/clan-app/ui/src/scene/cubes.tsx index 0626f9650..a316c318a 100644 --- a/pkgs/clan-app/ui/src/scene/cubes.tsx +++ b/pkgs/clan-app/ui/src/scene/cubes.tsx @@ -563,7 +563,7 @@ export function CubeScene(props: { const ambientLight = new THREE.AmbientLight(0xffffff, 1); scene.add(ambientLight); - const directionalLight = new THREE.DirectionalLight(0xffffff, 1); + const directionalLight = new THREE.DirectionalLight(0xffffff, 2); // scene.add(new THREE.DirectionalLightHelper(directionalLight)); // scene.add(new THREE.CameraHelper(directionalLight.shadow.camera)); @@ -587,7 +587,7 @@ export function CubeScene(props: { directionalLight.shadow.camera.far = 2000; directionalLight.shadow.mapSize.width = 4096; // Higher resolution for sharper shadows directionalLight.shadow.mapSize.height = 4096; - directionalLight.shadow.radius = 0; // Hard shadows (low radius) + directionalLight.shadow.radius = 1; // Hard shadows (low radius) directionalLight.shadow.blurSamples = 4; // Fewer samples for harder edges scene.add(directionalLight); scene.add(directionalLight.target); @@ -814,7 +814,7 @@ export function CubeScene(props: { nameDiv.textContent = `${userData.id}`; const nameLabel = new CSS2DObject(nameDiv); - nameLabel.position.set(0, CUBE_Y + CUBE_SIZE / 2 + 0.2, 0); + nameLabel.position.set(0, CUBE_Y + CUBE_SIZE / 2 - 0.2, 0); cubeMesh.add(nameLabel); // TODO: Destroy Group in onCleanup From f18e70dda6255b4177d82341c9ad8b25389985e4 Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Thu, 24 Jul 2025 10:23:43 +0100 Subject: [PATCH 06/15] fix(ui): increase z index for sidebar dropdown --- pkgs/clan-app/ui/src/components/Sidebar/SidebarHeader.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-app/ui/src/components/Sidebar/SidebarHeader.css b/pkgs/clan-app/ui/src/components/Sidebar/SidebarHeader.css index 8eb419e12..c1fcb541f 100644 --- a/pkgs/clan-app/ui/src/components/Sidebar/SidebarHeader.css +++ b/pkgs/clan-app/ui/src/components/Sidebar/SidebarHeader.css @@ -39,7 +39,7 @@ div.sidebar-header { } .sidebar-dropdown-content { - @apply flex flex-col w-full px-2 py-1.5; + @apply flex flex-col w-full px-2 py-1.5 z-10; @apply bg-def-1 rounded-bl-md rounded-br-md; @apply border border-def-2; From b74aa31b876830a1617d24c3c1cad0f9d7ac4a9d Mon Sep 17 00:00:00 2001 From: Qubasa Date: Thu, 24 Jul 2025 17:29:09 +0700 Subject: [PATCH 07/15] clan-lib: Fix missing logging for flake.select execution --- docs/site/developer/contributing/debugging.md | 3 +- pkgs/clan-cli/clan_lib/flake/flake.py | 62 +++++++------------ 2 files changed, 24 insertions(+), 41 deletions(-) diff --git a/docs/site/developer/contributing/debugging.md b/docs/site/developer/contributing/debugging.md index a20ea8bca..a26c3bd1e 100644 --- a/docs/site/developer/contributing/debugging.md +++ b/docs/site/developer/contributing/debugging.md @@ -85,7 +85,8 @@ export CLAN_DEBUG_COMMANDS=1 These options help you pinpoint the source and context of print messages and debug logs during development. - +!!! Note + `CLAN_DEBUG_NIX_SELECTORS` and `CLAN_DEBUG_NIX_PREFETCH` will only print the command on a cache miss! ## Analyzing Performance diff --git a/pkgs/clan-cli/clan_lib/flake/flake.py b/pkgs/clan-cli/clan_lib/flake/flake.py index 1f20731ae..914c2f876 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake.py +++ b/pkgs/clan-cli/clan_lib/flake/flake.py @@ -2,7 +2,6 @@ import json import logging import os import re -import textwrap from dataclasses import asdict, dataclass, field from enum import Enum from functools import cache @@ -729,7 +728,7 @@ class Flake: self.identifier, ] - trace_prefetch = os.environ.get("CLAN_DEBUG_NIX_PREFETCH", "0") == "1" + trace_prefetch = os.environ.get("CLAN_DEBUG_NIX_PREFETCH", False) == "1" if not trace_prefetch: log.debug(f"Prefetching flake {self.identifier}") flake_prefetch = run(nix_command(cmd), RunOpts(trace=trace_prefetch)) @@ -862,47 +861,11 @@ class Flake: ]; }} """ - if len(selectors) > 1 : - msg = textwrap.dedent(f""" - clan select "{selectors}" - """).lstrip("\n").rstrip("\n") - if os.environ.get("CLAN_DEBUG_NIX_SELECTORS"): - msg += textwrap.dedent(f""" - to debug run: - nix repl --expr 'rec {{ - flake = builtins.getFlake "{self.identifier}"; - selectLib = (builtins.getFlake "path:{select_source()}?narHash={select_hash}").lib; - query = [ - {" ".join( - [ - f"(selectLib.select ''{selector}'' flake)" - for selector in selectors - ] - )} - ]; - }}' - """).lstrip("\n") - log.debug(msg) - # fmt: on - elif len(selectors) == 1: - msg = textwrap.dedent(f""" - $ clan select "{selectors[0]}" - """).lstrip("\n").rstrip("\n") - if os.environ.get("CLAN_DEBUG_NIX_SELECTORS"): - msg += textwrap.dedent(f""" - to debug run: - nix repl --expr 'rec {{ - flake = builtins.getFlake "{self.identifier}"; - selectLib = (builtins.getFlake "path:{select_source()}?narHash={select_hash}").lib; - query = selectLib.select '"''{selectors[0]}''"' flake; - }}' - """).lstrip("\n") - log.debug(msg) - + trace = os.environ.get("CLAN_DEBUG_NIX_SELECTORS", False) == "1" build_output = Path( run( nix_build(["--expr", nix_code, *nix_options]), - RunOpts(log=Log.NONE, trace=False), + RunOpts(log=Log.NONE, trace=trace), ).stdout.strip() ) @@ -918,6 +881,22 @@ class Flake: if self.flake_cache_path: self._cache.save_to_file(self.flake_cache_path) + def log_selectors(self, selectors: list[str]) -> None: + if not selectors: + return + + if len(selectors) > 1: + log.debug("==== Printing multi selector command as multiple commands. ====") + + msg = "" + # Build base message + for selector in selectors: + msg += f'$ clan select "{selector}"' + if len(selectors) > 1: + msg += "\n" + + log.debug(msg) + def precache(self, selectors: list[str]) -> None: """ Ensures that the specified selectors are cached locally. @@ -929,6 +908,7 @@ class Flake: Args: selectors (list[str]): A list of attribute selectors to check and cache. """ + if self._cache is None: self.invalidate_cache() assert self._cache is not None @@ -956,6 +936,8 @@ class Flake: assert self._cache is not None assert self.flake_cache_path is not None + self.log_selectors([selector]) + if not self._cache.is_cached(selector): log.debug(f"Cache miss for {selector}") self.get_from_nix([selector]) From d3d1489829355bd4518be71623ae2cb3428709d6 Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Thu, 24 Jul 2025 11:40:54 +0100 Subject: [PATCH 08/15] feat(ui): animate sidebar pane entry/exit --- .../ui/src/components/Sidebar/SidebarPane.css | 68 ++++++++++++++++++- .../ui/src/components/Sidebar/SidebarPane.tsx | 14 +++- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.css b/pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.css index 793cc44a9..bfdb7ef39 100644 --- a/pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.css +++ b/pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.css @@ -1,5 +1,16 @@ div.sidebar-pane { - @apply w-full max-w-60 border-none z-10; + @apply border-none z-10; + + animation: sidebarPaneShow 250ms ease-in forwards; + + &.closing { + animation: sidebarPaneHide 250ms ease-out 300ms forwards; + + & > div.header > *, + & > div.body > * { + animation: sidebarFadeOut 250ms ease-out forwards; + } + } & > div.header { @apply flex items-center justify-between px-3 py-2 rounded-t-[0.5rem]; @@ -7,11 +18,17 @@ div.sidebar-pane { border-r-[1px] border-r-bg-inv-3 border-b-2 border-b-bg-inv-4 border-l-[1px] border-l-bg-inv-3; + background: linear-gradient( 90deg, theme(colors.bg.inv.3) 0%, theme(colors.bg.inv.4) 100% ); + + & > * { + @apply opacity-0; + animation: sidebarFadeIn 250ms ease-in 250ms forwards; + } } & > div.body { @@ -29,5 +46,54 @@ div.sidebar-pane { theme(colors.bg.inv.2) 0%, theme(colors.bg.inv.3) 100% ); + + & > * { + @apply opacity-0; + animation: sidebarFadeIn 250ms ease-in 350ms forwards; + } + } +} + +@keyframes sidebarPaneShow { + 0% { + @apply w-0; + @apply opacity-0; + } + 10% { + @apply w-8; + } + 30% { + @apply opacity-100; + } + 100% { + @apply w-60; + } +} + +@keyframes sidebarPaneHide { + 90% { + @apply w-8; + } + 100% { + @apply w-0; + @apply opacity-0; + } +} + +@keyframes sidebarFadeIn { + from { + opacity: 0; + } + to { + opacity: 1; + } +} + +@keyframes sidebarFadeOut { + from { + opacity: 1; + } + to { + opacity: 0; } } diff --git a/pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.tsx b/pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.tsx index 53bba06bb..f80863c3b 100644 --- a/pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.tsx +++ b/pkgs/clan-app/ui/src/components/Sidebar/SidebarPane.tsx @@ -1,8 +1,9 @@ -import { JSX } from "solid-js"; +import { createSignal, JSX } from "solid-js"; import "./SidebarPane.css"; import { Typography } from "@/src/components/Typography/Typography"; import Icon from "../Icon/Icon"; import { Button as KButton } from "@kobalte/core/button"; +import cx from "classnames"; export interface SidebarPaneProps { title: string; @@ -11,13 +12,20 @@ export interface SidebarPaneProps { } export const SidebarPane = (props: SidebarPaneProps) => { + const [closing, setClosing] = createSignal(false); + + const onClose = () => { + setClosing(true); + setTimeout(() => props.onClose(), 550); + }; + return ( -
+
{props.title} - +
From 7146c973626b5021914a78540203b2976f9e41ff Mon Sep 17 00:00:00 2001 From: Qubasa Date: Thu, 24 Jul 2025 18:27:40 +0700 Subject: [PATCH 09/15] clan_lib: Fix flake.select logging, now we log the first time select queries a path for the first time, it doesn't matter if it is cached or not. --- pkgs/clan-cli/clan_lib/flake/flake.py | 111 +++- .../clan_lib/flake/test_flake_access.py | 488 ++++++++++++++++++ 2 files changed, 581 insertions(+), 18 deletions(-) create mode 100644 pkgs/clan-cli/clan_lib/flake/test_flake_access.py diff --git a/pkgs/clan-cli/clan_lib/flake/flake.py b/pkgs/clan-cli/clan_lib/flake/flake.py index 914c2f876..98b02b33a 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake.py +++ b/pkgs/clan-cli/clan_lib/flake/flake.py @@ -2,6 +2,7 @@ import json import logging import os import re +import shlex from dataclasses import asdict, dataclass, field from enum import Enum from functools import cache @@ -306,6 +307,7 @@ class FlakeCacheEntry: is_list: bool = False exists: bool = True fetched_all: bool = False + _num_accessed: int = field(default=0, init=False) def insert( self, @@ -475,6 +477,80 @@ class FlakeCacheEntry: return False + def is_path_unaccessed(self, selectors: list[Selector]) -> bool: + """Check if the leaf entry has _num_accessed == 0""" + if selectors == []: + # This is the leaf, check if it's unaccessed + return self._num_accessed == 0 + + selector = selectors[0] + + # Navigate to leaf for string/maybe selectors + if ( + selector.type == SelectorType.STR or selector.type == SelectorType.MAYBE + ) and isinstance(self.value, dict): + assert isinstance(selector.value, str) + if selector.value in self.value: + return self.value[selector.value].is_path_unaccessed(selectors[1:]) + return True # Non-existent path is considered unaccessed + + # For set selectors, check if any leaf is unaccessed + if ( + selector.type == SelectorType.SET + and isinstance(selector.value, list) + and isinstance(self.value, dict) + ): + for subselector in selector.value: + if subselector.value in self.value: + if not self.value[subselector.value].is_path_unaccessed( + selectors[1:] + ): + return False + return True + + # For all selectors, check if any existing key is unaccessed + if selector.type == SelectorType.ALL and isinstance(self.value, dict): + for key in self.value: + if self.value[key].exists: + if not self.value[key].is_path_unaccessed(selectors[1:]): + return False + return True + + return True + + def mark_path_accessed(self, selectors: list[Selector]) -> None: + """Mark only the leaf entry as accessed""" + if selectors == []: + # This is the leaf, increment access count + self._num_accessed += 1 + return + + selector = selectors[0] + + # Navigate to leaf for string/maybe selectors + if ( + selector.type == SelectorType.STR or selector.type == SelectorType.MAYBE + ) and isinstance(self.value, dict): + assert isinstance(selector.value, str) + if selector.value in self.value: + self.value[selector.value].mark_path_accessed(selectors[1:]) + + # For set selectors, mark all leaf paths + elif ( + selector.type == SelectorType.SET + and isinstance(selector.value, list) + and isinstance(self.value, dict) + ): + for subselector in selector.value: + if subselector.value in self.value: + self.value[subselector.value].mark_path_accessed(selectors[1:]) + + # For all selectors, mark all existing keys + elif selector.type == SelectorType.ALL and isinstance(self.value, dict): + for key in self.value: + if self.value[key].exists: + self.value[key].mark_path_accessed(selectors[1:]) + def select(self, selectors: list[Selector]) -> Any: selector: Selector if selectors == []: @@ -592,6 +668,7 @@ class FlakeCacheEntry: entry = FlakeCacheEntry( value=value, is_list=is_list, exists=exists, fetched_all=fetched_all ) + entry._num_accessed = 0 return entry def __repr__(self) -> str: @@ -619,12 +696,21 @@ class FlakeCache: def select(self, selector_str: str) -> Any: selectors = parse_selector(selector_str) + self.mark_path_accessed(selectors) return self.cache.select(selectors) def is_cached(self, selector_str: str) -> bool: selectors = parse_selector(selector_str) + if self.is_path_unaccessed(selectors): + log.debug(f"$ clan select {shlex.quote(selector_str)}") return self.cache.is_cached(selectors) + def is_path_unaccessed(self, selectors: list[Selector]) -> bool: + return self.cache.is_path_unaccessed(selectors) + + def mark_path_accessed(self, selectors: list[Selector]) -> None: + self.cache.mark_path_accessed(selectors) + def save_to_file(self, path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) with NamedTemporaryFile(mode="w", dir=path.parent, delete=False) as temp_file: @@ -881,22 +967,6 @@ class Flake: if self.flake_cache_path: self._cache.save_to_file(self.flake_cache_path) - def log_selectors(self, selectors: list[str]) -> None: - if not selectors: - return - - if len(selectors) > 1: - log.debug("==== Printing multi selector command as multiple commands. ====") - - msg = "" - # Build base message - for selector in selectors: - msg += f'$ clan select "{selector}"' - if len(selectors) > 1: - msg += "\n" - - log.debug(msg) - def precache(self, selectors: list[str]) -> None: """ Ensures that the specified selectors are cached locally. @@ -915,8 +985,14 @@ class Flake: assert self.flake_cache_path is not None not_fetched_selectors = [] for selector in selectors: + parsed_selectors = parse_selector(selector) + if not self._cache.is_cached(selector): not_fetched_selectors.append(selector) + + # Mark path as accessed after checking + self._cache.mark_path_accessed(parsed_selectors) + if not_fetched_selectors: self.get_from_nix(not_fetched_selectors) @@ -936,11 +1012,10 @@ class Flake: assert self._cache is not None assert self.flake_cache_path is not None - self.log_selectors([selector]) - if not self._cache.is_cached(selector): log.debug(f"Cache miss for {selector}") self.get_from_nix([selector]) + value = self._cache.select(selector) return value diff --git a/pkgs/clan-cli/clan_lib/flake/test_flake_access.py b/pkgs/clan-cli/clan_lib/flake/test_flake_access.py new file mode 100644 index 000000000..6915ea331 --- /dev/null +++ b/pkgs/clan-cli/clan_lib/flake/test_flake_access.py @@ -0,0 +1,488 @@ +from clan_lib.flake.flake import FlakeCache, parse_selector + + +class TestFlakeAccessTracking: + """Test the leaf-only access tracking feature for FlakeCache""" + + def test_leaf_access_tracking_simple(self) -> None: + """Test that only leaf nodes have their access count incremented""" + cache = FlakeCache() + + # Insert some data + data = {"level1": {"level2": {"leaf": "value"}}} + cache.insert(data, "") + + # Initially, the leaf should be unaccessed + assert cache.is_path_unaccessed(parse_selector("level1.level2.leaf")) + + # Mark the path as accessed + cache.mark_path_accessed(parse_selector("level1.level2.leaf")) + + # Now the leaf should be accessed + assert not cache.is_path_unaccessed(parse_selector("level1.level2.leaf")) + + # But intermediate nodes should still have _num_accessed == 0 + # We'll need to check this directly on the cache entries + entry = cache.cache + assert isinstance(entry.value, dict) + assert entry._num_accessed == 0 # Root should not be incremented + + level1_entry = entry.value["level1"] + assert level1_entry._num_accessed == 0 # level1 should not be incremented + + level2_entry = level1_entry.value["level2"] + assert level2_entry._num_accessed == 0 # level2 should not be incremented + + leaf_entry = level2_entry.value["leaf"] + assert leaf_entry._num_accessed == 1 # Only leaf should be incremented + + def test_leaf_access_tracking_multiple_paths(self) -> None: + """Test access tracking with multiple paths""" + cache = FlakeCache() + + # Insert data with multiple branches + data = {"branch1": {"leaf1": "value1"}, "branch2": {"leaf2": "value2"}} + cache.insert(data, "") + + # Initially both leaves should be unaccessed + assert cache.is_path_unaccessed(parse_selector("branch1.leaf1")) + assert cache.is_path_unaccessed(parse_selector("branch2.leaf2")) + + # Access first leaf + cache.mark_path_accessed(parse_selector("branch1.leaf1")) + + # First leaf should be accessed, second should not + assert not cache.is_path_unaccessed(parse_selector("branch1.leaf1")) + assert cache.is_path_unaccessed(parse_selector("branch2.leaf2")) + + # Access second leaf + cache.mark_path_accessed(parse_selector("branch2.leaf2")) + + # Both should now be accessed + assert not cache.is_path_unaccessed(parse_selector("branch1.leaf1")) + assert not cache.is_path_unaccessed(parse_selector("branch2.leaf2")) + + def test_leaf_access_tracking_with_lists(self) -> None: + """Test access tracking with list entries""" + cache = FlakeCache() + + # Insert data with a list + data = {"items": ["item0", "item1", "item2"]} + cache.insert(data, "") + + # Check initial state + assert cache.is_path_unaccessed(parse_selector("items.0")) + assert cache.is_path_unaccessed(parse_selector("items.1")) + assert cache.is_path_unaccessed(parse_selector("items.2")) + + # Access one item + cache.mark_path_accessed(parse_selector("items.1")) + + # Only that item should be marked as accessed + assert cache.is_path_unaccessed(parse_selector("items.0")) + assert not cache.is_path_unaccessed(parse_selector("items.1")) + assert cache.is_path_unaccessed(parse_selector("items.2")) + + def test_leaf_access_tracking_nonexistent_path(self) -> None: + """Test that non-existent paths are considered unaccessed""" + cache = FlakeCache() + + # Insert minimal data + data = {"exists": "value"} + cache.insert(data, "") + + # Non-existent path should be considered unaccessed + assert cache.is_path_unaccessed(parse_selector("does.not.exist")) + + # Marking non-existent path should not crash + cache.mark_path_accessed(parse_selector("does.not.exist")) + + # It should still be considered unaccessed since it doesn't exist + assert cache.is_path_unaccessed(parse_selector("does.not.exist")) + + def test_leaf_access_tracking_integration_with_select(self) -> None: + """Test that the access tracking works correctly with the select operation""" + cache = FlakeCache() + + # Insert nested data + data = { + "config": { + "database": {"host": "localhost", "port": 5432}, + "cache": {"enabled": True}, + } + } + cache.insert(data, "") + + # Verify all paths are initially unaccessed + assert cache.is_path_unaccessed(parse_selector("config.database.host")) + assert cache.is_path_unaccessed(parse_selector("config.database.port")) + assert cache.is_path_unaccessed(parse_selector("config.cache.enabled")) + + # Select a value (this would happen in Flake.select) + value = cache.select("config.database.host") + assert value == "localhost" + + # Mark the path as accessed (simulating what Flake.select does) + cache.mark_path_accessed(parse_selector("config.database.host")) + + # Only the accessed path should be marked + assert not cache.is_path_unaccessed(parse_selector("config.database.host")) + assert cache.is_path_unaccessed(parse_selector("config.database.port")) + assert cache.is_path_unaccessed(parse_selector("config.cache.enabled")) + + def test_leaf_access_multiple_accesses(self) -> None: + """Test that multiple accesses increment the counter""" + cache = FlakeCache() + + data = {"key": "value"} + cache.insert(data, "") + + # Initially unaccessed + assert cache.is_path_unaccessed(parse_selector("key")) + + # Access once + cache.mark_path_accessed(parse_selector("key")) + assert not cache.is_path_unaccessed(parse_selector("key")) + + # Access again + cache.mark_path_accessed(parse_selector("key")) + assert not cache.is_path_unaccessed(parse_selector("key")) + + # Check the actual count + assert isinstance(cache.cache.value, dict) + entry = cache.cache.value["key"] + assert entry._num_accessed == 2 + + def test_leaf_access_tracking_with_set_selectors(self) -> None: + """Test access tracking with SET selectors (subselectors)""" + cache = FlakeCache() + + # Insert data with multiple keys at the same level + data = {"apps": {"web": "web-app", "api": "api-service", "db": "database"}} + cache.insert(data, "") + + # Initially all should be unaccessed + assert cache.is_path_unaccessed(parse_selector("apps.web")) + assert cache.is_path_unaccessed(parse_selector("apps.api")) + assert cache.is_path_unaccessed(parse_selector("apps.db")) + + # Use SET selector to access multiple keys at once + cache.mark_path_accessed(parse_selector("apps.{web,api}")) + + # Only the keys in the set should be marked as accessed + assert not cache.is_path_unaccessed(parse_selector("apps.web")) + assert not cache.is_path_unaccessed(parse_selector("apps.api")) + assert cache.is_path_unaccessed(parse_selector("apps.db")) # Not in set + + # Verify the intermediate node is not marked as accessed + assert isinstance(cache.cache.value, dict) + apps_entry = cache.cache.value["apps"] + assert ( + apps_entry._num_accessed == 0 + ) # Intermediate node should not be incremented + + def test_leaf_access_tracking_set_selectors_nested(self) -> None: + """Test SET selectors with nested paths""" + cache = FlakeCache() + + # Insert nested data + data = { + "services": { + "frontend": {"port": 3000, "host": "localhost"}, + "backend": {"port": 8080, "host": "0.0.0.0"}, + } + } + cache.insert(data, "") + + # Use SET selector at different nesting levels + cache.mark_path_accessed(parse_selector("services.{frontend,backend}.port")) + + # Both port leaves should be accessed + assert not cache.is_path_unaccessed(parse_selector("services.frontend.port")) + assert not cache.is_path_unaccessed(parse_selector("services.backend.port")) + + # Host leaves should not be accessed + assert cache.is_path_unaccessed(parse_selector("services.frontend.host")) + assert cache.is_path_unaccessed(parse_selector("services.backend.host")) + + def test_leaf_access_tracking_set_selectors_partial_match(self) -> None: + """Test SET selectors when some keys don't exist""" + cache = FlakeCache() + + # Insert data with only some of the keys that will be requested + data = { + "config": { + "database": "postgres", + "cache": "redis", + # Note: 'queue' key is missing + } + } + cache.insert(data, "") + + # Use SET selector that includes non-existent key + cache.mark_path_accessed(parse_selector("config.{database,cache,queue}")) + + # Existing keys should be marked as accessed + assert not cache.is_path_unaccessed(parse_selector("config.database")) + assert not cache.is_path_unaccessed(parse_selector("config.cache")) + + # Non-existent key should still be considered unaccessed + assert cache.is_path_unaccessed(parse_selector("config.queue")) + + def test_leaf_access_tracking_set_selectors_is_path_unaccessed(self) -> None: + """Test is_path_unaccessed with SET selectors""" + cache = FlakeCache() + + # Insert data + data = {"metrics": {"cpu": 50, "memory": 80, "disk": 30}} + cache.insert(data, "") + + # Initially, SET selector path should be unaccessed + assert cache.is_path_unaccessed(parse_selector("metrics.{cpu,memory}")) + + # Access one of the keys in the set + cache.mark_path_accessed(parse_selector("metrics.cpu")) + + # Now the SET selector should be considered accessed (since at least one key is accessed) + assert not cache.is_path_unaccessed(parse_selector("metrics.{cpu,memory}")) + + # But a different SET selector should still be unaccessed + assert cache.is_path_unaccessed(parse_selector("metrics.{memory,disk}")) + + def test_leaf_access_tracking_set_selectors_mixed_access(self) -> None: + """Test SET selectors with mixed individual and set access patterns""" + cache = FlakeCache() + + # Insert data + data = { + "endpoints": { + "users": "/api/users", + "posts": "/api/posts", + "comments": "/api/comments", + "auth": "/api/auth", + } + } + cache.insert(data, "") + + # Access individual key first + cache.mark_path_accessed(parse_selector("endpoints.users")) + + # Then access using SET selector + cache.mark_path_accessed(parse_selector("endpoints.{posts,comments}")) + + # Check individual access states + assert not cache.is_path_unaccessed( + parse_selector("endpoints.users") + ) # Individual access + assert not cache.is_path_unaccessed( + parse_selector("endpoints.posts") + ) # SET access + assert not cache.is_path_unaccessed( + parse_selector("endpoints.comments") + ) # SET access + assert cache.is_path_unaccessed( + parse_selector("endpoints.auth") + ) # Not accessed + + # Verify access counts + assert isinstance(cache.cache.value, dict) + endpoints_entry = cache.cache.value["endpoints"] + assert isinstance(endpoints_entry.value, dict) + + # Each accessed leaf should have _num_accessed == 1 + assert endpoints_entry.value["users"]._num_accessed == 1 + assert endpoints_entry.value["posts"]._num_accessed == 1 + assert endpoints_entry.value["comments"]._num_accessed == 1 + assert endpoints_entry.value["auth"]._num_accessed == 0 + + def test_leaf_access_tracking_with_all_selectors(self) -> None: + """Test access tracking with ALL selectors (*)""" + cache = FlakeCache() + + # Insert data with multiple keys at the same level + data = { + "packages": { + "python": "3.11", + "nodejs": "18.0", + "java": "17.0", + "rust": "1.70", + } + } + cache.insert(data, "") + + # Initially all should be unaccessed + assert cache.is_path_unaccessed(parse_selector("packages.python")) + assert cache.is_path_unaccessed(parse_selector("packages.nodejs")) + assert cache.is_path_unaccessed(parse_selector("packages.java")) + assert cache.is_path_unaccessed(parse_selector("packages.rust")) + + # Use ALL selector to access all keys at once + cache.mark_path_accessed(parse_selector("packages.*")) + + # All keys should now be marked as accessed + assert not cache.is_path_unaccessed(parse_selector("packages.python")) + assert not cache.is_path_unaccessed(parse_selector("packages.nodejs")) + assert not cache.is_path_unaccessed(parse_selector("packages.java")) + assert not cache.is_path_unaccessed(parse_selector("packages.rust")) + + # Verify the intermediate node is not marked as accessed + assert isinstance(cache.cache.value, dict) + packages_entry = cache.cache.value["packages"] + assert ( + packages_entry._num_accessed == 0 + ) # Intermediate node should not be incremented + + # Verify each leaf has been accessed exactly once + assert isinstance(packages_entry.value, dict) + assert packages_entry.value["python"]._num_accessed == 1 + assert packages_entry.value["nodejs"]._num_accessed == 1 + assert packages_entry.value["java"]._num_accessed == 1 + assert packages_entry.value["rust"]._num_accessed == 1 + + def test_leaf_access_tracking_all_selectors_nested(self) -> None: + """Test ALL selectors with nested paths""" + cache = FlakeCache() + + # Insert nested data + data = { + "environments": { + "dev": {"database": "dev-db", "cache": "dev-cache"}, + "prod": {"database": "prod-db", "cache": "prod-cache"}, + "test": {"database": "test-db", "cache": "test-cache"}, + } + } + cache.insert(data, "") + + # Use ALL selector at different nesting levels + cache.mark_path_accessed(parse_selector("environments.*.database")) + + # All database leaves should be accessed + assert not cache.is_path_unaccessed(parse_selector("environments.dev.database")) + assert not cache.is_path_unaccessed( + parse_selector("environments.prod.database") + ) + assert not cache.is_path_unaccessed( + parse_selector("environments.test.database") + ) + + # Cache leaves should not be accessed + assert cache.is_path_unaccessed(parse_selector("environments.dev.cache")) + assert cache.is_path_unaccessed(parse_selector("environments.prod.cache")) + assert cache.is_path_unaccessed(parse_selector("environments.test.cache")) + + def test_leaf_access_tracking_all_selectors_is_path_unaccessed(self) -> None: + """Test is_path_unaccessed with ALL selectors""" + cache = FlakeCache() + + # Insert data + data = {"services": {"web": "nginx", "api": "fastapi", "worker": "celery"}} + cache.insert(data, "") + + # Initially, ALL selector path should be unaccessed + assert cache.is_path_unaccessed(parse_selector("services.*")) + + # Access one of the keys individually + cache.mark_path_accessed(parse_selector("services.web")) + + # Now the ALL selector should be considered accessed (since at least one key is accessed) + assert not cache.is_path_unaccessed(parse_selector("services.*")) + + def test_leaf_access_tracking_comprehensive_mixed_selectors(self) -> None: + """Comprehensive test combining ALL, SET, individual, and list selectors""" + cache = FlakeCache() + + # Insert complex nested data structure + data = { + "cluster": { + "nodes": { + "master": {"cpu": 4, "memory": "8GB", "disk": "100GB"}, + "worker1": {"cpu": 2, "memory": "4GB", "disk": "50GB"}, + "worker2": {"cpu": 2, "memory": "4GB", "disk": "50GB"}, + }, + "config": { + "network": "10.0.0.0/8", + "storage": "nfs", + "monitoring": "prometheus", + }, + "services": ["api", "web", "database", "cache"], + "metadata": {"version": "1.0", "created": "2023-01-01"}, + } + } + cache.insert(data, "") + + # Phase 1: Use individual selector + cache.mark_path_accessed(parse_selector("cluster.config.network")) + assert not cache.is_path_unaccessed(parse_selector("cluster.config.network")) + assert cache.is_path_unaccessed(parse_selector("cluster.config.storage")) + + # Phase 2: Use SET selector + cache.mark_path_accessed(parse_selector("cluster.nodes.{master,worker1}.cpu")) + assert not cache.is_path_unaccessed(parse_selector("cluster.nodes.master.cpu")) + assert not cache.is_path_unaccessed(parse_selector("cluster.nodes.worker1.cpu")) + assert cache.is_path_unaccessed(parse_selector("cluster.nodes.worker2.cpu")) + + # Phase 3: Use ALL selector + cache.mark_path_accessed(parse_selector("cluster.metadata.*")) + assert not cache.is_path_unaccessed(parse_selector("cluster.metadata.version")) + assert not cache.is_path_unaccessed(parse_selector("cluster.metadata.created")) + + # Phase 4: Use list selector + cache.mark_path_accessed(parse_selector("cluster.services.1")) # "web" + assert not cache.is_path_unaccessed(parse_selector("cluster.services.1")) + assert cache.is_path_unaccessed(parse_selector("cluster.services.0")) # "api" + + # Phase 5: Complex combination - ALL selector followed by SET selector + cache.mark_path_accessed(parse_selector("cluster.nodes.*.{memory,disk}")) + + # All memory and disk leaves should be accessed + assert not cache.is_path_unaccessed( + parse_selector("cluster.nodes.master.memory") + ) + assert not cache.is_path_unaccessed(parse_selector("cluster.nodes.master.disk")) + assert not cache.is_path_unaccessed( + parse_selector("cluster.nodes.worker1.memory") + ) + assert not cache.is_path_unaccessed( + parse_selector("cluster.nodes.worker1.disk") + ) + assert not cache.is_path_unaccessed( + parse_selector("cluster.nodes.worker2.memory") + ) + assert not cache.is_path_unaccessed( + parse_selector("cluster.nodes.worker2.disk") + ) + + # Verify intermediate nodes are not accessed + assert isinstance(cache.cache.value, dict) + cluster_entry = cache.cache.value["cluster"] + assert cluster_entry._num_accessed == 0 + + assert isinstance(cluster_entry.value, dict) + nodes_entry = cluster_entry.value["nodes"] + assert nodes_entry._num_accessed == 0 + + config_entry = cluster_entry.value["config"] + assert config_entry._num_accessed == 0 + + # Verify specific leaf access counts + assert isinstance(nodes_entry.value, dict) + master_entry = nodes_entry.value["master"] + assert master_entry._num_accessed == 0 # Intermediate node + + assert isinstance(master_entry.value, dict) + # CPU was accessed once via SET selector + assert master_entry.value["cpu"]._num_accessed == 1 + # Memory and disk were accessed once via ALL+SET combination + assert master_entry.value["memory"]._num_accessed == 1 + assert master_entry.value["disk"]._num_accessed == 1 + + # Verify that mixed selector patterns work correctly + # Test a complex query that should be unaccessed + assert cache.is_path_unaccessed( + parse_selector("cluster.config.{storage,monitoring}") + ) + + # Test a complex query that should be accessed + assert not cache.is_path_unaccessed( + parse_selector("cluster.nodes.{master,worker2}.memory") + ) From 033f34c0b8ccb51126be8a6e40a8f572b2ebaf87 Mon Sep 17 00:00:00 2001 From: Qubasa Date: Thu, 24 Jul 2025 18:28:18 +0700 Subject: [PATCH 10/15] ruff: Ignore accessed internal variable error as it is needed in tests quite often remove incorrect doc change --- docs/site/developer/contributing/debugging.md | 3 +-- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/site/developer/contributing/debugging.md b/docs/site/developer/contributing/debugging.md index a26c3bd1e..a20ea8bca 100644 --- a/docs/site/developer/contributing/debugging.md +++ b/docs/site/developer/contributing/debugging.md @@ -85,8 +85,7 @@ export CLAN_DEBUG_COMMANDS=1 These options help you pinpoint the source and context of print messages and debug logs during development. -!!! Note - `CLAN_DEBUG_NIX_SELECTORS` and `CLAN_DEBUG_NIX_PREFETCH` will only print the command on a cache miss! + ## Analyzing Performance diff --git a/pyproject.toml b/pyproject.toml index 6abac8cd7..bf7088450 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ lint.ignore = [ "TRY301", "TRY300", "ANN401", + "SLF001", "RUF100", "TRY400", "E402", From caaf9dc4f33886bc8b8a52b8f6760a8aaeffcddb Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 24 Jul 2025 16:51:57 +0200 Subject: [PATCH 11/15] docs: unify documentation Strictly enforce diataxis Use resource driven approach Can extend later to add 'developer' link index page --- README.md | 2 +- clanModules/borgbackup-static/README.md | 2 +- clanModules/root-password/README.md | 2 +- clanModules/user-password/README.md | 2 +- docs/mkdocs.yml | 153 ++++++------ docs/nix/render_options/__init__.py | 133 +--------- docs/site/{developer => }/api.md | 0 docs/site/concepts/autoincludes.md | 15 ++ .../generators.md} | 9 +- docs/site/{guides => concepts}/inventory.md | 4 +- .../developer/extensions/clanModules/index.md | 229 ------------------ docs/site/developer/index.md | 25 -- docs/site/guides/clanServices.md | 4 +- .../contributing/CONTRIBUTING.md | 0 .../contributing/debugging.md | 0 .../contributing/testing.md | 0 .../disko-templates/community.md} | 0 docs/site/guides/flake-parts.md | 2 +- .../guides/getting-started/add-machines.md | 2 +- .../guides/getting-started/add-services.md | 2 +- docs/site/guides/macos.md | 2 +- .../migrations/migrate-inventory-services.md | 6 +- .../guides/migrations/migration-facts-vars.md | 2 +- docs/site/guides/more-machines.md | 50 ---- docs/site/guides/secrets.md | 4 +- .../index.md => guides/services/community.md} | 14 +- docs/site/index.md | 88 +++---- docs/site/reference/index.md | 4 +- lib/modules/inventoryClass/interface.nix | 5 +- nixosModules/clanCore/outputs.nix | 7 +- 30 files changed, 174 insertions(+), 594 deletions(-) rename docs/site/{developer => }/api.md (100%) create mode 100644 docs/site/concepts/autoincludes.md rename docs/site/{guides/vars-backend.md => concepts/generators.md} (96%) rename docs/site/{guides => concepts}/inventory.md (96%) delete mode 100644 docs/site/developer/extensions/clanModules/index.md delete mode 100644 docs/site/developer/index.md rename docs/site/{developer => guides}/contributing/CONTRIBUTING.md (100%) rename docs/site/{developer => guides}/contributing/debugging.md (100%) rename docs/site/{developer => guides}/contributing/testing.md (100%) rename docs/site/{developer/extensions/templates/disk/disko-templates.md => guides/disko-templates/community.md} (100%) delete mode 100644 docs/site/guides/more-machines.md rename docs/site/{developer/extensions/clanServices/index.md => guides/services/community.md} (93%) diff --git a/README.md b/README.md index 2b56ecfe2..6a7e71e3b 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ If you're new to Clan and eager to dive in, start with our quickstart guide and In the Clan ecosystem, security is paramount. Learn how to handle secrets effectively: -- **Secrets Management**: Securely manage secrets by consulting [Vars](https://docs.clan.lol/guides/vars-backend/). +- **Secrets Management**: Securely manage secrets by consulting [Vars](https://docs.clan.lol/concepts/generators/). ### Contributing to Clan diff --git a/clanModules/borgbackup-static/README.md b/clanModules/borgbackup-static/README.md index a0dda99da..d93c0f6e2 100644 --- a/clanModules/borgbackup-static/README.md +++ b/clanModules/borgbackup-static/README.md @@ -4,7 +4,7 @@ description = "Statically configure borgbackup with sane defaults." !!! Danger "Deprecated" Use [borgbackup](borgbackup.md) instead. - Don't use borgbackup-static through [inventory](../../guides/inventory.md). + Don't use borgbackup-static through [inventory](../../concepts/inventory.md). This module implements the `borgbackup` backend and implements sane defaults for backup management through `borgbackup` for members of the clan. diff --git a/clanModules/root-password/README.md b/clanModules/root-password/README.md index 58e7cf467..59e5e6be7 100644 --- a/clanModules/root-password/README.md +++ b/clanModules/root-password/README.md @@ -12,7 +12,7 @@ After the system was installed/deployed the following command can be used to dis clan vars get [machine_name] root-password/root-password ``` -See also: [Vars](../../guides/vars-backend.md) +See also: [Vars](../../concepts/generators.md) To regenerate the password run: ``` diff --git a/clanModules/user-password/README.md b/clanModules/user-password/README.md index 049e1a780..9106a8d81 100644 --- a/clanModules/user-password/README.md +++ b/clanModules/user-password/README.md @@ -16,7 +16,7 @@ After the system was installed/deployed the following command can be used to dis clan vars get [machine_name] root-password/root-password ``` -See also: [Vars](../../guides/vars-backend.md) +See also: [Vars](../../concepts/generators.md) To regenerate the password run: ``` diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index dbf4f90f4..ca867d620 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -55,29 +55,39 @@ nav: - Add Services: guides/getting-started/add-services.md - Deploy Machine: guides/getting-started/deploy.md - Continuous Integration: guides/getting-started/check.md - - Inventory: guides/inventory.md - Using Services: guides/clanServices.md - Backup & Restore: guides/backups.md - Disk Encryption: guides/disk-encryption.md - - Vars: guides/vars-backend.md - Age Plugins: guides/age-plugins.md - - Advanced Secrets: guides/secrets.md - - Machine Autoincludes: guides/more-machines.md + - Secrets management: guides/secrets.md - Target Host: guides/target-host.md - Zerotier VPN: guides/mesh-vpn.md - Secure Boot: guides/secure-boot.md - Flake-parts: guides/flake-parts.md + - macOS: guides/macos.md + - Contributing: + - Contributing: guides/contributing/CONTRIBUTING.md + - Debugging: guides/contributing/debugging.md + - Testing: guides/contributing/testing.md + + - Writing a Service Module: guides/services/community.md + - Writing a Disko Template: guides/disko-templates/community.md - Migrations: - Migrate existing Flakes: guides/migrations/migration-guide.md - Migrate inventory Services: guides/migrations/migrate-inventory-services.md - Facts Vars Migration: guides/migrations/migration-facts-vars.md - Disk id: guides/migrations/disk-id.md - - macOS: guides/macos.md + - Concepts: + - Inventory: concepts/inventory.md + - Generators: concepts/generators.md + - Autoincludes: concepts/autoincludes.md - Reference: - Overview: reference/index.md + - Clan Options: options.md - Services: - - List: - - Overview: reference/clanServices/index.md + - Overview: + - reference/clanServices/index.md + - reference/clanServices/admin.md - reference/clanServices/borgbackup.md - reference/clanServices/data-mesher.md @@ -94,66 +104,7 @@ nav: - reference/clanServices/wifi.md - reference/clanServices/zerotier.md - API: reference/clanServices/clan-service-author-interface.md - - Writing a Service Module: developer/extensions/clanServices/index.md - - Modules: - - List: - - Overview: reference/clanModules/index.md - - reference/clanModules/frontmatter/index.md - # TODO: display the docs of the clan.service modules - - reference/clanModules/admin.md - # This is the module overview and should stay at the top - - reference/clanModules/borgbackup-static.md - - reference/clanModules/data-mesher.md - - reference/clanModules/borgbackup.md - - reference/clanModules/deltachat.md - - reference/clanModules/disk-id.md - - reference/clanModules/dyndns.md - - reference/clanModules/ergochat.md - - reference/clanModules/garage.md - - reference/clanModules/heisenbridge.md - - reference/clanModules/importer.md - - reference/clanModules/iwd.md - - reference/clanModules/localbackup.md - - reference/clanModules/localsend.md - - reference/clanModules/matrix-synapse.md - - reference/clanModules/moonlight.md - - reference/clanModules/mumble.md - - reference/clanModules/mycelium.md - - reference/clanModules/nginx.md - - reference/clanModules/packages.md - - reference/clanModules/postgresql.md - - reference/clanModules/root-password.md - - reference/clanModules/single-disk.md - - reference/clanModules/sshd.md - - reference/clanModules/state-version.md - - reference/clanModules/static-hosts.md - - reference/clanModules/sunshine.md - - reference/clanModules/syncthing-static-peers.md - - reference/clanModules/syncthing.md - - reference/clanModules/thelounge.md - - reference/clanModules/trusted-nix-caches.md - - reference/clanModules/user-password.md - - reference/clanModules/auto-upgrade.md - - reference/clanModules/vaultwarden.md - - reference/clanModules/xfce.md - - reference/clanModules/zerotier-static-peers.md - - reference/clanModules/zerotier.md - - reference/clanModules/zt-tcp-relay.md - - Writing a Clan Module: developer/extensions/clanModules/index.md - - Nix API: - - inputs.clan-core.lib.clan: reference/nix-api/clan.md - - config.clan.core: - - Overview: reference/clan.core/index.md - - reference/clan.core/backups.md - - reference/clan.core/deployment.md - - reference/clan.core/facts.md - - reference/clan.core/networking.md - - reference/clan.core/settings.md - - reference/clan.core/sops.md - - reference/clan.core/state.md - - reference/clan.core/vars.md - - Inventory: reference/nix-api/inventory.md - CLI: - Overview: reference/cli/index.md @@ -170,8 +121,63 @@ nav: - reference/cli/templates.md - reference/cli/vars.md - reference/cli/vms.md + - Modules (deprecated): + - Overview: reference/clanModules/index.md + - reference/clanModules/frontmatter/index.md + # TODO: display the docs of the clan.service modules + - reference/clanModules/admin.md + # This is the module overview and should stay at the top + - reference/clanModules/borgbackup-static.md + - reference/clanModules/data-mesher.md + - reference/clanModules/borgbackup.md + - reference/clanModules/deltachat.md + - reference/clanModules/disk-id.md + - reference/clanModules/dyndns.md + - reference/clanModules/ergochat.md + - reference/clanModules/garage.md + - reference/clanModules/heisenbridge.md + - reference/clanModules/importer.md + - reference/clanModules/iwd.md + - reference/clanModules/localbackup.md + - reference/clanModules/localsend.md + - reference/clanModules/matrix-synapse.md + - reference/clanModules/moonlight.md + - reference/clanModules/mumble.md + - reference/clanModules/mycelium.md + - reference/clanModules/nginx.md + - reference/clanModules/packages.md + - reference/clanModules/postgresql.md + - reference/clanModules/root-password.md + - reference/clanModules/single-disk.md + - reference/clanModules/sshd.md + - reference/clanModules/state-version.md + - reference/clanModules/static-hosts.md + - reference/clanModules/sunshine.md + - reference/clanModules/syncthing-static-peers.md + - reference/clanModules/syncthing.md + - reference/clanModules/thelounge.md + - reference/clanModules/trusted-nix-caches.md + - reference/clanModules/user-password.md + - reference/clanModules/auto-upgrade.md + - reference/clanModules/vaultwarden.md + - reference/clanModules/xfce.md + - reference/clanModules/zerotier-static-peers.md + - reference/clanModules/zerotier.md + - reference/clanModules/zt-tcp-relay.md + + - clan.core (NixOS Options): + - Overview: reference/clan.core/index.md + - reference/clan.core/backups.md + - reference/clan.core/deployment.md + - reference/clan.core/facts.md + - reference/clan.core/networking.md + - reference/clan.core/settings.md + - reference/clan.core/sops.md + - reference/clan.core/state.md + - reference/clan.core/vars.md + + - Developer-api: api.md - - Glossary: reference/glossary.md - Decisions: - Architecture Decisions: decisions/README.md - 01-clanModules: decisions/01-ClanModules.md @@ -180,16 +186,7 @@ nav: - 04-fetching-nix-from-python: decisions/04-fetching-nix-from-python.md - 05-deployment-parameters: decisions/05-deployment-parameters.md - Template: decisions/_template.md - - Options: options.md - - Developer: - - Introduction: developer/index.md - - Dev Setup: developer/contributing/CONTRIBUTING.md - - Writing a Service Module: developer/extensions/clanServices/index.md - - Writing a Clan Module: developer/extensions/clanModules/index.md - - Writing a Disko Template: developer/extensions/templates/disk/disko-templates.md - - Debugging: developer/contributing/debugging.md - - Testing: developer/contributing/testing.md - - Python API: developer/api.md + - Glossary: reference/glossary.md docs_dir: site site_dir: out @@ -249,4 +246,4 @@ plugins: - redoc-tag - redirects: redirect_maps: - guides/getting-started/secrets.md: guides/vars-backend.md + guides/getting-started/secrets.md: concepts/generators.md diff --git a/docs/nix/render_options/__init__.py b/docs/nix/render_options/__init__.py index b057da5c0..2feae390a 100644 --- a/docs/nix/render_options/__init__.py +++ b/docs/nix/render_options/__init__.py @@ -193,7 +193,7 @@ def module_header(module_name: str, has_inventory_feature: bool = False) -> str: def module_nix_usage(module_name: str) -> str: return f"""## Usage via Nix -**This module can be also imported directly in your nixos configuration. Although it is recommended to use the [inventory](../../reference/nix-api/inventory.md) interface if available.** +**This module can be also imported directly in your nixos configuration. Although it is recommended to use the [inventory](../../concepts/inventory.md) interface if available.** Some modules are considered 'low-level' or 'expert modules' and are not available via the inventory interface. @@ -373,7 +373,7 @@ This module can be used via predefined roles """ Every role has its own configuration options, which are each listed below. -For more information, see the [inventory guide](../../guides/inventory.md). +For more information, see the [inventory guide](../../concepts/inventory.md). ??? Example For example the `admin` module adds the following options globally to all machines where it is used. @@ -402,7 +402,7 @@ certain option types restricted to enable configuration through a graphical interface. !!! note "🔹" - Modules with this indicator support the [inventory](../../guides/inventory.md) feature. + Modules with this indicator support the [inventory](../../concepts/inventory.md) feature. """ @@ -679,86 +679,6 @@ def build_option_card(module_name: str, frontmatter: Frontmatter) -> str: return f"{to_md_li(module_name, frontmatter)}\n\n" -def produce_build_clan_docs() -> None: - if not BUILD_CLAN_PATH: - msg = f"Environment variables are not set correctly: BUILD_CLAN_PATH={BUILD_CLAN_PATH}. Expected a path to the optionsJSON" - raise ClanError(msg) - - if not OUT: - msg = f"Environment variables are not set correctly: $out={OUT}" - raise ClanError(msg) - - output = """# Clan - -This provides an overview of the available arguments of the `clan` interface. - -Each attribute is documented below - -- **clan-core.lib.clan**: A function that takes an attribute set. - - ??? example "clan Example" - - ```nix - clan { - self = self; - machines = { - jon = { }; - sara = { }; - }; - }; - ``` - -- **clan with flake-parts**: Import the FlakeModule - - After importing the FlakeModule you can define your `clan` as a flake attribute - - All attribute can be defined via `clan.*` - - Further information see: [flake-parts](../../guides/flake-parts.md) guide. - - ??? example "flake-parts Example" - - ```nix - flake-parts.lib.mkFlake { inherit inputs; } ({ - systems = []; - imports = [ - clan-core.flakeModules.default - ]; - clan = { - machines = { - jon = { }; - sara = { }; - }; - }; - }); - ``` - -""" - with Path(BUILD_CLAN_PATH).open() as f: - options: dict[str, dict[str, Any]] = json.load(f) - - split = split_options_by_root(options) - for option_name, options in split.items(): - # Skip underscore options - if option_name.startswith("_"): - continue - # Skip inventory sub options - # Inventory model has its own chapter - if option_name.startswith("inventory."): - continue - - print(f"[build_clan_docs] Rendering option of {option_name}...") - root = options_to_tree(options) - - for option in root.suboptions: - output += options_docs_from_tree(option, init_level=2) - - outfile = Path(OUT) / "nix-api/clan.md" - outfile.parent.mkdir(parents=True, exist_ok=True) - with Path.open(outfile, "w") as of: - of.write(output) - - def split_options_by_root(options: dict[str, Any]) -> dict[str, dict[str, Any]]: """ Split the flat dictionary of options into a dict of which each entry will construct complete option trees. @@ -805,7 +725,7 @@ Typically needed by module authors to define roles, behavior and metadata for di !!! Note This is not a user-facing documentation, but rather meant as a reference for *module authors* - See: [clanService Authoring Guide](../../developer/extensions/clanServices/index.md) + See: [clanService Authoring Guide](../../guides/services/community.md) """ # Inventory options are already included under the clan attribute # We just omitted them in the clan docs, because we want a separate output for the inventory model @@ -834,48 +754,6 @@ class Option: suboptions: list["Option"] = field(default_factory=list) -def produce_inventory_docs() -> None: - if not BUILD_CLAN_PATH: - msg = f"Environment variables are not set correctly: BUILD_CLAN_PATH={BUILD_CLAN_PATH}. Expected a path to the optionsJSON" - raise ClanError(msg) - - if not OUT: - msg = f"Environment variables are not set correctly: $out={OUT}" - raise ClanError(msg) - - output = """# Inventory -This provides an overview of the available attributes of the `inventory` model. - -It can be set via the `inventory` attribute of the [`clan`](./clan.md#inventory) function, or via the [`clan.inventory`](./clan.md#inventory) attribute of flake-parts. - -""" - # Inventory options are already included under the clan attribute - # We just omitted them in the clan docs, because we want a separate output for the inventory model - with Path(BUILD_CLAN_PATH).open() as f: - options: dict[str, dict[str, Any]] = json.load(f) - - clan_root_option = options_to_tree(options) - # Find the inventory options - inventory_opt: None | Option = None - for opt in clan_root_option.suboptions: - if opt.name == "inventory": - inventory_opt = opt - break - - if not inventory_opt: - print("No inventory options found.") - exit(1) - # Render the inventory options - # This for loop excludes the root node - for option in inventory_opt.suboptions: - output += options_docs_from_tree(option, init_level=2) - - outfile = Path(OUT) / "nix-api/inventory.md" - outfile.parent.mkdir(parents=True, exist_ok=True) - with Path.open(outfile, "w") as of: - of.write(output) - - def option_short_name(option_name: str) -> str: parts = option_name.split(".") short_name = "" @@ -984,9 +862,6 @@ def options_docs_from_tree( if __name__ == "__main__": # produce_clan_core_docs() - produce_build_clan_docs() - produce_inventory_docs() - produce_clan_service_author_docs() produce_clan_modules_docs() diff --git a/docs/site/developer/api.md b/docs/site/api.md similarity index 100% rename from docs/site/developer/api.md rename to docs/site/api.md diff --git a/docs/site/concepts/autoincludes.md b/docs/site/concepts/autoincludes.md new file mode 100644 index 000000000..40f1631ff --- /dev/null +++ b/docs/site/concepts/autoincludes.md @@ -0,0 +1,15 @@ + +Clan automatically imports the following files from a directory and registers them. + +## Machine registration + +Every folder `machines/{machineName}` will be registered automatically as a Clan machine. + +!!! info "Automatically loaded files" + + The following files are loaded automatically for each Clan machine: + + - [x] `machines/{machineName}/configuration.nix` + - [x] `machines/{machineName}/hardware-configuration.nix` + - [x] `machines/{machineName}/facter.json` Automatically configured, for further information see [nixos-facter](https://clan.lol/blog/nixos-facter/) + - [x] `machines/{machineName}/disko.nix` Automatically loaded, for further information see the [disko docs](https://github.com/nix-community/disko/blob/master/docs/quickstart.md). diff --git a/docs/site/guides/vars-backend.md b/docs/site/concepts/generators.md similarity index 96% rename from docs/site/guides/vars-backend.md rename to docs/site/concepts/generators.md index 1d44b978f..759da6ad9 100644 --- a/docs/site/guides/vars-backend.md +++ b/docs/site/concepts/generators.md @@ -1,7 +1,4 @@ - -!!! Note - Vars is the new secret backend that will soon replace the Facts backend - +# Generators Defining a linux user's password via the nixos configuration previously required running `mkpasswd ...` and then copying the hash back into the nix configuration. @@ -11,7 +8,7 @@ For a more general explanation of what clan vars are and how it works, see the i This guide assumes - Clan is set up already (see [Getting Started](../guides/getting-started/index.md)) -- a machine has been added to the clan (see [Adding Machines](./more-machines.md)) +- a machine has been added to the clan (see [Adding Machines](../guides/getting-started/add-machines.md)) This section will walk you through the following steps: @@ -23,7 +20,7 @@ This section will walk you through the following steps: 6. share the root password between machines 7. change the password -## Declare the generator +## Declare a generator In this example, a `vars` `generator` is used to: diff --git a/docs/site/guides/inventory.md b/docs/site/concepts/inventory.md similarity index 96% rename from docs/site/guides/inventory.md rename to docs/site/concepts/inventory.md index 878a6cf57..e0712bf51 100644 --- a/docs/site/guides/inventory.md +++ b/docs/site/concepts/inventory.md @@ -9,8 +9,6 @@ The inventory logic will automatically derive the modules and configurations to The following tutorial will walk through setting up a Backup service where the terms `Service` and `Role` will become more clear. -See also: [Inventory API Documentation](../reference/nix-api/inventory.md) - !!! example "Experimental status" The inventory implementation is not considered stable yet. We are actively soliciting feedback from users. @@ -19,7 +17,7 @@ See also: [Inventory API Documentation](../reference/nix-api/inventory.md) ## Prerequisites -- [x] [Add multiple machines](./more-machines.md) to your Clan. +- [x] [Add some machines](../guides/getting-started/add-machines.md) to your Clan. ## Services diff --git a/docs/site/developer/extensions/clanModules/index.md b/docs/site/developer/extensions/clanModules/index.md deleted file mode 100644 index 6bbb559dc..000000000 --- a/docs/site/developer/extensions/clanModules/index.md +++ /dev/null @@ -1,229 +0,0 @@ -# Authoring a clanModule - -!!! Danger "Will get deprecated soon" - Please consider twice creating new modules in this format - - [`clan.service` module](../clanServices/index.md) will be the new standard soon. - -This site will guide you through authoring your first module. Explaining which conventions must be followed, such that others will have an enjoyable experience and the module can be used with minimal effort. - - -!!! Tip - External ClanModules can be ad-hoc loaded via [`clan.inventory.modules`](../../../reference/nix-api/inventory.md#inventory.modules) - -## Bootstrapping the `clanModule` - -A ClanModule is a specific subset of a [NixOS Module](https://nix.dev/tutorials/module-system/index.html), but it has some constraints and might be used via the [Inventory](../../../guides/inventory.md) interface. -In fact a `ClanModule` can be thought of as a layer of abstraction on-top of NixOS and/or other ClanModules. It may configure sane defaults and provide an ergonomic interface that is easy to use and can also be used via a UI that is under development currently. - -Because ClanModules should be configurable via `json`/`API` all of its interface (`options`) must be serializable. - -!!! Tip - ClanModules interface can be checked by running the json schema converter as follows. - - `nix build .#legacyPackages.x86_64-linux.schemas.inventory` - - If the build succeeds the module is compatible. - -## Directory structure - -Each module SHOULD be a directory of the following format: - -```sh -# Example: borgbackup -clanModules/borgbackup -├── README.md -└── roles - ├── client.nix - └── server.nix -``` - -!!! Tip - `README.md` is always required. See section [Readme](#readme) for further details. - - The `roles` folder is strictly required for `features = [ "inventory" ]`. - -## Registering the module - -=== "User module" - - If the module should be ad-hoc loaded. - It can be made available in any project via the [`clan.inventory.modules`](../../../reference/nix-api/inventory.md#inventory.modules) attribute. - - ```nix title="flake.nix" - # ... - # Sometimes this attribute set is defined in clan.nix - clan-core.lib.clan { - # 1. Add the module to the available clanModules with inventory support - inventory.modules = { - custom-module = ./modules/my_module; - }; - # 2. Use the module in the inventory - inventory.services = { - custom-module.instance_1 = { - roles.default.machines = [ "machineA" ]; - }; - }; - }; - ``` - -=== "Upstream module" - - If the module will be contributed to [`clan-core`](https://git.clan.lol/clan-core) - The clanModule must be registered within the `clanModules` attribute in `clan-core` - - ```nix title="clanModules/flake-module.nix" - --8<-- "clanModules/flake-module.nix:0:5" - # Register our new module here - # ... - ``` - -## Readme - -The `README.md` is a required file for all modules. It MUST contain frontmatter in [`toml`](https://toml.io) format. - -```markdown ---- -description = "Module A" ---- - -This is the example module that does xyz. -``` - -See the [Full Frontmatter reference](../../../reference/clanModules/frontmatter/index.md) further details and all supported attributes. - -## Roles - -If the module declares to implement `features = [ "inventory" ]` then it MUST contain a roles directory. - -Each `.nix` file in the `roles` directory is added as a role to the inventory service. - -Other files can also be placed alongside the `.nix` files - -```sh -└── roles - ├── client.nix - └── server.nix -``` - -Adds the roles: `client` and `server` - -??? Tip "Good to know" - Sometimes a `ClanModule` should be usable via both clan's `inventory` concept but also natively as a NixOS module. - - > In the long term, we want most modules to implement support for the inventory, - > but we are also aware that there are certain low-level modules that always serve as a backend for other higher-level `clanModules` with inventory support. - > These modules may not want to implement inventory interfaces as they are always used directly by other modules. - - This can be achieved by placing an additional `default.nix` into the root of the ClanModules directory as shown: - - ```sh - # ModuleA - ├── README.md - ├── default.nix - └── roles - └── default.nix - ``` - - ```nix title="default.nix" - {...}:{ - imports = [ ./roles/default.nix ]; - } - ``` - - By utilizing this pattern the module (`moduleA`) can then be imported into any regular NixOS module via: - - ```nix - {...}:{ - imports = [ clanModules.moduleA ]; - } - ``` - -## Adding configuration options - -While we recommend to keep the interface as minimal as possible and deriving all required information from the `roles` model it might sometimes be required or convenient to expose customization options beyond `roles`. - -The following shows how to add options to your module. - -**It is important to understand that every module has its own namespace where it should declare options** - -**`clan.{moduleName}`** - -???+ Example - The following example shows how to register options in the module interface - - and how it can be set via the inventory - - - ```nix title="/default.nix" - custom-module = ./modules/custom-module; - ``` - - Since the module is called `custom-module` all of its exposed options should be added to `options.clan.custom-module.*...*` - - ```nix title="custom-module/roles/default.nix" - { - options = { - clan.custom-module.foo = mkOption { - type = types.str; - default = "bar"; - }; - }; - } - ``` - - If the module is [registered](#registering-the-module). - Configuration can be set as follows. - - ```nix title="flake.nix" - # Sometimes this attribute set is defined in clan.nix - clan-core.lib.clan { - inventory.services = { - custom-module.instance_1 = { - roles.default.machines = [ "machineA" ]; - roles.default.config = { - # All configuration here is scoped to `clan.custom-module` - foo = "foobar"; - }; - }; - }; - } - ``` - -## Organizing the ClanModule - -Each `{role}.nix` is included into the machine if the machine is declared to have the role. - -For example - -```nix -roles.client.machines = ["MachineA"]; -``` - -Then `roles/client.nix` will be added to the machine `MachineA`. - -This behavior makes it possible to split the interface and common code paths when using multiple roles. -In the concrete example of `borgbackup` this allows a `server` to declare a different interface than the corresponding `client`. - -The client offers configuration option, to exclude certain local directories from being backed up: - -```nix title="roles/client.nix" -# Example client interface - options.clan.borgbackup.exclude = ... -``` - -The server doesn't offer any configuration option. Because everything is set-up automatically. - -```nix title="roles/server.nix" -# Example server interface - options.clan.borgbackup = {}; -``` - -Assuming that there is a common code path or a common interface between `server` and `client` this can be structured as: - -```nix title="roles/server.nix, roles/client.nix" -{...}: { - # ... - imports = [ ../common.nix ]; -} -``` diff --git a/docs/site/developer/index.md b/docs/site/developer/index.md deleted file mode 100644 index cbf897802..000000000 --- a/docs/site/developer/index.md +++ /dev/null @@ -1,25 +0,0 @@ -# Developer Documentation - -!!! Danger - This documentation is **not** intended for external users. It may contain low-level details and internal-only interfaces.* - -Welcome to the internal developer documentation. - -This section is intended for contributors, engineers, and internal stakeholders working directly with our system, tooling, and APIs. It provides a technical overview of core components, internal APIs, conventions, and patterns that support the platform. - -Our goal is to make the internal workings of the system **transparent, discoverable, and consistent** — helping you contribute confidently, troubleshoot effectively, and build faster. - -## What's Here? - -!!! note "docs migration ongoing" - -- [ ] **API Reference**: 🚧🚧🚧 Detailed documentation of internal API functions, inputs, and expected outputs. 🚧🚧🚧 -- [ ] **System Concepts**: Architectural overviews and domain-specific guides. -- [ ] **Development Guides**: How to test, extend, or integrate with key components. -- [ ] **Design Notes**: Rationales behind major design decisions or patterns. - -## Who is This For? - -* Developers contributing to the platform -* Engineers debugging or extending internal systems -* Anyone needing to understand **how** and **why** things work under the hood diff --git a/docs/site/guides/clanServices.md b/docs/site/guides/clanServices.md index 94df37f01..88f3a4962 100644 --- a/docs/site/guides/clanServices.md +++ b/docs/site/guides/clanServices.md @@ -138,7 +138,7 @@ You can use services exposed by Clan’s core module library, `clan-core`. You can also author your own `clanService` modules. -🔗 Learn how to write your own service: [Authoring a clanService](../developer/extensions/clanServices/index.md) +🔗 Learn how to write your own service: [Authoring a service](../guides/services/community.md) You might expose your service module from your flake — this makes it easy for other people to also use your module in their clan. @@ -154,6 +154,6 @@ You might expose your service module from your flake — this makes it easy for ## What’s Next? -* [Author your own clanService →](../developer/extensions/clanServices/index.md) +* [Author your own clanService →](../guides/services/community.md) * [Migrate from clanModules →](../guides/migrations/migrate-inventory-services.md) diff --git a/docs/site/developer/contributing/CONTRIBUTING.md b/docs/site/guides/contributing/CONTRIBUTING.md similarity index 100% rename from docs/site/developer/contributing/CONTRIBUTING.md rename to docs/site/guides/contributing/CONTRIBUTING.md diff --git a/docs/site/developer/contributing/debugging.md b/docs/site/guides/contributing/debugging.md similarity index 100% rename from docs/site/developer/contributing/debugging.md rename to docs/site/guides/contributing/debugging.md diff --git a/docs/site/developer/contributing/testing.md b/docs/site/guides/contributing/testing.md similarity index 100% rename from docs/site/developer/contributing/testing.md rename to docs/site/guides/contributing/testing.md diff --git a/docs/site/developer/extensions/templates/disk/disko-templates.md b/docs/site/guides/disko-templates/community.md similarity index 100% rename from docs/site/developer/extensions/templates/disk/disko-templates.md rename to docs/site/guides/disko-templates/community.md diff --git a/docs/site/guides/flake-parts.md b/docs/site/guides/flake-parts.md index 6a3c8bf04..7fd87da1b 100644 --- a/docs/site/guides/flake-parts.md +++ b/docs/site/guides/flake-parts.md @@ -27,7 +27,7 @@ inputs = { ## Import the Clan flake-parts Module -After updating your flake inputs, the next step is to import the Clan flake-parts module. This will make the [Clan options](../reference/nix-api/clan.md) available within `mkFlake`. +After updating your flake inputs, the next step is to import the Clan flake-parts module. This will make the [Clan options](../options.md) available within `mkFlake`. ```nix { diff --git a/docs/site/guides/getting-started/add-machines.md b/docs/site/guides/getting-started/add-machines.md index 5cf0ee197..b3d93ad29 100644 --- a/docs/site/guides/getting-started/add-machines.md +++ b/docs/site/guides/getting-started/add-machines.md @@ -6,7 +6,7 @@ Machines can be added using the following methods - Editing machines/`machine_name`/configuration.nix (automatically included if it exists) - `clan machines create` (imperative) -See the complete [list](../../guides/more-machines.md#automatic-registration) of auto-loaded files. +See the complete [list](../../concepts/autoincludes.md) of auto-loaded files. ## Create a machine diff --git a/docs/site/guides/getting-started/add-services.md b/docs/site/guides/getting-started/add-services.md index 5052f3dbd..8c561059c 100644 --- a/docs/site/guides/getting-started/add-services.md +++ b/docs/site/guides/getting-started/add-services.md @@ -41,7 +41,7 @@ To learn more: [Guide about clanService](../clanServices.md) ``` 1. See [reference/clanServices](../../reference/clanServices/index.md) for all available services and how to configure them. - Or read [authoring/clanServices](../../developer/extensions/clanServices/index.md) if you want to bring your own + Or read [authoring/clanServices](../../guides/services/community.md) if you want to bring your own 2. Replace `__YOUR_CONTROLLER_` with the *name* of your machine. diff --git a/docs/site/guides/macos.md b/docs/site/guides/macos.md index 0eccbca58..fce5ac2dd 100644 --- a/docs/site/guides/macos.md +++ b/docs/site/guides/macos.md @@ -7,7 +7,7 @@ This guide explains how to manage macOS machines using Clan. Currently, Clan supports the following features for macOS: - `clan machines update` for existing [nix-darwin](https://github.com/nix-darwin/nix-darwin) installations -- Support for [vars](../guides/vars-backend.md) +- Support for [vars](../concepts/generators.md) ## Add Your Machine to Your Clan Flake diff --git a/docs/site/guides/migrations/migrate-inventory-services.md b/docs/site/guides/migrations/migrate-inventory-services.md index ebeaa35d8..94716316c 100644 --- a/docs/site/guides/migrations/migrate-inventory-services.md +++ b/docs/site/guides/migrations/migrate-inventory-services.md @@ -1,7 +1,7 @@ # Migrating from using `clanModules` to `clanServices` **Audience**: This is a guide for **people using `clanModules`**. -If you are a **module author** and need to migrate your modules please consult our **new** [clanServices authoring guide](../../developer/extensions/clanServices/index.md) +If you are a **module author** and need to migrate your modules please consult our **new** [clanServices authoring guide](../../guides/services/community.md) ## What's Changing? @@ -329,6 +329,6 @@ instances = { ## Further reference -* [Authoring a 'clan.service' module](../../developer/extensions/clanServices/index.md) +* [Inventory Concept](../../concepts/inventory.md) +* [Authoring a 'clan.service' module](../../guides/services/community.md) * [ClanServices](../clanServices.md) -* [Inventory Reference](../../reference/nix-api/inventory.md) diff --git a/docs/site/guides/migrations/migration-facts-vars.md b/docs/site/guides/migrations/migration-facts-vars.md index 14e24aad7..6dcde4dbe 100644 --- a/docs/site/guides/migrations/migration-facts-vars.md +++ b/docs/site/guides/migrations/migration-facts-vars.md @@ -3,7 +3,7 @@ For a high level overview about `vars` see our [blog post](https://clan.lol/blog/vars/). This guide will help you migrate your modules that still use our [`facts`](../../guides/secrets.md) backend -to the [`vars`](../../guides/vars-backend.md) backend. +to the [`vars`](../../concepts/generators.md) backend. The `vars` [module](../../reference/clan.core/vars.md) and the clan [command](../../reference/cli/vars.md) work in tandem, they should ideally be kept in sync. diff --git a/docs/site/guides/more-machines.md b/docs/site/guides/more-machines.md deleted file mode 100644 index 5ce064626..000000000 --- a/docs/site/guides/more-machines.md +++ /dev/null @@ -1,50 +0,0 @@ - -Clan has two general methods of adding machines: - -- **Automatic**: Detects every folder in the `machines` folder. -- **Declarative**: Explicit declarations in Nix. - -## Automatic registration - -Every folder `machines/{machineName}` will be registered automatically as a Clan machine. - -!!! info "Automatically loaded files" - - The following files are loaded automatically for each Clan machine: - - - [x] `machines/{machineName}/configuration.nix` - - [x] `machines/{machineName}/hardware-configuration.nix` - - [x] `machines/{machineName}/facter.json` Automatically configured, for further information see [nixos-facter](https://clan.lol/blog/nixos-facter/) - - [x] `machines/{machineName}/disko.nix` Automatically loaded, for further information see the [disko docs](https://github.com/nix-community/disko/blob/master/docs/quickstart.md). - -## Manual declaration - -Machines can be added via [`clan.inventory.machines`](../guides/inventory.md) or in `clan.machines`, which allows for defining NixOS options. - -=== "**Individual Machine Configuration**" - - ```{.nix} - clan-core.lib.clan { - machines = { - "jon" = { - # Any valid nixos config - }; - }; - } - ``` - -=== "**Inventory Configuration**" - - ```{.nix} - clan-core.lib.clan { - inventory = { - machines = { - "jon" = { - # Inventory can set tags and other metadata - tags = [ "zone1" ]; - deploy.targetHost = "root@jon"; - }; - }; - }; - } - ``` diff --git a/docs/site/guides/secrets.md b/docs/site/guides/secrets.md index 4f3faee9b..4c78f0f0b 100644 --- a/docs/site/guides/secrets.md +++ b/docs/site/guides/secrets.md @@ -1,5 +1,5 @@ -This article provides an overview over the underlying secrets system which is used by [Vars](../guides/vars-backend.md). -Under most circumstances you should use [Vars](../guides/vars-backend.md) directly instead. +This article provides an overview over the underlying secrets system which is used by [Vars](../concepts/generators.md). +Under most circumstances you should use [Vars](../concepts/generators.md) directly instead. Consider using `clan secrets` only for managing admin users and groups, as well as a debugging tool. diff --git a/docs/site/developer/extensions/clanServices/index.md b/docs/site/guides/services/community.md similarity index 93% rename from docs/site/developer/extensions/clanServices/index.md rename to docs/site/guides/services/community.md index 3f0ab86b3..fa92d008f 100644 --- a/docs/site/developer/extensions/clanServices/index.md +++ b/docs/site/guides/services/community.md @@ -1,16 +1,16 @@ # Authoring a 'clan.service' module !!! Tip - This is the successor format to the older [clanModules](../clanModules/index.md) + This is the successor format to the older [clanModules](../../reference/clanModules/index.md) While some features might still be missing we recommend to adapt this format early and give feedback. ## Service Module Specification This section explains how to author a clan service module. -We discussed the initial architecture in [01-clan-service-modules](../../../decisions/01-ClanModules.md) and decided to rework the format. +We discussed the initial architecture in [01-clan-service-modules](../../decisions/01-ClanModules.md) and decided to rework the format. -For the full specification and current state see: **[Service Author Reference](../../../reference/clanServices/clan-service-author-interface.md)** +For the full specification and current state see: **[Service Author Reference](../../reference/clanServices/clan-service-author-interface.md)** ### A Minimal module @@ -52,7 +52,7 @@ The imported module file must fulfill at least the following requirements: } ``` -For more attributes see: **[Service Author Reference](../../../reference/clanServices/clan-service-author-interface.md)** +For more attributes see: **[Service Author Reference](../../reference/clanServices/clan-service-author-interface.md)** ### Adding functionality to the module @@ -266,6 +266,6 @@ The benefit of this approach is that downstream users can override the value of ## Further -- [Reference Documentation for Service Authors](../../../reference/clanServices/clan-service-author-interface.md) -- [Migration Guide from ClanModules to ClanServices](../../../guides/migrations/migrate-inventory-services.md) -- [Decision that lead to ClanServices](../../../decisions/01-ClanModules.md) +- [Reference Documentation for Service Authors](../../reference/clanServices/clan-service-author-interface.md) +- [Migration Guide from ClanModules to ClanServices](../../guides/migrations/migrate-inventory-services.md) +- [Decision that lead to ClanServices](../../decisions/01-ClanModules.md) diff --git a/docs/site/index.md b/docs/site/index.md index 178bcaa36..9a354c0c5 100644 --- a/docs/site/index.md +++ b/docs/site/index.md @@ -6,19 +6,26 @@ hide: # :material-home: What is Clan? -[Clan](https://clan.lol/) is a peer-to-peer computer management framework that -empowers you to reclaim control over your digital computing experience. Built on -NixOS, Clan provides a unified interface for managing networks of machines with -automated [secret management](./guides/secrets.md), secure [mesh VPN -connectivity](./guides/mesh-vpn.md), and customizable installation images. Whether -you're running a homelab or building decentralized computing infrastructure, -Clan simplifies configuration management while restoring your independence from -closed computing ecosystems. +## [Clan](https://clan.lol/) - Peer-to-Peer Computer Management Framework + +### Clan is a secure, peer-to-peer framework for managing personal or distributed computing networks + +- **Automated** [Secret Management](./guides/secrets.md): + + Effortlessly manage secrets across machines in your network using our automated tools + +- **Secure** [mesh VPN connectivity](./guides/mesh-vpn.md): + + Establish encrypted, peer-to-peer communication between machines — without centralized servers. + +- **Composable** [Clan Services](./reference/clanServices/index.md): + + Define and deploy reusable service modules that work seamlessly across machines. + +- **Custom** [Community services](./guides/services/community.md): + + Everyone can contribute to a growing ecosystem of open-source services. -At the heart of Clan are [Clan Services](./reference/clanServices/index.md) - the core -concept that enables you to add functionality across multiple machines in your -network. While Clan ships with essential core services, you can [create custom -services](./guides/clanServices.md) tailored to your specific needs. [Getting Started](./guides/getting-started/index.md){ .md-button } @@ -28,47 +35,49 @@ services](./guides/clanServices.md) tailored to your specific needs.
-- [Adding more machines](./guides/more-machines.md) +- [Create a Machine](./guides/getting-started/add-machines.md) --- - Learn how Clan automatically includes machines and Nix files. + How to create your first machine -- [Vars Backend](./guides/vars-backend.md) +- [macOS](./guides/macos.md) --- - Learn how to manage secrets with vars. + How to manage macOS machines with nix-darwin -- [Inventory](./guides/inventory.md) +- [Contribute](./guides/contributing/CONTRIBUTING.md) --- - Clan's declaration format for running **services** on one or multiple **machines**. + How to set up a development environment -- [Flake-parts](./guides/flake-parts.md) +
+ +## Concepts + +Explore the foundational ideas. + +
+ +- [Generators](./concepts/generators.md) --- - Use Clan with [https://flake.parts/]() + Learn about Generators -- [Contribute](./developer/contributing/CONTRIBUTING.md) +- [Inventory](./concepts/inventory.md) --- - Discover how to set up a development environment to contribute to Clan! - -- [macOS machines](./guides/macos.md) - - --- - - Manage macOS machines with nix-darwin + Learn about Inventory
## API Reference -**Reference API Documentation** +Technical reference for Clan's CLI and Nix modules
@@ -76,29 +85,22 @@ services](./guides/clanServices.md) tailored to your specific needs. --- - The `clan` CLI command + Command-line interface. + + Full reference for the `clan` CLI tool. - [Service Modules](./reference/clanServices/index.md) --- - An overview of available service modules + Overview of built-in service modules that provide composable functionality across machines. -- [Core](./reference/clan.core/index.md) +- [Core NixOS-module](./reference/clan.core/index.md) --- - The clan core nix module. - This is imported when using clan and is the basis of the extra functionality - that can be provided. - -- [(Legacy) Modules](./reference/clanModules/index.md) - - --- - - An overview of available clanModules - - !!! Example "These will be deprecated soon" + The foundation of Clan's functionality + Reference for the `clan-core` NixOS module — automatically part of any machine to enable Clan's core features.
diff --git a/docs/site/reference/index.md b/docs/site/reference/index.md index bb08355a8..c5c0eda8f 100644 --- a/docs/site/reference/index.md +++ b/docs/site/reference/index.md @@ -4,10 +4,10 @@ This section of the site provides an overview of available options and commands --- +- [Clan Configuration Option](../options.md) - for defining a Clan - Learn how to use the [Clan CLI](./cli/index.md) - Explore available services and application [modules](./clanModules/index.md) -- Discover [configuration options](./clan.core/index.md) that manage essential features -- Find descriptions of the [Nix interfaces](./nix-api/clan.md) for defining a Clan +- [NixOS Configuration Options](./clan.core/index.md) - Additional options avilable on a NixOS machine. --- diff --git a/lib/modules/inventoryClass/interface.nix b/lib/modules/inventoryClass/interface.nix index 9dd43d467..f3a7508bd 100644 --- a/lib/modules/inventoryClass/interface.nix +++ b/lib/modules/inventoryClass/interface.nix @@ -142,7 +142,7 @@ in - The module MUST have at least `features = [ "inventory" ]` in the frontmatter section. - The module MUST have a subfolder `roles` with at least one `{roleName}.nix` file. - For further information see: [Module Authoring Guide](../../developer/extensions/clanServices/index.md). + For further information see: [Module Authoring Guide](../../guides/services/community.md). ???+ example ```nix @@ -179,8 +179,7 @@ in map (m: "'${m}'") (lib.attrNames (lib.filterAttrs (n: _v: !builtins.elem n allowedNames) moduleSet)) )} - See: https://docs.clan.lol/developer/extensions/clanServices/ - And: https://docs.clan.lol/developer/extensions/clanServices/ + See: https://docs.clan.lol/guides/services/community/ '' moduleSet; }; diff --git a/nixosModules/clanCore/outputs.nix b/nixosModules/clanCore/outputs.nix index 9104926ae..20d549a2b 100644 --- a/nixosModules/clanCore/outputs.nix +++ b/nixosModules/clanCore/outputs.nix @@ -31,6 +31,7 @@ The deployment data is now accessed directly from the configuration instead of being written to a separate JSON file. ''; + defaultText = "error: deployment.json file generation has been removed in favor of direct selectors."; }; deployment.buildHost = lib.mkOption { type = lib.types.nullOr lib.types.str; @@ -54,10 +55,10 @@ deployment.nixosMobileWorkaround = lib.mkOption { type = lib.types.bool; description = '' - if true, the deployment will first do a nixos-rebuild switch + if true, the deployment will first do a nixos-rebuild switch to register the boot profile the command will fail applying it to the running system - which is why afterwards we execute a nixos-rebuild test to apply - the new config without having to reboot. + which is why afterwards we execute a nixos-rebuild test to apply + the new config without having to reboot. This is a nixos-mobile deployment bug and will be removed in the future ''; default = false; From b6a04e4f128ccd301d9fd2e54d1b1539449be240 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 24 Jul 2025 16:54:49 +0200 Subject: [PATCH 12/15] docs: restore index page --- docs/site/index.md | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/docs/site/index.md b/docs/site/index.md index 9a354c0c5..44081b7a9 100644 --- a/docs/site/index.md +++ b/docs/site/index.md @@ -6,26 +6,19 @@ hide: # :material-home: What is Clan? -## [Clan](https://clan.lol/) - Peer-to-Peer Computer Management Framework - -### Clan is a secure, peer-to-peer framework for managing personal or distributed computing networks - -- **Automated** [Secret Management](./guides/secrets.md): - - Effortlessly manage secrets across machines in your network using our automated tools - -- **Secure** [mesh VPN connectivity](./guides/mesh-vpn.md): - - Establish encrypted, peer-to-peer communication between machines — without centralized servers. - -- **Composable** [Clan Services](./reference/clanServices/index.md): - - Define and deploy reusable service modules that work seamlessly across machines. - -- **Custom** [Community services](./guides/services/community.md): - - Everyone can contribute to a growing ecosystem of open-source services. +[Clan](https://clan.lol/) is a peer-to-peer computer management framework that +empowers you to reclaim control over your digital computing experience. Built on +NixOS, Clan provides a unified interface for managing networks of machines with +automated [secret management](./guides/secrets.md), secure [mesh VPN +connectivity](./guides/mesh-vpn.md), and customizable installation images. Whether +you're running a homelab or building decentralized computing infrastructure, +Clan simplifies configuration management while restoring your independence from +closed computing ecosystems. +At the heart of Clan are [Clan Services](./reference/clanServices/index.md) - the core +concept that enables you to add functionality across multiple machines in your +network. While Clan ships with essential core services, you can [create custom +services](./guides/clanServices.md) tailored to your specific needs. [Getting Started](./guides/getting-started/index.md){ .md-button } From 7dd9e6b97c99267910b64e85bcbad7ad02e7d9df Mon Sep 17 00:00:00 2001 From: lassulus Date: Thu, 24 Jul 2025 22:24:15 +0200 Subject: [PATCH 13/15] clan-cli vars: show which var we are getting in debug log --- pkgs/clan-cli/clan_cli/vars/get.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/clan-cli/clan_cli/vars/get.py b/pkgs/clan-cli/clan_cli/vars/get.py index fe0e67d27..ac1a092a4 100644 --- a/pkgs/clan-cli/clan_cli/vars/get.py +++ b/pkgs/clan-cli/clan_cli/vars/get.py @@ -17,6 +17,7 @@ log = logging.getLogger(__name__) def get_machine_var(base_dir: str, machine_name: str, var_id: str) -> Var: + log.debug(f"getting var: {var_id} from machine: {machine_name}") vars_ = get_machine_vars(base_dir=base_dir, machine_name=machine_name) results = [] for var in vars_: From 9e85c64139055222a031ada142bfdd5d88c8024b Mon Sep 17 00:00:00 2001 From: lassulus Date: Thu, 24 Jul 2025 22:24:34 +0200 Subject: [PATCH 14/15] clan-cli flake: show cache file location --- pkgs/clan-cli/clan_lib/flake/flake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_lib/flake/flake.py b/pkgs/clan-cli/clan_lib/flake/flake.py index 98b02b33a..1c68b37d0 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake.py +++ b/pkgs/clan-cli/clan_lib/flake/flake.py @@ -721,7 +721,7 @@ class FlakeCache: def load_from_file(self, path: Path) -> None: with path.open("r") as f: - log.debug("Loading flake cache from file") + log.debug(f"Loading flake cache from file {path}") data = json.load(f) self.cache = FlakeCacheEntry.from_json(data["cache"]) From 1a5b77d47a51f936542c1e57bae0dcb69d0f37b0 Mon Sep 17 00:00:00 2001 From: lassulus Date: Thu, 24 Jul 2025 22:26:36 +0200 Subject: [PATCH 15/15] refactor: generalize Tor support to SOCKS5 proxy in network module - Replace Tor-specific implementation with generic SOCKS5 proxy support - Change `tor_socks` boolean to `socks_port` and `socks_wrapper` parameters - Move Tor functionality to clan_lib.network.tor submodule - Add connection context managers to NetworkTechnologyBase - Improve network abstraction with proper remote() and connection() methods - Update all callers to use new SOCKS5 proxy interface - Fix network ping command to properly handle connection contexts This allows for more flexible proxy configurations beyond just Tor, while maintaining backward compatibility for Tor usage. --- pkgs/clan-cli/clan_cli/machines/install.py | 6 +- pkgs/clan-cli/clan_cli/network/ping.py | 25 +- pkgs/clan-cli/clan_cli/ssh/deploy_info.py | 32 ++- .../clan-cli/clan_cli/ssh/test_deploy_info.py | 4 +- pkgs/clan-cli/clan_cli/ssh/tor.py | 233 ------------------ pkgs/clan-cli/clan_lib/errors/__init__.py | 10 - .../import_utils/import_utils_test.py | 40 ++- pkgs/clan-cli/clan_lib/machines/install.py | 11 +- pkgs/clan-cli/clan_lib/network/__init__.py | 3 + pkgs/clan-cli/clan_lib/network/check.py | 131 ++++++++++ pkgs/clan-cli/clan_lib/network/direct.py | 40 ++- pkgs/clan-cli/clan_lib/network/network.py | 113 +++++---- pkgs/clan-cli/clan_lib/network/tor.py | 20 -- .../clan-cli/clan_lib/network/tor/__init__.py | 62 +++++ pkgs/clan-cli/clan_lib/network/tor/lib.py | 98 ++++++++ pkgs/clan-cli/clan_lib/ssh/remote.py | 133 +++------- pkgs/clan-cli/clan_lib/tests/test_create.py | 8 +- 17 files changed, 504 insertions(+), 465 deletions(-) delete mode 100755 pkgs/clan-cli/clan_cli/ssh/tor.py create mode 100644 pkgs/clan-cli/clan_lib/network/__init__.py create mode 100644 pkgs/clan-cli/clan_lib/network/check.py delete mode 100644 pkgs/clan-cli/clan_lib/network/tor.py create mode 100644 pkgs/clan-cli/clan_lib/network/tor/__init__.py create mode 100755 pkgs/clan-cli/clan_lib/network/tor/lib.py diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index 6791ba425..99a5cc61e 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -35,7 +35,7 @@ def install_command(args: argparse.Namespace) -> None: use_tor = False if deploy_info: host = find_reachable_host(deploy_info) - if host is None or host.tor_socks: + if host is None or host.socks_port: use_tor = True target_host_str = deploy_info.tor.target else: @@ -74,7 +74,9 @@ def install_command(args: argparse.Namespace) -> None: target_host = target_host.override(password=password) if use_tor: - target_host = target_host.override(tor_socks=True) + target_host = target_host.override( + socks_port=9050, socks_wrapper=["torify"] + ) return run_machine_install( InstallOptions( diff --git a/pkgs/clan-cli/clan_cli/network/ping.py b/pkgs/clan-cli/clan_cli/network/ping.py index 942d25c5e..767a96f1d 100644 --- a/pkgs/clan-cli/clan_cli/network/ping.py +++ b/pkgs/clan-cli/clan_cli/network/ping.py @@ -27,28 +27,27 @@ def ping_command(args: argparse.Namespace) -> None: networks_to_check = sorted(networks.items(), key=lambda x: -x[1].priority) found = False - results = [] for net_name, network in networks_to_check: if machine in network.peers: found = True - # Check if network technology is running - if not network.is_running(): - results.append(f"{machine} ({net_name}): network not running") - continue - - # Check if peer is online - ping = network.ping(machine) - results.append(f"{machine} ({net_name}): {ping}") + with network.module.connection(network) as network: + log.info(f"Pinging '{machine}' in network '{net_name}' ...") + res = "" + # Check if peer is online + ping = network.ping(machine) + if ping is None: + res = "not reachable" + log.info(f"{machine} ({net_name}): {res}") + else: + res = f"reachable, ping: {ping:.2f} ms" + log.info(f"{machine} ({net_name}): {res}") + break if not found: msg = f"Machine '{machine}' not found in any network" raise ClanError(msg) - # Print all results - for result in results: - print(result) - def register_ping_parser(parser: argparse.ArgumentParser) -> None: parser.add_argument( diff --git a/pkgs/clan-cli/clan_cli/ssh/deploy_info.py b/pkgs/clan-cli/clan_cli/ssh/deploy_info.py index 9688a5c57..42f35fdb7 100644 --- a/pkgs/clan-cli/clan_cli/ssh/deploy_info.py +++ b/pkgs/clan-cli/clan_cli/ssh/deploy_info.py @@ -1,4 +1,5 @@ import argparse +import contextlib import json import logging import textwrap @@ -9,6 +10,7 @@ from typing import Any, get_args from clan_lib.cmd import run from clan_lib.errors import ClanError from clan_lib.machines.machines import Machine +from clan_lib.network.tor.lib import spawn_tor from clan_lib.nix import nix_shell from clan_lib.ssh.remote import HostKeyCheck, Remote @@ -16,7 +18,6 @@ from clan_cli.completions import ( add_dynamic_completer, complete_machines, ) -from clan_cli.ssh.tor import TorTarget, spawn_tor, ssh_tor_reachable log = logging.getLogger(__name__) @@ -27,15 +28,15 @@ class DeployInfo: @property def tor(self) -> Remote: - """Return a list of Remote objects that are configured for Tor.""" - addrs = [addr for addr in self.addrs if addr.tor_socks] + """Return a list of Remote objects that are configured for SOCKS5 proxy.""" + addrs = [addr for addr in self.addrs if addr.socks_port] if not addrs: - msg = "No tor address provided, please provide a tor address." + msg = "No socks5 proxy address provided, please provide a socks5 proxy address." raise ClanError(msg) if len(addrs) > 1: - msg = "Multiple tor addresses provided, expected only one." + msg = "Multiple socks5 proxy addresses provided, expected only one." raise ClanError(msg) return addrs[0] @@ -76,7 +77,12 @@ class DeployInfo: remote = Remote.from_ssh_uri( machine_name="clan-installer", address=tor_addr, - ).override(host_key_check=host_key_check, tor_socks=True, password=password) + ).override( + host_key_check=host_key_check, + socks_port=9050, + socks_wrapper=["torify"], + password=password, + ) addrs.append(remote) return DeployInfo(addrs=addrs) @@ -103,7 +109,8 @@ def find_reachable_host(deploy_info: DeployInfo) -> Remote | None: return deploy_info.addrs[0] for addr in deploy_info.addrs: - if addr.check_machine_ssh_reachable(): + with contextlib.suppress(ClanError): + addr.check_machine_ssh_reachable() return addr return None @@ -129,7 +136,7 @@ def ssh_shell_from_deploy( log.info("Could not reach host via clearnet 'addrs'") log.info(f"Trying to reach host via tor '{deploy_info}'") - tor_addrs = [addr for addr in deploy_info.addrs if addr.tor_socks] + tor_addrs = [addr for addr in deploy_info.addrs if addr.socks_port] if not tor_addrs: msg = "No tor address provided, please provide a tor address." raise ClanError(msg) @@ -137,11 +144,10 @@ def ssh_shell_from_deploy( with spawn_tor(): for tor_addr in tor_addrs: log.info(f"Trying to reach host via tor address: {tor_addr}") - if ssh_tor_reachable( - TorTarget( - onion=tor_addr.address, port=tor_addr.port if tor_addr.port else 22 - ) - ): + + with contextlib.suppress(ClanError): + tor_addr.check_machine_ssh_reachable() + log.info( "Host reachable via tor address, starting interactive ssh session." ) diff --git a/pkgs/clan-cli/clan_cli/ssh/test_deploy_info.py b/pkgs/clan-cli/clan_cli/ssh/test_deploy_info.py index 30d758311..10e7ecc8a 100644 --- a/pkgs/clan-cli/clan_cli/ssh/test_deploy_info.py +++ b/pkgs/clan-cli/clan_cli/ssh/test_deploy_info.py @@ -37,7 +37,7 @@ def test_qrcode_scan(temp_dir: Path) -> None: tor_host.address == "qjeerm4r6t55hcfum4pinnvscn5njlw2g3k7ilqfuu7cdt3ahaxhsbid.onion" ) - assert tor_host.tor_socks is True + assert tor_host.socks_port == 9050 assert tor_host.password == "scabbed-defender-headlock" assert tor_host.user == "root" assert ( @@ -59,7 +59,7 @@ def test_from_json() -> None: tor_host.address == "qjeerm4r6t55hcfum4pinnvscn5njlw2g3k7ilqfuu7cdt3ahaxhsbid.onion" ) - assert tor_host.tor_socks is True + assert tor_host.socks_port == 9050 assert tor_host.password == "scabbed-defender-headlock" assert tor_host.user == "root" assert ( diff --git a/pkgs/clan-cli/clan_cli/ssh/tor.py b/pkgs/clan-cli/clan_cli/ssh/tor.py deleted file mode 100755 index 992e731f5..000000000 --- a/pkgs/clan-cli/clan_cli/ssh/tor.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import logging -import socket -import struct -import time -from collections.abc import Iterator -from contextlib import contextmanager -from dataclasses import dataclass -from subprocess import Popen - -from clan_lib.errors import TorConnectionError, TorSocksError -from clan_lib.nix import nix_shell - -log = logging.getLogger(__name__) - - -@dataclass -class TorTarget: - onion: str - port: int - proxy_host: str = "127.0.0.1" - proxy_port: int = 9050 - - -def connect_to_tor_socks(sock: socket.socket, target: TorTarget) -> socket.socket: - """ - Connects to a .onion host through Tor's SOCKS5 proxy using the standard library. - - Args: - target (TorTarget): An object containing the .onion address, port, proxy host, and proxy port. - - Returns: - socket.socket: A socket connected to the .onion address via Tor. - """ - try: - # 1. Create a socket to the Tor SOCKS proxy - sock.connect((target.proxy_host, target.proxy_port)) - except ConnectionRefusedError as ex: - msg = f"Failed to connect to Tor SOCKS proxy at {target.proxy_host}:{target.proxy_port}: {ex}" - raise TorSocksError(msg) from ex - - # 2. SOCKS5 handshake - sock.sendall( - b"\x05\x01\x00" - ) # SOCKS version (0x05), number of authentication methods (0x01), no-authentication (0x00) - response = sock.recv(2) - - # Validate the SOCKS5 handshake response - if response != b"\x05\x00": # SOCKS version = 0x05, no-authentication = 0x00 - msg = f"SOCKS5 handshake failed, unexpected response: {response.hex()}" - raise TorSocksError(msg) - - # 3. Connection request - request = ( - b"\x05\x01\x00\x03" # SOCKS version, connect command, reserved, address type = domainname - + bytes([len(target.onion)]) - + target.onion.encode("utf-8") # Add domain name length and domain name - + struct.pack(">H", target.port) # Add destination port in network byte order - ) - sock.sendall(request) - - # Read the connection request response - response = sock.recv(10) - if response[1] != 0x00: # 0x00 indicates success - msg = f".onion address not reachable: {response[1]}" - raise TorConnectionError(msg) - - return sock - - -def fetch_onion_content(target: TorTarget) -> str: - """ - Fetches the HTTP response from a .onion service through a Tor SOCKS5 proxy. - - Args: - target (TorTarget): An object containing the .onion address, port, proxy host, and proxy port. - - Returns: - str: The HTTP response text, or an error message if something goes wrong. - """ - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - # Connect to the .onion service via the SOCKS proxy - sock = connect_to_tor_socks(sock, target) - - # 1. Send an HTTP GET request - request = f"GET / HTTP/1.1\r\nHost: {target.onion}\r\nConnection: close\r\n\r\n" - sock.sendall(request.encode("utf-8")) - - # 2. Read the HTTP response - response = b"" - while True: - chunk = sock.recv(4096) - if not chunk: - break - response += chunk - - return response.decode("utf-8", errors="replace") - - -def is_tor_running() -> bool: - """Checks if Tor is online.""" - try: - tor_online_test() - except TorSocksError: - return False - else: - return True - - -@contextmanager -def spawn_tor() -> Iterator[None]: - """ - Spawns a Tor process using `nix-shell` if Tor is not already running. - """ - - # Check if Tor is already running - if is_tor_running(): - log.info("Tor is running") - return - cmd_args = ["tor", "--HardwareAccel", "1"] - packages = ["tor"] - cmd = nix_shell(packages, cmd_args) - process = Popen(cmd) - try: - while not is_tor_running(): - log.debug("Waiting for Tor to start...") - time.sleep(0.2) - log.info("Tor is now running") - yield - finally: - log.info("Terminating Tor process...") - process.terminate() - process.wait() - log.info("Tor process terminated") - - -def tor_online_test() -> bool: - """ - Tests if Tor is online by attempting to fetch content from a known .onion service. - Returns True if successful, False otherwise. - """ - target = TorTarget( - onion="duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion", port=80 - ) - try: - response = fetch_onion_content(target) - except TorConnectionError: - return False - else: - return "duckduckgo" in response - - -def ssh_tor_reachable(target: TorTarget) -> bool: - """ - Tests if SSH is reachable via Tor by attempting to connect to a known .onion service. - Returns True if successful, False otherwise. - """ - try: - response = fetch_onion_content(target) - except TorConnectionError: - return False - else: - return "SSH-" in response - - -def main() -> None: - """ - Main function to handle command-line arguments and execute the script. - """ - parser = argparse.ArgumentParser( - description="Interact with a .onion service through Tor SOCKS5 proxy." - ) - parser.add_argument( - "onion_url", type=str, help=".onion URL to connect to (e.g., 'example.onion')" - ) - parser.add_argument( - "--port", type=int, help="Port to connect to on the .onion URL (default: 80)" - ) - parser.add_argument( - "--proxy-host", - type=str, - default="127.0.0.1", - help="Address of the Tor SOCKS5 proxy (default: 127.0.0.1)", - ) - parser.add_argument( - "--proxy-port", - type=int, - default=9050, - help="Port of the Tor SOCKS5 proxy (default: 9050)", - ) - parser.add_argument( - "--ssh-tor-reachable", - action="store_true", - help="Test if SSH is reachable via Tor", - ) - - args = parser.parse_args() - default_port = 22 if args.ssh_tor_reachable else 80 - - # Create a TorTarget instance - target = TorTarget( - onion=args.onion_url, - port=args.port or default_port, - proxy_host=args.proxy_host, - proxy_port=args.proxy_port, - ) - - if args.ssh_tor_reachable: - print(f"Testing if SSH is reachable via Tor for {target.onion}...") - reachable = ssh_tor_reachable(target) - print(f"SSH is {'reachable' if reachable else 'not reachable'} via Tor.") - return - - print( - f"Connecting to {target.onion} on port {target.port} via proxy {target.proxy_host}:{target.proxy_port}..." - ) - try: - response = fetch_onion_content(target) - print("Response:") - print(response) - except TorSocksError: - log.error("Failed to connect to the Tor SOCKS proxy.") - log.error( - "Is Tor running? If not, you can start it by running 'tor' in a nix-shell." - ) - except TorConnectionError: - log.error("The onion address is not reachable via Tor.") - - -if __name__ == "__main__": - main() diff --git a/pkgs/clan-cli/clan_lib/errors/__init__.py b/pkgs/clan-cli/clan_lib/errors/__init__.py index 31b3a489e..1c22a0c4c 100644 --- a/pkgs/clan-cli/clan_lib/errors/__init__.py +++ b/pkgs/clan-cli/clan_lib/errors/__init__.py @@ -189,13 +189,3 @@ class ClanCmdError(ClanError): def __repr__(self) -> str: return f"ClanCmdError({self.cmd})" - - -class TorSocksError(ClanError): - def __init__(self, msg: str) -> None: - super().__init__(msg) - - -class TorConnectionError(ClanError): - def __init__(self, msg: str) -> None: - super().__init__(msg) diff --git a/pkgs/clan-cli/clan_lib/import_utils/import_utils_test.py b/pkgs/clan-cli/clan_lib/import_utils/import_utils_test.py index fd34b8c71..7a0579212 100644 --- a/pkgs/clan-cli/clan_lib/import_utils/import_utils_test.py +++ b/pkgs/clan-cli/clan_lib/import_utils/import_utils_test.py @@ -22,7 +22,13 @@ def test_import_with_source(tmp_path: Path) -> None: test_module_path = module_dir / "test_tech.py" test_module_path.write_text( dedent(""" - from clan_lib.network.network import NetworkTechnologyBase + from clan_lib.network.network import NetworkTechnologyBase, Peer, Network + from contextlib import contextmanager + from collections.abc import Iterator + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from clan_lib.ssh.remote import Remote class NetworkTechnology(NetworkTechnologyBase): def __init__(self, source): @@ -31,6 +37,17 @@ def test_import_with_source(tmp_path: Path) -> None: def is_running(self) -> bool: return True + + def remote(self, peer: Peer) -> "Remote": + from clan_lib.ssh.remote import Remote + return Remote(host=peer.host) + + def ping(self, peer: Peer) -> None | float: + return 0.1 + + @contextmanager + def connection(self, network: Network) -> Iterator[Network]: + yield network """) ) @@ -57,7 +74,7 @@ def test_import_with_source(tmp_path: Path) -> None: assert instance.source.module_name == "test_module.test_tech" assert instance.source.file_path.name == "test_tech.py" assert instance.source.object_name == "NetworkTechnology" - assert instance.source.line_number == 4 # Line where class is defined + assert instance.source.line_number == 10 # Line where class is defined # Test string representations str_repr = str(instance) @@ -81,7 +98,13 @@ def test_import_with_source_with_args() -> None: with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: f.write( dedent(""" - from clan_lib.network.network import NetworkTechnologyBase + from clan_lib.network.network import NetworkTechnologyBase, Peer, Network + from contextlib import contextmanager + from collections.abc import Iterator + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from clan_lib.ssh.remote import Remote class NetworkTechnology(NetworkTechnologyBase): def __init__(self, source, extra_arg, keyword_arg=None): @@ -91,6 +114,17 @@ def test_import_with_source_with_args() -> None: def is_running(self) -> bool: return False + + def remote(self, peer: Peer) -> "Remote": + from clan_lib.ssh.remote import Remote + return Remote(host=peer.host) + + def ping(self, peer: Peer) -> None | float: + return None + + @contextmanager + def connection(self, network: Network) -> Iterator[Network]: + yield network """) ) temp_file = Path(f.name) diff --git a/pkgs/clan-cli/clan_lib/machines/install.py b/pkgs/clan-cli/clan_lib/machines/install.py index 949f826ad..6a758a1b2 100644 --- a/pkgs/clan-cli/clan_lib/machines/install.py +++ b/pkgs/clan-cli/clan_lib/machines/install.py @@ -133,16 +133,15 @@ def run_machine_install(opts: InstallOptions, target_host: Remote) -> None: cmd.extend(opts.machine.flake.nix_options or []) cmd.append(target_host.target) - if target_host.tor_socks: - # nix copy does not support tor socks proxy - # cmd.append("--ssh-option") - # cmd.append("ProxyCommand=nc -x 127.0.0.1:9050 -X 5 %h %p") + if target_host.socks_port: + # nix copy does not support socks5 proxy, use wrapper command + wrapper_cmd = target_host.socks_wrapper or ["torify"] cmd = nix_shell( [ "nixos-anywhere", - "tor", + *wrapper_cmd, ], - ["torify", *cmd], + [*wrapper_cmd, *cmd], ) else: cmd = nix_shell( diff --git a/pkgs/clan-cli/clan_lib/network/__init__.py b/pkgs/clan-cli/clan_lib/network/__init__.py new file mode 100644 index 000000000..416376463 --- /dev/null +++ b/pkgs/clan-cli/clan_lib/network/__init__.py @@ -0,0 +1,3 @@ +from .network import Network, NetworkTechnologyBase, Peer + +__all__ = ["Network", "NetworkTechnologyBase", "Peer"] diff --git a/pkgs/clan-cli/clan_lib/network/check.py b/pkgs/clan-cli/clan_lib/network/check.py new file mode 100644 index 000000000..44af02fce --- /dev/null +++ b/pkgs/clan-cli/clan_lib/network/check.py @@ -0,0 +1,131 @@ +import logging +from dataclasses import dataclass + +from clan_lib.api import API +from clan_lib.cmd import ClanCmdError, ClanCmdTimeoutError, RunOpts, run +from clan_lib.errors import ClanError # Assuming these are available +from clan_lib.ssh.remote import Remote + +cmdlog = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ConnectionOptions: + timeout: int = 20 + + +@API.register +def check_machine_ssh_login( + remote: Remote, opts: ConnectionOptions | None = None +) -> None: + """Checks if a remote machine is reachable via SSH by attempting to run a simple command. + Args: + remote (Remote): The remote host to check for SSH login. + opts (ConnectionOptions, optional): Connection options such as timeout. + If not provided, default values are used. + Usage: + result = check_machine_ssh_login(remote) + if result.ok: + print("SSH login successful") + else: + print(f"SSH login failed: {result.reason}") + Raises: + ClanError: If the SSH login fails. + """ + if opts is None: + opts = ConnectionOptions() + + with remote.ssh_control_master() as ssh: + try: + ssh.run( + ["true"], + RunOpts(timeout=opts.timeout, needs_user_terminal=True), + ) + return + except ClanCmdTimeoutError as e: + msg = f"SSH login timeout after {opts.timeout}s" + raise ClanError(msg) from e + except ClanCmdError as e: + if "Host key verification failed." in e.cmd.stderr: + raise ClanError(e.cmd.stderr.strip()) from e + msg = f"SSH login failed: {e}" + raise ClanError(msg) from e + + +@API.register +def check_machine_ssh_reachable( + remote: Remote, opts: ConnectionOptions | None = None +) -> None: + """ + Checks if a remote machine is reachable via SSH by attempting to open a TCP connection + to the specified address and port. + Args: + remote (Remote): The remote host to check for SSH reachability. + opts (ConnectionOptions, optional): Connection options such as timeout. + If not provided, default values are used. + Returns: + CheckResult: An object indicating whether the SSH port is reachable (`ok=True`) or not (`ok=False`), + and a reason if the check failed. + Usage: + result = check_machine_ssh_reachable(remote) + if result.ok: + print("SSH port is reachable") + print(f"SSH port is not reachable: {result.reason}") + """ + if opts is None: + opts = ConnectionOptions() + + cmdlog.debug( + f"Checking SSH reachability for {remote.target} on port {remote.port or 22}", + ) + + # Use ssh with ProxyCommand to check through SOCKS5 + cmd = [ + "ssh", + ] + + # If using SOCKS5 proxy, add ProxyCommand + if remote.socks_port: + cmd.extend( + [ + "-o", + f"ProxyCommand=nc -X 5 -x localhost:{remote.socks_port} %h %p", + ] + ) + + cmd.extend( + [ + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + f"ConnectTimeout={opts.timeout}", + "-o", + "PreferredAuthentications=none", + "-p", + str(remote.port or 22), + f"dummy@{remote.address.strip()}", + "true", + ] + ) + + try: + res = run(cmd, options=RunOpts(timeout=opts.timeout, check=False)) + + # SSH will fail with authentication error if server is reachable + # Check for SSH-related errors in stderr + if ( + "Permission denied" in res.stderr + or "No supported authentication" in res.stderr + ): + return # Server is reachable, auth failed as expected + + msg = "Connection failed: SSH server not reachable" + raise ClanError(msg) + + except ClanCmdTimeoutError as e: + msg = f"Connection timeout after {opts.timeout}s" + raise ClanError(msg) from e diff --git a/pkgs/clan-cli/clan_lib/network/direct.py b/pkgs/clan-cli/clan_lib/network/direct.py index 489c98329..94fd5bd36 100644 --- a/pkgs/clan-cli/clan_lib/network/direct.py +++ b/pkgs/clan-cli/clan_lib/network/direct.py @@ -1,9 +1,47 @@ -from clan_lib.network.network import NetworkTechnologyBase +import logging +import time +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass + +from clan_lib.errors import ClanError +from clan_lib.network.network import Network, NetworkTechnologyBase, Peer +from clan_lib.ssh.remote import Remote + +log = logging.getLogger(__name__) +@dataclass(frozen=True) class NetworkTechnology(NetworkTechnologyBase): """Direct network connection technology - checks SSH connectivity""" def is_running(self) -> bool: """Direct connections are always 'running' as they don't require a daemon""" return True + + def ping(self, peer: Peer) -> None | float: + if self.is_running(): + try: + # Parse the peer's host address to create a Remote object, use peer here since we don't have the machine_name here + remote = Remote.from_ssh_uri(machine_name="peer", address=peer.host) + + # Use the existing SSH reachability check + now = time.time() + + remote.check_machine_ssh_reachable() + + return (time.time() - now) * 1000 + + except ClanError as e: + log.debug(f"Error checking peer {peer.host}: {e}") + return None + return None + + @contextmanager + def connection(self, network: Network) -> Iterator[Network]: + # direct connections are always online and don't use SOCKS, so we just return the original network + # TODO maybe we want to setup jumphosts for network access? but sounds complicated + yield network + + def remote(self, peer: Peer) -> Remote: + return Remote.from_ssh_uri(machine_name=peer.name, address=peer.host) diff --git a/pkgs/clan-cli/clan_lib/network/network.py b/pkgs/clan-cli/clan_lib/network/network.py index 477aaed4e..b61f38da9 100644 --- a/pkgs/clan-cli/clan_lib/network/network.py +++ b/pkgs/clan-cli/clan_lib/network/network.py @@ -1,23 +1,27 @@ import logging import textwrap -import time from abc import ABC, abstractmethod +from collections.abc import Iterator +from contextlib import contextmanager from dataclasses import dataclass from functools import cached_property -from typing import Any +from typing import TYPE_CHECKING, Any from clan_cli.vars.get import get_machine_var + from clan_lib.errors import ClanError from clan_lib.flake import Flake from clan_lib.import_utils import ClassSource, import_with_source -from clan_lib.ssh.parse import parse_ssh_uri -from clan_lib.ssh.remote import Remote, check_machine_ssh_reachable + +if TYPE_CHECKING: + from clan_lib.ssh.remote import Remote log = logging.getLogger(__name__) @dataclass(frozen=True) class Peer: + name: str _host: dict[str, str | dict[str, str]] flake: Flake @@ -30,7 +34,9 @@ class Peer: machine_name = _var["machine"] generator = _var["generator"] var = get_machine_var( - str(self.flake), + str( + self.flake + ), # TODO we should really pass the flake instance here instead of a str representation machine_name, f"{generator}/{_var['file']}", ) @@ -50,35 +56,6 @@ class Peer: raise ClanError(msg) -@dataclass -class NetworkTechnologyBase(ABC): - source: ClassSource - - @abstractmethod - def is_running(self) -> bool: - pass - - # TODO this will depend on the network implementation if we do user networking at some point, so it should be abstractmethod - def ping(self, peer: Peer) -> None | float: - if self.is_running(): - try: - # Parse the peer's host address to create a Remote object, use peer here since we don't have the machine_name here - remote = parse_ssh_uri(machine_name="peer", address=peer.host) - - # Use the existing SSH reachability check - now = time.time() - result = check_machine_ssh_reachable(remote) - - if result.ok: - return (time.time() - now) * 1000 - return None - - except Exception as e: - log.debug(f"Error checking peer {peer.host}: {e}") - return None - return None - - @dataclass(frozen=True) class Network: peers: dict[str, Peer] @@ -86,7 +63,7 @@ class Network: priority: int = 1000 @cached_property - def module(self) -> NetworkTechnologyBase: + def module(self) -> "NetworkTechnologyBase": res = import_with_source( self.module_name, "NetworkTechnology", @@ -100,15 +77,49 @@ class Network: def ping(self, peer: str) -> float | None: return self.module.ping(self.peers[peer]) + def remote(self, peer: str) -> "Remote": + # TODO raise exception if peer is not in peers + return self.module.remote(self.peers[peer]) + + +@dataclass(frozen=True) +class NetworkTechnologyBase(ABC): + source: ClassSource + + @abstractmethod + def is_running(self) -> bool: + pass + + @abstractmethod + def remote(self, peer: Peer) -> "Remote": + pass + + @abstractmethod + def ping(self, peer: Peer) -> None | float: + pass + + @contextmanager + @abstractmethod + def connection(self, network: Network) -> Iterator[Network]: + pass + def networks_from_flake(flake: Flake) -> dict[str, Network]: + # TODO more precaching, for example for vars + flake.precache( + [ + "clan.exports.instances.*.networking", + ] + ) networks: dict[str, Network] = {} networks_ = flake.select("clan.exports.instances.*.networking") for network_name, network in networks_.items(): if network: peers: dict[str, Peer] = {} for _peer in network["peers"].values(): - peers[_peer["name"]] = Peer(_host=_peer["host"], flake=flake) + peers[_peer["name"]] = Peer( + name=_peer["name"], _host=_peer["host"], flake=flake + ) networks[network_name] = Network( peers=peers, module_name=network["module"], @@ -117,17 +128,14 @@ def networks_from_flake(flake: Flake) -> dict[str, Network]: return networks -def get_best_remote(machine_name: str, networks: dict[str, Network]) -> Remote | None: +def get_best_network(machine_name: str, networks: dict[str, Network]) -> Network | None: for network_name, network in sorted( networks.items(), key=lambda network: -network[1].priority ): if machine_name in network.peers: if network.is_running() and network.ping(machine_name): print(f"connecting via {network_name}") - return Remote.from_ssh_uri( - machine_name=machine_name, - address=network.peers[machine_name].host, - ) + return network return None @@ -137,20 +145,19 @@ def get_network_overview(networks: dict[str, Network]) -> dict: result[network_name] = {} result[network_name]["status"] = None result[network_name]["peers"] = {} - network_online = False module = network.module log.debug(f"Using network module: {module}") if module.is_running(): result[network_name]["status"] = True - network_online = True - for peer_name in network.peers: - if network_online: - try: - result[network_name]["peers"][peer_name] = network.ping(peer_name) - except ClanError: - log.warning( - f"getting host for machine: {peer_name} in network: {network_name} failed" - ) - else: - result[network_name]["peers"][peer_name] = None + else: + with module.connection(network) as network: + for peer_name in network.peers: + try: + result[network_name]["peers"][peer_name] = network.ping( + peer_name + ) + except ClanError: + log.warning( + f"getting host for machine: {peer_name} in network: {network_name} failed" + ) return result diff --git a/pkgs/clan-cli/clan_lib/network/tor.py b/pkgs/clan-cli/clan_lib/network/tor.py deleted file mode 100644 index 9aedf01e3..000000000 --- a/pkgs/clan-cli/clan_lib/network/tor.py +++ /dev/null @@ -1,20 +0,0 @@ -from urllib.error import HTTPError -from urllib.request import urlopen - -from .network import NetworkTechnologyBase - - -class NetworkTechnology(NetworkTechnologyBase): - socks_port: int - command_port: int - - def is_running(self) -> bool: - """Check if Tor is running by sending HTTP request to SOCKS port.""" - try: - response = urlopen("http://127.0.0.1:9050", timeout=5) - content = response.read().decode("utf-8", errors="ignore") - return "tor" in content.lower() - except HTTPError as e: - return "tor" in str(e).lower() - except Exception: - return False diff --git a/pkgs/clan-cli/clan_lib/network/tor/__init__.py b/pkgs/clan-cli/clan_lib/network/tor/__init__.py new file mode 100644 index 000000000..120f6741d --- /dev/null +++ b/pkgs/clan-cli/clan_lib/network/tor/__init__.py @@ -0,0 +1,62 @@ +import logging +import time +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from clan_lib.errors import ClanError +from clan_lib.network import Network, NetworkTechnologyBase, Peer +from clan_lib.network.tor.lib import is_tor_running, spawn_tor + +if TYPE_CHECKING: + from clan_lib.ssh.remote import Remote + + +log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class NetworkTechnology(NetworkTechnologyBase): + @property + def proxy(self) -> int: + """Return the SOCKS5 proxy port for this network technology.""" + return 9050 + + def is_running(self) -> bool: + """Check if Tor is running by sending HTTP request to SOCKS port.""" + return is_tor_running(self.proxy) + + def ping(self, peer: Peer) -> None | float: + if self.is_running(): + try: + remote = self.remote(peer) + + # Use the existing SSH reachability check + now = time.time() + remote.check_machine_ssh_reachable() + + return (time.time() - now) * 1000 + + except ClanError as e: + log.debug(f"Error checking peer {peer.host}: {e}") + return None + return None + + @contextmanager + def connection(self, network: Network) -> Iterator[Network]: + if self.is_running(): + yield network + else: + with spawn_tor() as _: + yield network + + def remote(self, peer: Peer) -> "Remote": + from clan_lib.ssh.remote import Remote + + return Remote( + address=peer.host, + command_prefix=peer.name, + socks_port=self.proxy, + socks_wrapper=["torify"], + ) diff --git a/pkgs/clan-cli/clan_lib/network/tor/lib.py b/pkgs/clan-cli/clan_lib/network/tor/lib.py new file mode 100755 index 000000000..6b0ea0df7 --- /dev/null +++ b/pkgs/clan-cli/clan_lib/network/tor/lib.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 + +import logging +import time +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from subprocess import Popen + +from clan_lib.errors import ClanError +from clan_lib.nix import nix_shell + +log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class TorTarget: + onion: str + port: int + + +def is_tor_running(proxy_port: int | None = None) -> bool: + """Checks if Tor is online.""" + if proxy_port is None: + proxy_port = 9050 + try: + tor_online_test(proxy_port) + except ClanError as err: + log.debug(f"Tor is not running: {err}") + return False + + return True + + +# TODO: Move this to network technology tor module +@contextmanager +def spawn_tor() -> Iterator[None]: + """ + Spawns a Tor process using `nix-shell` if Tor is not already running. + """ + + # Check if Tor is already running + if is_tor_running(): + log.info("Tor is running") + return + cmd_args = ["tor", "--HardwareAccel", "1"] + packages = ["tor"] + cmd = nix_shell(packages, cmd_args) + process = Popen(cmd) + try: + while not is_tor_running(): + log.debug("Waiting for Tor to start...") + time.sleep(0.2) + log.info("Tor is now running") + yield + finally: + log.info("Terminating Tor process...") + process.terminate() + process.wait() + log.info("Tor process terminated") + + +@dataclass(frozen=True) +class TorCheck: + onion: str + expected_content: str + port: int = 80 + + +def tor_online_test(proxy_port: int) -> None: + """ + Tests if Tor is online by checking if we can establish a SOCKS5 connection. + """ + import socket + + # Try to establish a SOCKS5 handshake with the Tor proxy + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(2.0) # Short timeout for local connection + + try: + # Connect to the SOCKS5 proxy + sock.connect(("localhost", proxy_port)) + + # Send SOCKS5 handshake + sock.sendall(b"\x05\x01\x00") # SOCKS5, 1 auth method, no auth + response = sock.recv(2) + + # Check if we got a valid SOCKS5 response + if response == b"\x05\x00": # SOCKS5, no auth accepted + return + msg = f"Invalid SOCKS5 response from Tor: {response.hex()}" + raise ClanError(msg) + + except (TimeoutError, ConnectionRefusedError, OSError) as e: + msg = f"Cannot connect to Tor SOCKS5 proxy at localhost:{proxy_port}: {e}" + raise ClanError(msg) from e + finally: + sock.close() diff --git a/pkgs/clan-cli/clan_lib/ssh/remote.py b/pkgs/clan-cli/clan_lib/ssh/remote.py index b850fd8b1..b7cdb74e4 100644 --- a/pkgs/clan-cli/clan_lib/ssh/remote.py +++ b/pkgs/clan-cli/clan_lib/ssh/remote.py @@ -2,19 +2,17 @@ import ipaddress import logging import os import shlex -import socket import subprocess import sys -import time from collections.abc import Iterator from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path from shlex import quote from tempfile import TemporaryDirectory +from typing import TYPE_CHECKING -from clan_lib.api import API -from clan_lib.cmd import ClanCmdError, ClanCmdTimeoutError, CmdOut, RunOpts, run +from clan_lib.cmd import CmdOut, RunOpts, run from clan_lib.colors import AnsiColor from clan_lib.errors import ClanError, indent_command # Assuming these are available from clan_lib.nix import nix_shell @@ -22,6 +20,9 @@ from clan_lib.ssh.host_key import HostKeyCheck, hostkey_to_ssh_opts from clan_lib.ssh.parse import parse_ssh_uri from clan_lib.ssh.sudo_askpass_proxy import SudoAskpassProxy +if TYPE_CHECKING: + from clan_lib.network.check import ConnectionOptions + cmdlog = logging.getLogger(__name__) # Seconds until a message is printed when _run produces no output. @@ -40,7 +41,8 @@ class Remote: host_key_check: HostKeyCheck = "ask" verbose_ssh: bool = False ssh_options: dict[str, str] = field(default_factory=dict) - tor_socks: bool = False + socks_port: int | None = None + socks_wrapper: list[str] | None = None _control_path_dir: Path | None = None _askpass_path: str | None = None @@ -60,7 +62,8 @@ class Remote: host_key_check: HostKeyCheck | None = None, private_key: Path | None = None, password: str | None = None, - tor_socks: bool | None = None, + socks_port: int | None = None, + socks_wrapper: list[str] | None = None, command_prefix: str | None = None, port: int | None = None, ssh_options: dict[str, str] | None = None, @@ -81,7 +84,10 @@ class Remote: ), verbose_ssh=self.verbose_ssh, ssh_options=ssh_options or self.ssh_options, - tor_socks=tor_socks if tor_socks is not None else self.tor_socks, + socks_port=socks_port if socks_port is not None else self.socks_port, + socks_wrapper=socks_wrapper + if socks_wrapper is not None + else self.socks_wrapper, _control_path_dir=self._control_path_dir, _askpass_path=self._askpass_path, ) @@ -152,7 +158,7 @@ class Remote: host_key_check=self.host_key_check, verbose_ssh=self.verbose_ssh, ssh_options=self.ssh_options, - tor_socks=self.tor_socks, + socks_port=self.socks_port, _control_path_dir=Path(temp_dir), _askpass_path=self._askpass_path, ) @@ -220,7 +226,7 @@ class Remote: host_key_check=self.host_key_check, verbose_ssh=self.verbose_ssh, ssh_options=self.ssh_options, - tor_socks=self.tor_socks, + socks_port=self.socks_port, _control_path_dir=self._control_path_dir, _askpass_path=askpass_path, ) @@ -373,10 +379,13 @@ class Remote: if tty: current_ssh_opts.extend(["-t"]) - if self.tor_socks: + if self.socks_port: packages.append("netcat") current_ssh_opts.extend( - ["-o", "ProxyCommand=nc -x 127.0.0.1:9050 -X 5 %h %p"] + [ + "-o", + f"ProxyCommand=nc -x localhost:{self.socks_port} -X 5 %h %p", + ] ) cmd = [ @@ -447,100 +456,14 @@ class Remote: if self.password: self.check_sshpass_errorcode(res) - def check_machine_ssh_reachable(self) -> bool: - return check_machine_ssh_reachable(self).ok + def check_machine_ssh_reachable( + self, opts: "ConnectionOptions | None" = None + ) -> None: + from clan_lib.network.check import check_machine_ssh_reachable + return check_machine_ssh_reachable(self, opts) -@dataclass(frozen=True) -class ConnectionOptions: - timeout: int = 2 - retries: int = 5 + def check_machine_ssh_login(self) -> None: + from clan_lib.network.check import check_machine_ssh_login - -@dataclass -class CheckResult: - ok: bool - reason: str | None = None - - -@API.register -def check_machine_ssh_login( - remote: Remote, opts: ConnectionOptions | None = None -) -> CheckResult: - """Checks if a remote machine is reachable via SSH by attempting to run a simple command. - Args: - remote (Remote): The remote host to check for SSH login. - opts (ConnectionOptions, optional): Connection options such as timeout and number of retries. - If not provided, default values are used. - Returns: - CheckResult: An object indicating whether the SSH login is successful (`ok=True`) or not (`ok=False`), - and a reason if the check failed. - Usage: - result = check_machine_ssh_login(remote) - if result.ok: - print("SSH login successful") - else: - print(f"SSH login failed: {result.reason}") - """ - if opts is None: - opts = ConnectionOptions() - - for _ in range(opts.retries): - with remote.ssh_control_master() as ssh: - try: - res = ssh.run( - ["true"], - RunOpts(timeout=opts.timeout, needs_user_terminal=True), - ) - return CheckResult(True) - except ClanCmdTimeoutError: - pass - except ClanCmdError as e: - if "Host key verification failed." in e.cmd.stderr: - raise ClanError(res.stderr.strip()) from e - else: - time.sleep(opts.timeout) - - return CheckResult(False, f"failed after {opts.retries} attempts") - - -@API.register -def check_machine_ssh_reachable( - remote: Remote, opts: ConnectionOptions | None = None -) -> CheckResult: - """ - Checks if a remote machine is reachable via SSH by attempting to open a TCP connection - to the specified address and port. - Args: - remote (Remote): The remote host to check for SSH reachability. - opts (ConnectionOptions, optional): Connection options such as timeout and number of retries. - If not provided, default values are used. - Returns: - CheckResult: An object indicating whether the SSH port is reachable (`ok=True`) or not (`ok=False`), - and a reason if the check failed. - Usage: - result = check_machine_ssh_reachable(remote) - if result.ok: - print("SSH port is reachable") - print(f"SSH port is not reachable: {result.reason}") - """ - if opts is None: - opts = ConnectionOptions() - - cmdlog.debug( - f"Checking SSH reachability for {remote.target} on port {remote.port or 22}", - ) - - address_family = socket.AF_INET6 if remote.is_ipv6() else socket.AF_INET - for _ in range(opts.retries): - with socket.socket(address_family, socket.SOCK_STREAM) as sock: - sock.settimeout(opts.timeout) - try: - sock.connect((remote.address, remote.port or 22)) - return CheckResult(True) - except (TimeoutError, OSError): - pass - else: - time.sleep(opts.timeout) - - return CheckResult(False, f"failed after {opts.retries} attempts") + return check_machine_ssh_login(self) diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index d5b1498e8..f0d4df399 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -32,7 +32,7 @@ from clan_lib.nix_models.clan import InventoryMachineDeploy as MachineDeploy from clan_lib.persist.inventory_store import InventoryStore from clan_lib.persist.util import set_value_by_path from clan_lib.services.modules import list_service_modules -from clan_lib.ssh.remote import Remote, check_machine_ssh_login +from clan_lib.ssh.remote import Remote from clan_lib.templates.disk import hw_main_disk_options, set_machine_disk_schema log = logging.getLogger(__name__) @@ -189,9 +189,9 @@ def test_clan_create_api( target_host = machine.target_host().override( private_key=private_key, host_key_check="none" ) - assert check_machine_ssh_login(target_host).ok, ( - f"Machine {machine.name} is not online" - ) + + target_host.check_machine_ssh_reachable() + target_host.check_machine_ssh_login() ssh_keys = [ SSHKeyPair(