diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index 8e77efc0f..c7c1fbcb4 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -5,7 +5,7 @@ from types import ModuleType from typing import Optional from . import config, flakes, join, machines, secrets, vms, webui -from .custom_logger import register +from .custom_logger import setup_logging from .ssh import cli as ssh_cli log = logging.getLogger(__name__) @@ -57,7 +57,7 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser: vms.register_parser(parser_vms) # if args.debug: - register(logging.DEBUG) + setup_logging(logging.DEBUG) log.debug("Debug log activated") if argcomplete: diff --git a/pkgs/clan-cli/clan_cli/async_cmd.py b/pkgs/clan-cli/clan_cli/async_cmd.py index 9befc98bc..06abb8a8a 100644 --- a/pkgs/clan-cli/clan_cli/async_cmd.py +++ b/pkgs/clan-cli/clan_cli/async_cmd.py @@ -4,6 +4,7 @@ import shlex from pathlib import Path from typing import Any, Callable, Coroutine, Dict, NamedTuple, Optional +from .custom_logger import get_caller from .errors import ClanError log = logging.getLogger(__name__) @@ -16,7 +17,6 @@ class CmdOut(NamedTuple): async def run(cmd: list[str], cwd: Optional[Path] = None) -> CmdOut: - log.debug(f"$: {shlex.join(cmd)}") cwd_res = None if cwd is not None: if not cwd.exists(): @@ -24,7 +24,9 @@ async def run(cmd: list[str], cwd: Optional[Path] = None) -> CmdOut: if not cwd.is_dir(): raise ClanError(f"Working directory {cwd} is not a directory") cwd_res = cwd.resolve() - log.debug(f"Working directory: {cwd_res}") + log.debug( + f"Command: {shlex.join(cmd)}\nWorking directory: {cwd_res}\nCaller : {get_caller()}" + ) proc = await asyncio.create_subprocess_exec( *cmd, stdout=asyncio.subprocess.PIPE, diff --git a/pkgs/clan-cli/clan_cli/clan_modules.py b/pkgs/clan-cli/clan_cli/clan_modules.py index 926feeb0d..54d0900d9 100644 --- a/pkgs/clan-cli/clan_cli/clan_modules.py +++ b/pkgs/clan-cli/clan_cli/clan_modules.py @@ -1,20 +1,20 @@ import json import subprocess -from pathlib import Path from typing import Optional -from clan_cli.dirs import get_clan_flake_toplevel from clan_cli.nix import nix_eval +from .dirs import specific_flake_dir +from .types import FlakeName + def get_clan_module_names( - flake: Optional[Path] = None, + flake_name: FlakeName, ) -> tuple[list[str], Optional[str]]: """ Get the list of clan modules from the clan-core flake input """ - if flake is None: - flake = get_clan_flake_toplevel() + flake = specific_flake_dir(flake_name) proc = subprocess.run( nix_eval( [ diff --git a/pkgs/clan-cli/clan_cli/config/machine.py b/pkgs/clan-cli/clan_cli/config/machine.py index b04c87b8b..c7433f7a5 100644 --- a/pkgs/clan-cli/clan_cli/config/machine.py +++ b/pkgs/clan-cli/clan_cli/config/machine.py @@ -3,6 +3,8 @@ import os import subprocess import sys from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Optional from fastapi import HTTPException @@ -19,31 +21,35 @@ from ..types import FlakeName def verify_machine_config( - machine_name: str, config: Optional[dict] = None, flake: Optional[Path] = None + flake_name: FlakeName, + machine_name: str, + config: Optional[dict] = None, + flake: Optional[Path] = None, ) -> Optional[str]: """ Verify that the machine evaluates successfully Returns a tuple of (success, error_message) """ if config is None: - config = config_for_machine(machine_name) - if flake is None: - flake = get_clan_flake_toplevel() - with NamedTemporaryFile(mode="w") as clan_machine_settings_file: + config = config_for_machine(flake_name, machine_name) + flake = specific_flake_dir(flake_name) + with NamedTemporaryFile(mode="w", dir=flake) as clan_machine_settings_file: json.dump(config, clan_machine_settings_file, indent=2) clan_machine_settings_file.seek(0) env = os.environ.copy() env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name + cmd = nix_eval( + flags=[ + "--impure", + "--show-trace", + "--show-trace", + "--impure", # needed to access CLAN_MACHINE_SETTINGS_FILE + f".#nixosConfigurations.{machine_name}.config.system.build.toplevel.outPath", + ], + ) + # repro_env_break(work_dir=flake, env=env, cmd=cmd) proc = subprocess.run( - nix_eval( - flags=[ - "--impure", - "--show-trace", - "--show-trace", - "--impure", # needed to access CLAN_MACHINE_SETTINGS_FILE - f".#nixosConfigurations.{machine_name}.config.system.build.toplevel.outPath", - ], - ), + cmd, capture_output=True, text=True, cwd=flake, @@ -54,7 +60,6 @@ def verify_machine_config( return None - def config_for_machine(flake_name: FlakeName, machine_name: str) -> dict: # read the config from a json file located at {flake}/machines/{machine_name}/settings.json if not specific_machine_dir(flake_name, machine_name).exists(): @@ -88,76 +93,49 @@ def set_config_for_machine( commit_file(settings_path, repo_dir) -def schema_for_machine(flake_name: FlakeName, machine_name: str) -> dict: +def schema_for_machine( + flake_name: FlakeName, machine_name: str, config: Optional[dict] = None +) -> dict: flake = specific_flake_dir(flake_name) - - # use nix eval to lib.evalModules .#nixosModules.machine-{machine_name} - proc = subprocess.run( - nix_eval( - flags=[ - "--impure", - "--show-trace", - "--expr", - f""" - let - flake = builtins.getFlake (toString {flake}); - lib = import {nixpkgs_source()}/lib; - options = flake.nixosConfigurations.{machine_name}.options; - clanOptions = options.clan; - jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }}; - jsonschema = jsonschemaLib.parseOptions clanOptions; - in - jsonschema - """, - ], - ), - capture_output=True, - text=True, - ) -# def schema_for_machine( -# machine_name: str, config: Optional[dict] = None, flake: Optional[Path] = None -# ) -> dict: -# if flake is None: -# flake = get_clan_flake_toplevel() -# # use nix eval to lib.evalModules .#nixosConfigurations..options.clan -# with NamedTemporaryFile(mode="w") as clan_machine_settings_file: -# env = os.environ.copy() -# inject_config_flags = [] -# if config is not None: -# json.dump(config, clan_machine_settings_file, indent=2) -# clan_machine_settings_file.seek(0) -# env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name -# inject_config_flags = [ -# "--impure", # needed to access CLAN_MACHINE_SETTINGS_FILE -# ] -# proc = subprocess.run( -# nix_eval( -# flags=inject_config_flags -# + [ -# "--impure", -# "--show-trace", -# "--expr", -# f""" -# let -# flake = builtins.getFlake (toString {flake}); -# lib = import {nixpkgs_source()}/lib; -# options = flake.nixosConfigurations.{machine_name}.options; -# clanOptions = options.clan; -# jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }}; -# jsonschema = jsonschemaLib.parseOptions clanOptions; -# in -# jsonschema -# """, -# ], -# ), -# capture_output=True, -# text=True, -# cwd=flake, -# env=env, -# ) -# if proc.returncode != 0: -# print(proc.stderr, file=sys.stderr) -# raise Exception( -# f"Failed to read schema for machine {machine_name}:\n{proc.stderr}" -# ) -# return json.loads(proc.stdout) + # use nix eval to lib.evalModules .#nixosConfigurations..options.clan + with NamedTemporaryFile(mode="w", dir=flake) as clan_machine_settings_file: + env = os.environ.copy() + inject_config_flags = [] + if config is not None: + json.dump(config, clan_machine_settings_file, indent=2) + clan_machine_settings_file.seek(0) + env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name + inject_config_flags = [ + "--impure", # needed to access CLAN_MACHINE_SETTINGS_FILE + ] + proc = subprocess.run( + nix_eval( + flags=inject_config_flags + + [ + "--impure", + "--show-trace", + "--expr", + f""" + let + flake = builtins.getFlake (toString {flake}); + lib = import {nixpkgs_source()}/lib; + options = flake.nixosConfigurations.{machine_name}.options; + clanOptions = options.clan; + jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }}; + jsonschema = jsonschemaLib.parseOptions clanOptions; + in + jsonschema + """, + ], + ), + capture_output=True, + text=True, + cwd=flake, + env=env, + ) + if proc.returncode != 0: + print(proc.stderr, file=sys.stderr) + raise Exception( + f"Failed to read schema for machine {machine_name}:\n{proc.stderr}" + ) + return json.loads(proc.stdout) diff --git a/pkgs/clan-cli/clan_cli/custom_logger.py b/pkgs/clan-cli/clan_cli/custom_logger.py index f9f324e1a..8c91a10ac 100644 --- a/pkgs/clan-cli/clan_cli/custom_logger.py +++ b/pkgs/clan-cli/clan_cli/custom_logger.py @@ -61,10 +61,12 @@ def get_caller() -> str: return ret -def register(level: Any) -> None: +def setup_logging(level: Any) -> None: handler = logging.StreamHandler() handler.setLevel(level) handler.setFormatter(CustomFormatter()) logger = logging.getLogger("registerHandler") + logging.getLogger("asyncio").setLevel(logging.INFO) + logging.getLogger("httpx").setLevel(level=logging.WARNING) logger.addHandler(handler) # logging.basicConfig(level=level, handlers=[handler]) diff --git a/pkgs/clan-cli/clan_cli/machines/create.py b/pkgs/clan-cli/clan_cli/machines/create.py index 9a2e39bef..1c5810e37 100644 --- a/pkgs/clan-cli/clan_cli/machines/create.py +++ b/pkgs/clan-cli/clan_cli/machines/create.py @@ -32,7 +32,6 @@ async def create_machine(flake_name: FlakeName, machine_name: str) -> Dict[str, cwd=folder, ) response["git commit"] = out - return response diff --git a/pkgs/clan-cli/clan_cli/webui/routers/clan_modules.py b/pkgs/clan-cli/clan_cli/webui/routers/clan_modules.py index fe97e2825..1721546fa 100644 --- a/pkgs/clan-cli/clan_cli/webui/routers/clan_modules.py +++ b/pkgs/clan-cli/clan_cli/webui/routers/clan_modules.py @@ -4,8 +4,9 @@ import logging from fastapi import APIRouter, HTTPException from clan_cli.clan_modules import get_clan_module_names +from clan_cli.types import FlakeName -from ..schemas import ( +from ..api_outputs import ( ClanModulesResponse, ) @@ -13,9 +14,9 @@ log = logging.getLogger(__name__) router = APIRouter() -@router.get("/api/clan_modules") -async def list_clan_modules() -> ClanModulesResponse: - module_names, error = get_clan_module_names() +@router.get("/api/{flake_name}clan_modules") +async def list_clan_modules(flake_name: FlakeName) -> ClanModulesResponse: + module_names, error = get_clan_module_names(flake_name) if error is not None: raise HTTPException(status_code=400, detail=error) return ClanModulesResponse(clan_modules=module_names) diff --git a/pkgs/clan-cli/clan_cli/webui/routers/machines.py b/pkgs/clan-cli/clan_cli/webui/routers/machines.py index b636a8007..3c2dffd97 100644 --- a/pkgs/clan-cli/clan_cli/webui/routers/machines.py +++ b/pkgs/clan-cli/clan_cli/webui/routers/machines.py @@ -41,8 +41,7 @@ async def list_machines(flake_name: FlakeName) -> MachinesResponse: async def create_machine( flake_name: FlakeName, machine: Annotated[MachineCreate, Body()] ) -> MachineResponse: - out = await _create_machine(flake_name, machine.name) - log.debug(out) + await _create_machine(flake_name, machine.name) return MachineResponse(machine=Machine(name=machine.name, status=Status.UNKNOWN)) @@ -72,26 +71,29 @@ async def get_machine_schema(flake_name: FlakeName, name: str) -> SchemaResponse return SchemaResponse(schema=schema) -@router.put("/api/machines/{name}/schema") +@router.put("/api/{flake_name}/machines/{name}/schema") async def set_machine_schema( - name: str, config: Annotated[dict, Body()] + flake_name: FlakeName, name: str, config: Annotated[dict, Body()] ) -> SchemaResponse: - schema = schema_for_machine(name, config) + schema = schema_for_machine(flake_name, name, config) return SchemaResponse(schema=schema) -@router.get("/api/machines/{name}/verify") -async def get_verify_machine_config(name: str) -> VerifyMachineResponse: - error = verify_machine_config(name) +@router.get("/api/{flake_name}/machines/{name}/verify") +async def get_verify_machine_config( + flake_name: FlakeName, name: str +) -> VerifyMachineResponse: + error = verify_machine_config(flake_name, name) success = error is None return VerifyMachineResponse(success=success, error=error) -@router.put("/api/machines/{name}/verify") +@router.put("/api/{flake_name}/machines/{name}/verify") async def put_verify_machine_config( + flake_name: FlakeName, name: str, config: Annotated[dict, Body()], ) -> VerifyMachineResponse: - error = verify_machine_config(name, config) + error = verify_machine_config(flake_name, name, config) success = error is None return VerifyMachineResponse(success=success, error=error) diff --git a/pkgs/clan-cli/pyproject.toml b/pkgs/clan-cli/pyproject.toml index 7d61dd8e6..d0f2d1427 100644 --- a/pkgs/clan-cli/pyproject.toml +++ b/pkgs/clan-cli/pyproject.toml @@ -20,7 +20,7 @@ clan_cli = [ "config/jsonschema/*", "webui/assets/**/*"] testpaths = "tests" faulthandler_timeout = 60 log_level = "DEBUG" -log_format = "%(levelname)s: %(message)s" +log_format = "%(levelname)s: %(message)s\n %(pathname)s:%(lineno)d::%(funcName)s" addopts = "--cov . --cov-report term --cov-report html:.reports/html --no-cov-on-fail --durations 5 --color=yes --new-first --maxfail=1" # Add --pdb for debugging norecursedirs = "tests/helpers" markers = [ "impure" ] diff --git a/pkgs/clan-cli/tests/api.py b/pkgs/clan-cli/tests/api.py index 78d35e98a..8f7ff1929 100644 --- a/pkgs/clan-cli/tests/api.py +++ b/pkgs/clan-cli/tests/api.py @@ -1,3 +1,5 @@ +import logging + import pytest from fastapi.testclient import TestClient @@ -7,4 +9,6 @@ from clan_cli.webui.app import app # TODO: Why stateful @pytest.fixture(scope="session") def api() -> TestClient: + logging.getLogger("httpx").setLevel(level=logging.WARNING) + logging.getLogger("asyncio").setLevel(logging.INFO) return TestClient(app) diff --git a/pkgs/clan-cli/tests/temporary_dir.py b/pkgs/clan-cli/tests/temporary_dir.py index e7cbfa0d5..0b16c3fc1 100644 --- a/pkgs/clan-cli/tests/temporary_dir.py +++ b/pkgs/clan-cli/tests/temporary_dir.py @@ -19,7 +19,6 @@ def temporary_home(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: monkeypatch.chdir(str(path)) yield path else: - log.debug("TEST_TEMPORARY_DIR not set, using TemporaryDirectory") with tempfile.TemporaryDirectory(prefix="pytest-") as dirpath: monkeypatch.setenv("HOME", str(dirpath)) monkeypatch.chdir(str(dirpath)) diff --git a/pkgs/clan-cli/tests/test_machines_api.py b/pkgs/clan-cli/tests/test_machines_api.py index 1249949db..8a9fe0ed5 100644 --- a/pkgs/clan-cli/tests/test_machines_api.py +++ b/pkgs/clan-cli/tests/test_machines_api.py @@ -56,7 +56,7 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: # verify an invalid config (fileSystems missing) fails response = api.put( - "/api/machines/machine1/verify", + f"/api/{test_flake.name}/machines/machine1/verify", json=invalid_config, ) assert response.status_code == 200 @@ -67,13 +67,13 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: # set come invalid config (fileSystems missing) response = api.put( - "/api/machines/machine1/config", + f"/api/{test_flake.name}/machines/machine1/config", json=invalid_config, ) assert response.status_code == 200 # ensure the config has actually been updated - response = api.get("/api/machines/machine1/config") + response = api.get(f"/api/{test_flake.name}/machines/machine1/config") assert response.status_code == 200 assert response.json() == {"config": invalid_config} @@ -91,13 +91,8 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: devices=["/dev/fake_disk"], ), ), - - json=dict( - clan=dict( - jitsi=True, - ) ), - )) + ) # set some valid config config2 = dict( @@ -108,8 +103,9 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: ), **fs_config, ) + response = api.put( - "/api/machines/machine1/config", + f"/api/{test_flake.name}/machines/machine1/config", json=config2, ) assert response.status_code == 200 @@ -124,20 +120,21 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: # For example, this should not result in the boot.loader.grub.devices being # set twice (eg. merged) response = api.put( - "/api/machines/machine1/config", + f"/api/{test_flake.name}/machines/machine1/config", json=config2, ) assert response.status_code == 200 assert response.json() == {"config": config2} # verify the machine config evaluates - response = api.get("/api/machines/machine1/verify") + response = api.get(f"/api/{test_flake.name}/machines/machine1/verify") assert response.status_code == 200 + assert response.json() == {"success": True, "error": None} # get the schema with an extra module imported response = api.put( - "/api/machines/machine1/schema", + f"/api/{test_flake.name}/machines/machine1/schema", json={"clanImports": ["fake-module"]}, ) # expect the result schema to contain the fake-module.fake-flag option @@ -162,7 +159,7 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: # set the fake-module.fake-flag option to true response = api.put( - "/api/machines/machine1/config", + f"/api/{test_flake.name}/machines/machine1/config", json=config_with_imports, ) assert response.status_code == 200 @@ -184,7 +181,7 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None: **fs_config, ) response = api.put( - "/api/machines/machine1/config", + f"/api/{test_flake.name}/machines/machine1/config", json=config_with_empty_imports, ) assert response.status_code == 200