Updated to main branch. Removed cluttering asyncio and httpx log messages

pkgs/clan-cli/clan_cli/__init__.py

@@ -5,7 +5,7 @@ from types import ModuleType
 from typing import Optional
 
 from . import config, flakes, join, machines, secrets, vms, webui
-from .custom_logger import register
+from .custom_logger import setup_logging
 from .ssh import cli as ssh_cli
 
 log = logging.getLogger(__name__)
@@ -57,7 +57,7 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser:
     vms.register_parser(parser_vms)
 
     #    if args.debug:
-    register(logging.DEBUG)
+    setup_logging(logging.DEBUG)
     log.debug("Debug log activated")
 
     if argcomplete:

pkgs/clan-cli/clan_cli/async_cmd.py

@@ -4,6 +4,7 @@ import shlex
 from pathlib import Path
 from typing import Any, Callable, Coroutine, Dict, NamedTuple, Optional
 
+from .custom_logger import get_caller
 from .errors import ClanError
 
 log = logging.getLogger(__name__)
@@ -16,7 +17,6 @@ class CmdOut(NamedTuple):
 
 
 async def run(cmd: list[str], cwd: Optional[Path] = None) -> CmdOut:
-    log.debug(f"$: {shlex.join(cmd)}")
     cwd_res = None
     if cwd is not None:
         if not cwd.exists():
@@ -24,7 +24,9 @@ async def run(cmd: list[str], cwd: Optional[Path] = None) -> CmdOut:
         if not cwd.is_dir():
             raise ClanError(f"Working directory {cwd} is not a directory")
         cwd_res = cwd.resolve()
-        log.debug(f"Working directory: {cwd_res}")
+    log.debug(
+        f"Command: {shlex.join(cmd)}\nWorking directory: {cwd_res}\nCaller : {get_caller()}"
+    )
     proc = await asyncio.create_subprocess_exec(
         *cmd,
         stdout=asyncio.subprocess.PIPE,

pkgs/clan-cli/clan_cli/clan_modules.py

@@ -1,20 +1,20 @@
 import json
 import subprocess
-from pathlib import Path
 from typing import Optional
 
-from clan_cli.dirs import get_clan_flake_toplevel
 from clan_cli.nix import nix_eval
 
+from .dirs import specific_flake_dir
+from .types import FlakeName
+
 
 def get_clan_module_names(
-    flake: Optional[Path] = None,
+    flake_name: FlakeName,
 ) -> tuple[list[str], Optional[str]]:
     """
     Get the list of clan modules from the clan-core flake input
     """
-    if flake is None:
+    flake = specific_flake_dir(flake_name)
-        flake = get_clan_flake_toplevel()
     proc = subprocess.run(
         nix_eval(
             [

pkgs/clan-cli/clan_cli/config/machine.py

@@ -3,6 +3,8 @@ import os
 import subprocess
 import sys
 from pathlib import Path
+from tempfile import NamedTemporaryFile
+from typing import Optional
 
 from fastapi import HTTPException
 
@@ -19,31 +21,35 @@ from ..types import FlakeName
 
 
 def verify_machine_config(
-    machine_name: str, config: Optional[dict] = None, flake: Optional[Path] = None
+    flake_name: FlakeName,
+    machine_name: str,
+    config: Optional[dict] = None,
+    flake: Optional[Path] = None,
 ) -> Optional[str]:
     """
     Verify that the machine evaluates successfully
     Returns a tuple of (success, error_message)
     """
     if config is None:
-        config = config_for_machine(machine_name)
+        config = config_for_machine(flake_name, machine_name)
-    if flake is None:
+    flake = specific_flake_dir(flake_name)
-        flake = get_clan_flake_toplevel()
+    with NamedTemporaryFile(mode="w", dir=flake) as clan_machine_settings_file:
-    with NamedTemporaryFile(mode="w") as clan_machine_settings_file:
         json.dump(config, clan_machine_settings_file, indent=2)
         clan_machine_settings_file.seek(0)
         env = os.environ.copy()
         env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name
+        cmd = nix_eval(
+            flags=[
+                "--impure",
+                "--show-trace",
+                "--show-trace",
+                "--impure",  # needed to access CLAN_MACHINE_SETTINGS_FILE
+                f".#nixosConfigurations.{machine_name}.config.system.build.toplevel.outPath",
+            ],
+        )
+        # repro_env_break(work_dir=flake, env=env, cmd=cmd)
         proc = subprocess.run(
-            nix_eval(
+            cmd,
-                flags=[
-                    "--impure",
-                    "--show-trace",
-                    "--show-trace",
-                    "--impure",  # needed to access CLAN_MACHINE_SETTINGS_FILE
-                    f".#nixosConfigurations.{machine_name}.config.system.build.toplevel.outPath",
-                ],
-            ),
             capture_output=True,
             text=True,
             cwd=flake,
@@ -54,7 +60,6 @@ def verify_machine_config(
     return None
 
 
-
 def config_for_machine(flake_name: FlakeName, machine_name: str) -> dict:
     # read the config from a json file located at {flake}/machines/{machine_name}/settings.json
     if not specific_machine_dir(flake_name, machine_name).exists():
@@ -88,76 +93,49 @@ def set_config_for_machine(
         commit_file(settings_path, repo_dir)
 
 
-def schema_for_machine(flake_name: FlakeName, machine_name: str) -> dict:
+def schema_for_machine(
+    flake_name: FlakeName, machine_name: str, config: Optional[dict] = None
+) -> dict:
     flake = specific_flake_dir(flake_name)
-
+    # use nix eval to lib.evalModules .#nixosConfigurations.<machine_name>.options.clan
-    # use nix eval to lib.evalModules .#nixosModules.machine-{machine_name}
+    with NamedTemporaryFile(mode="w", dir=flake) as clan_machine_settings_file:
-    proc = subprocess.run(
+        env = os.environ.copy()
-        nix_eval(
+        inject_config_flags = []
-            flags=[
+        if config is not None:
-                "--impure",
+            json.dump(config, clan_machine_settings_file, indent=2)
-                "--show-trace",
+            clan_machine_settings_file.seek(0)
-                "--expr",
+            env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name
-                f"""
+            inject_config_flags = [
-                let
+                "--impure",  # needed to access CLAN_MACHINE_SETTINGS_FILE
-                    flake = builtins.getFlake (toString {flake});
+            ]
-                    lib = import {nixpkgs_source()}/lib;
+        proc = subprocess.run(
-                    options = flake.nixosConfigurations.{machine_name}.options;
+            nix_eval(
-                    clanOptions = options.clan;
+                flags=inject_config_flags
-                    jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }};
+                + [
-                    jsonschema = jsonschemaLib.parseOptions clanOptions;
+                    "--impure",
-                in
+                    "--show-trace",
-                    jsonschema
+                    "--expr",
-                """,
+                    f"""
-            ],
+                    let
-        ),
+                        flake = builtins.getFlake (toString {flake});
-        capture_output=True,
+                        lib = import {nixpkgs_source()}/lib;
-        text=True,
+                        options = flake.nixosConfigurations.{machine_name}.options;
-    )
+                        clanOptions = options.clan;
-# def schema_for_machine(
+                        jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }};
-#     machine_name: str, config: Optional[dict] = None, flake: Optional[Path] = None
+                        jsonschema = jsonschemaLib.parseOptions clanOptions;
-# ) -> dict:
+                    in
-#     if flake is None:
+                        jsonschema
-#         flake = get_clan_flake_toplevel()
+                    """,
-#     # use nix eval to lib.evalModules .#nixosConfigurations.<machine_name>.options.clan
+                ],
-#     with NamedTemporaryFile(mode="w") as clan_machine_settings_file:
+            ),
-#         env = os.environ.copy()
+            capture_output=True,
-#         inject_config_flags = []
+            text=True,
-#         if config is not None:
+            cwd=flake,
-#             json.dump(config, clan_machine_settings_file, indent=2)
+            env=env,
-#             clan_machine_settings_file.seek(0)
+        )
-#             env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name
+    if proc.returncode != 0:
-#             inject_config_flags = [
+        print(proc.stderr, file=sys.stderr)
-#                 "--impure",  # needed to access CLAN_MACHINE_SETTINGS_FILE
+        raise Exception(
-#             ]
+            f"Failed to read schema for machine {machine_name}:\n{proc.stderr}"
-#         proc = subprocess.run(
+        )
-#             nix_eval(
+    return json.loads(proc.stdout)
-#                 flags=inject_config_flags
-#                 + [
-#                     "--impure",
-#                     "--show-trace",
-#                     "--expr",
-#                     f"""
-#                     let
-#                         flake = builtins.getFlake (toString {flake});
-#                         lib = import {nixpkgs_source()}/lib;
-#                         options = flake.nixosConfigurations.{machine_name}.options;
-#                         clanOptions = options.clan;
-#                         jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }};
-#                         jsonschema = jsonschemaLib.parseOptions clanOptions;
-#                     in
-#                         jsonschema
-#                     """,
-#                 ],
-#             ),
-#             capture_output=True,
-#             text=True,
-#             cwd=flake,
-#             env=env,
-#         )
-#     if proc.returncode != 0:
-#         print(proc.stderr, file=sys.stderr)
-#         raise Exception(
-#             f"Failed to read schema for machine {machine_name}:\n{proc.stderr}"
-#         )
-#     return json.loads(proc.stdout)

pkgs/clan-cli/clan_cli/custom_logger.py

@@ -61,10 +61,12 @@ def get_caller() -> str:
     return ret
 
 
-def register(level: Any) -> None:
+def setup_logging(level: Any) -> None:
     handler = logging.StreamHandler()
     handler.setLevel(level)
     handler.setFormatter(CustomFormatter())
     logger = logging.getLogger("registerHandler")
+    logging.getLogger("asyncio").setLevel(logging.INFO)
+    logging.getLogger("httpx").setLevel(level=logging.WARNING)
     logger.addHandler(handler)
     # logging.basicConfig(level=level, handlers=[handler])

pkgs/clan-cli/clan_cli/machines/create.py

@@ -32,7 +32,6 @@ async def create_machine(flake_name: FlakeName, machine_name: str) -> Dict[str,
         cwd=folder,
     )
     response["git commit"] = out
-
     return response
 
 

pkgs/clan-cli/clan_cli/webui/routers/clan_modules.py

@@ -4,8 +4,9 @@ import logging
 from fastapi import APIRouter, HTTPException
 
 from clan_cli.clan_modules import get_clan_module_names
+from clan_cli.types import FlakeName
 
-from ..schemas import (
+from ..api_outputs import (
     ClanModulesResponse,
 )
 
@@ -13,9 +14,9 @@ log = logging.getLogger(__name__)
 router = APIRouter()
 
 
-@router.get("/api/clan_modules")
+@router.get("/api/{flake_name}clan_modules")
-async def list_clan_modules() -> ClanModulesResponse:
+async def list_clan_modules(flake_name: FlakeName) -> ClanModulesResponse:
-    module_names, error = get_clan_module_names()
+    module_names, error = get_clan_module_names(flake_name)
     if error is not None:
         raise HTTPException(status_code=400, detail=error)
     return ClanModulesResponse(clan_modules=module_names)

pkgs/clan-cli/clan_cli/webui/routers/machines.py

@@ -41,8 +41,7 @@ async def list_machines(flake_name: FlakeName) -> MachinesResponse:
 async def create_machine(
     flake_name: FlakeName, machine: Annotated[MachineCreate, Body()]
 ) -> MachineResponse:
-    out = await _create_machine(flake_name, machine.name)
+    await _create_machine(flake_name, machine.name)
-    log.debug(out)
     return MachineResponse(machine=Machine(name=machine.name, status=Status.UNKNOWN))
 
 
@@ -72,26 +71,29 @@ async def get_machine_schema(flake_name: FlakeName, name: str) -> SchemaResponse
     return SchemaResponse(schema=schema)
 
 
-@router.put("/api/machines/{name}/schema")
+@router.put("/api/{flake_name}/machines/{name}/schema")
 async def set_machine_schema(
-    name: str, config: Annotated[dict, Body()]
+    flake_name: FlakeName, name: str, config: Annotated[dict, Body()]
 ) -> SchemaResponse:
-    schema = schema_for_machine(name, config)
+    schema = schema_for_machine(flake_name, name, config)
     return SchemaResponse(schema=schema)
 
 
-@router.get("/api/machines/{name}/verify")
+@router.get("/api/{flake_name}/machines/{name}/verify")
-async def get_verify_machine_config(name: str) -> VerifyMachineResponse:
+async def get_verify_machine_config(
-    error = verify_machine_config(name)
+    flake_name: FlakeName, name: str
+) -> VerifyMachineResponse:
+    error = verify_machine_config(flake_name, name)
     success = error is None
     return VerifyMachineResponse(success=success, error=error)
 
 
-@router.put("/api/machines/{name}/verify")
+@router.put("/api/{flake_name}/machines/{name}/verify")
 async def put_verify_machine_config(
+    flake_name: FlakeName,
     name: str,
     config: Annotated[dict, Body()],
 ) -> VerifyMachineResponse:
-    error = verify_machine_config(name, config)
+    error = verify_machine_config(flake_name, name, config)
     success = error is None
     return VerifyMachineResponse(success=success, error=error)

pkgs/clan-cli/pyproject.toml

@@ -20,7 +20,7 @@ clan_cli = [ "config/jsonschema/*", "webui/assets/**/*"]
 testpaths = "tests"
 faulthandler_timeout = 60
 log_level = "DEBUG"
-log_format = "%(levelname)s: %(message)s"
+log_format = "%(levelname)s: %(message)s\n       %(pathname)s:%(lineno)d::%(funcName)s"
 addopts = "--cov . --cov-report term --cov-report html:.reports/html --no-cov-on-fail --durations 5 --color=yes --new-first --maxfail=1" # Add --pdb for debugging
 norecursedirs = "tests/helpers"
 markers = [ "impure" ]

pkgs/clan-cli/tests/api.py

@@ -1,3 +1,5 @@
+import logging
+
 import pytest
 from fastapi.testclient import TestClient
 
@@ -7,4 +9,6 @@ from clan_cli.webui.app import app
 # TODO: Why stateful
 @pytest.fixture(scope="session")
 def api() -> TestClient:
+    logging.getLogger("httpx").setLevel(level=logging.WARNING)
+    logging.getLogger("asyncio").setLevel(logging.INFO)
     return TestClient(app)

pkgs/clan-cli/tests/temporary_dir.py

@@ -19,7 +19,6 @@ def temporary_home(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
         monkeypatch.chdir(str(path))
         yield path
     else:
-        log.debug("TEST_TEMPORARY_DIR not set, using TemporaryDirectory")
         with tempfile.TemporaryDirectory(prefix="pytest-") as dirpath:
             monkeypatch.setenv("HOME", str(dirpath))
             monkeypatch.chdir(str(dirpath))

pkgs/clan-cli/tests/test_machines_api.py

@@ -56,7 +56,7 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
 
     # verify an invalid config (fileSystems missing) fails
     response = api.put(
-        "/api/machines/machine1/verify",
+        f"/api/{test_flake.name}/machines/machine1/verify",
         json=invalid_config,
     )
     assert response.status_code == 200
@@ -67,13 +67,13 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
 
     # set come invalid config (fileSystems missing)
     response = api.put(
-        "/api/machines/machine1/config",
+        f"/api/{test_flake.name}/machines/machine1/config",
         json=invalid_config,
     )
     assert response.status_code == 200
 
     # ensure the config has actually been updated
-    response = api.get("/api/machines/machine1/config")
+    response = api.get(f"/api/{test_flake.name}/machines/machine1/config")
     assert response.status_code == 200
     assert response.json() == {"config": invalid_config}
 
@@ -91,13 +91,8 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
                     devices=["/dev/fake_disk"],
                 ),
             ),
-        
-        json=dict(
-            clan=dict(
-                jitsi=True,
-            )
         ),
-    ))
+    )
 
     # set some valid config
     config2 = dict(
@@ -108,8 +103,9 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
         ),
         **fs_config,
     )
+
     response = api.put(
-        "/api/machines/machine1/config",
+        f"/api/{test_flake.name}/machines/machine1/config",
         json=config2,
     )
     assert response.status_code == 200
@@ -124,20 +120,21 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
     # For example, this should not result in the boot.loader.grub.devices being
     #   set twice (eg. merged)
     response = api.put(
-        "/api/machines/machine1/config",
+        f"/api/{test_flake.name}/machines/machine1/config",
         json=config2,
     )
     assert response.status_code == 200
     assert response.json() == {"config": config2}
 
     # verify the machine config evaluates
-    response = api.get("/api/machines/machine1/verify")
+    response = api.get(f"/api/{test_flake.name}/machines/machine1/verify")
     assert response.status_code == 200
+
     assert response.json() == {"success": True, "error": None}
 
     # get the schema with an extra module imported
     response = api.put(
-        "/api/machines/machine1/schema",
+        f"/api/{test_flake.name}/machines/machine1/schema",
         json={"clanImports": ["fake-module"]},
     )
     # expect the result schema to contain the fake-module.fake-flag option
@@ -162,7 +159,7 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
 
     # set the fake-module.fake-flag option to true
     response = api.put(
-        "/api/machines/machine1/config",
+        f"/api/{test_flake.name}/machines/machine1/config",
         json=config_with_imports,
     )
     assert response.status_code == 200
@@ -184,7 +181,7 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
         **fs_config,
     )
     response = api.put(
-        "/api/machines/machine1/config",
+        f"/api/{test_flake.name}/machines/machine1/config",
         json=config_with_empty_imports,
     )
     assert response.status_code == 200