yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Updated to main branch. Removed cluttering asyncio and httpx log messages

Browse Source
This commit is contained in:
Qubasa
2023-10-27 23:36:45 +02:00
parent 97a20053e7
commit e389c7cfe7
12 changed files with 113 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkgs/clan-cli/clan_cli/__init__.py
View File

@@ -5,7 +5,7 @@ from types import ModuleType
from typing import Optional
from . import config, flakes, join, machines, secrets, vms, webui
from .custom_logger import register
from .custom_logger import setup_logging
from .ssh import cli as ssh_cli
log = logging.getLogger(__name__)
@@ -57,7 +57,7 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser:
vms.register_parser(parser_vms)
# if args.debug:
register(logging.DEBUG)
setup_logging(logging.DEBUG)
log.debug("Debug log activated")
if argcomplete:

6
pkgs/clan-cli/clan_cli/async_cmd.py
View File

@@ -4,6 +4,7 @@ import shlex
from pathlib import Path
from typing import Any, Callable, Coroutine, Dict, NamedTuple, Optional
from .custom_logger import get_caller
from .errors import ClanError
log = logging.getLogger(__name__)
@@ -16,7 +17,6 @@ class CmdOut(NamedTuple):
async def run(cmd: list[str], cwd: Optional[Path] = None) -> CmdOut:
log.debug(f"$: {shlex.join(cmd)}")
cwd_res = None
if cwd is not None:
if not cwd.exists():
@@ -24,7 +24,9 @@ async def run(cmd: list[str], cwd: Optional[Path] = None) -> CmdOut:
if not cwd.is_dir():
raise ClanError(f"Working directory {cwd} is not a directory")
cwd_res = cwd.resolve()
log.debug(f"Working directory: {cwd_res}")
log.debug(
f"Command: {shlex.join(cmd)}\nWorking directory: {cwd_res}\nCaller : {get_caller()}"
)
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,

10
pkgs/clan-cli/clan_cli/clan_modules.py
View File

@@ -1,20 +1,20 @@
import json
import subprocess
from pathlib import Path
from typing import Optional
from clan_cli.dirs import get_clan_flake_toplevel
from clan_cli.nix import nix_eval
from .dirs import specific_flake_dir
from .types import FlakeName
def get_clan_module_names(
flake: Optional[Path] = None,
flake_name: FlakeName,
) -> tuple[list[str], Optional[str]]:
"""
Get the list of clan modules from the clan-core flake input
"""
if flake is None:
flake = get_clan_flake_toplevel()
flake = specific_flake_dir(flake_name)
proc = subprocess.run(
nix_eval(
[

152
pkgs/clan-cli/clan_cli/config/machine.py
View File

@@ -3,6 +3,8 @@ import os
import subprocess
import sys
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Optional
from fastapi import HTTPException
@@ -19,31 +21,35 @@ from ..types import FlakeName
def verify_machine_config(
machine_name: str, config: Optional[dict] = None, flake: Optional[Path] = None
flake_name: FlakeName,
machine_name: str,
config: Optional[dict] = None,
flake: Optional[Path] = None,
) -> Optional[str]:
"""
Verify that the machine evaluates successfully
Returns a tuple of (success, error_message)
"""
if config is None:
config = config_for_machine(machine_name)
if flake is None:
flake = get_clan_flake_toplevel()
with NamedTemporaryFile(mode="w") as clan_machine_settings_file:
config = config_for_machine(flake_name, machine_name)
flake = specific_flake_dir(flake_name)
with NamedTemporaryFile(mode="w", dir=flake) as clan_machine_settings_file:
json.dump(config, clan_machine_settings_file, indent=2)
clan_machine_settings_file.seek(0)
env = os.environ.copy()
env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name
cmd = nix_eval(
flags=[
"--impure",
"--show-trace",
"--show-trace",
"--impure", # needed to access CLAN_MACHINE_SETTINGS_FILE
f".#nixosConfigurations.{machine_name}.config.system.build.toplevel.outPath",
],
)
# repro_env_break(work_dir=flake, env=env, cmd=cmd)
proc = subprocess.run(
nix_eval(
flags=[
"--impure",
"--show-trace",
"--show-trace",
"--impure", # needed to access CLAN_MACHINE_SETTINGS_FILE
f".#nixosConfigurations.{machine_name}.config.system.build.toplevel.outPath",
],
),
cmd,
capture_output=True,
text=True,
cwd=flake,
@@ -54,7 +60,6 @@ def verify_machine_config(
return None
def config_for_machine(flake_name: FlakeName, machine_name: str) -> dict:
# read the config from a json file located at {flake}/machines/{machine_name}/settings.json
if not specific_machine_dir(flake_name, machine_name).exists():
@@ -88,76 +93,49 @@ def set_config_for_machine(
commit_file(settings_path, repo_dir)
def schema_for_machine(flake_name: FlakeName, machine_name: str) -> dict:
def schema_for_machine(
flake_name: FlakeName, machine_name: str, config: Optional[dict] = None
) -> dict:
flake = specific_flake_dir(flake_name)
# use nix eval to lib.evalModules .#nixosModules.machine-{machine_name}
proc = subprocess.run(
nix_eval(
flags=[
"--impure",
"--show-trace",
"--expr",
f"""
let
flake = builtins.getFlake (toString {flake});
lib = import {nixpkgs_source()}/lib;
options = flake.nixosConfigurations.{machine_name}.options;
clanOptions = options.clan;
jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }};
jsonschema = jsonschemaLib.parseOptions clanOptions;
in
jsonschema
""",
],
),
capture_output=True,
text=True,
)
# def schema_for_machine(
# machine_name: str, config: Optional[dict] = None, flake: Optional[Path] = None
# ) -> dict:
# if flake is None:
# flake = get_clan_flake_toplevel()
# # use nix eval to lib.evalModules .#nixosConfigurations.<machine_name>.options.clan
# with NamedTemporaryFile(mode="w") as clan_machine_settings_file:
# env = os.environ.copy()
# inject_config_flags = []
# if config is not None:
# json.dump(config, clan_machine_settings_file, indent=2)
# clan_machine_settings_file.seek(0)
# env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name
# inject_config_flags = [
# "--impure", # needed to access CLAN_MACHINE_SETTINGS_FILE
# ]
# proc = subprocess.run(
# nix_eval(
# flags=inject_config_flags
# + [
# "--impure",
# "--show-trace",
# "--expr",
# f"""
# let
# flake = builtins.getFlake (toString {flake});
# lib = import {nixpkgs_source()}/lib;
# options = flake.nixosConfigurations.{machine_name}.options;
# clanOptions = options.clan;
# jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }};
# jsonschema = jsonschemaLib.parseOptions clanOptions;
# in
# jsonschema
# """,
# ],
# ),
# capture_output=True,
# text=True,
# cwd=flake,
# env=env,
# )
# if proc.returncode != 0:
# print(proc.stderr, file=sys.stderr)
# raise Exception(
# f"Failed to read schema for machine {machine_name}:\n{proc.stderr}"
# )
# return json.loads(proc.stdout)
# use nix eval to lib.evalModules .#nixosConfigurations.<machine_name>.options.clan
with NamedTemporaryFile(mode="w", dir=flake) as clan_machine_settings_file:
env = os.environ.copy()
inject_config_flags = []
if config is not None:
json.dump(config, clan_machine_settings_file, indent=2)
clan_machine_settings_file.seek(0)
env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name
inject_config_flags = [
"--impure", # needed to access CLAN_MACHINE_SETTINGS_FILE
]
proc = subprocess.run(
nix_eval(
flags=inject_config_flags
+ [
"--impure",
"--show-trace",
"--expr",
f"""
let
flake = builtins.getFlake (toString {flake});
lib = import {nixpkgs_source()}/lib;
options = flake.nixosConfigurations.{machine_name}.options;
clanOptions = options.clan;
jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }};
jsonschema = jsonschemaLib.parseOptions clanOptions;
in
jsonschema
""",
],
),
capture_output=True,
text=True,
cwd=flake,
env=env,
)
if proc.returncode != 0:
print(proc.stderr, file=sys.stderr)
raise Exception(
f"Failed to read schema for machine {machine_name}:\n{proc.stderr}"
)
return json.loads(proc.stdout)

4
pkgs/clan-cli/clan_cli/custom_logger.py
View File

@@ -61,10 +61,12 @@ def get_caller() -> str:
return ret
def register(level: Any) -> None:
def setup_logging(level: Any) -> None:
handler = logging.StreamHandler()
handler.setLevel(level)
handler.setFormatter(CustomFormatter())
logger = logging.getLogger("registerHandler")
logging.getLogger("asyncio").setLevel(logging.INFO)
logging.getLogger("httpx").setLevel(level=logging.WARNING)
logger.addHandler(handler)
# logging.basicConfig(level=level, handlers=[handler])

1
pkgs/clan-cli/clan_cli/machines/create.py
View File

@@ -32,7 +32,6 @@ async def create_machine(flake_name: FlakeName, machine_name: str) -> Dict[str,
cwd=folder,
)
response["git commit"] = out
return response

9
pkgs/clan-cli/clan_cli/webui/routers/clan_modules.py
View File

@@ -4,8 +4,9 @@ import logging
from fastapi import APIRouter, HTTPException
from clan_cli.clan_modules import get_clan_module_names
from clan_cli.types import FlakeName
from ..schemas import (
from ..api_outputs import (
ClanModulesResponse,
)
@@ -13,9 +14,9 @@ log = logging.getLogger(__name__)
router = APIRouter()
@router.get("/api/clan_modules")
async def list_clan_modules() -> ClanModulesResponse:
module_names, error = get_clan_module_names()
@router.get("/api/{flake_name}clan_modules")
async def list_clan_modules(flake_name: FlakeName) -> ClanModulesResponse:
module_names, error = get_clan_module_names(flake_name)
if error is not None:
raise HTTPException(status_code=400, detail=error)
return ClanModulesResponse(clan_modules=module_names)

22
pkgs/clan-cli/clan_cli/webui/routers/machines.py
View File

@@ -41,8 +41,7 @@ async def list_machines(flake_name: FlakeName) -> MachinesResponse:
async def create_machine(
flake_name: FlakeName, machine: Annotated[MachineCreate, Body()]
) -> MachineResponse:
out = await _create_machine(flake_name, machine.name)
log.debug(out)
await _create_machine(flake_name, machine.name)
return MachineResponse(machine=Machine(name=machine.name, status=Status.UNKNOWN))
@@ -72,26 +71,29 @@ async def get_machine_schema(flake_name: FlakeName, name: str) -> SchemaResponse
return SchemaResponse(schema=schema)
@router.put("/api/machines/{name}/schema")
@router.put("/api/{flake_name}/machines/{name}/schema")
async def set_machine_schema(
name: str, config: Annotated[dict, Body()]
flake_name: FlakeName, name: str, config: Annotated[dict, Body()]
) -> SchemaResponse:
schema = schema_for_machine(name, config)
schema = schema_for_machine(flake_name, name, config)
return SchemaResponse(schema=schema)
@router.get("/api/machines/{name}/verify")
async def get_verify_machine_config(name: str) -> VerifyMachineResponse:
error = verify_machine_config(name)
@router.get("/api/{flake_name}/machines/{name}/verify")
async def get_verify_machine_config(
flake_name: FlakeName, name: str
) -> VerifyMachineResponse:
error = verify_machine_config(flake_name, name)
success = error is None
return VerifyMachineResponse(success=success, error=error)
@router.put("/api/machines/{name}/verify")
@router.put("/api/{flake_name}/machines/{name}/verify")
async def put_verify_machine_config(
flake_name: FlakeName,
name: str,
config: Annotated[dict, Body()],
) -> VerifyMachineResponse:
error = verify_machine_config(name, config)
error = verify_machine_config(flake_name, name, config)
success = error is None
return VerifyMachineResponse(success=success, error=error)

2
pkgs/clan-cli/pyproject.toml
View File

@@ -20,7 +20,7 @@ clan_cli = [ "config/jsonschema/*", "webui/assets/**/*"]
testpaths = "tests"
faulthandler_timeout = 60
log_level = "DEBUG"
log_format = "%(levelname)s: %(message)s"
log_format = "%(levelname)s: %(message)s\n %(pathname)s:%(lineno)d::%(funcName)s"
addopts = "--cov . --cov-report term --cov-report html:.reports/html --no-cov-on-fail --durations 5 --color=yes --new-first --maxfail=1" # Add --pdb for debugging
norecursedirs = "tests/helpers"
markers = [ "impure" ]

4
pkgs/clan-cli/tests/api.py
View File

@@ -1,3 +1,5 @@
import logging
import pytest
from fastapi.testclient import TestClient
@@ -7,4 +9,6 @@ from clan_cli.webui.app import app
# TODO: Why stateful
@pytest.fixture(scope="session")
def api() -> TestClient:
logging.getLogger("httpx").setLevel(level=logging.WARNING)
logging.getLogger("asyncio").setLevel(logging.INFO)
return TestClient(app)

1
pkgs/clan-cli/tests/temporary_dir.py
View File

@@ -19,7 +19,6 @@ def temporary_home(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
monkeypatch.chdir(str(path))
yield path
else:
log.debug("TEST_TEMPORARY_DIR not set, using TemporaryDirectory")
with tempfile.TemporaryDirectory(prefix="pytest-") as dirpath:
monkeypatch.setenv("HOME", str(dirpath))
monkeypatch.chdir(str(dirpath))

27
pkgs/clan-cli/tests/test_machines_api.py
View File

@@ -56,7 +56,7 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
# verify an invalid config (fileSystems missing) fails
response = api.put(
"/api/machines/machine1/verify",
f"/api/{test_flake.name}/machines/machine1/verify",
json=invalid_config,
)
assert response.status_code == 200
@@ -67,13 +67,13 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
# set come invalid config (fileSystems missing)
response = api.put(
"/api/machines/machine1/config",
f"/api/{test_flake.name}/machines/machine1/config",
json=invalid_config,
)
assert response.status_code == 200
# ensure the config has actually been updated
response = api.get("/api/machines/machine1/config")
response = api.get(f"/api/{test_flake.name}/machines/machine1/config")
assert response.status_code == 200
assert response.json() == {"config": invalid_config}
@@ -91,13 +91,8 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
devices=["/dev/fake_disk"],
),
),
json=dict(
clan=dict(
jitsi=True,
)
),
))
)
# set some valid config
config2 = dict(
@@ -108,8 +103,9 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
),
**fs_config,
)
response = api.put(
"/api/machines/machine1/config",
f"/api/{test_flake.name}/machines/machine1/config",
json=config2,
)
assert response.status_code == 200
@@ -124,20 +120,21 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
# For example, this should not result in the boot.loader.grub.devices being
# set twice (eg. merged)
response = api.put(
"/api/machines/machine1/config",
f"/api/{test_flake.name}/machines/machine1/config",
json=config2,
)
assert response.status_code == 200
assert response.json() == {"config": config2}
# verify the machine config evaluates
response = api.get("/api/machines/machine1/verify")
response = api.get(f"/api/{test_flake.name}/machines/machine1/verify")
assert response.status_code == 200
assert response.json() == {"success": True, "error": None}
# get the schema with an extra module imported
response = api.put(
"/api/machines/machine1/schema",
f"/api/{test_flake.name}/machines/machine1/schema",
json={"clanImports": ["fake-module"]},
)
# expect the result schema to contain the fake-module.fake-flag option
@@ -162,7 +159,7 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
# set the fake-module.fake-flag option to true
response = api.put(
"/api/machines/machine1/config",
f"/api/{test_flake.name}/machines/machine1/config",
json=config_with_imports,
)
assert response.status_code == 200
@@ -184,7 +181,7 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
**fs_config,
)
response = api.put(
"/api/machines/machine1/config",
f"/api/{test_flake.name}/machines/machine1/config",
json=config_with_empty_imports,
)
assert response.status_code == 200