Updated to main

pkgs/clan-cli/.vscode/launch.json

@@ -12,6 +12,15 @@
             "justMyCode": false,
             "args": [ "--reload", "--no-open", "--log-level", "debug" ],
 
+        },
+        {
+            "name": "Clan Cli VMs",
+            "type": "python",
+            "request": "launch",
+            "module": "clan_cli",
+            "justMyCode": false,
+            "args": [ "vms" ],
+
         }
     ]
 }

pkgs/clan-cli/clan_cli/__init__.py

@@ -1,12 +1,15 @@
 import argparse
+import logging
 import sys
 from types import ModuleType
 from typing import Optional
 
-from . import config, create, machines, secrets, vms, webui
+from . import config, create, custom_logger, machines, secrets, vms, webui
 from .errors import ClanError
 from .ssh import cli as ssh_cli
 
+log = logging.getLogger(__name__)
+
 argcomplete: Optional[ModuleType] = None
 try:
     import argcomplete  # type: ignore[no-redef]
@@ -62,14 +65,20 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser:
 def main() -> None:
     parser = create_parser()
     args = parser.parse_args()
+
+    if args.debug:
+        custom_logger.register(logging.DEBUG)
+        log.debug("Debug logging enabled")
+    else:
+        custom_logger.register(logging.INFO)
+
     if not hasattr(args, "func"):
+        log.error("No argparse function registered")
         return
     try:
         args.func(args)
     except ClanError as e:
-        if args.debug:
+        log.exception(e)
-            raise
-        print(f"{sys.argv[0]}: {e}")
         sys.exit(1)
 
 

pkgs/clan-cli/clan_cli/__main__.py

@@ -0,0 +1,4 @@
+from . import main
+
+if __name__ == "__main__":
+    main()

pkgs/clan-cli/clan_cli/machines/list.py

@@ -1,12 +1,16 @@
 import argparse
+import logging
 import os
 
 from .folders import machines_folder
 from .types import validate_hostname
 
+log = logging.getLogger(__name__)
+
 
 def list_machines() -> list[str]:
     path = machines_folder()
+    log.debug(f"Listing machines in {path}")
     if not path.exists():
         return []
     objs: list[str] = []

pkgs/clan-cli/clan_cli/vms/create.py

@@ -1,99 +1,23 @@
 import argparse
-import json
+import asyncio
-import subprocess
-import tempfile
-from pathlib import Path
 
 from ..dirs import get_clan_flake_toplevel
-from ..nix import nix_build, nix_shell
+from ..webui.routers import vms
-
+from ..webui.schemas import VmConfig
-
-def get_vm_create_info(machine: str) -> dict:
-    clan_dir = get_clan_flake_toplevel().as_posix()
-
-    # config = nix_config()
-    # system = config["system"]
-
-    vm_json = subprocess.run(
-        nix_build(
-            [
-                # f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.clan.virtualisation.createJSON' # TODO use this
-                f'{clan_dir}#nixosConfigurations."{machine}".config.system.clan.vm.create'
-            ]
-        ),
-        stdout=subprocess.PIPE,
-        check=True,
-        text=True,
-    ).stdout.strip()
-    with open(vm_json) as f:
-        return json.load(f)
 
 
 def create(args: argparse.Namespace) -> None:
-    print(f"Creating VM for {args.machine}")
+    clan_dir = get_clan_flake_toplevel().as_posix()
-    machine = args.machine
+    vm = VmConfig(
-    vm_config = get_vm_create_info(machine)
+        flake_url=clan_dir,
-    with tempfile.TemporaryDirectory() as tmpdir_:
+        flake_attr=args.machine,
-        xchg_dir = Path(tmpdir_) / "xchg"
+        cores=0,
-        xchg_dir.mkdir()
+        graphics=False,
-        disk_img = f"{tmpdir_}/disk.img"
+        memory_size=0,
-        subprocess.run(
-            nix_shell(
-                ["qemu"],
-                [
-                    "qemu-img",
-                    "create",
-                    "-f",
-                    "raw",
-                    disk_img,
-                    "1024M",
-                ],
-            ),
-            stdout=subprocess.PIPE,
-            check=True,
-            text=True,
-        )
-        subprocess.run(
-            [
-                "mkfs.ext4",
-                "-L",
-                "nixos",
-                disk_img,
-            ],
-            stdout=subprocess.PIPE,
-            check=True,
-            text=True,
     )
 
-        subprocess.run(
+    res = asyncio.run(vms.create_vm(vm))
-            nix_shell(
+    print(res.json())
-                ["qemu"],
-                [
-                    # fmt: off
-                    "qemu-kvm",
-                    "-name", machine,
-                    "-m", f'{vm_config["memorySize"]}M',
-                    "-smp", str(vm_config["cores"]),
-                    "-device", "virtio-rng-pci",
-                    "-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0",
-                    "-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store",
-                    "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
-                    "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
-                    "-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report',
-                    "-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
-                    "-device", "virtio-keyboard",
-                    "-usb",
-                    "-device", "usb-tablet,bus=usb-bus.0",
-                    "-kernel", f'{vm_config["toplevel"]}/kernel',
-                    "-initrd", vm_config["initrd"],
-                    "-append", f'{(Path(vm_config["toplevel"]) / "kernel-params").read_text()} init={vm_config["toplevel"]}/init regInfo={vm_config["regInfo"]}/registration console=ttyS0,115200n8 console=tty0',
-                    # fmt: on
-                ],
-            ),
-            stdout=subprocess.PIPE,
-            check=True,
-            text=True,
-        )
 
 
 def register_create_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/vms/inspect.py

@@ -1,36 +1,14 @@
 import argparse
-import json
+import asyncio
-import subprocess
 
 from ..dirs import get_clan_flake_toplevel
-from ..nix import nix_eval
+from ..webui.routers import vms
-
-
-def get_vm_inspect_info(machine: str) -> dict:
-    clan_dir = get_clan_flake_toplevel().as_posix()
-
-    # config = nix_config()
-    # system = config["system"]
-
-    return json.loads(
-        subprocess.run(
-            nix_eval(
-                [
-                    # f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.clan.virtualisation' # TODO use this
-                    f'{clan_dir}#nixosConfigurations."{machine}".config.system.clan.vm.config'
-                ]
-            ),
-            stdout=subprocess.PIPE,
-            check=True,
-            text=True,
-        ).stdout
-    )
 
 
 def inspect(args: argparse.Namespace) -> None:
-    print(f"Creating VM for {args.machine}")
+    clan_dir = get_clan_flake_toplevel().as_posix()
-    machine = args.machine
+    res = asyncio.run(vms.inspect_vm(flake_url=clan_dir, flake_attr=args.machine))
-    print(get_vm_inspect_info(machine))
+    print(res.json())
 
 
 def register_inspect_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/webui/__init__.py

@@ -45,6 +45,8 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
         help="Log level",
         choices=["critical", "error", "warning", "info", "debug", "trace"],
     )
+
+    # Set the args.func variable in args
     if start_server is None:
         parser.set_defaults(func=fastapi_is_not_installed)
     else:

pkgs/clan-cli/clan_cli/webui/__main__.py

@@ -5,6 +5,11 @@ from . import register_parser
 if __name__ == "__main__":
     # this is use in our integration test
     parser = argparse.ArgumentParser()
+    # call the register_parser function, which adds arguments to the parser
     register_parser(parser)
     args = parser.parse_args()
+
+    # call the function that is stored
+    # in the func attribute of args, and pass args as the argument
+    # look into register_parser to see how this is done
     args.func(args)

pkgs/clan-cli/clan_cli/webui/app.py

@@ -5,7 +5,6 @@ from fastapi.middleware.cors import CORSMiddleware
 from fastapi.routing import APIRoute
 from fastapi.staticfiles import StaticFiles
 
-from .. import custom_logger
 from .assets import asset_path
 from .routers import flake, health, machines, root, utils, vms
 
@@ -43,15 +42,11 @@ def setup_app() -> FastAPI:
         if isinstance(route, APIRoute):
             route.operation_id = route.name  # in this case, 'read_items'
         log.debug(f"Registered route: {route}")
+
+    for i in app.exception_handlers.items():
+        log.debug(f"Registered exception handler: {i}")
+
     return app
 
 
-# TODO: How do I get the log level from the command line in here?
-custom_logger.register(logging.DEBUG)
 app = setup_app()
-
-for i in app.exception_handlers.items():
-    log.info(f"Registered exception handler: {i}")
-
-log.warning("log warn")
-log.debug("log debug")

pkgs/clan-cli/clan_cli/webui/routers/vms.py

@@ -1,12 +1,16 @@
 import json
 import logging
+import tempfile
+from pathlib import Path
 from typing import Annotated, Iterator
 from uuid import UUID
 
+from fastapi import APIRouter, Body
 from fastapi import APIRouter, BackgroundTasks, Body, status
 from fastapi.exceptions import HTTPException
 from fastapi.responses import StreamingResponse
 
+from ...nix import nix_build, nix_eval, nix_shell
 from clan_cli.webui.routers.flake import get_attrs
 
 from ...nix import nix_build, nix_eval
@@ -39,23 +43,94 @@ class BuildVmTask(BaseTask):
         super().__init__(uuid)
         self.vm = vm
 
-    def run(self) -> None:
+    def get_vm_create_info(self) -> dict:
-        try:
+        clan_dir = self.vm.flake_url
-            self.log.debug(f"BuildVM with uuid {self.uuid} started")
+        machine = self.vm.flake_attr
-            cmd = nix_build_vm_cmd(self.vm.flake_attr, flake_url=self.vm.flake_url)
+        cmd_state = self.run_cmd(
+            nix_build(
+                [
+                    # f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.clan.virtualisation.createJSON' # TODO use this
+                    f'{clan_dir}#nixosConfigurations."{machine}".config.system.clan.vm.create'
+                ]
+            )
+        )
+        vm_json = "".join(cmd_state.stdout)
+        self.log.debug(f"VM JSON path: {vm_json}")
+        with open(vm_json) as f:
+            return json.load(f)
 
-            proc = self.run_cmd(cmd)
+    def task_run(self) -> None:
-            self.log.debug(f"stdout: {proc.stdout}")
+        machine = self.vm.flake_attr
+        self.log.debug(f"Creating VM for {machine}")
+        vm_config = self.get_vm_create_info()
+        with tempfile.TemporaryDirectory() as tmpdir_:
+            xchg_dir = Path(tmpdir_) / "xchg"
+            xchg_dir.mkdir()
+            disk_img = f"{tmpdir_}/disk.img"
+            cmd = nix_shell(
+                ["qemu"],
+                [
+                    "qemu" "qemu-img",
+                    "create",
+                    "-f",
+                    "raw",
+                    disk_img,
+                    "1024M",
+                ],
+            )
+            self.run_cmd(cmd)
 
-            vm_path = f"{''.join(proc.stdout[0])}/bin/run-nixos-vm"
+            cmd = [
-            self.log.debug(f"vm_path: {vm_path}")
+                "mkfs.ext4",
+                "-L",
+                "nixos",
+                disk_img,
+            ]
+            self.run_cmd(cmd)
 
-            self.run_cmd([vm_path])
+            cmd = nix_shell(
-            self.finished = True
+                ["qemu"],
-        except Exception as e:
+                [
-            self.failed = True
+                    # fmt: off
-            self.finished = True
+                        "qemu-kvm",
-            log.exception(e)
+                        "-name", machine,
+                        "-m", f'{vm_config["memorySize"]}M',
+                        "-smp", str(vm_config["cores"]),
+                        "-device", "virtio-rng-pci",
+                        "-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0",
+                        "-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store",
+                        "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
+                        "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
+                        "-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report',
+                        "-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
+                        "-device", "virtio-keyboard",
+                        "-usb",
+                        "-device", "usb-tablet,bus=usb-bus.0",
+                        "-kernel", f'{vm_config["toplevel"]}/kernel',
+                        "-initrd", vm_config["initrd"],
+                        "-append", f'{(Path(vm_config["toplevel"]) / "kernel-params").read_text()} init={vm_config["toplevel"]}/init regInfo={vm_config["regInfo"]}/registration console=ttyS0,115200n8 console=tty0',
+                    # fmt: on
+                ],
+            )
+            self.run_cmd(cmd)
+
+    # def run(self) -> None:
+    #     try:
+    #         self.log.debug(f"BuildVM with uuid {self.uuid} started")
+    #         cmd = nix_build_vm_cmd(self.vm.flake_attr, flake_url=self.vm.flake_url)
+
+    #         proc = self.run_cmd(cmd)
+    #         self.log.debug(f"stdout: {proc.stdout}")
+
+    #         vm_path = f"{''.join(proc.stdout[0])}/bin/run-nixos-vm"
+    #         self.log.debug(f"vm_path: {vm_path}")
+
+    #         self.run_cmd([vm_path])
+    #         self.finished = True
+    # except Exception as e:
+    #     self.failed = True
+    #     self.finished = True
+    #     log.exception(e)
 
 
 @router.post("/api/vms/inspect")

pkgs/clan-cli/clan_cli/webui/server.py

@@ -1,6 +1,11 @@
 import argparse
 import logging
+import multiprocessing as mp
+import os
+import socket
 import subprocess
+import sys
+import syslog
 import time
 import urllib.request
 import webbrowser
@@ -90,3 +95,98 @@ def start_server(args: argparse.Namespace) -> None:
             access_log=args.log_level == "debug",
             headers=headers,
         )
+
+
+# Define a function that takes the path of the file socket as input and returns True if it is served, False otherwise
+def is_served(file_socket: Path) -> bool:
+    # Create a Unix stream socket
+    client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    # Try to connect to the file socket
+    try:
+        client.connect(str(file_socket))
+        # Connection succeeded, return True
+        return True
+    except OSError:
+        # Connection failed, return False
+        return False
+    finally:
+        # Close the client socket
+        client.close()
+
+
+def set_out_to_syslog() -> None:  # type: ignore
+    # Define some constants for convenience
+    log_levels = {
+        "emerg": syslog.LOG_EMERG,
+        "alert": syslog.LOG_ALERT,
+        "crit": syslog.LOG_CRIT,
+        "err": syslog.LOG_ERR,
+        "warning": syslog.LOG_WARNING,
+        "notice": syslog.LOG_NOTICE,
+        "info": syslog.LOG_INFO,
+        "debug": syslog.LOG_DEBUG,
+    }
+    facility = syslog.LOG_USER  # Use user facility for custom applications
+
+    # Open a connection to the system logger
+    syslog.openlog("clan-cli", 0, facility)  # Use "myapp" as the prefix for messages
+
+    # Define a custom write function that sends messages to syslog
+    def write(message: str) -> int:
+        # Strip the newline character from the message
+        message = message.rstrip("\n")
+        # Check if the message is not empty
+        if message:
+            # Send the message to syslog with the appropriate level
+            if message.startswith("ERROR:"):
+                # Use error level for messages that start with "ERROR:"
+                syslog.syslog(log_levels["err"], message)
+            else:
+                # Use info level for other messages
+                syslog.syslog(log_levels["info"], message)
+        return 0
+
+    # Assign the custom write function to sys.stdout and sys.stderr
+    setattr(sys.stdout, "write", write)
+    setattr(sys.stderr, "write", write)
+
+    # Define a dummy flush function to prevent errors
+    def flush() -> None:
+        pass
+
+    # Assign the dummy flush function to sys.stdout and sys.stderr
+    setattr(sys.stdout, "flush", flush)
+    setattr(sys.stderr, "flush", flush)
+
+
+def _run_socketfile(socket_file: Path, debug: bool) -> None:
+    set_out_to_syslog()
+    run(
+        "clan_cli.webui.app:app",
+        uds=str(socket_file),
+        access_log=debug,
+        reload=False,
+        log_level="debug" if debug else "info",
+    )
+
+
+@contextmanager
+def api_server(debug: bool) -> Iterator[Path]:
+    runtime_dir = os.getenv("XDG_RUNTIME_DIR")
+    if runtime_dir is None:
+        raise RuntimeError("XDG_RUNTIME_DIR not set")
+    socket_path = Path(runtime_dir) / "clan.sock"
+    socket_path = socket_path.resolve()
+
+    log.debug("Socketfile lies at %s", socket_path)
+
+    if not is_served(socket_path):
+        log.debug("Starting api server...")
+        mp.set_start_method(method="spawn")
+        proc = mp.Process(target=_run_socketfile, args=(socket_path, debug))
+        proc.start()
+    else:
+        log.info("Api server is already running on %s", socket_path)
+
+    yield socket_path
+    proc.terminate()

pkgs/clan-cli/clan_cli/webui/task_manager.py

@@ -33,8 +33,17 @@ class BaseTask(threading.Thread):
         self.finished: bool = False
 
     def run(self) -> None:
+        try:
+            self.task_run()
+        except Exception as e:
+            self.failed = True
+            self.log.exception(e)
+        finally:
             self.finished = True
 
+    def task_run(self) -> None:
+        raise NotImplementedError
+
     def run_cmd(self, cmd: list[str]) -> CmdState:
         cwd = os.getcwd()
         self.log.debug(f"Working directory: {cwd}")