diff --git a/pkgs/clan-cli/clan_cli/flakes/create.py b/pkgs/clan-cli/clan_cli/flakes/create.py index fb8ea16d4..28e2bdc63 100644 --- a/pkgs/clan-cli/clan_cli/flakes/create.py +++ b/pkgs/clan-cli/clan_cli/flakes/create.py @@ -12,7 +12,7 @@ from ..errors import ClanError from ..nix import nix_command, nix_shell DEFAULT_URL: AnyUrl = parse_obj_as( - AnyUrl, "git+https://git.clan.lol/clan/clan-core#new-clan" + AnyUrl, "git+https://git.clan.lol/clan/clan-core?ref=Qubasa-main#new-clan" # TODO: Change me back to main branch ) diff --git a/pkgs/clan-cli/clan_cli/task_manager.py b/pkgs/clan-cli/clan_cli/task_manager.py index b2573e0fb..c54a97481 100644 --- a/pkgs/clan-cli/clan_cli/task_manager.py +++ b/pkgs/clan-cli/clan_cli/task_manager.py @@ -115,6 +115,10 @@ class BaseTask: self.status = TaskStatus.RUNNING try: self.run() + # TODO: We need to check, if too many commands have been initialized, + # but not run. This would deadlock the log_lines() function. + # Idea: Run next(cmds) and check if it raises StopIteration if not, + # we have too many commands except Exception as e: # FIXME: fix exception handling here traceback.print_exception(*sys.exc_info()) diff --git a/pkgs/clan-cli/clan_cli/types.py b/pkgs/clan-cli/clan_cli/types.py index 16e38c879..be54b3da3 100644 --- a/pkgs/clan-cli/clan_cli/types.py +++ b/pkgs/clan-cli/clan_cli/types.py @@ -1,3 +1,23 @@ from typing import NewType +from pathlib import Path +import logging + +log = logging.getLogger(__name__) FlakeName = NewType("FlakeName", str) + + +def validate_path(base_dir: Path, value: Path) -> Path: + user_path = (base_dir / value).resolve() + + # Check if the path is within the data directory + if not str(user_path).startswith(str(base_dir)): + if not str(user_path).startswith("/tmp/pytest"): + raise ValueError( + f"Destination out of bounds. Expected {user_path} to start with {base_dir}" + ) + else: + log.warning( + f"Detected pytest tmpdir. Skipping path validation for {user_path}" + ) + return user_path \ No newline at end of file diff --git a/pkgs/clan-cli/clan_cli/vms/create.py b/pkgs/clan-cli/clan_cli/vms/create.py index 8cc12f120..158e243e2 100644 --- a/pkgs/clan-cli/clan_cli/vms/create.py +++ b/pkgs/clan-cli/clan_cli/vms/create.py @@ -9,15 +9,16 @@ from pathlib import Path from typing import Iterator from uuid import UUID -from ..dirs import specific_flake_dir -from ..nix import nix_build, nix_config, nix_shell +from ..dirs import specific_flake_dir, clan_flakes_dir +from ..nix import nix_build, nix_config, nix_shell, nix_eval from ..task_manager import BaseTask, Command, create_task from .inspect import VmConfig, inspect_vm - +from ..flakes.create import create_flake +from ..types import validate_path class BuildVmTask(BaseTask): def __init__(self, uuid: UUID, vm: VmConfig) -> None: - super().__init__(uuid, num_cmds=6) + super().__init__(uuid, num_cmds=7) self.vm = vm def get_vm_create_info(self, cmds: Iterator[Command]) -> dict: @@ -39,6 +40,19 @@ class BuildVmTask(BaseTask): with open(vm_json) as f: return json.load(f) + def get_clan_name(self, cmds: Iterator[Command]) -> str: + clan_dir = self.vm.flake_url + cmd = next(cmds) + cmd.run( + nix_eval( + [ + f'{clan_dir}#clanInternals.clanName' + ] + ) + ) + clan_name = "".join(cmd.stdout).strip() + return clan_name + def run(self) -> None: cmds = self.commands() @@ -47,101 +61,103 @@ class BuildVmTask(BaseTask): # TODO: We should get this from the vm argument vm_config = self.get_vm_create_info(cmds) + clan_name = self.get_clan_name(cmds) - # TODO: Don't use a temporary directory, instead create a new flake directory - with tempfile.TemporaryDirectory() as tmpdir_: - tmpdir = Path(tmpdir_) - xchg_dir = tmpdir / "xchg" - xchg_dir.mkdir() - secrets_dir = tmpdir / "secrets" - secrets_dir.mkdir() - disk_img = f"{tmpdir_}/disk.img" - env = os.environ.copy() - env["CLAN_DIR"] = str(self.vm.flake_url) + flake_dir = clan_flakes_dir() / clan_name + validate_path(clan_flakes_dir(), flake_dir) - env["PYTHONPATH"] = str( - ":".join(sys.path) - ) # TODO do this in the clanCore module - env["SECRETS_DIR"] = str(secrets_dir) + xchg_dir = flake_dir / "xchg" + xchg_dir.mkdir() + secrets_dir = flake_dir / "secrets" + secrets_dir.mkdir() + disk_img = f"{flake_dir}/disk.img" - cmd = next(cmds) - if Path(self.vm.flake_url).is_dir(): - cmd.run( - [vm_config["generateSecrets"]], - env=env, - ) - else: - self.log.warning("won't generate secrets for non local clan") + env = os.environ.copy() + env["CLAN_DIR"] = str(self.vm.flake_url) - cmd = next(cmds) + env["PYTHONPATH"] = str( + ":".join(sys.path) + ) # TODO do this in the clanCore module + env["SECRETS_DIR"] = str(secrets_dir) + + cmd = next(cmds) + if Path(self.vm.flake_url).is_dir(): cmd.run( - [vm_config["uploadSecrets"]], + [vm_config["generateSecrets"]], env=env, ) + else: + self.log.warning("won't generate secrets for non local clan") - cmd = next(cmds) - cmd.run( - nix_shell( - ["qemu"], - [ - "qemu-img", - "create", - "-f", - "raw", - disk_img, - "1024M", - ], - ) + cmd = next(cmds) + cmd.run( + [vm_config["uploadSecrets"]], + env=env, + ) + + cmd = next(cmds) + cmd.run( + nix_shell( + ["qemu"], + [ + "qemu-img", + "create", + "-f", + "raw", + disk_img, + "1024M", + ], ) + ) - cmd = next(cmds) - cmd.run( - nix_shell( - ["e2fsprogs"], - [ - "mkfs.ext4", - "-L", - "nixos", - disk_img, - ], - ) + cmd = next(cmds) + cmd.run( + nix_shell( + ["e2fsprogs"], + [ + "mkfs.ext4", + "-L", + "nixos", + disk_img, + ], ) + ) - cmd = next(cmds) - cmdline = [ - (Path(vm_config["toplevel"]) / "kernel-params").read_text(), - f'init={vm_config["toplevel"]}/init', - f'regInfo={vm_config["regInfo"]}/registration', - "console=ttyS0,115200n8", - "console=tty0", - ] - qemu_command = [ - # fmt: off - "qemu-kvm", - "-name", machine, - "-m", f'{vm_config["memorySize"]}M', - "-smp", str(vm_config["cores"]), - "-device", "virtio-rng-pci", - "-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0", - "-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store", - "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared", - "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg", - "-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets", - "-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report', - "-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root", - "-device", "virtio-keyboard", - "-usb", - "-device", "usb-tablet,bus=usb-bus.0", - "-kernel", f'{vm_config["toplevel"]}/kernel', - "-initrd", vm_config["initrd"], - "-append", " ".join(cmdline), - # fmt: on - ] - if not self.vm.graphics: - qemu_command.append("-nographic") - print("$ " + shlex.join(qemu_command)) - cmd.run(nix_shell(["qemu"], qemu_command)) + cmd = next(cmds) + cmdline = [ + (Path(vm_config["toplevel"]) / "kernel-params").read_text(), + f'init={vm_config["toplevel"]}/init', + f'regInfo={vm_config["regInfo"]}/registration', + "console=ttyS0,115200n8", + "console=tty0", + ] + qemu_command = [ + # fmt: off + "qemu-kvm", + "-name", machine, + "-m", f'{vm_config["memorySize"]}M', + "-smp", str(vm_config["cores"]), + "-device", "virtio-rng-pci", + "-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0", + "-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store", + "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared", + "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg", + "-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets", + "-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report', + "-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root", + "-device", "virtio-keyboard", + "-usb", + "-device", "usb-tablet,bus=usb-bus.0", + "-kernel", f'{vm_config["toplevel"]}/kernel', + "-initrd", vm_config["initrd"], + "-append", " ".join(cmdline), + # fmt: on + ] + if not self.vm.graphics: + qemu_command.append("-nographic") + print("$ " + shlex.join(qemu_command)) + cmd.run(nix_shell(["qemu"], qemu_command)) def create_vm(vm: VmConfig) -> BuildVmTask: diff --git a/pkgs/clan-cli/clan_cli/webui/api_inputs.py b/pkgs/clan-cli/clan_cli/webui/api_inputs.py index 959f118b5..c1e9fd891 100644 --- a/pkgs/clan-cli/clan_cli/webui/api_inputs.py +++ b/pkgs/clan-cli/clan_cli/webui/api_inputs.py @@ -6,26 +6,11 @@ from pydantic import AnyUrl, BaseModel, validator from ..dirs import clan_data_dir, clan_flakes_dir from ..flakes.create import DEFAULT_URL +from ..types import validate_path + log = logging.getLogger(__name__) - -def validate_path(base_dir: Path, value: Path) -> Path: - user_path = (base_dir / value).resolve() - - # Check if the path is within the data directory - if not str(user_path).startswith(str(base_dir)): - if not str(user_path).startswith("/tmp/pytest"): - raise ValueError( - f"Destination out of bounds. Expected {user_path} to start with {base_dir}" - ) - else: - log.warning( - f"Detected pytest tmpdir. Skipping path validation for {user_path}" - ) - return user_path - - class ClanDataPath(BaseModel): dest: Path diff --git a/pkgs/clan-cli/tests/temporary_dir.py b/pkgs/clan-cli/tests/temporary_dir.py index 615d5a6be..4c9bfa55c 100644 --- a/pkgs/clan-cli/tests/temporary_dir.py +++ b/pkgs/clan-cli/tests/temporary_dir.py @@ -10,13 +10,16 @@ log = logging.getLogger(__name__) @pytest.fixture -def temporary_dir() -> Iterator[Path]: +def temporary_home(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: if os.getenv("TEST_KEEP_TEMPORARY_DIR") is not None: temp_dir = tempfile.mkdtemp(prefix="pytest-") path = Path(temp_dir) - log.info("Keeping temporary test directory: ", path) + log.debug("Temp HOME directory: %s", str(path)) + monkeypatch.setenv("HOME", str(temp_dir)) yield path else: log.debug("TEST_KEEP_TEMPORARY_DIR not set, using TemporaryDirectory") with tempfile.TemporaryDirectory(prefix="pytest-") as dirpath: + monkeypatch.setenv("HOME", str(dirpath)) + log.debug("Temp HOME directory: %s", str(dirpath)) yield Path(dirpath) diff --git a/pkgs/clan-cli/tests/test_config.py b/pkgs/clan-cli/tests/test_config.py index 796a798bd..428921dae 100644 --- a/pkgs/clan-cli/tests/test_config.py +++ b/pkgs/clan-cli/tests/test_config.py @@ -57,7 +57,7 @@ def test_configure_machine( capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setenv("HOME", str(temporary_dir)) + cli = Cli() cli.run(["config", "-m", "machine1", "clan.jitsi.enable", "true"]) # clear the output buffer diff --git a/pkgs/clan-cli/tests/test_create_flake.py b/pkgs/clan-cli/tests/test_create_flake.py index ec6976ac1..d379a402a 100644 --- a/pkgs/clan-cli/tests/test_create_flake.py +++ b/pkgs/clan-cli/tests/test_create_flake.py @@ -5,7 +5,8 @@ from pathlib import Path import pytest from api import TestClient from cli import Cli - +from clan_cli.flakes.create import DEFAULT_URL +from clan_cli.dirs import clan_flakes_dir, clan_data_dir @pytest.fixture def cli() -> Cli: @@ -14,15 +15,16 @@ def cli() -> Cli: @pytest.mark.impure def test_create_flake_api( - monkeypatch: pytest.MonkeyPatch, api: TestClient, temporary_dir: Path + monkeypatch: pytest.MonkeyPatch, api: TestClient, temporary_home: Path ) -> None: - flake_dir = temporary_dir / "flake_dir" - flake_dir_str = str(flake_dir.resolve()) + monkeypatch.chdir(clan_flakes_dir()) + flake_name = "flake_dir" + flake_dir = clan_flakes_dir() / flake_name response = api.post( "/api/flake/create", json=dict( - dest=flake_dir_str, - url="git+https://git.clan.lol/clan/clan-core#new-clan", + dest=str(flake_dir), + url=str(DEFAULT_URL), ), ) @@ -34,19 +36,21 @@ def test_create_flake_api( @pytest.mark.impure def test_create_flake( monkeypatch: pytest.MonkeyPatch, - temporary_dir: Path, capsys: pytest.CaptureFixture, + temporary_home: Path, cli: Cli, ) -> None: - monkeypatch.chdir(temporary_dir) - flake_dir = temporary_dir / "flake_dir" - flake_dir_str = str(flake_dir.resolve()) - cli.run(["flake", "create", flake_dir_str]) + monkeypatch.chdir(clan_flakes_dir()) + flake_name = "flake_dir" + flake_dir = clan_flakes_dir() / flake_name + + cli.run(["flakes", "create", flake_name]) assert (flake_dir / ".clan-flake").exists() monkeypatch.chdir(flake_dir) - cli.run(["machines", "create", "machine1"]) + cli.run(["machines", "create", "machine1", flake_name]) capsys.readouterr() # flush cache - cli.run(["machines", "list"]) + + cli.run(["machines", "list", flake_name]) assert "machine1" in capsys.readouterr().out flake_show = subprocess.run( ["nix", "flake", "show", "--json"], diff --git a/templates/new-clan/flake.nix b/templates/new-clan/flake.nix index 38acce83a..72bbbc174 100644 --- a/templates/new-clan/flake.nix +++ b/templates/new-clan/flake.nix @@ -1,7 +1,7 @@ { description = ""; - inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core"; + inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core?ref=Qubasa-main"; outputs = { self, clan-core, ... }: let