yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Added ref to Qubasa-main in template/new-clan/flake.nix

Browse Source
This commit is contained in:
Qubasa
2023-10-20 01:11:32 +02:00
parent a71584d9d2
commit 26bfb793b1
9 changed files with 153 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkgs/clan-cli/clan_cli/flakes/create.py
View File

@@ -12,7 +12,7 @@ from ..errors import ClanError
from ..nix import nix_command, nix_shell from ..nix import nix_command, nix_shell
DEFAULT_URL: AnyUrl = parse_obj_as( DEFAULT_URL: AnyUrl = parse_obj_as(
AnyUrl, "git+https://git.clan.lol/clan/clan-core#new-clan" AnyUrl, "git+https://git.clan.lol/clan/clan-core?ref=Qubasa-main#new-clan" # TODO: Change me back to main branch
) )

4
pkgs/clan-cli/clan_cli/task_manager.py
View File

@@ -115,6 +115,10 @@ class BaseTask:
self.status = TaskStatus.RUNNING self.status = TaskStatus.RUNNING
try: try:
self.run() self.run()
# TODO: We need to check, if too many commands have been initialized,
# but not run. This would deadlock the log_lines() function.
# Idea: Run next(cmds) and check if it raises StopIteration if not,
# we have too many commands
except Exception as e: except Exception as e:
# FIXME: fix exception handling here # FIXME: fix exception handling here
traceback.print_exception(*sys.exc_info()) traceback.print_exception(*sys.exc_info())

20
pkgs/clan-cli/clan_cli/types.py
View File

@@ -1,3 +1,23 @@
from typing import NewType from typing import NewType
from pathlib import Path
import logging
log = logging.getLogger(__name__)
FlakeName = NewType("FlakeName", str) FlakeName = NewType("FlakeName", str)
def validate_path(base_dir: Path, value: Path) -> Path:
user_path = (base_dir / value).resolve()
# Check if the path is within the data directory
if not str(user_path).startswith(str(base_dir)):
if not str(user_path).startswith("/tmp/pytest"):
raise ValueError(
f"Destination out of bounds. Expected {user_path} to start with {base_dir}"
)
else:
log.warning(
f"Detected pytest tmpdir. Skipping path validation for {user_path}"
)
return user_path

188
pkgs/clan-cli/clan_cli/vms/create.py
View File

@@ -9,15 +9,16 @@ from pathlib import Path
from typing import Iterator from typing import Iterator
from uuid import UUID from uuid import UUID
from ..dirs import specific_flake_dir from ..dirs import specific_flake_dir, clan_flakes_dir
from ..nix import nix_build, nix_config, nix_shell from ..nix import nix_build, nix_config, nix_shell, nix_eval
from ..task_manager import BaseTask, Command, create_task from ..task_manager import BaseTask, Command, create_task
from .inspect import VmConfig, inspect_vm from .inspect import VmConfig, inspect_vm
from ..flakes.create import create_flake
from ..types import validate_path
class BuildVmTask(BaseTask): class BuildVmTask(BaseTask):
def __init__(self, uuid: UUID, vm: VmConfig) -> None: def __init__(self, uuid: UUID, vm: VmConfig) -> None:
super().__init__(uuid, num_cmds=6) super().__init__(uuid, num_cmds=7)
self.vm = vm self.vm = vm
def get_vm_create_info(self, cmds: Iterator[Command]) -> dict: def get_vm_create_info(self, cmds: Iterator[Command]) -> dict:
@@ -39,6 +40,19 @@ class BuildVmTask(BaseTask):
with open(vm_json) as f: with open(vm_json) as f:
return json.load(f) return json.load(f)
def get_clan_name(self, cmds: Iterator[Command]) -> str:
clan_dir = self.vm.flake_url
cmd = next(cmds)
cmd.run(
nix_eval(
[
f'{clan_dir}#clanInternals.clanName'
]
)
)
clan_name = "".join(cmd.stdout).strip()
return clan_name
def run(self) -> None: def run(self) -> None:
cmds = self.commands() cmds = self.commands()
@@ -47,101 +61,103 @@ class BuildVmTask(BaseTask):
# TODO: We should get this from the vm argument # TODO: We should get this from the vm argument
vm_config = self.get_vm_create_info(cmds) vm_config = self.get_vm_create_info(cmds)
clan_name = self.get_clan_name(cmds)
# TODO: Don't use a temporary directory, instead create a new flake directory
with tempfile.TemporaryDirectory() as tmpdir_:
tmpdir = Path(tmpdir_)
xchg_dir = tmpdir / "xchg"
xchg_dir.mkdir()
secrets_dir = tmpdir / "secrets"
secrets_dir.mkdir()
disk_img = f"{tmpdir_}/disk.img"
env = os.environ.copy() flake_dir = clan_flakes_dir() / clan_name
env["CLAN_DIR"] = str(self.vm.flake_url) validate_path(clan_flakes_dir(), flake_dir)
env["PYTHONPATH"] = str( xchg_dir = flake_dir / "xchg"
":".join(sys.path) xchg_dir.mkdir()
) # TODO do this in the clanCore module secrets_dir = flake_dir / "secrets"
env["SECRETS_DIR"] = str(secrets_dir) secrets_dir.mkdir()
disk_img = f"{flake_dir}/disk.img"
cmd = next(cmds) env = os.environ.copy()
if Path(self.vm.flake_url).is_dir(): env["CLAN_DIR"] = str(self.vm.flake_url)
cmd.run(
[vm_config["generateSecrets"]],
env=env,
)
else:
self.log.warning("won't generate secrets for non local clan")
cmd = next(cmds) env["PYTHONPATH"] = str(
":".join(sys.path)
) # TODO do this in the clanCore module
env["SECRETS_DIR"] = str(secrets_dir)
cmd = next(cmds)
if Path(self.vm.flake_url).is_dir():
cmd.run( cmd.run(
[vm_config["uploadSecrets"]], [vm_config["generateSecrets"]],
env=env, env=env,
) )
else:
self.log.warning("won't generate secrets for non local clan")
cmd = next(cmds) cmd = next(cmds)
cmd.run( cmd.run(
nix_shell( [vm_config["uploadSecrets"]],
["qemu"], env=env,
[ )
"qemu-img",
"create", cmd = next(cmds)
"-f", cmd.run(
"raw", nix_shell(
disk_img, ["qemu"],
"1024M", [
], "qemu-img",
) "create",
"-f",
"raw",
disk_img,
"1024M",
],
) )
)
cmd = next(cmds) cmd = next(cmds)
cmd.run( cmd.run(
nix_shell( nix_shell(
["e2fsprogs"], ["e2fsprogs"],
[ [
"mkfs.ext4", "mkfs.ext4",
"-L", "-L",
"nixos", "nixos",
disk_img, disk_img,
], ],
)
) )
)
cmd = next(cmds) cmd = next(cmds)
cmdline = [ cmdline = [
(Path(vm_config["toplevel"]) / "kernel-params").read_text(), (Path(vm_config["toplevel"]) / "kernel-params").read_text(),
f'init={vm_config["toplevel"]}/init', f'init={vm_config["toplevel"]}/init',
f'regInfo={vm_config["regInfo"]}/registration', f'regInfo={vm_config["regInfo"]}/registration',
"console=ttyS0,115200n8", "console=ttyS0,115200n8",
"console=tty0", "console=tty0",
] ]
qemu_command = [ qemu_command = [
# fmt: off # fmt: off
"qemu-kvm", "qemu-kvm",
"-name", machine, "-name", machine,
"-m", f'{vm_config["memorySize"]}M', "-m", f'{vm_config["memorySize"]}M',
"-smp", str(vm_config["cores"]), "-smp", str(vm_config["cores"]),
"-device", "virtio-rng-pci", "-device", "virtio-rng-pci",
"-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0", "-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0",
"-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store", "-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared", "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg", "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
"-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets", "-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
"-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report', "-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report',
"-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root", "-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
"-device", "virtio-keyboard", "-device", "virtio-keyboard",
"-usb", "-usb",
"-device", "usb-tablet,bus=usb-bus.0", "-device", "usb-tablet,bus=usb-bus.0",
"-kernel", f'{vm_config["toplevel"]}/kernel', "-kernel", f'{vm_config["toplevel"]}/kernel',
"-initrd", vm_config["initrd"], "-initrd", vm_config["initrd"],
"-append", " ".join(cmdline), "-append", " ".join(cmdline),
# fmt: on # fmt: on
] ]
if not self.vm.graphics: if not self.vm.graphics:
qemu_command.append("-nographic") qemu_command.append("-nographic")
print("$ " + shlex.join(qemu_command)) print("$ " + shlex.join(qemu_command))
cmd.run(nix_shell(["qemu"], qemu_command)) cmd.run(nix_shell(["qemu"], qemu_command))
def create_vm(vm: VmConfig) -> BuildVmTask: def create_vm(vm: VmConfig) -> BuildVmTask:

19
pkgs/clan-cli/clan_cli/webui/api_inputs.py
View File

@@ -6,26 +6,11 @@ from pydantic import AnyUrl, BaseModel, validator
from ..dirs import clan_data_dir, clan_flakes_dir from ..dirs import clan_data_dir, clan_flakes_dir
from ..flakes.create import DEFAULT_URL from ..flakes.create import DEFAULT_URL
from ..types import validate_path
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
def validate_path(base_dir: Path, value: Path) -> Path:
user_path = (base_dir / value).resolve()
# Check if the path is within the data directory
if not str(user_path).startswith(str(base_dir)):
if not str(user_path).startswith("/tmp/pytest"):
raise ValueError(
f"Destination out of bounds. Expected {user_path} to start with {base_dir}"
)
else:
log.warning(
f"Detected pytest tmpdir. Skipping path validation for {user_path}"
)
return user_path
class ClanDataPath(BaseModel): class ClanDataPath(BaseModel):
dest: Path dest: Path

7
pkgs/clan-cli/tests/temporary_dir.py
View File

@@ -10,13 +10,16 @@ log = logging.getLogger(__name__)
@pytest.fixture @pytest.fixture
def temporary_dir() -> Iterator[Path]: def temporary_home(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
if os.getenv("TEST_KEEP_TEMPORARY_DIR") is not None: if os.getenv("TEST_KEEP_TEMPORARY_DIR") is not None:
temp_dir = tempfile.mkdtemp(prefix="pytest-") temp_dir = tempfile.mkdtemp(prefix="pytest-")
path = Path(temp_dir) path = Path(temp_dir)
log.info("Keeping temporary test directory: ", path) log.debug("Temp HOME directory: %s", str(path))
monkeypatch.setenv("HOME", str(temp_dir))
yield path yield path
else: else:
log.debug("TEST_KEEP_TEMPORARY_DIR not set, using TemporaryDirectory") log.debug("TEST_KEEP_TEMPORARY_DIR not set, using TemporaryDirectory")
with tempfile.TemporaryDirectory(prefix="pytest-") as dirpath: with tempfile.TemporaryDirectory(prefix="pytest-") as dirpath:
monkeypatch.setenv("HOME", str(dirpath))
log.debug("Temp HOME directory: %s", str(dirpath))
yield Path(dirpath) yield Path(dirpath)

2
pkgs/clan-cli/tests/test_config.py
View File

@@ -57,7 +57,7 @@ def test_configure_machine(
capsys: pytest.CaptureFixture, capsys: pytest.CaptureFixture,
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
) -> None: ) -> None:
monkeypatch.setenv("HOME", str(temporary_dir))
cli = Cli() cli = Cli()
cli.run(["config", "-m", "machine1", "clan.jitsi.enable", "true"]) cli.run(["config", "-m", "machine1", "clan.jitsi.enable", "true"])
# clear the output buffer # clear the output buffer

30
pkgs/clan-cli/tests/test_create_flake.py
View File

@@ -5,7 +5,8 @@ from pathlib import Path
import pytest import pytest
from api import TestClient from api import TestClient
from cli import Cli from cli import Cli
from clan_cli.flakes.create import DEFAULT_URL
from clan_cli.dirs import clan_flakes_dir, clan_data_dir
@pytest.fixture @pytest.fixture
def cli() -> Cli: def cli() -> Cli:
@@ -14,15 +15,16 @@ def cli() -> Cli:
@pytest.mark.impure @pytest.mark.impure
def test_create_flake_api( def test_create_flake_api(
monkeypatch: pytest.MonkeyPatch, api: TestClient, temporary_dir: Path monkeypatch: pytest.MonkeyPatch, api: TestClient, temporary_home: Path
) -> None: ) -> None:
flake_dir = temporary_dir / "flake_dir" monkeypatch.chdir(clan_flakes_dir())
flake_dir_str = str(flake_dir.resolve()) flake_name = "flake_dir"
flake_dir = clan_flakes_dir() / flake_name
response = api.post( response = api.post(
"/api/flake/create", "/api/flake/create",
json=dict( json=dict(
dest=flake_dir_str, dest=str(flake_dir),
url="git+https://git.clan.lol/clan/clan-core#new-clan", url=str(DEFAULT_URL),
), ),
) )
@@ -34,19 +36,21 @@ def test_create_flake_api(
@pytest.mark.impure @pytest.mark.impure
def test_create_flake( def test_create_flake(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
temporary_dir: Path,
capsys: pytest.CaptureFixture, capsys: pytest.CaptureFixture,
temporary_home: Path,
cli: Cli, cli: Cli,
) -> None: ) -> None:
monkeypatch.chdir(temporary_dir) monkeypatch.chdir(clan_flakes_dir())
flake_dir = temporary_dir / "flake_dir" flake_name = "flake_dir"
flake_dir_str = str(flake_dir.resolve()) flake_dir = clan_flakes_dir() / flake_name
cli.run(["flake", "create", flake_dir_str])
cli.run(["flakes", "create", flake_name])
assert (flake_dir / ".clan-flake").exists() assert (flake_dir / ".clan-flake").exists()
monkeypatch.chdir(flake_dir) monkeypatch.chdir(flake_dir)
cli.run(["machines", "create", "machine1"]) cli.run(["machines", "create", "machine1", flake_name])
capsys.readouterr() # flush cache capsys.readouterr() # flush cache
cli.run(["machines", "list"])
cli.run(["machines", "list", flake_name])
assert "machine1" in capsys.readouterr().out assert "machine1" in capsys.readouterr().out
flake_show = subprocess.run( flake_show = subprocess.run(
["nix", "flake", "show", "--json"], ["nix", "flake", "show", "--json"],

2
templates/new-clan/flake.nix
View File

@@ -1,7 +1,7 @@
{ {
description = "<Put your description here>"; description = "<Put your description here>";
inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core"; inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core?ref=Qubasa-main";
outputs = { self, clan-core, ... }: outputs = { self, clan-core, ... }:
let let