Merge pull request 'contributing.md: Fixed missing direnv install step' (#457) from Qubasa-main into main

docs/contributing.md

@@ -10,6 +10,11 @@ Welcome to our website template repository! This template is designed to help yo
 
 **Dependency Management**: We use the [Nix package manager](https://nixos.org/) to manage dependencies and ensure reproducibility, making your development process more robust.
 
+## Supported Operating Systems
+
+- Linux
+- macOS
+
 # Getting Started with the Development Environment
 
 Let's get your development environment up and running:
@@ -28,11 +33,20 @@ Let's get your development environment up and running:
      curl -sfL https://direnv.net/install.sh | bash
      ```
 
-3. **Clone the Repository and Navigate**:
+3. **Add direnv to your shell**:
+
+   - Direnv needs to [hook into your shell](https://direnv.net/docs/hook.html) to work.
+     You can do this by executing following command:
+
+   ```bash
+   echo 'eval "$(direnv hook zsh)"' >> ~/.zshrc && echo 'eval "$(direnv hook bash)"' >> ~/.bashrc && eval "$SHELL"
+   ```
+
+4. **Clone the Repository and Navigate**:
 
    - Clone this repository and navigate to it.
 
-4. **Allow .envrc**:
+5. **Allow .envrc**:
 
    - When you enter the directory, you'll receive an error message like this:
      ```bash
@@ -40,7 +54,7 @@ Let's get your development environment up and running:
      ```
    - Execute `direnv allow` to automatically execute the shell script `.envrc` when entering the directory.
 
-5. **Build the Backend**:
+6. **Build the Backend**:
 
    - Go to the `pkgs/clan-cli` directory and execute:
      ```bash
@@ -48,7 +62,7 @@ Let's get your development environment up and running:
      ```
    - Wait for the backend to build.
 
-6. **Start the Backend Server**:
+7. **Start the Backend Server**:
 
    - To start the backend server, execute:
      ```bash
@@ -56,7 +70,7 @@ Let's get your development environment up and running:
      ```
    - The server will automatically restart if any Python files change.
 
-7. **Build the Frontend**:
+8. **Build the Frontend**:
 
    - In a different shell, navigate to the `pkgs/ui` directory and execute:
      ```bash
@@ -64,7 +78,7 @@ Let's get your development environment up and running:
      ```
    - Wait for the frontend to build.
 
-8. **Start the Frontend**:
+9. **Start the Frontend**:
    - To start the frontend, execute:
      ```bash
      npm run dev
@@ -194,4 +208,4 @@ To make the most of this template:
    - Set the option to "Delete pull request branch after merge by default."
    - Also, set the default merge style to "Rebase then create merge commit."
 
-With this template, you're well-equipped to build and collaborate on high-quality websites efficiently. Happy coding!
+With this template, you're well-equipped to build and collaborate on high-quality websites efficiently. Happy coding!.

pkgs/clan-cli/clan_cli/flakes/__init__.py

@@ -2,7 +2,7 @@
 import argparse
 
 from .create import register_create_parser
-from .list import register_list_parser
+from .list_flakes import register_list_parser
 
 
 # takes a (sub)parser and configures it

pkgs/clan-cli/clan_cli/types.py

@@ -1,11 +1,15 @@
 import logging
 from pathlib import Path
-from typing import NewType
+from typing import NewType, Union
+
+from pydantic import AnyUrl
 
 log = logging.getLogger(__name__)
 
 FlakeName = NewType("FlakeName", str)
 
+FlakeUrl = Union[AnyUrl, Path]
+
 
 def validate_path(base_dir: Path, value: Path) -> Path:
     user_path = (base_dir / value).resolve()

pkgs/clan-cli/clan_cli/vms/create.py

@@ -11,23 +11,16 @@ from typing import Iterator
 from uuid import UUID
 
 from ..dirs import clan_flakes_dir, specific_flake_dir
-from ..errors import ClanError
 from ..nix import nix_build, nix_config, nix_eval, nix_shell
 from ..task_manager import BaseTask, Command, create_task
 from ..types import validate_path
 from .inspect import VmConfig, inspect_vm
 
 
-def is_path_or_url(s: str) -> str | None:
+def is_flake_url(s: str) -> bool:
-    # check if s is a valid path
+    if re.match(r"^http.?://[a-zA-Z0-9.-]+/[a-zA-Z0-9.-]+", s) is not None:
-    if os.path.exists(s):
+        return True
-        return "path"
+    return False
-    # check if s is a valid URL
-    elif re.match(r"^https?://[a-zA-Z0-9.-]+/[a-zA-Z0-9.-]+", s):
-        return "URL"
-    # otherwise, return None
-    else:
-        return None
 
 
 class BuildVmTask(BaseTask):
@@ -95,12 +88,8 @@ class BuildVmTask(BaseTask):
             )  # TODO do this in the clanCore module
             env["SECRETS_DIR"] = str(secrets_dir)
 
-            res = is_path_or_url(str(self.vm.flake_url))
+            # Only generate secrets for local clans
-            if res is None:
+            if not is_flake_url(str(self.vm.flake_url)):
-                raise ClanError(
-                    f"flake_url must be a valid path or URL, got {self.vm.flake_url}"
-                )
-            elif res == "path":  # Only generate secrets for local clans
                 cmd = next(cmds)
                 if Path(self.vm.flake_url).is_dir():
                     cmd.run(
@@ -203,8 +192,10 @@ def create_vm(vm: VmConfig, nix_options: list[str] = []) -> BuildVmTask:
 
 
 def create_command(args: argparse.Namespace) -> None:
-    clan_dir = specific_flake_dir(args.flake)
+    flake_url = args.flake
-    vm = asyncio.run(inspect_vm(flake_url=clan_dir, flake_attr=args.machine))
+    if not is_flake_url(args.flake):
+        flake_url = specific_flake_dir(args.flake)
+    vm = asyncio.run(inspect_vm(flake_url=flake_url, flake_attr=args.machine))
 
     task = create_vm(vm, args.option)
     for line in task.log_lines():

pkgs/clan-cli/clan_cli/webui/api_outputs.py

@@ -70,6 +70,10 @@ class FlakeAction(BaseModel):
     uri: str
 
 
+class FlakeListResponse(BaseModel):
+    flakes: list[str]
+
+
 class FlakeCreateResponse(BaseModel):
     cmd_out: Dict[str, CmdOut]
 

pkgs/clan-cli/clan_cli/webui/app.py

@@ -9,6 +9,7 @@ from ..errors import ClanError
 from .assets import asset_path
 from .error_handlers import clan_error_handler
 from .routers import clan_modules, flake, health, machines, root, vms
+from .tags import tags_metadata
 
 origins = [
     "http://localhost:3000",
@@ -39,6 +40,9 @@ def setup_app() -> FastAPI:
 
     app.mount("/static", StaticFiles(directory=asset_path()), name="static")
 
+    # Add tag descriptions to the OpenAPI schema
+    app.openapi_tags = tags_metadata
+
     for route in app.routes:
         if isinstance(route, APIRoute):
             route.operation_id = route.name  # in this case, 'read_items'

pkgs/clan-cli/clan_cli/webui/routers/clan_modules.py

@@ -9,12 +9,13 @@ from clan_cli.types import FlakeName
 from ..api_outputs import (
     ClanModulesResponse,
 )
+from ..tags import Tags
 
 log = logging.getLogger(__name__)
 router = APIRouter()
 
 
-@router.get("/api/{flake_name}/clan_modules")
+@router.get("/api/{flake_name}/clan_modules", tags=[Tags.modules])
 async def list_clan_modules(flake_name: FlakeName) -> ClanModulesResponse:
     module_names, error = get_clan_module_names(flake_name)
     if error is not None:

pkgs/clan-cli/clan_cli/webui/routers/flake.py

@@ -13,12 +13,14 @@ from clan_cli.webui.api_outputs import (
     FlakeAction,
     FlakeAttrResponse,
     FlakeCreateResponse,
+    FlakeListResponse,
     FlakeResponse,
 )
 
 from ...async_cmd import run
-from ...flakes import create
+from ...flakes import create, list_flakes
 from ...nix import nix_command, nix_flake_show
+from ..tags import Tags
 
 router = APIRouter()
 
@@ -45,13 +47,13 @@ async def get_attrs(url: AnyUrl | Path) -> list[str]:
 
 
 # TODO: Check for directory traversal
-@router.get("/api/flake/attrs")
+@router.get("/api/flake/attrs", tags=[Tags.flake])
 async def inspect_flake_attrs(url: AnyUrl | Path) -> FlakeAttrResponse:
     return FlakeAttrResponse(flake_attrs=await get_attrs(url))
 
 
 # TODO: Check for directory traversal
-@router.get("/api/flake")
+@router.get("/api/flake/inspect", tags=[Tags.flake])
 async def inspect_flake(
     url: AnyUrl | Path,
 ) -> FlakeResponse:
@@ -76,7 +78,15 @@ async def inspect_flake(
     return FlakeResponse(content=content, actions=actions)
 
 
-@router.post("/api/flake/create", status_code=status.HTTP_201_CREATED)
+@router.get("/api/flake/list", tags=[Tags.flake])
+async def list_all_flakes() -> FlakeListResponse:
+    flakes = list_flakes.list_flakes()
+    return FlakeListResponse(flakes=flakes)
+
+
+@router.post(
+    "/api/flake/create", tags=[Tags.flake], status_code=status.HTTP_201_CREATED
+)
 async def create_flake(
     args: Annotated[FlakeCreateInput, Body()],
 ) -> FlakeCreateResponse:

pkgs/clan-cli/clan_cli/webui/routers/machines.py

@@ -23,12 +23,13 @@ from ..api_outputs import (
     Status,
     VerifyMachineResponse,
 )
+from ..tags import Tags
 
 log = logging.getLogger(__name__)
 router = APIRouter()
 
 
-@router.get("/api/{flake_name}/machines")
+@router.get("/api/{flake_name}/machines", tags=[Tags.machine])
 async def list_machines(flake_name: FlakeName) -> MachinesResponse:
     machines = []
     for m in _list_machines(flake_name):
@@ -37,7 +38,7 @@ async def list_machines(flake_name: FlakeName) -> MachinesResponse:
     return MachinesResponse(machines=machines)
 
 
-@router.post("/api/{flake_name}/machines", status_code=201)
+@router.post("/api/{flake_name}/machines", tags=[Tags.machine], status_code=201)
 async def create_machine(
     flake_name: FlakeName, machine: Annotated[MachineCreate, Body()]
 ) -> MachineResponse:
@@ -45,19 +46,19 @@ async def create_machine(
     return MachineResponse(machine=Machine(name=machine.name, status=Status.UNKNOWN))
 
 
-@router.get("/api/{flake_name}/machines/{name}")
+@router.get("/api/{flake_name}/machines/{name}", tags=[Tags.machine])
 async def get_machine(flake_name: FlakeName, name: str) -> MachineResponse:
     log.error("TODO")
     return MachineResponse(machine=Machine(name=name, status=Status.UNKNOWN))
 
 
-@router.get("/api/{flake_name}/machines/{name}/config")
+@router.get("/api/{flake_name}/machines/{name}/config", tags=[Tags.machine])
 async def get_machine_config(flake_name: FlakeName, name: str) -> ConfigResponse:
     config = config_for_machine(flake_name, name)
     return ConfigResponse(config=config)
 
 
-@router.put("/api/{flake_name}/machines/{name}/config")
+@router.put("/api/{flake_name}/machines/{name}/config", tags=[Tags.machine])
 async def set_machine_config(
     flake_name: FlakeName, name: str, config: Annotated[dict, Body()]
 ) -> ConfigResponse:
@@ -65,13 +66,13 @@ async def set_machine_config(
     return ConfigResponse(config=config)
 
 
-@router.get("/api/{flake_name}/machines/{name}/schema")
+@router.get("/api/{flake_name}/machines/{name}/schema", tags=[Tags.machine])
 async def get_machine_schema(flake_name: FlakeName, name: str) -> SchemaResponse:
     schema = schema_for_machine(flake_name, name)
     return SchemaResponse(schema=schema)
 
 
-@router.put("/api/{flake_name}/machines/{name}/schema")
+@router.put("/api/{flake_name}/machines/{name}/schema", tags=[Tags.machine])
 async def set_machine_schema(
     flake_name: FlakeName, name: str, config: Annotated[dict, Body()]
 ) -> SchemaResponse:
@@ -79,7 +80,7 @@ async def set_machine_schema(
     return SchemaResponse(schema=schema)
 
 
-@router.get("/api/{flake_name}/machines/{name}/verify")
+@router.get("/api/{flake_name}/machines/{name}/verify", tags=[Tags.machine])
 async def get_verify_machine_config(
     flake_name: FlakeName, name: str
 ) -> VerifyMachineResponse:
@@ -88,7 +89,7 @@ async def get_verify_machine_config(
     return VerifyMachineResponse(success=success, error=error)
 
 
-@router.put("/api/{flake_name}/machines/{name}/verify")
+@router.put("/api/{flake_name}/machines/{name}/verify", tags=[Tags.machine])
 async def put_verify_machine_config(
     flake_name: FlakeName,
     name: str,

pkgs/clan-cli/clan_cli/webui/routers/root.py

@@ -6,13 +6,14 @@ from pathlib import Path
 from fastapi import APIRouter, Response
 
 from ..assets import asset_path
+from ..tags import Tags
 
 router = APIRouter()
 
 log = logging.getLogger(__name__)
 
 
-@router.get("/{path_name:path}")
+@router.get("/{path_name:path}", tags=[Tags.root])
 async def root(path_name: str) -> Response:
     if path_name == "":
         path_name = "index.html"

pkgs/clan-cli/clan_cli/webui/routers/vms.py

@@ -18,13 +18,14 @@ from ..api_outputs import (
     VmInspectResponse,
     VmStatusResponse,
 )
+from ..tags import Tags
 
 log = logging.getLogger(__name__)
 router = APIRouter()
 
 
 # TODO: Check for directory traversal
-@router.post("/api/vms/inspect")
+@router.post("/api/vms/inspect", tags=[Tags.vm])
 async def inspect_vm(
     flake_url: Annotated[AnyUrl | Path, Body()], flake_attr: Annotated[str, Body()]
 ) -> VmInspectResponse:
@@ -32,7 +33,7 @@ async def inspect_vm(
     return VmInspectResponse(config=config)
 
 
-@router.get("/api/vms/{uuid}/status")
+@router.get("/api/vms/{uuid}/status", tags=[Tags.vm])
 async def get_vm_status(uuid: UUID) -> VmStatusResponse:
     task = get_task(uuid)
     log.debug(msg=f"error: {task.error}, task.status: {task.status}")
@@ -40,7 +41,7 @@ async def get_vm_status(uuid: UUID) -> VmStatusResponse:
     return VmStatusResponse(status=task.status, error=error)
 
 
-@router.get("/api/vms/{uuid}/logs")
+@router.get("/api/vms/{uuid}/logs", tags=[Tags.vm])
 async def get_vm_logs(uuid: UUID) -> StreamingResponse:
     # Generator function that yields log lines as they are available
     def stream_logs() -> Iterator[str]:
@@ -55,7 +56,7 @@ async def get_vm_logs(uuid: UUID) -> StreamingResponse:
 
 
 # TODO: Check for directory traversal
-@router.post("/api/vms/create")
+@router.post("/api/vms/create", tags=[Tags.vm])
 async def create_vm(vm: Annotated[VmConfig, Body()]) -> VmCreateResponse:
     flake_attrs = await get_attrs(vm.flake_url)
     if vm.flake_attr not in flake_attrs:

pkgs/clan-cli/clan_cli/webui/tags.py

@@ -0,0 +1,41 @@
+from enum import Enum
+from typing import Any, Dict, List
+
+
+class Tags(Enum):
+    flake = "flake"
+    machine = "machine"
+    vm = "vm"
+    modules = "modules"
+    root = "root"
+
+    def __str__(self) -> str:
+        return self.value
+
+
+tags_metadata: List[Dict[str, Any]] = [
+    {
+        "name": str(Tags.flake),
+        "description": "Operations on a flake.",
+        "externalDocs": {
+            "description": "What is a flake?",
+            "url": "https://www.tweag.io/blog/2020-05-25-flakes/",
+        },
+    },
+    {
+        "name": str(Tags.machine),
+        "description": "Manage physical machines. Instances of a flake",
+    },
+    {
+        "name": str(Tags.vm),
+        "description": "Manage virtual machines. Instances of a flake",
+    },
+    {
+        "name": str(Tags.modules),
+        "description": "Manage cLAN modules of a flake",
+    },
+    {
+        "name": str(Tags.root),
+        "description": "This serves as the frontend delivery",
+    },
+]

pkgs/clan-cli/tests/fixtures_flakes.py

@@ -8,6 +8,8 @@ from pathlib import Path
 from typing import Iterator, NamedTuple
 
 import pytest
+from pydantic import AnyUrl
+from pydantic.tools import parse_obj_as
 from root import CLAN_CORE
 
 from clan_cli.dirs import nixpkgs_source
@@ -117,6 +119,16 @@ def test_flake_with_core(
     )
 
 
+@pytest.fixture
+def test_democlan_url(
+    monkeypatch: pytest.MonkeyPatch, temporary_home: Path
+) -> Iterator[AnyUrl]:
+    yield parse_obj_as(
+        AnyUrl,
+        "https://git.clan.lol/clan/democlan/archive/main.tar.gz",
+    )
+
+
 @pytest.fixture
 def test_flake_with_core_and_pass(
     monkeypatch: pytest.MonkeyPatch, temporary_home: Path

pkgs/clan-cli/tests/test_flake_api.py

@@ -8,6 +8,15 @@ from fixtures_flakes import FlakeForTest
 log = logging.getLogger(__name__)
 
 
+@pytest.mark.impure
+def test_list_flakes(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
+    response = api.get("/api/flake/list")
+    assert response.status_code == 200, "Failed to list flakes"
+    data = response.json()
+    print("Data: ", data)
+    assert data.get("flakes") == ["test_flake_with_core"]
+
+
 @pytest.mark.impure
 def test_inspect_ok(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
     params = {"url": str(test_flake_with_core.path)}
@@ -38,7 +47,7 @@ def test_inspect_err(api: TestClient) -> None:
 def test_inspect_flake(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
     params = {"url": str(test_flake_with_core.path)}
     response = api.get(
-        "/api/flake",
+        "/api/flake/inspect",
         params=params,
     )
     assert response.status_code == 200, "Failed to inspect vm"

pkgs/clan-cli/tests/test_vms_api_create.py

@@ -7,6 +7,7 @@ from api import TestClient
 from cli import Cli
 from fixtures_flakes import FlakeForTest, create_flake
 from httpx import SyncByteStream
+from pydantic import AnyUrl
 from root import CLAN_CORE
 
 from clan_cli.types import FlakeName
@@ -42,7 +43,7 @@ def remote_flake_with_vm_without_secrets(
     )
 
 
-def generic_create_vm_test(api: TestClient, flake: Path, vm: str) -> None:
+def generic_create_vm_test(api: TestClient, flake: Path | AnyUrl, vm: str) -> None:
     print(f"flake_url: {flake} ")
     response = api.post(
         "/api/vms/create",
@@ -113,3 +114,18 @@ def test_create_remote(
     generic_create_vm_test(
         api, remote_flake_with_vm_without_secrets.path, "vm_without_secrets"
     )
+
+
+# TODO: We need a test that creates the same VM twice, and checks that the second time it fails
+
+
+# TODO: Democlan needs a machine called testVM, which is headless and gets executed by this test below
+# pytest -n0 -s tests/test_vms_api_create.py::test_create_from_democlan
+# @pytest.mark.skipif(not os.path.exists("/dev/kvm"), reason="Requires KVM")
+# @pytest.mark.impure
+# def test_create_from_democlan(
+#     api: TestClient,
+#     test_democlan_url: AnyUrl) -> None:
+#         generic_create_vm_test(
+#             api, test_democlan_url, "defaultVM"
+#         )

pkgs/ui/shell.nix

@@ -25,7 +25,7 @@ pkgs.mkShell {
 
 
     # re-generate the api code 
-    rm -rf api openapi.json
+    rm -rf src/api openapi.json
     cp ${clanPkgs.clan-openapi}/openapi.json . 
     orval
   '';

pkgs/ui/src/components/createMachineForm/customConfig.tsx

@@ -1,5 +1,5 @@
 "use client";
-import { useGetMachineSchema } from "@/api/default/default";
+import { useGetMachineSchema } from "@/api/machine/machine";
 import { Check, Error } from "@mui/icons-material";
 import {
   Box,

pkgs/ui/src/components/hooks/useAppContext.tsx

@@ -1,4 +1,4 @@
-import { useListMachines } from "@/api/default/default";
+import { useListMachines } from "@/api/machine/machine";
 import { MachinesResponse } from "@/api/model";
 import { AxiosError, AxiosResponse } from "axios";
 import React, {

pkgs/ui/src/components/hooks/useMachines.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { useListMachines } from "@/api/default/default";
+import { useListMachines } from "@/api/machine/machine";
 import { Machine, MachinesResponse } from "@/api/model";
 import { AxiosError, AxiosResponse } from "axios";
 import React, {

pkgs/ui/src/components/hooks/useVms.tsx

@@ -1,4 +1,4 @@
-import { inspectVm } from "@/api/default/default";
+import { inspectVm } from "@/api/vm/vm";
 import { HTTPValidationError, VmConfig } from "@/api/model";
 import { AxiosError } from "axios";
 import { useEffect, useState } from "react";

pkgs/ui/src/components/join/configureVM.tsx

@@ -10,7 +10,8 @@ import {
 } from "@mui/material";
 import { Controller, SubmitHandler, UseFormReturn } from "react-hook-form";
 import { FlakeBadge } from "../flakeBadge/flakeBadge";
-import { createVm, useInspectFlakeAttrs } from "@/api/default/default";
+import { createVm } from "@/api/vm/vm";
+import { useInspectFlakeAttrs } from "@/api/flake/flake";
 import { VmConfig } from "@/api/model";
 import { Dispatch, SetStateAction, useEffect, useState } from "react";
 import { toast } from "react-hot-toast";

pkgs/ui/src/components/join/confirm.tsx

@@ -7,7 +7,7 @@ import { Typography, Button } from "@mui/material";
 import { ConfirmVM } from "./confirmVM";
 import { Log } from "./log";
 import GppMaybeIcon from "@mui/icons-material/GppMaybe";
-import { useInspectFlake } from "@/api/default/default";
+import { useInspectFlake } from "@/api/flake/flake";
 
 interface ConfirmProps {
   flakeUrl: string;

pkgs/ui/src/components/join/vmBuildLogs.tsx

@@ -1,5 +1,5 @@
 "use client";
-import { useGetVmLogs } from "@/api/default/default";
+import { useGetVmLogs } from "@/api/vm/vm";
 import { Log } from "./log";
 import { LoadingOverlay } from "./loadingOverlay";