API: Added /api/flake/list. Added API grouping and description for better visibility

pkgs/clan-cli/clan_cli/flakes/__init__.py

@@ -2,7 +2,7 @@
 import argparse
 
 from .create import register_create_parser
-from .list import register_list_parser
+from .list_flakes import register_list_parser
 
 
 # takes a (sub)parser and configures it

pkgs/clan-cli/clan_cli/types.py

@@ -1,11 +1,15 @@
 import logging
 from pathlib import Path
-from typing import NewType
+from typing import NewType, Union
+
+from pydantic import AnyUrl
 
 log = logging.getLogger(__name__)
 
 FlakeName = NewType("FlakeName", str)
 
+FlakeUrl = Union[AnyUrl, Path]
+
 
 def validate_path(base_dir: Path, value: Path) -> Path:
     user_path = (base_dir / value).resolve()

pkgs/clan-cli/clan_cli/vms/create.py

@@ -11,23 +11,16 @@ from typing import Iterator
 from uuid import UUID
 
 from ..dirs import clan_flakes_dir, specific_flake_dir
-from ..errors import ClanError
 from ..nix import nix_build, nix_config, nix_eval, nix_shell
 from ..task_manager import BaseTask, Command, create_task
 from ..types import validate_path
 from .inspect import VmConfig, inspect_vm
 
 
-def is_path_or_url(s: str) -> str | None:
-    # check if s is a valid path
-    if os.path.exists(s):
-        return "path"
-    # check if s is a valid URL
-    elif re.match(r"^https?://[a-zA-Z0-9.-]+/[a-zA-Z0-9.-]+", s):
-        return "URL"
-    # otherwise, return None
-    else:
-        return None
+def is_flake_url(s: str) -> bool:
+    if re.match(r"^http.?://[a-zA-Z0-9.-]+/[a-zA-Z0-9.-]+", s) is not None:
+        return True
+    return False
 
 
 class BuildVmTask(BaseTask):
@@ -95,12 +88,8 @@ class BuildVmTask(BaseTask):
             )  # TODO do this in the clanCore module
             env["SECRETS_DIR"] = str(secrets_dir)
 
-            res = is_path_or_url(str(self.vm.flake_url))
-            if res is None:
-                raise ClanError(
-                    f"flake_url must be a valid path or URL, got {self.vm.flake_url}"
-                )
-            elif res == "path":  # Only generate secrets for local clans
+            # Only generate secrets for local clans
+            if not is_flake_url(str(self.vm.flake_url)):
                 cmd = next(cmds)
                 if Path(self.vm.flake_url).is_dir():
                     cmd.run(
@@ -203,8 +192,10 @@ def create_vm(vm: VmConfig, nix_options: list[str] = []) -> BuildVmTask:
 
 
 def create_command(args: argparse.Namespace) -> None:
-    clan_dir = specific_flake_dir(args.flake)
-    vm = asyncio.run(inspect_vm(flake_url=clan_dir, flake_attr=args.machine))
+    flake_url = args.flake
+    if not is_flake_url(args.flake):
+        flake_url = specific_flake_dir(args.flake)
+    vm = asyncio.run(inspect_vm(flake_url=flake_url, flake_attr=args.machine))
 
     task = create_vm(vm, args.option)
     for line in task.log_lines():

pkgs/clan-cli/clan_cli/webui/api_outputs.py

@@ -70,6 +70,10 @@ class FlakeAction(BaseModel):
     uri: str
 
 
+class FlakeListResponse(BaseModel):
+    flakes: list[str]
+
+
 class FlakeCreateResponse(BaseModel):
     cmd_out: Dict[str, CmdOut]
 

pkgs/clan-cli/clan_cli/webui/app.py

@@ -9,6 +9,7 @@ from ..errors import ClanError
 from .assets import asset_path
 from .error_handlers import clan_error_handler
 from .routers import clan_modules, flake, health, machines, root, vms
+from .tags import tags_metadata
 
 origins = [
     "http://localhost:3000",
@@ -39,6 +40,9 @@ def setup_app() -> FastAPI:
 
     app.mount("/static", StaticFiles(directory=asset_path()), name="static")
 
+    # Add tag descriptions to the OpenAPI schema
+    app.openapi_tags = tags_metadata
+
     for route in app.routes:
         if isinstance(route, APIRoute):
             route.operation_id = route.name  # in this case, 'read_items'

pkgs/clan-cli/clan_cli/webui/routers/clan_modules.py

@@ -9,12 +9,13 @@ from clan_cli.types import FlakeName
 from ..api_outputs import (
     ClanModulesResponse,
 )
+from ..tags import Tags
 
 log = logging.getLogger(__name__)
 router = APIRouter()
 
 
-@router.get("/api/{flake_name}/clan_modules")
+@router.get("/api/{flake_name}/clan_modules", tags=[Tags.modules])
 async def list_clan_modules(flake_name: FlakeName) -> ClanModulesResponse:
     module_names, error = get_clan_module_names(flake_name)
     if error is not None:

pkgs/clan-cli/clan_cli/webui/routers/flake.py

@@ -13,12 +13,14 @@ from clan_cli.webui.api_outputs import (
     FlakeAction,
     FlakeAttrResponse,
     FlakeCreateResponse,
+    FlakeListResponse,
     FlakeResponse,
 )
 
 from ...async_cmd import run
-from ...flakes import create
+from ...flakes import create, list_flakes
 from ...nix import nix_command, nix_flake_show
+from ..tags import Tags
 
 router = APIRouter()
 
@@ -45,13 +47,13 @@ async def get_attrs(url: AnyUrl | Path) -> list[str]:
 
 
 # TODO: Check for directory traversal
-@router.get("/api/flake/attrs")
+@router.get("/api/flake/attrs", tags=[Tags.flake])
 async def inspect_flake_attrs(url: AnyUrl | Path) -> FlakeAttrResponse:
     return FlakeAttrResponse(flake_attrs=await get_attrs(url))
 
 
 # TODO: Check for directory traversal
-@router.get("/api/flake")
+@router.get("/api/flake/inspect", tags=[Tags.flake])
 async def inspect_flake(
     url: AnyUrl | Path,
 ) -> FlakeResponse:
@@ -76,7 +78,15 @@ async def inspect_flake(
     return FlakeResponse(content=content, actions=actions)
 
 
-@router.post("/api/flake/create", status_code=status.HTTP_201_CREATED)
+@router.get("/api/flake/list", tags=[Tags.flake])
+async def list_all_flakes() -> FlakeListResponse:
+    flakes = list_flakes.list_flakes()
+    return FlakeListResponse(flakes=flakes)
+
+
+@router.post(
+    "/api/flake/create", tags=[Tags.flake], status_code=status.HTTP_201_CREATED
+)
 async def create_flake(
     args: Annotated[FlakeCreateInput, Body()],
 ) -> FlakeCreateResponse:

pkgs/clan-cli/clan_cli/webui/routers/machines.py

@@ -23,12 +23,13 @@ from ..api_outputs import (
     Status,
     VerifyMachineResponse,
 )
+from ..tags import Tags
 
 log = logging.getLogger(__name__)
 router = APIRouter()
 
 
-@router.get("/api/{flake_name}/machines")
+@router.get("/api/{flake_name}/machines", tags=[Tags.machine])
 async def list_machines(flake_name: FlakeName) -> MachinesResponse:
     machines = []
     for m in _list_machines(flake_name):
@@ -37,7 +38,7 @@ async def list_machines(flake_name: FlakeName) -> MachinesResponse:
     return MachinesResponse(machines=machines)
 
 
-@router.post("/api/{flake_name}/machines", status_code=201)
+@router.post("/api/{flake_name}/machines", tags=[Tags.machine], status_code=201)
 async def create_machine(
     flake_name: FlakeName, machine: Annotated[MachineCreate, Body()]
 ) -> MachineResponse:
@@ -45,19 +46,19 @@ async def create_machine(
     return MachineResponse(machine=Machine(name=machine.name, status=Status.UNKNOWN))
 
 
-@router.get("/api/{flake_name}/machines/{name}")
+@router.get("/api/{flake_name}/machines/{name}", tags=[Tags.machine])
 async def get_machine(flake_name: FlakeName, name: str) -> MachineResponse:
     log.error("TODO")
     return MachineResponse(machine=Machine(name=name, status=Status.UNKNOWN))
 
 
-@router.get("/api/{flake_name}/machines/{name}/config")
+@router.get("/api/{flake_name}/machines/{name}/config", tags=[Tags.machine])
 async def get_machine_config(flake_name: FlakeName, name: str) -> ConfigResponse:
     config = config_for_machine(flake_name, name)
     return ConfigResponse(config=config)
 
 
-@router.put("/api/{flake_name}/machines/{name}/config")
+@router.put("/api/{flake_name}/machines/{name}/config", tags=[Tags.machine])
 async def set_machine_config(
     flake_name: FlakeName, name: str, config: Annotated[dict, Body()]
 ) -> ConfigResponse:
@@ -65,13 +66,13 @@ async def set_machine_config(
     return ConfigResponse(config=config)
 
 
-@router.get("/api/{flake_name}/machines/{name}/schema")
+@router.get("/api/{flake_name}/machines/{name}/schema", tags=[Tags.machine])
 async def get_machine_schema(flake_name: FlakeName, name: str) -> SchemaResponse:
     schema = schema_for_machine(flake_name, name)
     return SchemaResponse(schema=schema)
 
 
-@router.put("/api/{flake_name}/machines/{name}/schema")
+@router.put("/api/{flake_name}/machines/{name}/schema", tags=[Tags.machine])
 async def set_machine_schema(
     flake_name: FlakeName, name: str, config: Annotated[dict, Body()]
 ) -> SchemaResponse:
@@ -79,7 +80,7 @@ async def set_machine_schema(
     return SchemaResponse(schema=schema)
 
 
-@router.get("/api/{flake_name}/machines/{name}/verify")
+@router.get("/api/{flake_name}/machines/{name}/verify", tags=[Tags.machine])
 async def get_verify_machine_config(
     flake_name: FlakeName, name: str
 ) -> VerifyMachineResponse:
@@ -88,7 +89,7 @@ async def get_verify_machine_config(
     return VerifyMachineResponse(success=success, error=error)
 
 
-@router.put("/api/{flake_name}/machines/{name}/verify")
+@router.put("/api/{flake_name}/machines/{name}/verify", tags=[Tags.machine])
 async def put_verify_machine_config(
     flake_name: FlakeName,
     name: str,

pkgs/clan-cli/clan_cli/webui/routers/root.py

@@ -6,13 +6,14 @@ from pathlib import Path
 from fastapi import APIRouter, Response
 
 from ..assets import asset_path
+from ..tags import Tags
 
 router = APIRouter()
 
 log = logging.getLogger(__name__)
 
 
-@router.get("/{path_name:path}")
+@router.get("/{path_name:path}", tags=[Tags.root])
 async def root(path_name: str) -> Response:
     if path_name == "":
         path_name = "index.html"

pkgs/clan-cli/clan_cli/webui/routers/vms.py

@@ -18,13 +18,14 @@ from ..api_outputs import (
     VmInspectResponse,
     VmStatusResponse,
 )
+from ..tags import Tags
 
 log = logging.getLogger(__name__)
 router = APIRouter()
 
 
 # TODO: Check for directory traversal
-@router.post("/api/vms/inspect")
+@router.post("/api/vms/inspect", tags=[Tags.vm])
 async def inspect_vm(
     flake_url: Annotated[AnyUrl | Path, Body()], flake_attr: Annotated[str, Body()]
 ) -> VmInspectResponse:
@@ -32,7 +33,7 @@ async def inspect_vm(
     return VmInspectResponse(config=config)
 
 
-@router.get("/api/vms/{uuid}/status")
+@router.get("/api/vms/{uuid}/status", tags=[Tags.vm])
 async def get_vm_status(uuid: UUID) -> VmStatusResponse:
     task = get_task(uuid)
     log.debug(msg=f"error: {task.error}, task.status: {task.status}")
@@ -40,7 +41,7 @@ async def get_vm_status(uuid: UUID) -> VmStatusResponse:
     return VmStatusResponse(status=task.status, error=error)
 
 
-@router.get("/api/vms/{uuid}/logs")
+@router.get("/api/vms/{uuid}/logs", tags=[Tags.vm])
 async def get_vm_logs(uuid: UUID) -> StreamingResponse:
     # Generator function that yields log lines as they are available
     def stream_logs() -> Iterator[str]:
@@ -55,7 +56,7 @@ async def get_vm_logs(uuid: UUID) -> StreamingResponse:
 
 
 # TODO: Check for directory traversal
-@router.post("/api/vms/create")
+@router.post("/api/vms/create", tags=[Tags.vm])
 async def create_vm(vm: Annotated[VmConfig, Body()]) -> VmCreateResponse:
     flake_attrs = await get_attrs(vm.flake_url)
     if vm.flake_attr not in flake_attrs:

pkgs/clan-cli/clan_cli/webui/tags.py

@@ -0,0 +1,41 @@
+from enum import Enum
+from typing import Any, Dict, List
+
+
+class Tags(Enum):
+    flake = "flake"
+    machine = "machine"
+    vm = "vm"
+    modules = "modules"
+    root = "root"
+
+    def __str__(self) -> str:
+        return self.value
+
+
+tags_metadata: List[Dict[str, Any]] = [
+    {
+        "name": str(Tags.flake),
+        "description": "Operations on a flake.",
+        "externalDocs": {
+            "description": "What is a flake?",
+            "url": "https://www.tweag.io/blog/2020-05-25-flakes/",
+        },
+    },
+    {
+        "name": str(Tags.machine),
+        "description": "Manage physical machines. Instances of a flake",
+    },
+    {
+        "name": str(Tags.vm),
+        "description": "Manage virtual machines. Instances of a flake",
+    },
+    {
+        "name": str(Tags.modules),
+        "description": "Manage cLAN modules of a flake",
+    },
+    {
+        "name": str(Tags.root),
+        "description": "This serves as the frontend delivery",
+    },
+]

pkgs/clan-cli/tests/fixtures_flakes.py

@@ -8,6 +8,8 @@ from pathlib import Path
 from typing import Iterator, NamedTuple
 
 import pytest
+from pydantic import AnyUrl
+from pydantic.tools import parse_obj_as
 from root import CLAN_CORE
 
 from clan_cli.dirs import nixpkgs_source
@@ -117,6 +119,16 @@ def test_flake_with_core(
     )
 
 
+@pytest.fixture
+def test_democlan_url(
+    monkeypatch: pytest.MonkeyPatch, temporary_home: Path
+) -> Iterator[AnyUrl]:
+    yield parse_obj_as(
+        AnyUrl,
+        "https://git.clan.lol/clan/democlan/archive/main.tar.gz",
+    )
+
+
 @pytest.fixture
 def test_flake_with_core_and_pass(
     monkeypatch: pytest.MonkeyPatch, temporary_home: Path

pkgs/clan-cli/tests/test_flake_api.py

@@ -8,6 +8,14 @@ from fixtures_flakes import FlakeForTest
 log = logging.getLogger(__name__)
 
 
+def test_list_flakes(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
+    response = api.get("/api/flake/list")
+    assert response.status_code == 200, "Failed to list flakes"
+    data = response.json()
+    print("Data: ", data)
+    assert data.get("flakes") == ["test_flake_with_core"]
+
+
 @pytest.mark.impure
 def test_inspect_ok(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
     params = {"url": str(test_flake_with_core.path)}
@@ -38,7 +46,7 @@ def test_inspect_err(api: TestClient) -> None:
 def test_inspect_flake(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
     params = {"url": str(test_flake_with_core.path)}
     response = api.get(
-        "/api/flake",
+        "/api/flake/inspect",
         params=params,
     )
     assert response.status_code == 200, "Failed to inspect vm"

pkgs/clan-cli/tests/test_vms_api_create.py

@@ -7,6 +7,7 @@ from api import TestClient
 from cli import Cli
 from fixtures_flakes import FlakeForTest, create_flake
 from httpx import SyncByteStream
+from pydantic import AnyUrl
 from root import CLAN_CORE
 
 from clan_cli.types import FlakeName
@@ -42,7 +43,7 @@ def remote_flake_with_vm_without_secrets(
     )
 
 
-def generic_create_vm_test(api: TestClient, flake: Path, vm: str) -> None:
+def generic_create_vm_test(api: TestClient, flake: Path | AnyUrl, vm: str) -> None:
     print(f"flake_url: {flake} ")
     response = api.post(
         "/api/vms/create",
@@ -113,3 +114,18 @@ def test_create_remote(
     generic_create_vm_test(
         api, remote_flake_with_vm_without_secrets.path, "vm_without_secrets"
     )
+
+
+# TODO: We need a test that creates the same VM twice, and checks that the second time it fails
+
+
+# TODO: Democlan needs a machine called testVM, which is headless and gets executed by this test below
+# pytest -n0 -s tests/test_vms_api_create.py::test_create_from_democlan
+# @pytest.mark.skipif(not os.path.exists("/dev/kvm"), reason="Requires KVM")
+# @pytest.mark.impure
+# def test_create_from_democlan(
+#     api: TestClient,
+#     test_democlan_url: AnyUrl) -> None:
+#         generic_create_vm_test(
+#             api, test_democlan_url, "defaultVM"
+#         )