merge main

This commit is contained in:
Johannes Kirschbauer
2023-11-04 09:15:15 +01:00
39 changed files with 430 additions and 359 deletions

View File

@@ -0,0 +1,24 @@
(import ../lib/container-test.nix) ({ pkgs, ... }: {
name = "secrets";
nodes.machine = { self, ... }: {
imports = [
self.clanModules.deltachat
self.nixosModules.clanCore
{
clanCore.machineName = "machine";
clanCore.clanDir = ./.;
}
];
};
testScript = ''
start_all()
machine.wait_for_unit("maddy")
# imap
machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 143")
# smtp submission
machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 587")
# smtp
machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 25")
'';
})

View File

@@ -15,6 +15,7 @@
# import our test
secrets = import ./secrets nixosTestArgs;
container = import ./container nixosTestArgs;
deltachat = import ./deltachat nixosTestArgs;
};
schemaTests = pkgs.callPackages ./schemas.nix {
inherit self;

View File

@@ -1,8 +1,8 @@
{ extraPythonPackages, buildPythonApplication, self, setuptools, util-linux, systemd }:
{ extraPythonPackages, python3Packages, buildPythonApplication, setuptools, util-linux, systemd }:
buildPythonApplication {
pname = "test-driver";
version = "0.0.1";
propagatedBuildInputs = [ util-linux systemd ] ++ extraPythonPackages self;
propagatedBuildInputs = [ util-linux systemd ] ++ extraPythonPackages python3Packages;
nativeBuildInputs = [ setuptools ];
format = "pyproject";
src = ./.;

View File

@@ -21,11 +21,6 @@ line-length = 88
select = ["E", "F", "I", "U", "N"]
ignore = ["E501"]
[tool.black]
line-length = 88
target-version = ['py39']
include = '\.pyi?$'
[tool.mypy]
python_version = "3.10"
warn_redundant_casts = true

140
clanModules/deltachat.nix Normal file
View File

@@ -0,0 +1,140 @@
{ config, pkgs, ... }: {
networking.firewall.interfaces."zt+".allowedTCPPorts = [ 25 ]; # smtp with other hosts
environment.systemPackages = [ pkgs.deltachat-desktop ];
services.maddy = {
enable = true;
primaryDomain = "${config.clanCore.machineName}.local";
config = ''
# Minimal configuration with TLS disabled, adapted from upstream example
# configuration here https://github.com/foxcpp/maddy/blob/master/maddy.conf
# Do not use this in unencrypted networks!
auth.pass_table local_authdb {
table sql_table {
driver sqlite3
dsn credentials.db
table_name passwords
}
}
storage.imapsql local_mailboxes {
driver sqlite3
dsn imapsql.db
}
table.chain local_rewrites {
optional_step regexp "(.+)\+(.+)@(.+)" "$1@$3"
optional_step static {
entry postmaster postmaster@$(primary_domain)
}
optional_step file /etc/maddy/aliases
}
msgpipeline local_routing {
destination postmaster $(local_domains) {
modify {
replace_rcpt &local_rewrites
}
deliver_to &local_mailboxes
}
default_destination {
reject 550 5.1.1 "User doesn't exist"
}
}
smtp tcp://[::]:25 {
limits {
all rate 20 1s
all concurrency 10
}
dmarc yes
check {
require_mx_record
dkim
spf
}
source $(local_domains) {
reject 501 5.1.8 "Use Submission for outgoing SMTP"
}
default_source {
destination postmaster $(local_domains) {
deliver_to &local_routing
}
default_destination {
reject 550 5.1.1 "User doesn't exist"
}
}
}
submission tcp://[::1]:587 {
limits {
all rate 50 1s
}
auth &local_authdb
source $(local_domains) {
check {
authorize_sender {
prepare_email &local_rewrites
user_to_email identity
}
}
destination postmaster $(local_domains) {
deliver_to &local_routing
}
default_destination {
modify {
dkim $(primary_domain) $(local_domains) default
}
deliver_to &remote_queue
}
}
default_source {
reject 501 5.1.8 "Non-local sender domain"
}
}
target.remote outbound_delivery {
limits {
destination rate 20 1s
destination concurrency 10
}
mx_auth {
dane
mtasts {
cache fs
fs_dir mtasts_cache/
}
local_policy {
min_tls_level encrypted
min_mx_level none
}
}
}
target.queue remote_queue {
target &outbound_delivery
autogenerated_msg_domain $(primary_domain)
bounce {
destination postmaster $(local_domains) {
deliver_to &local_routing
}
default_destination {
reject 550 5.0.0 "Refusing to send DSNs to non-local addresses"
}
}
}
imap tcp://[::1]:143 {
auth &local_authdb
storage &local_mailboxes
}
'';
ensureAccounts = [
"user@${config.clanCore.machineName}.local"
];
ensureCredentials = {
"user@${config.clanCore.machineName}.local".passwordFile = pkgs.writeText "dummy" "foobar";
};
};
}

View File

@@ -1,3 +0,0 @@
{ pkgs, ... }: {
environment.systemPackages = [ pkgs.dino ];
}

View File

@@ -1,199 +0,0 @@
{ config
, ...
}: {
services.ejabberd = {
enable = true;
configFile = "/etc/ejabberd.yml";
};
environment.etc."ejabberd.yml" = {
user = "ejabberd";
mode = "0600";
text = ''
loglevel: 4
default_db: sql
new_sql_schema: true
sql_type: sqlite
sql_database: "/var/lib/ejabberd/db.sqlite"
hosts:
- ${config.clanCore.machineName}.local
listen:
-
port: 5222
ip: "::1"
module: ejabberd_c2s
max_stanza_size: 262144
shaper: c2s_shaper
access: c2s
starttls_required: false
-
port: 5269
ip: "::"
module: ejabberd_s2s_in
max_stanza_size: 524288
auth_method: [anonymous]
anonymous_protocol: login_anon
acl:
local:
user_regexp: ""
loopback:
ip:
- 127.0.0.0/8
- ::1/128
access_rules:
local:
allow: local
c2s:
deny: blocked
allow: all
s2s:
- allow
announce:
allow: admin
configure:
allow: admin
muc_create:
allow: all
pubsub_createnode:
allow: local
trusted_network:
allow: loopback
api_permissions:
"console commands":
from:
- ejabberd_ctl
who: all
what: "*"
"admin access":
who:
access:
allow:
acl: loopback
acl: admin
oauth:
scope: "ejabberd:admin"
access:
allow:
acl: loopback
acl: admin
what:
- "*"
- "!stop"
- "!start"
"public commands":
who:
ip: 127.0.0.1/8
what:
- status
- connected_users_number
shaper:
normal: 1000
fast: 50000
shaper_rules:
max_user_sessions: 10
max_user_offline_messages:
5000: admin
100: all
c2s_shaper:
none: admin
normal: all
s2s_shaper: fast
modules:
mod_adhoc: {}
mod_admin_extra: {}
mod_announce:
access: announce
mod_avatar: {}
mod_blocking: {}
mod_bosh: {}
mod_caps: {}
mod_carboncopy: {}
mod_client_state: {}
mod_configure: {}
mod_disco: {}
mod_fail2ban: {}
mod_http_api: {}
mod_http_upload:
put_url: https://@HOST@:5443/upload
mod_last: {}
mod_mam:
## Mnesia is limited to 2GB, better to use an SQL backend
## For small servers SQLite is a good fit and is very easy
## to configure. Uncomment this when you have SQL configured:
## db_type: sql
assume_mam_usage: true
default: always
mod_mqtt:
access_publish:
"homeassistant/#":
- allow: hass_publisher
- deny
"#":
- deny
access_subscribe:
"homeassistant/#":
- allow: hass_subscriber
- deny
"#":
- deny
mod_muc:
host: "muc.@HOST@"
access:
- allow
access_admin:
- allow: admin
access_create: muc_create
access_persistent: muc_create
access_mam:
- allow
default_room_options:
mam: true
mod_muc_admin: {}
mod_offline:
access_max_user_messages: max_user_offline_messages
mod_ping: {}
mod_privacy: {}
mod_private: {}
mod_proxy65:
access: local
max_connections: 5
mod_pubsub:
access_createnode: pubsub_createnode
plugins:
- flat
- pep
force_node_config:
## Avoid buggy clients to make their bookmarks public
storage:bookmarks:
access_model: whitelist
mod_push: {}
mod_push_keepalive: {}
mod_register:
## Only accept registration requests from the "trusted"
## network (see access_rules section above).
## Think twice before enabling registration from any
## address. See the Jabber SPAM Manifesto for details:
## https://github.com/ge0rg/jabber-spam-fighting-manifesto
ip_access: trusted_network
mod_roster:
versioning: true
mod_s2s_dialback: {}
mod_shared_roster: {}
mod_stream_mgmt:
resend_on_timeout: if_offline
mod_vcard: {}
mod_vcard_xupdate: {}
mod_version:
show_os: false
'';
};
networking.firewall.allowedTCPPorts = [
5269 # xmpp-server
];
}

View File

@@ -8,8 +8,7 @@
];
})
(builtins.readDir ./diskLayouts);
ejabberd = ./ejabberd.nix;
dino = ./dino.nix;
deltachat = ./deltachat.nix;
xfce = ./xfce.nix;
};
}

View File

@@ -10,6 +10,11 @@ Welcome to our website template repository! This template is designed to help yo
**Dependency Management**: We use the [Nix package manager](https://nixos.org/) to manage dependencies and ensure reproducibility, making your development process more robust.
## Supported Operating Systems
- Linux
- macOS
# Getting Started with the Development Environment
Let's get your development environment up and running:
@@ -28,11 +33,20 @@ Let's get your development environment up and running:
curl -sfL https://direnv.net/install.sh | bash
```
3. **Clone the Repository and Navigate**:
3. **Add direnv to your shell**:
- Direnv needs to [hook into your shell](https://direnv.net/docs/hook.html) to work.
You can do this by executing following command:
```bash
echo 'eval "$(direnv hook zsh)"' >> ~/.zshrc && echo 'eval "$(direnv hook bash)"' >> ~/.bashrc && eval "$SHELL"
```
4. **Clone the Repository and Navigate**:
- Clone this repository and navigate to it.
4. **Allow .envrc**:
5. **Allow .envrc**:
- When you enter the directory, you'll receive an error message like this:
```bash
@@ -40,7 +54,7 @@ Let's get your development environment up and running:
```
- Execute `direnv allow` to automatically execute the shell script `.envrc` when entering the directory.
5. **Build the Backend**:
6. **Build the Backend**:
- Go to the `pkgs/clan-cli` directory and execute:
```bash
@@ -48,7 +62,7 @@ Let's get your development environment up and running:
```
- Wait for the backend to build.
6. **Start the Backend Server**:
7. **Start the Backend Server**:
- To start the backend server, execute:
```bash
@@ -56,7 +70,7 @@ Let's get your development environment up and running:
```
- The server will automatically restart if any Python files change.
7. **Build the Frontend**:
8. **Build the Frontend**:
- In a different shell, navigate to the `pkgs/ui` directory and execute:
```bash
@@ -64,7 +78,7 @@ Let's get your development environment up and running:
```
- Wait for the frontend to build.
8. **Start the Frontend**:
9. **Start the Frontend**:
- To start the frontend, execute:
```bash
npm run dev
@@ -194,4 +208,4 @@ To make the most of this template:
- Set the option to "Delete pull request branch after merge by default."
- Also, set the default merge style to "Rebase then create merge commit."
With this template, you're well-equipped to build and collaborate on high-quality websites efficiently. Happy coding!
With this template, you're well-equipped to build and collaborate on high-quality websites efficiently. Happy coding!.

32
flake.lock generated
View File

@@ -7,11 +7,11 @@
]
},
"locked": {
"lastModified": 1697995812,
"narHash": "sha256-UDlK6p/6vAiVOQ92PR0ySDZBS3yiryrlJpSOw3b9Ito=",
"lastModified": 1698422527,
"narHash": "sha256-SDu3Xg263t3oXIyTaH0buOvFnKIDeZsvKDBtOz+jRbs=",
"owner": "nix-community",
"repo": "disko",
"rev": "4122a18340094151d7911e838237ec7627f0d0c5",
"rev": "944d338d24a9d043a3f7461c30ee6cfe4f9cca30",
"type": "github"
},
"original": {
@@ -27,11 +27,11 @@
]
},
"locked": {
"lastModified": 1696343447,
"narHash": "sha256-B2xAZKLkkeRFG5XcHHSXXcP7To9Xzr59KXeZiRf4vdQ=",
"lastModified": 1698882062,
"narHash": "sha256-HkhafUayIqxXyHH1X8d9RDl1M2CkFgZLjKD3MzabiEo=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "c9afaba3dfa4085dbd2ccb38dfade5141e33d9d4",
"rev": "8c9fa2545007b49a5db5f650ae91f227672c3877",
"type": "github"
},
"original": {
@@ -98,16 +98,16 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1696051733,
"narHash": "sha256-fEC8/6wJOWgCSvBjPwMBdaYtp57OUfQd3dJgp0D/It4=",
"lastModified": 1699007274,
"narHash": "sha256-m0NH2trnW8cOhona6m3hWkeDZ28BV/wAGPd/YWik23g=",
"owner": "Mic92",
"repo": "nixpkgs",
"rev": "c3bd4f19ef0062d4462444aa413e26c917187ae9",
"rev": "fcb19bae00e9d3fd5ecf4a1f80cf33248bf7f714",
"type": "github"
},
"original": {
"owner": "Mic92",
"ref": "fakeroot",
"ref": "deltachat",
"repo": "nixpkgs",
"type": "github"
}
@@ -131,11 +131,11 @@
"nixpkgs-stable": []
},
"locked": {
"lastModified": 1697943852,
"narHash": "sha256-DaBxUPaZhQ3yLCmAATshYB7qo7NwcMvSFWz9T3bjYYY=",
"lastModified": 1699021419,
"narHash": "sha256-oy2j2OHXYcckifASMeZzpmbDLSvobMGt0V/RvoDotF4=",
"owner": "Mic92",
"repo": "sops-nix",
"rev": "30a0ba4a20703b4bfe047fe5def1fc24978e322c",
"rev": "275b28593ef3a1b9d05b6eeda3ddce2f45f5c06f",
"type": "github"
},
"original": {
@@ -151,11 +151,11 @@
]
},
"locked": {
"lastModified": 1697388351,
"narHash": "sha256-63N2eBpKaziIy4R44vjpUu8Nz5fCJY7okKrkixvDQmY=",
"lastModified": 1698438538,
"narHash": "sha256-AWxaKTDL3MtxaVTVU5lYBvSnlspOS0Fjt8GxBgnU0Do=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "aae39f64f5ecbe89792d05eacea5cb241891292a",
"rev": "5deb8dc125a9f83b65ca86cf0c8167c46593e0b1",
"type": "github"
},
"original": {

View File

@@ -5,9 +5,9 @@
nixConfig.extra-trusted-public-keys = [ "cache.clan.lol-1:3KztgSAB5R1M+Dz7vzkBGzXdodizbgLXGXKXlcQLA28=" ];
inputs = {
#nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
# https://github.com/NixOS/nixpkgs/pull/257462
nixpkgs.url = "github:Mic92/nixpkgs/fakeroot";
#nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
# https://github.com/NixOS/nixpkgs/pull/265024
nixpkgs.url = "github:Mic92/nixpkgs/deltachat";
floco.url = "github:aakropotkin/floco";
floco.inputs.nixpkgs.follows = "nixpkgs";
disko.url = "github:nix-community/disko";

View File

@@ -46,7 +46,7 @@
"-eucx"
''
${lib.getExe pkgs.ruff} --fix "$@"
${lib.getExe pkgs.black} "$@"
${lib.getExe pkgs.ruff} format "$@"
''
"--" # this argument is ignored by bash
];

View File

@@ -2,7 +2,7 @@ import argparse
import logging
import sys
from types import ModuleType
from typing import Optional
from typing import Any, Optional, Sequence
from . import config, flakes, join, machines, secrets, vms, webui
from .custom_logger import setup_logging
@@ -17,6 +17,24 @@ except ImportError:
pass
class AppendOptionAction(argparse.Action):
def __init__(self, option_strings: str, dest: str, **kwargs: Any) -> None:
super().__init__(option_strings, dest, **kwargs)
def __call__(
self,
parser: argparse.ArgumentParser,
namespace: argparse.Namespace,
values: str | Sequence[str] | None,
option_string: Optional[str] = None,
) -> None:
lst = getattr(namespace, self.dest)
lst.append("--option")
assert isinstance(values, list), "values must be a list"
lst.append(values[0])
lst.append(values[1])
def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog=prog, description="cLAN tool")
@@ -26,6 +44,15 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser:
action="store_true",
)
parser.add_argument(
"--option",
help="Nix option to set",
nargs=2,
metavar=("name", "value"),
action=AppendOptionAction,
default=[],
)
subparsers = parser.add_subparsers()
parser_flake = subparsers.add_parser(

View File

@@ -2,7 +2,7 @@
import argparse
from .create import register_create_parser
from .list import register_list_parser
from .list_flakes import register_list_parser
# takes a (sub)parser and configures it

View File

@@ -1,11 +1,15 @@
import logging
from pathlib import Path
from typing import NewType
from typing import NewType, Union
from pydantic import AnyUrl
log = logging.getLogger(__name__)
FlakeName = NewType("FlakeName", str)
FlakeUrl = Union[AnyUrl, Path]
def validate_path(base_dir: Path, value: Path) -> Path:
user_path = (base_dir / value).resolve()

View File

@@ -11,29 +11,23 @@ from typing import Iterator
from uuid import UUID
from ..dirs import clan_flakes_dir, specific_flake_dir
from ..errors import ClanError
from ..nix import nix_build, nix_config, nix_eval, nix_shell
from ..task_manager import BaseTask, Command, create_task
from ..types import validate_path
from .inspect import VmConfig, inspect_vm
def is_path_or_url(s: str) -> str | None:
# check if s is a valid path
if os.path.exists(s):
return "path"
# check if s is a valid URL
elif re.match(r"^https?://[a-zA-Z0-9.-]+/[a-zA-Z0-9.-]+", s):
return "URL"
# otherwise, return None
else:
return None
def is_flake_url(s: str) -> bool:
if re.match(r"^http.?://[a-zA-Z0-9.-]+/[a-zA-Z0-9.-]+", s) is not None:
return True
return False
class BuildVmTask(BaseTask):
def __init__(self, uuid: UUID, vm: VmConfig) -> None:
def __init__(self, uuid: UUID, vm: VmConfig, nix_options: list[str] = []) -> None:
super().__init__(uuid, num_cmds=7)
self.vm = vm
self.nix_options = nix_options
def get_vm_create_info(self, cmds: Iterator[Command]) -> dict:
config = nix_config()
@@ -47,6 +41,7 @@ class BuildVmTask(BaseTask):
[
f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.system.clan.vm.create'
]
+ self.nix_options
)
)
vm_json = "".join(cmd.stdout).strip()
@@ -57,7 +52,7 @@ class BuildVmTask(BaseTask):
def get_clan_name(self, cmds: Iterator[Command]) -> str:
clan_dir = self.vm.flake_url
cmd = next(cmds)
cmd.run(nix_eval([f"{clan_dir}#clanInternals.clanName"]))
cmd.run(nix_eval([f"{clan_dir}#clanInternals.clanName"]) + self.nix_options)
clan_name = cmd.stdout[0].strip().strip('"')
return clan_name
@@ -93,12 +88,8 @@ class BuildVmTask(BaseTask):
) # TODO do this in the clanCore module
env["SECRETS_DIR"] = str(secrets_dir)
res = is_path_or_url(str(self.vm.flake_url))
if res is None:
raise ClanError(
f"flake_url must be a valid path or URL, got {self.vm.flake_url}"
)
elif res == "path": # Only generate secrets for local clans
# Only generate secrets for local clans
if not is_flake_url(str(self.vm.flake_url)):
cmd = next(cmds)
if Path(self.vm.flake_url).is_dir():
cmd.run(
@@ -151,27 +142,44 @@ class BuildVmTask(BaseTask):
"console=tty0",
]
qemu_command = [
# fmt: off
"qemu-kvm",
"-name", machine,
"-m", f'{vm_config["memorySize"]}M',
"-smp", str(vm_config["cores"]),
"-device", "virtio-rng-pci",
"-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0",
"-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
"-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
"-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report',
"-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
"-device", "virtio-keyboard",
"-vga", "virtio",
"-name",
machine,
"-m",
f'{vm_config["memorySize"]}M',
"-smp",
str(vm_config["cores"]),
"-device",
"virtio-rng-pci",
"-net",
"nic,netdev=user.0,model=virtio",
"-netdev",
"user,id=user.0",
"-virtfs",
"local,path=/nix/store,security_model=none,mount_tag=nix-store",
"-virtfs",
f"local,path={xchg_dir},security_model=none,mount_tag=shared",
"-virtfs",
f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
"-virtfs",
f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
"-drive",
f"cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report",
"-device",
"virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
"-device",
"virtio-keyboard",
"-vga",
"virtio",
"-usb",
"-device", "usb-tablet,bus=usb-bus.0",
"-kernel", f'{vm_config["toplevel"]}/kernel',
"-initrd", vm_config["initrd"],
"-append", " ".join(cmdline),
# fmt: on
"-device",
"usb-tablet,bus=usb-bus.0",
"-kernel",
f'{vm_config["toplevel"]}/kernel',
"-initrd",
vm_config["initrd"],
"-append",
" ".join(cmdline),
]
if not self.vm.graphics:
qemu_command.append("-nographic")
@@ -179,15 +187,17 @@ class BuildVmTask(BaseTask):
cmd.run(nix_shell(["qemu"], qemu_command))
def create_vm(vm: VmConfig) -> BuildVmTask:
return create_task(BuildVmTask, vm)
def create_vm(vm: VmConfig, nix_options: list[str] = []) -> BuildVmTask:
return create_task(BuildVmTask, vm, nix_options)
def create_command(args: argparse.Namespace) -> None:
clan_dir = specific_flake_dir(args.flake)
vm = asyncio.run(inspect_vm(flake_url=clan_dir, flake_attr=args.machine))
flake_url = args.flake
if not is_flake_url(args.flake):
flake_url = specific_flake_dir(args.flake)
vm = asyncio.run(inspect_vm(flake_url=flake_url, flake_attr=args.machine))
task = create_vm(vm)
task = create_vm(vm, args.option)
for line in task.log_lines():
print(line, end="")

View File

@@ -70,6 +70,10 @@ class FlakeAction(BaseModel):
uri: str
class FlakeListResponse(BaseModel):
flakes: list[str]
class FlakeCreateResponse(BaseModel):
cmd_out: Dict[str, CmdOut]

View File

@@ -9,6 +9,7 @@ from ..errors import ClanError
from .assets import asset_path
from .error_handlers import clan_error_handler
from .routers import clan_modules, flake, health, machines, root, vms
from .tags import tags_metadata
origins = [
"http://localhost:3000",
@@ -39,6 +40,9 @@ def setup_app() -> FastAPI:
app.mount("/static", StaticFiles(directory=asset_path()), name="static")
# Add tag descriptions to the OpenAPI schema
app.openapi_tags = tags_metadata
for route in app.routes:
if isinstance(route, APIRoute):
route.operation_id = route.name # in this case, 'read_items'

View File

@@ -9,7 +9,8 @@ from ..errors import ClanError
log = logging.getLogger(__name__)
def clan_error_handler(request: Request, exc: ClanError) -> JSONResponse:
def clan_error_handler(request: Request, exc: Exception) -> JSONResponse:
assert isinstance(exc, ClanError)
log.error("ClanError: %s", exc)
detail = [
{

View File

@@ -9,12 +9,13 @@ from clan_cli.types import FlakeName
from ..api_outputs import (
ClanModulesResponse,
)
from ..tags import Tags
log = logging.getLogger(__name__)
router = APIRouter()
@router.get("/api/{flake_name}/clan_modules")
@router.get("/api/{flake_name}/clan_modules", tags=[Tags.modules])
async def list_clan_modules(flake_name: FlakeName) -> ClanModulesResponse:
module_names, error = get_clan_module_names(flake_name)
if error is not None:

View File

@@ -13,12 +13,14 @@ from clan_cli.webui.api_outputs import (
FlakeAction,
FlakeAttrResponse,
FlakeCreateResponse,
FlakeListResponse,
FlakeResponse,
)
from ...async_cmd import run
from ...flakes import create
from ...flakes import create, list_flakes
from ...nix import nix_command, nix_flake_show
from ..tags import Tags
router = APIRouter()
@@ -45,13 +47,13 @@ async def get_attrs(url: AnyUrl | Path) -> list[str]:
# TODO: Check for directory traversal
@router.get("/api/flake/attrs")
@router.get("/api/flake/attrs", tags=[Tags.flake])
async def inspect_flake_attrs(url: AnyUrl | Path) -> FlakeAttrResponse:
return FlakeAttrResponse(flake_attrs=await get_attrs(url))
# TODO: Check for directory traversal
@router.get("/api/flake")
@router.get("/api/flake/inspect", tags=[Tags.flake])
async def inspect_flake(
url: AnyUrl | Path,
) -> FlakeResponse:
@@ -76,7 +78,15 @@ async def inspect_flake(
return FlakeResponse(content=content, actions=actions)
@router.post("/api/flake/create", status_code=status.HTTP_201_CREATED)
@router.get("/api/flake/list", tags=[Tags.flake])
async def list_all_flakes() -> FlakeListResponse:
flakes = list_flakes.list_flakes()
return FlakeListResponse(flakes=flakes)
@router.post(
"/api/flake/create", tags=[Tags.flake], status_code=status.HTTP_201_CREATED
)
async def create_flake(
args: Annotated[FlakeCreateInput, Body()],
) -> FlakeCreateResponse:

View File

@@ -23,12 +23,13 @@ from ..api_outputs import (
Status,
VerifyMachineResponse,
)
from ..tags import Tags
log = logging.getLogger(__name__)
router = APIRouter()
@router.get("/api/{flake_name}/machines")
@router.get("/api/{flake_name}/machines", tags=[Tags.machine])
async def list_machines(flake_name: FlakeName) -> MachinesResponse:
machines = []
for m in _list_machines(flake_name):
@@ -37,7 +38,7 @@ async def list_machines(flake_name: FlakeName) -> MachinesResponse:
return MachinesResponse(machines=machines)
@router.post("/api/{flake_name}/machines", status_code=201)
@router.post("/api/{flake_name}/machines", tags=[Tags.machine], status_code=201)
async def create_machine(
flake_name: FlakeName, machine: Annotated[MachineCreate, Body()]
) -> MachineResponse:
@@ -45,19 +46,19 @@ async def create_machine(
return MachineResponse(machine=Machine(name=machine.name, status=Status.UNKNOWN))
@router.get("/api/{flake_name}/machines/{name}")
@router.get("/api/{flake_name}/machines/{name}", tags=[Tags.machine])
async def get_machine(flake_name: FlakeName, name: str) -> MachineResponse:
log.error("TODO")
return MachineResponse(machine=Machine(name=name, status=Status.UNKNOWN))
@router.get("/api/{flake_name}/machines/{name}/config")
@router.get("/api/{flake_name}/machines/{name}/config", tags=[Tags.machine])
async def get_machine_config(flake_name: FlakeName, name: str) -> ConfigResponse:
config = config_for_machine(flake_name, name)
return ConfigResponse(config=config)
@router.put("/api/{flake_name}/machines/{name}/config")
@router.put("/api/{flake_name}/machines/{name}/config", tags=[Tags.machine])
async def set_machine_config(
flake_name: FlakeName, name: str, config: Annotated[dict, Body()]
) -> ConfigResponse:
@@ -65,13 +66,13 @@ async def set_machine_config(
return ConfigResponse(config=config)
@router.get("/api/{flake_name}/machines/{name}/schema")
@router.get("/api/{flake_name}/machines/{name}/schema", tags=[Tags.machine])
async def get_machine_schema(flake_name: FlakeName, name: str) -> SchemaResponse:
schema = schema_for_machine(flake_name, name)
return SchemaResponse(schema=schema)
@router.put("/api/{flake_name}/machines/{name}/schema")
@router.put("/api/{flake_name}/machines/{name}/schema", tags=[Tags.machine])
async def set_machine_schema(
flake_name: FlakeName, name: str, config: Annotated[dict, Body()]
) -> SchemaResponse:
@@ -79,7 +80,7 @@ async def set_machine_schema(
return SchemaResponse(schema=schema)
@router.get("/api/{flake_name}/machines/{name}/verify")
@router.get("/api/{flake_name}/machines/{name}/verify", tags=[Tags.machine])
async def get_verify_machine_config(
flake_name: FlakeName, name: str
) -> VerifyMachineResponse:
@@ -88,7 +89,7 @@ async def get_verify_machine_config(
return VerifyMachineResponse(success=success, error=error)
@router.put("/api/{flake_name}/machines/{name}/verify")
@router.put("/api/{flake_name}/machines/{name}/verify", tags=[Tags.machine])
async def put_verify_machine_config(
flake_name: FlakeName,
name: str,

View File

@@ -6,13 +6,14 @@ from pathlib import Path
from fastapi import APIRouter, Response
from ..assets import asset_path
from ..tags import Tags
router = APIRouter()
log = logging.getLogger(__name__)
@router.get("/{path_name:path}")
@router.get("/{path_name:path}", tags=[Tags.root])
async def root(path_name: str) -> Response:
if path_name == "":
path_name = "index.html"

View File

@@ -18,13 +18,14 @@ from ..api_outputs import (
VmInspectResponse,
VmStatusResponse,
)
from ..tags import Tags
log = logging.getLogger(__name__)
router = APIRouter()
# TODO: Check for directory traversal
@router.post("/api/vms/inspect")
@router.post("/api/vms/inspect", tags=[Tags.vm])
async def inspect_vm(
flake_url: Annotated[AnyUrl | Path, Body()], flake_attr: Annotated[str, Body()]
) -> VmInspectResponse:
@@ -32,7 +33,7 @@ async def inspect_vm(
return VmInspectResponse(config=config)
@router.get("/api/vms/{uuid}/status")
@router.get("/api/vms/{uuid}/status", tags=[Tags.vm])
async def get_vm_status(uuid: UUID) -> VmStatusResponse:
task = get_task(uuid)
log.debug(msg=f"error: {task.error}, task.status: {task.status}")
@@ -40,7 +41,7 @@ async def get_vm_status(uuid: UUID) -> VmStatusResponse:
return VmStatusResponse(status=task.status, error=error)
@router.get("/api/vms/{uuid}/logs")
@router.get("/api/vms/{uuid}/logs", tags=[Tags.vm])
async def get_vm_logs(uuid: UUID) -> StreamingResponse:
# Generator function that yields log lines as they are available
def stream_logs() -> Iterator[str]:
@@ -55,7 +56,7 @@ async def get_vm_logs(uuid: UUID) -> StreamingResponse:
# TODO: Check for directory traversal
@router.post("/api/vms/create")
@router.post("/api/vms/create", tags=[Tags.vm])
async def create_vm(vm: Annotated[VmConfig, Body()]) -> VmCreateResponse:
flake_attrs = await get_attrs(vm.flake_url)
if vm.flake_attr not in flake_attrs:

View File

@@ -0,0 +1,41 @@
from enum import Enum
from typing import Any, Dict, List
class Tags(Enum):
flake = "flake"
machine = "machine"
vm = "vm"
modules = "modules"
root = "root"
def __str__(self) -> str:
return self.value
tags_metadata: List[Dict[str, Any]] = [
{
"name": str(Tags.flake),
"description": "Operations on a flake.",
"externalDocs": {
"description": "What is a flake?",
"url": "https://www.tweag.io/blog/2020-05-25-flakes/",
},
},
{
"name": str(Tags.machine),
"description": "Manage physical machines. Instances of a flake",
},
{
"name": str(Tags.vm),
"description": "Manage virtual machines. Instances of a flake",
},
{
"name": str(Tags.modules),
"description": "Manage cLAN modules of a flake",
},
{
"name": str(Tags.root),
"description": "This serves as the frontend delivery",
},
]

View File

@@ -58,25 +58,3 @@ line-length = 88
select = [ "E", "F", "I", "N"]
ignore = [ "E501" ]
[tool.black]
line-length = 88
target-version = [ "py310" ]
include = "\\.pyi?$"
exclude = '''
/(
\.git
| \.hg
| \.mypy_cache
| \.tox
| \.venv
| _build
| buck-out
| build
| dist
# The following are specific to Black, you probably don't want those.
| blib2to3
| tests/data
| profiling
)/
'''

View File

@@ -8,6 +8,8 @@ from pathlib import Path
from typing import Iterator, NamedTuple
import pytest
from pydantic import AnyUrl
from pydantic.tools import parse_obj_as
from root import CLAN_CORE
from clan_cli.dirs import nixpkgs_source
@@ -117,6 +119,16 @@ def test_flake_with_core(
)
@pytest.fixture
def test_democlan_url(
monkeypatch: pytest.MonkeyPatch, temporary_home: Path
) -> Iterator[AnyUrl]:
yield parse_obj_as(
AnyUrl,
"https://git.clan.lol/clan/democlan/archive/main.tar.gz",
)
@pytest.fixture
def test_flake_with_core_and_pass(
monkeypatch: pytest.MonkeyPatch, temporary_home: Path

View File

@@ -8,6 +8,15 @@ from fixtures_flakes import FlakeForTest
log = logging.getLogger(__name__)
@pytest.mark.impure
def test_list_flakes(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
response = api.get("/api/flake/list")
assert response.status_code == 200, "Failed to list flakes"
data = response.json()
print("Data: ", data)
assert data.get("flakes") == ["test_flake_with_core"]
@pytest.mark.impure
def test_inspect_ok(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
params = {"url": str(test_flake_with_core.path)}
@@ -38,7 +47,7 @@ def test_inspect_err(api: TestClient) -> None:
def test_inspect_flake(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
params = {"url": str(test_flake_with_core.path)}
response = api.get(
"/api/flake",
"/api/flake/inspect",
params=params,
)
assert response.status_code == 200, "Failed to inspect vm"

View File

@@ -7,6 +7,7 @@ from api import TestClient
from cli import Cli
from fixtures_flakes import FlakeForTest, create_flake
from httpx import SyncByteStream
from pydantic import AnyUrl
from root import CLAN_CORE
from clan_cli.types import FlakeName
@@ -42,7 +43,7 @@ def remote_flake_with_vm_without_secrets(
)
def generic_create_vm_test(api: TestClient, flake: Path, vm: str) -> None:
def generic_create_vm_test(api: TestClient, flake: Path | AnyUrl, vm: str) -> None:
print(f"flake_url: {flake} ")
response = api.post(
"/api/vms/create",
@@ -113,3 +114,18 @@ def test_create_remote(
generic_create_vm_test(
api, remote_flake_with_vm_without_secrets.path, "vm_without_secrets"
)
# TODO: We need a test that creates the same VM twice, and checks that the second time it fails
# TODO: Democlan needs a machine called testVM, which is headless and gets executed by this test below
# pytest -n0 -s tests/test_vms_api_create.py::test_create_from_democlan
# @pytest.mark.skipif(not os.path.exists("/dev/kvm"), reason="Requires KVM")
# @pytest.mark.impure
# def test_create_from_democlan(
# api: TestClient,
# test_democlan_url: AnyUrl) -> None:
# generic_create_vm_test(
# api, test_democlan_url, "defaultVM"
# )

View File

@@ -1,5 +1,5 @@
{ fetchzip }:
fetchzip {
url = "https://git.clan.lol/api/packages/clan/generic/ui/01mhnbcfx4b5as3fz2mx45izf6k3x2idahf0p8vxvhnh9d6h8sdj/assets.tar.gz";
sha256 = "01mhnbcfx4b5as3fz2mx45izf6k3x2idahf0p8vxvhnh9d6h8sdj";
url = "https://git.clan.lol/api/packages/clan/generic/ui/0vfv15ff7ja9j6a3hj4365gyzcjy0f4zwy38igdyr0smis7a0qj4/assets.tar.gz";
sha256 = "0vfv15ff7ja9j6a3hj4365gyzcjy0f4zwy38igdyr0smis7a0qj4";
}

View File

@@ -25,7 +25,7 @@ pkgs.mkShell {
# re-generate the api code
rm -rf api openapi.json
rm -rf src/api openapi.json
cp ${clanPkgs.clan-openapi}/openapi.json .
orval
'';

View File

@@ -1,5 +1,5 @@
"use client";
import { useGetMachineSchema } from "@/api/default/default";
import { useGetMachineSchema } from "@/api/machine/machine";
import { Check, Error } from "@mui/icons-material";
import {
Box,

View File

@@ -1,6 +1,6 @@
"use client";
import { useListMachines } from "@/api/default/default";
import { useListMachines } from "@/api/machine/machine";
import { Machine, MachinesResponse } from "@/api/model";
import { AxiosError, AxiosResponse } from "axios";
import React, {

View File

@@ -1,4 +1,4 @@
import { inspectVm } from "@/api/default/default";
import { inspectVm } from "@/api/vm/vm";
import { HTTPValidationError, VmConfig } from "@/api/model";
import { AxiosError } from "axios";
import { useEffect, useState } from "react";

View File

@@ -10,7 +10,8 @@ import {
} from "@mui/material";
import { Controller, SubmitHandler, UseFormReturn } from "react-hook-form";
import { FlakeBadge } from "../flakeBadge/flakeBadge";
import { createVm, useInspectFlakeAttrs } from "@/api/default/default";
import { createVm } from "@/api/vm/vm";
import { useInspectFlakeAttrs } from "@/api/flake/flake";
import { VmConfig } from "@/api/model";
import { Dispatch, SetStateAction, useEffect, useState } from "react";
import { toast } from "react-hot-toast";

View File

@@ -7,7 +7,7 @@ import { Typography, Button } from "@mui/material";
import { ConfirmVM } from "./confirmVM";
import { Log } from "./log";
import GppMaybeIcon from "@mui/icons-material/GppMaybe";
import { useInspectFlake } from "@/api/default/default";
import { useInspectFlake } from "@/api/flake/flake";
interface ConfirmProps {
flakeUrl: string;

View File

@@ -1,5 +1,5 @@
"use client";
import { useGetVmLogs } from "@/api/default/default";
import { useGetVmLogs } from "@/api/vm/vm";
import { Log } from "./log";
import { LoadingOverlay } from "./loadingOverlay";

View File

@@ -1,5 +1,6 @@
[tool.mypy]
python_version = "3.10"
pretty = true
warn_redundant_casts = true
disallow_untyped_calls = true
disallow_untyped_defs = true
@@ -11,25 +12,3 @@ line-length = 88
select = [ "E", "F", "I", "U", "N"]
ignore = [ "E501" ]
[tool.black]
line-length = 88
target-version = [ "py310" ]
include = "\\.pyi?$"
exclude = '''
/(
\.git
| \.hg
| \.mypy_cache
| \.tox
| \.venv
| _build
| buck-out
| build
| dist
# The following are specific to Black, you probably don't want those.
| blib2to3
| tests/data
| profiling
)/
'''