merge main

checks/deltachat/default.nix

@@ -0,0 +1,24 @@
+(import ../lib/container-test.nix) ({ pkgs, ... }: {
+  name = "secrets";
+
+  nodes.machine = { self, ... }: {
+    imports = [
+      self.clanModules.deltachat
+      self.nixosModules.clanCore
+      {
+        clanCore.machineName = "machine";
+        clanCore.clanDir = ./.;
+      }
+    ];
+  };
+  testScript = ''
+    start_all()
+    machine.wait_for_unit("maddy")
+    # imap
+    machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 143")
+    # smtp submission
+    machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 587")
+    # smtp
+    machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 25")
+  '';
+})

checks/flake-module.nix

@@ -15,6 +15,7 @@
           # import our test
           secrets = import ./secrets nixosTestArgs;
           container = import ./container nixosTestArgs;
+          deltachat = import ./deltachat nixosTestArgs;
         };
         schemaTests = pkgs.callPackages ./schemas.nix {
           inherit self;

checks/lib/container-driver/package.nix

@@ -1,8 +1,8 @@
-{ extraPythonPackages, buildPythonApplication, self, setuptools, util-linux, systemd }:
+{ extraPythonPackages, python3Packages, buildPythonApplication, setuptools, util-linux, systemd }:
 buildPythonApplication {
   pname = "test-driver";
   version = "0.0.1";
-  propagatedBuildInputs = [ util-linux systemd ] ++ extraPythonPackages self;
+  propagatedBuildInputs = [ util-linux systemd ] ++ extraPythonPackages python3Packages;
   nativeBuildInputs = [ setuptools ];
   format = "pyproject";
   src = ./.;

checks/lib/container-driver/pyproject.toml

@@ -21,11 +21,6 @@ line-length = 88
 select = ["E", "F", "I", "U", "N"]
 ignore = ["E501"]
 
-[tool.black]
-line-length = 88
-target-version = ['py39']
-include = '\.pyi?$'
-
 [tool.mypy]
 python_version = "3.10"
 warn_redundant_casts = true

clanModules/deltachat.nix

@@ -0,0 +1,140 @@
+{ config, pkgs, ... }: {
+  networking.firewall.interfaces."zt+".allowedTCPPorts = [ 25 ]; # smtp with other hosts
+  environment.systemPackages = [ pkgs.deltachat-desktop ];
+
+  services.maddy = {
+    enable = true;
+    primaryDomain = "${config.clanCore.machineName}.local";
+    config = ''
+      # Minimal configuration with TLS disabled, adapted from upstream example
+      # configuration here https://github.com/foxcpp/maddy/blob/master/maddy.conf
+      # Do not use this in unencrypted networks!
+
+      auth.pass_table local_authdb {
+        table sql_table {
+          driver sqlite3
+          dsn credentials.db
+          table_name passwords
+        }
+      }
+
+      storage.imapsql local_mailboxes {
+        driver sqlite3
+        dsn imapsql.db
+      }
+
+      table.chain local_rewrites {
+        optional_step regexp "(.+)\+(.+)@(.+)" "$1@$3"
+        optional_step static {
+          entry postmaster postmaster@$(primary_domain)
+        }
+        optional_step file /etc/maddy/aliases
+      }
+
+      msgpipeline local_routing {
+        destination postmaster $(local_domains) {
+          modify {
+            replace_rcpt &local_rewrites
+          }
+          deliver_to &local_mailboxes
+        }
+        default_destination {
+          reject 550 5.1.1 "User doesn't exist"
+        }
+      }
+
+      smtp tcp://[::]:25 {
+        limits {
+          all rate 20 1s
+          all concurrency 10
+        }
+        dmarc yes
+        check {
+          require_mx_record
+          dkim
+          spf
+        }
+        source $(local_domains) {
+          reject 501 5.1.8 "Use Submission for outgoing SMTP"
+        }
+        default_source {
+          destination postmaster $(local_domains) {
+            deliver_to &local_routing
+          }
+          default_destination {
+            reject 550 5.1.1 "User doesn't exist"
+          }
+        }
+      }
+
+      submission tcp://[::1]:587 {
+        limits {
+          all rate 50 1s
+        }
+        auth &local_authdb
+        source $(local_domains) {
+          check {
+              authorize_sender {
+                  prepare_email &local_rewrites
+                  user_to_email identity
+              }
+          }
+          destination postmaster $(local_domains) {
+              deliver_to &local_routing
+          }
+          default_destination {
+              modify {
+                  dkim $(primary_domain) $(local_domains) default
+              }
+              deliver_to &remote_queue
+          }
+        }
+        default_source {
+          reject 501 5.1.8 "Non-local sender domain"
+        }
+      }
+
+      target.remote outbound_delivery {
+        limits {
+          destination rate 20 1s
+          destination concurrency 10
+        }
+        mx_auth {
+          dane
+          mtasts {
+            cache fs
+            fs_dir mtasts_cache/
+          }
+          local_policy {
+              min_tls_level encrypted
+              min_mx_level none
+          }
+        }
+      }
+
+      target.queue remote_queue {
+        target &outbound_delivery
+        autogenerated_msg_domain $(primary_domain)
+        bounce {
+          destination postmaster $(local_domains) {
+            deliver_to &local_routing
+          }
+          default_destination {
+              reject 550 5.0.0 "Refusing to send DSNs to non-local addresses"
+          }
+        }
+      }
+
+      imap tcp://[::1]:143 {
+        auth &local_authdb
+        storage &local_mailboxes
+      }
+    '';
+    ensureAccounts = [
+      "user@${config.clanCore.machineName}.local"
+    ];
+    ensureCredentials = {
+      "user@${config.clanCore.machineName}.local".passwordFile = pkgs.writeText "dummy" "foobar";
+    };
+  };
+}

clanModules/dino.nix

@@ -1,3 +0,0 @@
-{ pkgs, ... }: {
-  environment.systemPackages = [ pkgs.dino ];
-}

clanModules/ejabberd.nix

@@ -1,199 +0,0 @@
-{ config
-, ...
-}: {
-  services.ejabberd = {
-    enable = true;
-    configFile = "/etc/ejabberd.yml";
-  };
-
-  environment.etc."ejabberd.yml" = {
-    user = "ejabberd";
-    mode = "0600";
-    text = ''
-      loglevel: 4
-
-      default_db: sql
-      new_sql_schema: true
-      sql_type: sqlite
-      sql_database: "/var/lib/ejabberd/db.sqlite"
-
-      hosts:
-      - ${config.clanCore.machineName}.local
-
-      listen:
-      -
-        port: 5222
-        ip: "::1"
-        module: ejabberd_c2s
-        max_stanza_size: 262144
-        shaper: c2s_shaper
-        access: c2s
-        starttls_required: false
-      -
-        port: 5269
-        ip: "::"
-        module: ejabberd_s2s_in
-        max_stanza_size: 524288
-
-      auth_method: [anonymous]
-      anonymous_protocol: login_anon
-      acl:
-        local:
-          user_regexp: ""
-        loopback:
-          ip:
-            - 127.0.0.0/8
-            - ::1/128
-      access_rules:
-        local:
-          allow: local
-        c2s:
-          deny: blocked
-          allow: all
-        s2s:
-          - allow
-        announce:
-          allow: admin
-        configure:
-          allow: admin
-        muc_create:
-          allow: all
-        pubsub_createnode:
-          allow: local
-        trusted_network:
-          allow: loopback
-      api_permissions:
-        "console commands":
-          from:
-            - ejabberd_ctl
-          who: all
-          what: "*"
-        "admin access":
-          who:
-            access:
-              allow:
-                acl: loopback
-                acl: admin
-            oauth:
-              scope: "ejabberd:admin"
-              access:
-                allow:
-                  acl: loopback
-                  acl: admin
-          what:
-            - "*"
-            - "!stop"
-            - "!start"
-        "public commands":
-          who:
-            ip: 127.0.0.1/8
-          what:
-            - status
-            - connected_users_number
-      shaper:
-        normal: 1000
-        fast: 50000
-
-      shaper_rules:
-        max_user_sessions: 10
-        max_user_offline_messages:
-          5000: admin
-          100: all
-        c2s_shaper:
-          none: admin
-          normal: all
-        s2s_shaper: fast
-      modules:
-        mod_adhoc: {}
-        mod_admin_extra: {}
-        mod_announce:
-          access: announce
-        mod_avatar: {}
-        mod_blocking: {}
-        mod_bosh: {}
-        mod_caps: {}
-        mod_carboncopy: {}
-        mod_client_state: {}
-        mod_configure: {}
-        mod_disco: {}
-        mod_fail2ban: {}
-        mod_http_api: {}
-        mod_http_upload:
-          put_url: https://@HOST@:5443/upload
-        mod_last: {}
-        mod_mam:
-          ## Mnesia is limited to 2GB, better to use an SQL backend
-          ## For small servers SQLite is a good fit and is very easy
-          ## to configure. Uncomment this when you have SQL configured:
-          ## db_type: sql
-          assume_mam_usage: true
-          default: always
-        mod_mqtt:
-          access_publish:
-            "homeassistant/#":
-              - allow: hass_publisher
-              - deny
-            "#":
-              - deny
-          access_subscribe:
-            "homeassistant/#":
-              - allow: hass_subscriber
-              - deny
-            "#":
-              - deny
-        mod_muc:
-          host: "muc.@HOST@"
-          access:
-            - allow
-          access_admin:
-            - allow: admin
-          access_create: muc_create
-          access_persistent: muc_create
-          access_mam:
-            - allow
-          default_room_options:
-            mam: true
-        mod_muc_admin: {}
-        mod_offline:
-          access_max_user_messages: max_user_offline_messages
-        mod_ping: {}
-        mod_privacy: {}
-        mod_private: {}
-        mod_proxy65:
-          access: local
-          max_connections: 5
-        mod_pubsub:
-          access_createnode: pubsub_createnode
-          plugins:
-            - flat
-            - pep
-          force_node_config:
-            ## Avoid buggy clients to make their bookmarks public
-            storage:bookmarks:
-              access_model: whitelist
-        mod_push: {}
-        mod_push_keepalive: {}
-        mod_register:
-          ## Only accept registration requests from the "trusted"
-          ## network (see access_rules section above).
-          ## Think twice before enabling registration from any
-          ## address. See the Jabber SPAM Manifesto for details:
-          ## https://github.com/ge0rg/jabber-spam-fighting-manifesto
-          ip_access: trusted_network
-        mod_roster:
-          versioning: true
-        mod_s2s_dialback: {}
-        mod_shared_roster: {}
-        mod_stream_mgmt:
-          resend_on_timeout: if_offline
-        mod_vcard: {}
-        mod_vcard_xupdate: {}
-        mod_version:
-          show_os: false
-    '';
-  };
-
-  networking.firewall.allowedTCPPorts = [
-    5269 # xmpp-server
-  ];
-}

clanModules/flake-module.nix

@@ -8,8 +8,7 @@
         ];
       })
       (builtins.readDir ./diskLayouts);
-    ejabberd = ./ejabberd.nix;
+    deltachat = ./deltachat.nix;
-    dino = ./dino.nix;
     xfce = ./xfce.nix;
   };
 }

docs/contributing.md

@@ -10,6 +10,11 @@ Welcome to our website template repository! This template is designed to help yo
 
 **Dependency Management**: We use the [Nix package manager](https://nixos.org/) to manage dependencies and ensure reproducibility, making your development process more robust.
 
+## Supported Operating Systems
+
+- Linux
+- macOS
+
 # Getting Started with the Development Environment
 
 Let's get your development environment up and running:
@@ -28,11 +33,20 @@ Let's get your development environment up and running:
      curl -sfL https://direnv.net/install.sh | bash
      ```
 
-3. **Clone the Repository and Navigate**:
+3. **Add direnv to your shell**:
+
+   - Direnv needs to [hook into your shell](https://direnv.net/docs/hook.html) to work.
+     You can do this by executing following command:
+
+   ```bash
+   echo 'eval "$(direnv hook zsh)"' >> ~/.zshrc && echo 'eval "$(direnv hook bash)"' >> ~/.bashrc && eval "$SHELL"
+   ```
+
+4. **Clone the Repository and Navigate**:
 
    - Clone this repository and navigate to it.
 
-4. **Allow .envrc**:
+5. **Allow .envrc**:
 
    - When you enter the directory, you'll receive an error message like this:
      ```bash
@@ -40,7 +54,7 @@ Let's get your development environment up and running:
      ```
    - Execute `direnv allow` to automatically execute the shell script `.envrc` when entering the directory.
 
-5. **Build the Backend**:
+6. **Build the Backend**:
 
    - Go to the `pkgs/clan-cli` directory and execute:
      ```bash
@@ -48,7 +62,7 @@ Let's get your development environment up and running:
      ```
    - Wait for the backend to build.
 
-6. **Start the Backend Server**:
+7. **Start the Backend Server**:
 
    - To start the backend server, execute:
      ```bash
@@ -56,7 +70,7 @@ Let's get your development environment up and running:
      ```
    - The server will automatically restart if any Python files change.
 
-7. **Build the Frontend**:
+8. **Build the Frontend**:
 
    - In a different shell, navigate to the `pkgs/ui` directory and execute:
      ```bash
@@ -64,7 +78,7 @@ Let's get your development environment up and running:
      ```
    - Wait for the frontend to build.
 
-8. **Start the Frontend**:
+9. **Start the Frontend**:
    - To start the frontend, execute:
      ```bash
      npm run dev
@@ -194,4 +208,4 @@ To make the most of this template:
    - Set the option to "Delete pull request branch after merge by default."
    - Also, set the default merge style to "Rebase then create merge commit."
 
-With this template, you're well-equipped to build and collaborate on high-quality websites efficiently. Happy coding!
+With this template, you're well-equipped to build and collaborate on high-quality websites efficiently. Happy coding!.

flake.lock

@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1697995812,
+        "lastModified": 1698422527,
-        "narHash": "sha256-UDlK6p/6vAiVOQ92PR0ySDZBS3yiryrlJpSOw3b9Ito=",
+        "narHash": "sha256-SDu3Xg263t3oXIyTaH0buOvFnKIDeZsvKDBtOz+jRbs=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "4122a18340094151d7911e838237ec7627f0d0c5",
+        "rev": "944d338d24a9d043a3f7461c30ee6cfe4f9cca30",
         "type": "github"
       },
       "original": {
@@ -27,11 +27,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1696343447,
+        "lastModified": 1698882062,
-        "narHash": "sha256-B2xAZKLkkeRFG5XcHHSXXcP7To9Xzr59KXeZiRf4vdQ=",
+        "narHash": "sha256-HkhafUayIqxXyHH1X8d9RDl1M2CkFgZLjKD3MzabiEo=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "c9afaba3dfa4085dbd2ccb38dfade5141e33d9d4",
+        "rev": "8c9fa2545007b49a5db5f650ae91f227672c3877",
         "type": "github"
       },
       "original": {
@@ -98,16 +98,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1696051733,
+        "lastModified": 1699007274,
-        "narHash": "sha256-fEC8/6wJOWgCSvBjPwMBdaYtp57OUfQd3dJgp0D/It4=",
+        "narHash": "sha256-m0NH2trnW8cOhona6m3hWkeDZ28BV/wAGPd/YWik23g=",
         "owner": "Mic92",
         "repo": "nixpkgs",
-        "rev": "c3bd4f19ef0062d4462444aa413e26c917187ae9",
+        "rev": "fcb19bae00e9d3fd5ecf4a1f80cf33248bf7f714",
         "type": "github"
       },
       "original": {
         "owner": "Mic92",
-        "ref": "fakeroot",
+        "ref": "deltachat",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -131,11 +131,11 @@
         "nixpkgs-stable": []
       },
       "locked": {
-        "lastModified": 1697943852,
+        "lastModified": 1699021419,
-        "narHash": "sha256-DaBxUPaZhQ3yLCmAATshYB7qo7NwcMvSFWz9T3bjYYY=",
+        "narHash": "sha256-oy2j2OHXYcckifASMeZzpmbDLSvobMGt0V/RvoDotF4=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "30a0ba4a20703b4bfe047fe5def1fc24978e322c",
+        "rev": "275b28593ef3a1b9d05b6eeda3ddce2f45f5c06f",
         "type": "github"
       },
       "original": {
@@ -151,11 +151,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1697388351,
+        "lastModified": 1698438538,
-        "narHash": "sha256-63N2eBpKaziIy4R44vjpUu8Nz5fCJY7okKrkixvDQmY=",
+        "narHash": "sha256-AWxaKTDL3MtxaVTVU5lYBvSnlspOS0Fjt8GxBgnU0Do=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "aae39f64f5ecbe89792d05eacea5cb241891292a",
+        "rev": "5deb8dc125a9f83b65ca86cf0c8167c46593e0b1",
         "type": "github"
       },
       "original": {

flake.nix

@@ -5,9 +5,9 @@
   nixConfig.extra-trusted-public-keys = [ "cache.clan.lol-1:3KztgSAB5R1M+Dz7vzkBGzXdodizbgLXGXKXlcQLA28=" ];
 
   inputs = {
-    #nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    #nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
-    # https://github.com/NixOS/nixpkgs/pull/257462
+    # https://github.com/NixOS/nixpkgs/pull/265024
-    nixpkgs.url = "github:Mic92/nixpkgs/fakeroot";
+    nixpkgs.url = "github:Mic92/nixpkgs/deltachat";
     floco.url = "github:aakropotkin/floco";
     floco.inputs.nixpkgs.follows = "nixpkgs";
     disko.url = "github:nix-community/disko";

formatter.nix

@@ -46,7 +46,7 @@
         "-eucx"
         ''
           ${lib.getExe pkgs.ruff} --fix "$@"
-          ${lib.getExe pkgs.black} "$@"
+          ${lib.getExe pkgs.ruff} format "$@"
         ''
         "--" # this argument is ignored by bash
       ];

pkgs/clan-cli/clan_cli/__init__.py

@@ -2,7 +2,7 @@ import argparse
 import logging
 import sys
 from types import ModuleType
-from typing import Optional
+from typing import Any, Optional, Sequence
 
 from . import config, flakes, join, machines, secrets, vms, webui
 from .custom_logger import setup_logging
@@ -17,6 +17,24 @@ except ImportError:
     pass
 
 
+class AppendOptionAction(argparse.Action):
+    def __init__(self, option_strings: str, dest: str, **kwargs: Any) -> None:
+        super().__init__(option_strings, dest, **kwargs)
+
+    def __call__(
+        self,
+        parser: argparse.ArgumentParser,
+        namespace: argparse.Namespace,
+        values: str | Sequence[str] | None,
+        option_string: Optional[str] = None,
+    ) -> None:
+        lst = getattr(namespace, self.dest)
+        lst.append("--option")
+        assert isinstance(values, list), "values must be a list"
+        lst.append(values[0])
+        lst.append(values[1])
+
+
 def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(prog=prog, description="cLAN tool")
 
@@ -26,6 +44,15 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser:
         action="store_true",
     )
 
+    parser.add_argument(
+        "--option",
+        help="Nix option to set",
+        nargs=2,
+        metavar=("name", "value"),
+        action=AppendOptionAction,
+        default=[],
+    )
+
     subparsers = parser.add_subparsers()
 
     parser_flake = subparsers.add_parser(

pkgs/clan-cli/clan_cli/flakes/__init__.py

@@ -2,7 +2,7 @@
 import argparse
 
 from .create import register_create_parser
-from .list import register_list_parser
+from .list_flakes import register_list_parser
 
 
 # takes a (sub)parser and configures it

pkgs/clan-cli/clan_cli/types.py

@@ -1,11 +1,15 @@
 import logging
 from pathlib import Path
-from typing import NewType
+from typing import NewType, Union
+
+from pydantic import AnyUrl
 
 log = logging.getLogger(__name__)
 
 FlakeName = NewType("FlakeName", str)
 
+FlakeUrl = Union[AnyUrl, Path]
+
 
 def validate_path(base_dir: Path, value: Path) -> Path:
     user_path = (base_dir / value).resolve()

pkgs/clan-cli/clan_cli/vms/create.py

@@ -11,29 +11,23 @@ from typing import Iterator
 from uuid import UUID
 
 from ..dirs import clan_flakes_dir, specific_flake_dir
-from ..errors import ClanError
 from ..nix import nix_build, nix_config, nix_eval, nix_shell
 from ..task_manager import BaseTask, Command, create_task
 from ..types import validate_path
 from .inspect import VmConfig, inspect_vm
 
 
-def is_path_or_url(s: str) -> str | None:
+def is_flake_url(s: str) -> bool:
-    # check if s is a valid path
+    if re.match(r"^http.?://[a-zA-Z0-9.-]+/[a-zA-Z0-9.-]+", s) is not None:
-    if os.path.exists(s):
+        return True
-        return "path"
+    return False
-    # check if s is a valid URL
-    elif re.match(r"^https?://[a-zA-Z0-9.-]+/[a-zA-Z0-9.-]+", s):
-        return "URL"
-    # otherwise, return None
-    else:
-        return None
 
 
 class BuildVmTask(BaseTask):
-    def __init__(self, uuid: UUID, vm: VmConfig) -> None:
+    def __init__(self, uuid: UUID, vm: VmConfig, nix_options: list[str] = []) -> None:
         super().__init__(uuid, num_cmds=7)
         self.vm = vm
+        self.nix_options = nix_options
 
     def get_vm_create_info(self, cmds: Iterator[Command]) -> dict:
         config = nix_config()
@@ -47,6 +41,7 @@ class BuildVmTask(BaseTask):
                 [
                     f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.system.clan.vm.create'
                 ]
+                + self.nix_options
             )
         )
         vm_json = "".join(cmd.stdout).strip()
@@ -57,7 +52,7 @@ class BuildVmTask(BaseTask):
     def get_clan_name(self, cmds: Iterator[Command]) -> str:
         clan_dir = self.vm.flake_url
         cmd = next(cmds)
-        cmd.run(nix_eval([f"{clan_dir}#clanInternals.clanName"]))
+        cmd.run(nix_eval([f"{clan_dir}#clanInternals.clanName"]) + self.nix_options)
         clan_name = cmd.stdout[0].strip().strip('"')
         return clan_name
 
@@ -93,12 +88,8 @@ class BuildVmTask(BaseTask):
             )  # TODO do this in the clanCore module
             env["SECRETS_DIR"] = str(secrets_dir)
 
-            res = is_path_or_url(str(self.vm.flake_url))
+            # Only generate secrets for local clans
-            if res is None:
+            if not is_flake_url(str(self.vm.flake_url)):
-                raise ClanError(
-                    f"flake_url must be a valid path or URL, got {self.vm.flake_url}"
-                )
-            elif res == "path":  # Only generate secrets for local clans
                 cmd = next(cmds)
                 if Path(self.vm.flake_url).is_dir():
                     cmd.run(
@@ -151,27 +142,44 @@ class BuildVmTask(BaseTask):
                 "console=tty0",
             ]
             qemu_command = [
-                # fmt: off
                 "qemu-kvm",
-                "-name", machine,
+                "-name",
-                "-m", f'{vm_config["memorySize"]}M',
+                machine,
-                "-smp", str(vm_config["cores"]),
+                "-m",
-                "-device", "virtio-rng-pci",
+                f'{vm_config["memorySize"]}M',
-                "-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0",
+                "-smp",
-                "-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store",
+                str(vm_config["cores"]),
-                "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
+                "-device",
-                "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
+                "virtio-rng-pci",
-                "-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
+                "-net",
-                "-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report',
+                "nic,netdev=user.0,model=virtio",
-                "-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
+                "-netdev",
-                "-device", "virtio-keyboard",
+                "user,id=user.0",
-                "-vga", "virtio",
+                "-virtfs",
+                "local,path=/nix/store,security_model=none,mount_tag=nix-store",
+                "-virtfs",
+                f"local,path={xchg_dir},security_model=none,mount_tag=shared",
+                "-virtfs",
+                f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
+                "-virtfs",
+                f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
+                "-drive",
+                f"cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report",
+                "-device",
+                "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
+                "-device",
+                "virtio-keyboard",
+                "-vga",
+                "virtio",
                 "-usb",
-                "-device", "usb-tablet,bus=usb-bus.0",
+                "-device",
-                "-kernel", f'{vm_config["toplevel"]}/kernel',
+                "usb-tablet,bus=usb-bus.0",
-                "-initrd", vm_config["initrd"],
+                "-kernel",
-                "-append", " ".join(cmdline),
+                f'{vm_config["toplevel"]}/kernel',
-                # fmt: on
+                "-initrd",
+                vm_config["initrd"],
+                "-append",
+                " ".join(cmdline),
             ]
             if not self.vm.graphics:
                 qemu_command.append("-nographic")
@@ -179,15 +187,17 @@ class BuildVmTask(BaseTask):
             cmd.run(nix_shell(["qemu"], qemu_command))
 
 
-def create_vm(vm: VmConfig) -> BuildVmTask:
+def create_vm(vm: VmConfig, nix_options: list[str] = []) -> BuildVmTask:
-    return create_task(BuildVmTask, vm)
+    return create_task(BuildVmTask, vm, nix_options)
 
 
 def create_command(args: argparse.Namespace) -> None:
-    clan_dir = specific_flake_dir(args.flake)
+    flake_url = args.flake
-    vm = asyncio.run(inspect_vm(flake_url=clan_dir, flake_attr=args.machine))
+    if not is_flake_url(args.flake):
+        flake_url = specific_flake_dir(args.flake)
+    vm = asyncio.run(inspect_vm(flake_url=flake_url, flake_attr=args.machine))
 
-    task = create_vm(vm)
+    task = create_vm(vm, args.option)
     for line in task.log_lines():
         print(line, end="")
 

pkgs/clan-cli/clan_cli/webui/api_outputs.py

@@ -70,6 +70,10 @@ class FlakeAction(BaseModel):
     uri: str
 
 
+class FlakeListResponse(BaseModel):
+    flakes: list[str]
+
+
 class FlakeCreateResponse(BaseModel):
     cmd_out: Dict[str, CmdOut]
 

pkgs/clan-cli/clan_cli/webui/app.py

@@ -9,6 +9,7 @@ from ..errors import ClanError
 from .assets import asset_path
 from .error_handlers import clan_error_handler
 from .routers import clan_modules, flake, health, machines, root, vms
+from .tags import tags_metadata
 
 origins = [
     "http://localhost:3000",
@@ -39,6 +40,9 @@ def setup_app() -> FastAPI:
 
     app.mount("/static", StaticFiles(directory=asset_path()), name="static")
 
+    # Add tag descriptions to the OpenAPI schema
+    app.openapi_tags = tags_metadata
+
     for route in app.routes:
         if isinstance(route, APIRoute):
             route.operation_id = route.name  # in this case, 'read_items'

pkgs/clan-cli/clan_cli/webui/error_handlers.py

@@ -9,7 +9,8 @@ from ..errors import ClanError
 log = logging.getLogger(__name__)
 
 
-def clan_error_handler(request: Request, exc: ClanError) -> JSONResponse:
+def clan_error_handler(request: Request, exc: Exception) -> JSONResponse:
+    assert isinstance(exc, ClanError)
     log.error("ClanError: %s", exc)
     detail = [
         {

pkgs/clan-cli/clan_cli/webui/routers/clan_modules.py

@@ -9,12 +9,13 @@ from clan_cli.types import FlakeName
 from ..api_outputs import (
     ClanModulesResponse,
 )
+from ..tags import Tags
 
 log = logging.getLogger(__name__)
 router = APIRouter()
 
 
-@router.get("/api/{flake_name}/clan_modules")
+@router.get("/api/{flake_name}/clan_modules", tags=[Tags.modules])
 async def list_clan_modules(flake_name: FlakeName) -> ClanModulesResponse:
     module_names, error = get_clan_module_names(flake_name)
     if error is not None:

pkgs/clan-cli/clan_cli/webui/routers/flake.py

@@ -13,12 +13,14 @@ from clan_cli.webui.api_outputs import (
     FlakeAction,
     FlakeAttrResponse,
     FlakeCreateResponse,
+    FlakeListResponse,
     FlakeResponse,
 )
 
 from ...async_cmd import run
-from ...flakes import create
+from ...flakes import create, list_flakes
 from ...nix import nix_command, nix_flake_show
+from ..tags import Tags
 
 router = APIRouter()
 
@@ -45,13 +47,13 @@ async def get_attrs(url: AnyUrl | Path) -> list[str]:
 
 
 # TODO: Check for directory traversal
-@router.get("/api/flake/attrs")
+@router.get("/api/flake/attrs", tags=[Tags.flake])
 async def inspect_flake_attrs(url: AnyUrl | Path) -> FlakeAttrResponse:
     return FlakeAttrResponse(flake_attrs=await get_attrs(url))
 
 
 # TODO: Check for directory traversal
-@router.get("/api/flake")
+@router.get("/api/flake/inspect", tags=[Tags.flake])
 async def inspect_flake(
     url: AnyUrl | Path,
 ) -> FlakeResponse:
@@ -76,7 +78,15 @@ async def inspect_flake(
     return FlakeResponse(content=content, actions=actions)
 
 
-@router.post("/api/flake/create", status_code=status.HTTP_201_CREATED)
+@router.get("/api/flake/list", tags=[Tags.flake])
+async def list_all_flakes() -> FlakeListResponse:
+    flakes = list_flakes.list_flakes()
+    return FlakeListResponse(flakes=flakes)
+
+
+@router.post(
+    "/api/flake/create", tags=[Tags.flake], status_code=status.HTTP_201_CREATED
+)
 async def create_flake(
     args: Annotated[FlakeCreateInput, Body()],
 ) -> FlakeCreateResponse:

pkgs/clan-cli/clan_cli/webui/routers/machines.py

@@ -23,12 +23,13 @@ from ..api_outputs import (
     Status,
     VerifyMachineResponse,
 )
+from ..tags import Tags
 
 log = logging.getLogger(__name__)
 router = APIRouter()
 
 
-@router.get("/api/{flake_name}/machines")
+@router.get("/api/{flake_name}/machines", tags=[Tags.machine])
 async def list_machines(flake_name: FlakeName) -> MachinesResponse:
     machines = []
     for m in _list_machines(flake_name):
@@ -37,7 +38,7 @@ async def list_machines(flake_name: FlakeName) -> MachinesResponse:
     return MachinesResponse(machines=machines)
 
 
-@router.post("/api/{flake_name}/machines", status_code=201)
+@router.post("/api/{flake_name}/machines", tags=[Tags.machine], status_code=201)
 async def create_machine(
     flake_name: FlakeName, machine: Annotated[MachineCreate, Body()]
 ) -> MachineResponse:
@@ -45,19 +46,19 @@ async def create_machine(
     return MachineResponse(machine=Machine(name=machine.name, status=Status.UNKNOWN))
 
 
-@router.get("/api/{flake_name}/machines/{name}")
+@router.get("/api/{flake_name}/machines/{name}", tags=[Tags.machine])
 async def get_machine(flake_name: FlakeName, name: str) -> MachineResponse:
     log.error("TODO")
     return MachineResponse(machine=Machine(name=name, status=Status.UNKNOWN))
 
 
-@router.get("/api/{flake_name}/machines/{name}/config")
+@router.get("/api/{flake_name}/machines/{name}/config", tags=[Tags.machine])
 async def get_machine_config(flake_name: FlakeName, name: str) -> ConfigResponse:
     config = config_for_machine(flake_name, name)
     return ConfigResponse(config=config)
 
 
-@router.put("/api/{flake_name}/machines/{name}/config")
+@router.put("/api/{flake_name}/machines/{name}/config", tags=[Tags.machine])
 async def set_machine_config(
     flake_name: FlakeName, name: str, config: Annotated[dict, Body()]
 ) -> ConfigResponse:
@@ -65,13 +66,13 @@ async def set_machine_config(
     return ConfigResponse(config=config)
 
 
-@router.get("/api/{flake_name}/machines/{name}/schema")
+@router.get("/api/{flake_name}/machines/{name}/schema", tags=[Tags.machine])
 async def get_machine_schema(flake_name: FlakeName, name: str) -> SchemaResponse:
     schema = schema_for_machine(flake_name, name)
     return SchemaResponse(schema=schema)
 
 
-@router.put("/api/{flake_name}/machines/{name}/schema")
+@router.put("/api/{flake_name}/machines/{name}/schema", tags=[Tags.machine])
 async def set_machine_schema(
     flake_name: FlakeName, name: str, config: Annotated[dict, Body()]
 ) -> SchemaResponse:
@@ -79,7 +80,7 @@ async def set_machine_schema(
     return SchemaResponse(schema=schema)
 
 
-@router.get("/api/{flake_name}/machines/{name}/verify")
+@router.get("/api/{flake_name}/machines/{name}/verify", tags=[Tags.machine])
 async def get_verify_machine_config(
     flake_name: FlakeName, name: str
 ) -> VerifyMachineResponse:
@@ -88,7 +89,7 @@ async def get_verify_machine_config(
     return VerifyMachineResponse(success=success, error=error)
 
 
-@router.put("/api/{flake_name}/machines/{name}/verify")
+@router.put("/api/{flake_name}/machines/{name}/verify", tags=[Tags.machine])
 async def put_verify_machine_config(
     flake_name: FlakeName,
     name: str,

pkgs/clan-cli/clan_cli/webui/routers/root.py

@@ -6,13 +6,14 @@ from pathlib import Path
 from fastapi import APIRouter, Response
 
 from ..assets import asset_path
+from ..tags import Tags
 
 router = APIRouter()
 
 log = logging.getLogger(__name__)
 
 
-@router.get("/{path_name:path}")
+@router.get("/{path_name:path}", tags=[Tags.root])
 async def root(path_name: str) -> Response:
     if path_name == "":
         path_name = "index.html"

pkgs/clan-cli/clan_cli/webui/routers/vms.py

@@ -18,13 +18,14 @@ from ..api_outputs import (
     VmInspectResponse,
     VmStatusResponse,
 )
+from ..tags import Tags
 
 log = logging.getLogger(__name__)
 router = APIRouter()
 
 
 # TODO: Check for directory traversal
-@router.post("/api/vms/inspect")
+@router.post("/api/vms/inspect", tags=[Tags.vm])
 async def inspect_vm(
     flake_url: Annotated[AnyUrl | Path, Body()], flake_attr: Annotated[str, Body()]
 ) -> VmInspectResponse:
@@ -32,7 +33,7 @@ async def inspect_vm(
     return VmInspectResponse(config=config)
 
 
-@router.get("/api/vms/{uuid}/status")
+@router.get("/api/vms/{uuid}/status", tags=[Tags.vm])
 async def get_vm_status(uuid: UUID) -> VmStatusResponse:
     task = get_task(uuid)
     log.debug(msg=f"error: {task.error}, task.status: {task.status}")
@@ -40,7 +41,7 @@ async def get_vm_status(uuid: UUID) -> VmStatusResponse:
     return VmStatusResponse(status=task.status, error=error)
 
 
-@router.get("/api/vms/{uuid}/logs")
+@router.get("/api/vms/{uuid}/logs", tags=[Tags.vm])
 async def get_vm_logs(uuid: UUID) -> StreamingResponse:
     # Generator function that yields log lines as they are available
     def stream_logs() -> Iterator[str]:
@@ -55,7 +56,7 @@ async def get_vm_logs(uuid: UUID) -> StreamingResponse:
 
 
 # TODO: Check for directory traversal
-@router.post("/api/vms/create")
+@router.post("/api/vms/create", tags=[Tags.vm])
 async def create_vm(vm: Annotated[VmConfig, Body()]) -> VmCreateResponse:
     flake_attrs = await get_attrs(vm.flake_url)
     if vm.flake_attr not in flake_attrs:

pkgs/clan-cli/clan_cli/webui/tags.py

@@ -0,0 +1,41 @@
+from enum import Enum
+from typing import Any, Dict, List
+
+
+class Tags(Enum):
+    flake = "flake"
+    machine = "machine"
+    vm = "vm"
+    modules = "modules"
+    root = "root"
+
+    def __str__(self) -> str:
+        return self.value
+
+
+tags_metadata: List[Dict[str, Any]] = [
+    {
+        "name": str(Tags.flake),
+        "description": "Operations on a flake.",
+        "externalDocs": {
+            "description": "What is a flake?",
+            "url": "https://www.tweag.io/blog/2020-05-25-flakes/",
+        },
+    },
+    {
+        "name": str(Tags.machine),
+        "description": "Manage physical machines. Instances of a flake",
+    },
+    {
+        "name": str(Tags.vm),
+        "description": "Manage virtual machines. Instances of a flake",
+    },
+    {
+        "name": str(Tags.modules),
+        "description": "Manage cLAN modules of a flake",
+    },
+    {
+        "name": str(Tags.root),
+        "description": "This serves as the frontend delivery",
+    },
+]

pkgs/clan-cli/pyproject.toml

@@ -58,25 +58,3 @@ line-length = 88
 
 select = [ "E", "F", "I",  "N"]
 ignore = [ "E501" ]
-
-[tool.black]
-line-length = 88
-target-version = [ "py310" ]
-include = "\\.pyi?$"
-exclude = '''
-/(
-    \.git
-  | \.hg
-  | \.mypy_cache
-  | \.tox
-  | \.venv
-  | _build
-  | buck-out
-  | build
-  | dist
-  # The following are specific to Black, you probably don't want those.
-  | blib2to3
-  | tests/data
-  | profiling
-)/
-'''

pkgs/clan-cli/tests/fixtures_flakes.py

@@ -8,6 +8,8 @@ from pathlib import Path
 from typing import Iterator, NamedTuple
 
 import pytest
+from pydantic import AnyUrl
+from pydantic.tools import parse_obj_as
 from root import CLAN_CORE
 
 from clan_cli.dirs import nixpkgs_source
@@ -117,6 +119,16 @@ def test_flake_with_core(
     )
 
 
+@pytest.fixture
+def test_democlan_url(
+    monkeypatch: pytest.MonkeyPatch, temporary_home: Path
+) -> Iterator[AnyUrl]:
+    yield parse_obj_as(
+        AnyUrl,
+        "https://git.clan.lol/clan/democlan/archive/main.tar.gz",
+    )
+
+
 @pytest.fixture
 def test_flake_with_core_and_pass(
     monkeypatch: pytest.MonkeyPatch, temporary_home: Path

pkgs/clan-cli/tests/test_flake_api.py

@@ -8,6 +8,15 @@ from fixtures_flakes import FlakeForTest
 log = logging.getLogger(__name__)
 
 
+@pytest.mark.impure
+def test_list_flakes(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
+    response = api.get("/api/flake/list")
+    assert response.status_code == 200, "Failed to list flakes"
+    data = response.json()
+    print("Data: ", data)
+    assert data.get("flakes") == ["test_flake_with_core"]
+
+
 @pytest.mark.impure
 def test_inspect_ok(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
     params = {"url": str(test_flake_with_core.path)}
@@ -38,7 +47,7 @@ def test_inspect_err(api: TestClient) -> None:
 def test_inspect_flake(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
     params = {"url": str(test_flake_with_core.path)}
     response = api.get(
-        "/api/flake",
+        "/api/flake/inspect",
         params=params,
     )
     assert response.status_code == 200, "Failed to inspect vm"

pkgs/clan-cli/tests/test_vms_api_create.py

@@ -7,6 +7,7 @@ from api import TestClient
 from cli import Cli
 from fixtures_flakes import FlakeForTest, create_flake
 from httpx import SyncByteStream
+from pydantic import AnyUrl
 from root import CLAN_CORE
 
 from clan_cli.types import FlakeName
@@ -42,7 +43,7 @@ def remote_flake_with_vm_without_secrets(
     )
 
 
-def generic_create_vm_test(api: TestClient, flake: Path, vm: str) -> None:
+def generic_create_vm_test(api: TestClient, flake: Path | AnyUrl, vm: str) -> None:
     print(f"flake_url: {flake} ")
     response = api.post(
         "/api/vms/create",
@@ -113,3 +114,18 @@ def test_create_remote(
     generic_create_vm_test(
         api, remote_flake_with_vm_without_secrets.path, "vm_without_secrets"
     )
+
+
+# TODO: We need a test that creates the same VM twice, and checks that the second time it fails
+
+
+# TODO: Democlan needs a machine called testVM, which is headless and gets executed by this test below
+# pytest -n0 -s tests/test_vms_api_create.py::test_create_from_democlan
+# @pytest.mark.skipif(not os.path.exists("/dev/kvm"), reason="Requires KVM")
+# @pytest.mark.impure
+# def test_create_from_democlan(
+#     api: TestClient,
+#     test_democlan_url: AnyUrl) -> None:
+#         generic_create_vm_test(
+#             api, test_democlan_url, "defaultVM"
+#         )

pkgs/ui/nix/ui-assets.nix

@@ -1,5 +1,5 @@
 { fetchzip }:
 fetchzip {
-  url = "https://git.clan.lol/api/packages/clan/generic/ui/01mhnbcfx4b5as3fz2mx45izf6k3x2idahf0p8vxvhnh9d6h8sdj/assets.tar.gz";
+  url = "https://git.clan.lol/api/packages/clan/generic/ui/0vfv15ff7ja9j6a3hj4365gyzcjy0f4zwy38igdyr0smis7a0qj4/assets.tar.gz";
-  sha256 = "01mhnbcfx4b5as3fz2mx45izf6k3x2idahf0p8vxvhnh9d6h8sdj";
+  sha256 = "0vfv15ff7ja9j6a3hj4365gyzcjy0f4zwy38igdyr0smis7a0qj4";
 }

pkgs/ui/shell.nix

@@ -25,7 +25,7 @@ pkgs.mkShell {
 
 
     # re-generate the api code 
-    rm -rf api openapi.json
+    rm -rf src/api openapi.json
     cp ${clanPkgs.clan-openapi}/openapi.json . 
     orval
   '';

pkgs/ui/src/components/createMachineForm/customConfig.tsx

@@ -1,5 +1,5 @@
 "use client";
-import { useGetMachineSchema } from "@/api/default/default";
+import { useGetMachineSchema } from "@/api/machine/machine";
 import { Check, Error } from "@mui/icons-material";
 import {
   Box,

pkgs/ui/src/components/hooks/useMachines.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { useListMachines } from "@/api/default/default";
+import { useListMachines } from "@/api/machine/machine";
 import { Machine, MachinesResponse } from "@/api/model";
 import { AxiosError, AxiosResponse } from "axios";
 import React, {

pkgs/ui/src/components/hooks/useVms.tsx

@@ -1,4 +1,4 @@
-import { inspectVm } from "@/api/default/default";
+import { inspectVm } from "@/api/vm/vm";
 import { HTTPValidationError, VmConfig } from "@/api/model";
 import { AxiosError } from "axios";
 import { useEffect, useState } from "react";

pkgs/ui/src/components/join/configureVM.tsx

@@ -10,7 +10,8 @@ import {
 } from "@mui/material";
 import { Controller, SubmitHandler, UseFormReturn } from "react-hook-form";
 import { FlakeBadge } from "../flakeBadge/flakeBadge";
-import { createVm, useInspectFlakeAttrs } from "@/api/default/default";
+import { createVm } from "@/api/vm/vm";
+import { useInspectFlakeAttrs } from "@/api/flake/flake";
 import { VmConfig } from "@/api/model";
 import { Dispatch, SetStateAction, useEffect, useState } from "react";
 import { toast } from "react-hot-toast";

pkgs/ui/src/components/join/confirm.tsx

@@ -7,7 +7,7 @@ import { Typography, Button } from "@mui/material";
 import { ConfirmVM } from "./confirmVM";
 import { Log } from "./log";
 import GppMaybeIcon from "@mui/icons-material/GppMaybe";
-import { useInspectFlake } from "@/api/default/default";
+import { useInspectFlake } from "@/api/flake/flake";
 
 interface ConfirmProps {
   flakeUrl: string;

pkgs/ui/src/components/join/vmBuildLogs.tsx

@@ -1,5 +1,5 @@
 "use client";
-import { useGetVmLogs } from "@/api/default/default";
+import { useGetVmLogs } from "@/api/vm/vm";
 import { Log } from "./log";
 import { LoadingOverlay } from "./loadingOverlay";
 

pyproject.toml

@@ -1,5 +1,6 @@
 [tool.mypy]
 python_version = "3.10"
+pretty = true
 warn_redundant_casts = true
 disallow_untyped_calls = true
 disallow_untyped_defs = true
@@ -11,25 +12,3 @@ line-length = 88
 
 select = [ "E", "F", "I", "U", "N"]
 ignore = [ "E501" ]
-
-[tool.black]
-line-length = 88
-target-version = [ "py310" ]
-include = "\\.pyi?$"
-exclude = '''
-/(
-    \.git
-  | \.hg
-  | \.mypy_cache
-  | \.tox
-  | \.venv
-  | _build
-  | buck-out
-  | build
-  | dist
-  # The following are specific to Black, you probably don't want those.
-  | blib2to3
-  | tests/data
-  | profiling
-)/
-'''