From b316597bf4a232767dc593d656f35977ac00511e Mon Sep 17 00:00:00 2001 From: lassulus Date: Sun, 29 Oct 2023 20:48:09 +0100 Subject: [PATCH 1/7] cli machines update: find flake dir automatic if not provided, support flake as path --- pkgs/clan-cli/clan_cli/dirs.py | 35 ++++++++++++----------- pkgs/clan-cli/clan_cli/machines/update.py | 10 +++++-- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/dirs.py b/pkgs/clan-cli/clan_cli/dirs.py index c99545aa3..89f16bd71 100644 --- a/pkgs/clan-cli/clan_cli/dirs.py +++ b/pkgs/clan-cli/clan_cli/dirs.py @@ -2,6 +2,7 @@ import logging import os import sys from pathlib import Path +from typing import Optional from .errors import ClanError from .types import FlakeName @@ -9,27 +10,27 @@ from .types import FlakeName log = logging.getLogger(__name__) -# def _get_clan_flake_toplevel() -> Path: -# return find_toplevel([".clan-flake", ".git", ".hg", ".svn", "flake.nix"]) +def get_clan_flake_toplevel() -> Path: + return find_toplevel([".clan-flake", ".git", ".hg", ".svn", "flake.nix"]) -# def find_git_repo_root() -> Optional[Path]: -# try: -# return find_toplevel([".git"]) -# except ClanError: -# return None +def find_git_repo_root() -> Optional[Path]: + try: + return find_toplevel([".git"]) + except ClanError: + return None -# def find_toplevel(top_level_files: list[str]) -> Path: -# """Returns the path to the toplevel of the clan flake""" -# for project_file in top_level_files: -# initial_path = Path(os.getcwd()) -# path = Path(initial_path) -# while path.parent != path: -# if (path / project_file).exists(): -# return path -# path = path.parent -# raise ClanError("Could not find clan flake toplevel directory") +def find_toplevel(top_level_files: list[str]) -> Path: + """Returns the path to the toplevel of the clan flake""" + for project_file in top_level_files: + initial_path = Path(os.getcwd()) + path = Path(initial_path) + while path.parent != path: + if (path / project_file).exists(): + return path + path = path.parent + raise ClanError("Could not find clan flake toplevel directory") def user_config_dir() -> Path: diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index ac0bcf03d..5bcaf3f7c 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -4,7 +4,7 @@ import os import subprocess from pathlib import Path -from ..dirs import specific_flake_dir +from ..dirs import get_clan_flake_toplevel from ..machines.machines import Machine from ..nix import nix_build, nix_command, nix_config from ..secrets.generate import generate_secrets @@ -116,7 +116,10 @@ def get_selected_machines(machine_names: list[str], flake_dir: Path) -> HostGrou # FIXME: we want some kind of inventory here. def update(args: argparse.Namespace) -> None: - flake_dir = specific_flake_dir(args.flake) + if args.flake is None: + flake_dir = get_clan_flake_toplevel() + else: + flake_dir = args.flake if len(args.machines) == 1 and args.target_host is not None: machine = Machine(name=args.machines[0], flake_dir=flake_dir) machine.deployment_address = args.target_host @@ -148,9 +151,10 @@ def register_update_parser(parser: argparse.ArgumentParser) -> None: default=[], ) parser.add_argument( - "flake", + "--flake", type=str, help="name of the flake to update machine for", + default=None, ) parser.add_argument( "--target-host", From fb20b6040cfaca00866e706e3ee2d2005b9ace29 Mon Sep 17 00:00:00 2001 From: lassulus Date: Tue, 31 Oct 2023 13:01:16 +0100 Subject: [PATCH 2/7] switch flake to main branch again --- pkgs/clan-cli/clan_cli/flakes/create.py | 2 +- templates/new-clan/flake.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/flakes/create.py b/pkgs/clan-cli/clan_cli/flakes/create.py index 2c46cdad1..f5e7e1957 100644 --- a/pkgs/clan-cli/clan_cli/flakes/create.py +++ b/pkgs/clan-cli/clan_cli/flakes/create.py @@ -13,7 +13,7 @@ from ..nix import nix_command, nix_shell DEFAULT_URL: AnyUrl = parse_obj_as( AnyUrl, - "git+https://git.clan.lol/clan/clan-core?ref=Qubasa-main#new-clan", # TODO: Change me back to main branch + "git+https://git.clan.lol/clan/clan-core?new-clan", ) diff --git a/templates/new-clan/flake.nix b/templates/new-clan/flake.nix index 72bbbc174..38acce83a 100644 --- a/templates/new-clan/flake.nix +++ b/templates/new-clan/flake.nix @@ -1,7 +1,7 @@ { description = ""; - inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core?ref=Qubasa-main"; + inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core"; outputs = { self, clan-core, ... }: let From 724ba27f2f8faf4479f686b5bae289c05f22e6e2 Mon Sep 17 00:00:00 2001 From: Qubasa Date: Tue, 31 Oct 2023 12:57:28 +0100 Subject: [PATCH 3/7] Added contributing.md and improved README.md --- README.md | 26 ++++-- docs/contributing.md | 197 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 217 insertions(+), 6 deletions(-) create mode 100644 docs/contributing.md diff --git a/README.md b/README.md index e1f5ad046..6995b6cad 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,23 @@ -# clan.lol core +# cLAN Core Repository -This is the monorepo of the clan.lol project -In here are all the packages we use, all the nixosModules we use/expose, the CLI and tests for everything. +Welcome to the cLAN Core Repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo houses all the essential packages, NixOS modules, CLI tools, and tests you need to contribute and work with the cLAN project. -## cLAN config tool +## Getting Started -- The quickstart guide can be found here: [here](/clan/clan-core/src/branch/main/docs/quickstart.md) -- Find the docs [here](/clan/clan-core/src/branch/main/docs/clan-config.md) +If you're new to cLAN and eager to dive in, start with our quickstart guide: + +- **Quickstart Guide**: Check out [quickstart.md](docs/quickstart.md) to get up and running with cLAN in no time. + +## Managing Secrets + +Security is paramount, and cLAN provides guidelines for handling secrets effectively: + +- **Secrets Management**: Learn how to manage secrets securely by reading [secrets-management.md](docs/secrets-management.md). + +## Contributing to cLAN + +We welcome contributions from the community, and we've prepared a comprehensive guide to help you get started: + +- **Contribution Guidelines**: Find out how to contribute and make a meaningful impact on the cLAN project by reading [contributing.md](docs/contributing.md). + +Whether you're a newcomer or a seasoned developer, we look forward to your contributions and collaboration on the cLAN project. Let's build amazing things together! \ No newline at end of file diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 000000000..5a6f8358e --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,197 @@ +# Website Template + +Welcome to our website template repository! This template is designed to help you and your team build high-quality websites efficiently. We've carefully chosen the technologies to make development smooth and enjoyable. Here's what you can expect from this template: + +**Frontend**: Our frontend is powered by [React NextJS](https://nextjs.org/), a popular and versatile framework for building web applications. + +**Backend**: For the backend, we use Python along with the [FastAPI framework](https://fastapi.tiangolo.com/). To ensure seamless communication between the frontend and backend, we generate an `openapi.json` file from the Python code, which defines the REST API. This file is then used with [Orval](https://orval.dev/) to generate TypeScript bindings for the REST API. We're committed to code correctness, so we use [mypy](https://mypy-lang.org/) to ensure that our Python code is statically typed correctly. For backend testing, we rely on [pytest](https://docs.pytest.org/en/7.4.x/). + +**Continuous Integration (CI)**: We've set up a CI bot that rigorously checks your code using the quality assurance (QA) tools mentioned above. If any errors are detected, it will block pull requests until they're resolved. + +**Dependency Management**: We use the [Nix package manager](https://nixos.org/) to manage dependencies and ensure reproducibility, making your development process more robust. + +# Getting Started with the Development Environment + +Let's get your development environment up and running: + +1. **Install Nix Package Manager**: + + - You can install the Nix package manager by either [downloading the Nix installer](https://github.com/DeterminateSystems/nix-installer/releases) or running this command: + ```bash + curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install + ``` + +2. **Install direnv**: + + - Download the direnv package from [here](https://direnv.net/docs/installation.html) or run the following command: + ```bash + curl -sfL https://direnv.net/install.sh | bash + ``` + +3. **Clone the Repository and Navigate**: + + - Clone this repository and navigate to it. + +4. **Allow .envrc**: + + - When you enter the directory, you'll receive an error message like this: + ```bash + direnv: error .envrc is blocked. Run `direnv allow` to approve its content + ``` + - Execute `direnv allow` to automatically execute the shell script `.envrc` when entering the directory. + +5. **Build the Backend**: + + - Go to the `pkgs/clan-cli` directory and execute: + ```bash + direnv allow + ``` + - Wait for the backend to build. + +6. **Start the Backend Server**: + + - To start the backend server, execute: + ```bash + clan webui --reload --no-open --log-level debug + ``` + - The server will automatically restart if any Python files change. + +7. **Build the Frontend**: + + - In a different shell, navigate to the `pkgs/ui` directory and execute: + ```bash + direnv allow + ``` + - Wait for the frontend to build. + +8. **Start the Frontend**: + - To start the frontend, execute: + ```bash + npm run dev + ``` + - Access the website by going to [http://localhost:3000](http://localhost:3000). + +# Setting Up Your Git Workflow + +Let's set up your Git workflow to collaborate effectively: + +1. **Register Your Gitea Account Locally**: + + - Execute the following command to add your Gitea account locally: + ```bash + tea login add + ``` + - Fill out the prompt as follows: + - URL of Gitea instance: `https://gitea.gchq.icu` + - Name of new Login [gitea.gchq.icu]: `gitea.gchq.icu:7171` + - Do you have an access token? No + - Username: YourUsername + - Password: YourPassword + - Set Optional settings: No + +2. **Git Workflow**: + + 1. Add your changes to Git using `git add `. + 2. Run `nix fmt` to lint your files. + 3. Commit your changes with a descriptive message: `git commit -a -m "My descriptive commit message"`. + 4. Make sure your branch has the latest changes from upstream by executing: + ```bash + git fetch && git rebase origin/main --autostash + ``` + 5. Use `git status` to check for merge conflicts. + 6. If conflicts exist, resolve them. Here's a tutorial for resolving conflicts in [VSCode](https://code.visualstudio.com/docs/sourcecontrol/overview#_merge-conflicts). + 7. After resolving conflicts, execute `git merge --continue` and repeat step 5 until there are no conflicts. + +3. **Create a Pull Request**: + + - To automatically open a pull request that gets merged if all tests pass, execute: + ```bash + merge-after-ci + ``` + +4. **Review Your Pull Request**: + + - Visit https://gitea.gchq.icu and go to the project page. Check under "Pull Requests" for any issues with your pull request. + +5. **Push Your Changes**: + - If there are issues, fix them and redo step 2. Afterward, execute: + ```bash + git push origin HEAD:YourUsername-main + ``` + - This will directly push to your open pull request. + +# Debugging + +When working on the backend of your project, debugging is an essential part of the development process. Here are some methods for debugging and testing the backend of your application: + +## Test Backend Locally in Devshell with Breakpoints + +To test the backend locally in a development environment and set breakpoints for debugging, follow these steps: + +1. Run the following command to execute your tests and allow for debugging with breakpoints: + ```bash + pytest -n0 -s --maxfail=1 + ``` + You can place `breakpoint()` in your Python code where you want to trigger a breakpoint for debugging. + +## Test Backend Locally in a Nix Sandbox + +To run your backend tests in a Nix sandbox, you have two options depending on whether your test functions have been marked as impure or not: + +### Running Tests Marked as Impure + +If your test functions need to execute `nix build` and have been marked as impure because you can't execute `nix build` inside a Nix sandbox, use the following command: + +```bash +nix run .#impure-checks +``` + +This command will run the impure test functions. + +### Running Pure Tests + +For test functions that have not been marked as impure and don't require executing `nix build`, you can use the following command: + +```bash +nix build .#checks.x86_64-linux.clan-pytest --rebuild +``` + +This command will run all pure test functions. + +### Inspecting the Nix Sandbox + +If you need to inspect the Nix sandbox while running tests, follow these steps: + +1. Insert an endless sleep into your test code where you want to pause the execution. For example: + + ```python + import time + time.sleep(3600) # Sleep for one hour + ``` + +2. Use `cntr` and `psgrep` to attach to the Nix sandbox. This allows you to interactively debug your code while it's paused. For example: + + ```bash + cntr exec -w your_sandbox_name + psgrep -a -x your_python_process_name + ``` + +These debugging and testing methods will help you identify and fix issues in your backend code efficiently, ensuring the reliability and robustness of your application. + +# Using this Template + +To make the most of this template: + +1. Set up a new Gitea account named `ui-asset-bot`. Generate an access token with all access permissions and set it under `settings/actions/secrets` as a secret called `BOT_ACCESS_TOKEN`. + + - Also, edit the file `.gitea/workflows/ui_assets.yaml` and change the `BOT_EMAIL` variable to match the email you set for that account. Gitea matches commits to accounts by their email address, so this step is essential. + +2. Create a second Gitea account named `merge-bot`. Edit the file `pkgs/merge-after-ci/default.nix` if the name should be different. Under "Branches," set the main branch to be protected and add `merge-bot` to the whitelisted users for pushing. Set the unprotected file pattern to `**/ui-assets.nix`. + + - Enable the status check for "build / test (pull_request)." + +3. Add both `merge-bot` and `ui-asset-bot` as collaborators. + - Set the option to "Delete pull request branch after merge by default." + - Also, set the default merge style to "Rebase then create merge commit." + +With this template, you're well-equipped to build and collaborate on high-quality websites efficiently. Happy coding! From da2e48cab50be4617189b54ff711d6c5a02ecf6c Mon Sep 17 00:00:00 2001 From: Qubasa Date: Tue, 31 Oct 2023 13:00:48 +0100 Subject: [PATCH 4/7] nix fmt --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6995b6cad..63c3f01d9 100644 --- a/README.md +++ b/README.md @@ -20,4 +20,4 @@ We welcome contributions from the community, and we've prepared a comprehensive - **Contribution Guidelines**: Find out how to contribute and make a meaningful impact on the cLAN project by reading [contributing.md](docs/contributing.md). -Whether you're a newcomer or a seasoned developer, we look forward to your contributions and collaboration on the cLAN project. Let's build amazing things together! \ No newline at end of file +Whether you're a newcomer or a seasoned developer, we look forward to your contributions and collaboration on the cLAN project. Let's build amazing things together! From 48561f8c7072a091665c357e26a99e89df065fd9 Mon Sep 17 00:00:00 2001 From: Qubasa Date: Tue, 31 Oct 2023 13:16:19 +0100 Subject: [PATCH 5/7] Fixing tea-create-pr --- pkgs/tea-create-pr/script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/tea-create-pr/script.sh b/pkgs/tea-create-pr/script.sh index e85d5ea4c..8216c027e 100644 --- a/pkgs/tea-create-pr/script.sh +++ b/pkgs/tea-create-pr/script.sh @@ -6,7 +6,7 @@ targetBranch="${2:-main}" shift && shift TMPDIR="$(mktemp -d)" currentBranch="$(git rev-parse --abbrev-ref HEAD)" -user="$(tea login list -o simple | cut -d" " -f4)" +user="$(tea login list -o simple | cut -d" " -f4 | head -n1)" tempRemoteBranch="$user-$currentBranch" nix fmt -- --fail-on-change From ac754e1b44d3781bfbbecfd5c382ecbb7d202922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 31 Oct 2023 14:29:07 +0100 Subject: [PATCH 6/7] vms/create: try virtio for vga seems to suck less than std. --- pkgs/clan-cli/clan_cli/vms/create.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/clan-cli/clan_cli/vms/create.py b/pkgs/clan-cli/clan_cli/vms/create.py index 8a075e1e5..39d368330 100644 --- a/pkgs/clan-cli/clan_cli/vms/create.py +++ b/pkgs/clan-cli/clan_cli/vms/create.py @@ -165,6 +165,7 @@ class BuildVmTask(BaseTask): "-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report', "-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root", "-device", "virtio-keyboard", + "-vga", "virtio", "-usb", "-device", "usb-tablet,bus=usb-bus.0", "-kernel", f'{vm_config["toplevel"]}/kernel', From 53b5e989ee95f939096818a98ce6df2b0732ff9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 31 Oct 2023 15:34:27 +0100 Subject: [PATCH 7/7] zerotier: enable networkd --- nixosModules/clanCore/zerotier/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixosModules/clanCore/zerotier/default.nix b/nixosModules/clanCore/zerotier/default.nix index a4149de43..a732cb942 100644 --- a/nixosModules/clanCore/zerotier/default.nix +++ b/nixosModules/clanCore/zerotier/default.nix @@ -70,6 +70,8 @@ in services.zerotierone.package = lib.mkDefault (pkgs.zerotierone.overrideAttrs (_old: { meta = { }; })); }) (lib.mkIf (cfg.networkId != null) { + systemd.network.enable = true; + networking.useNetworkd = true; systemd.network.networks.zerotier = { matchConfig.Name = "zt*"; networkConfig = {