diff --git a/pkgs/clan-cli/README.md b/pkgs/clan-cli/README.md index edfd8a08a..13ca55da6 100644 --- a/pkgs/clan-cli/README.md +++ b/pkgs/clan-cli/README.md @@ -70,7 +70,7 @@ pytest -n0 -s tests/test_secrets_cli.py::test_users ``` ## Debugging functions + Debugging functions can be found under `src/debug.py` quite interesting is the function repro_env_break() which drops you into a shell with the test environment loaded. - diff --git a/pkgs/clan-cli/clan_cli/debug.py b/pkgs/clan-cli/clan_cli/debug.py index 79271e338..c2231ce01 100644 --- a/pkgs/clan-cli/clan_cli/debug.py +++ b/pkgs/clan-cli/clan_cli/debug.py @@ -1,23 +1,27 @@ -from typing import Dict, Optional, Tuple, Callable, Any, Mapping, List -from pathlib import Path -import ipdb +import logging +import multiprocessing as mp import os +import shlex import stat import subprocess -from .dirs import find_git_repo_root -import multiprocessing as mp -from .types import FlakeName -import logging import sys -import shlex -import time +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +import ipdb log = logging.getLogger(__name__) -def command_exec(cmd: List[str], work_dir:Path, env: Dict[str, str]) -> None: + +def command_exec(cmd: List[str], work_dir: Path, env: Dict[str, str]) -> None: subprocess.run(cmd, check=True, env=env, cwd=work_dir.resolve()) -def repro_env_break(work_dir: Path, env: Optional[Dict[str, str]] = None, cmd: Optional[List[str]] = None) -> None: + +def repro_env_break( + work_dir: Path, + env: Optional[Dict[str, str]] = None, + cmd: Optional[List[str]] = None, +) -> None: if env is None: env = os.environ.copy() else: @@ -36,14 +40,16 @@ def repro_env_break(work_dir: Path, env: Optional[Dict[str, str]] = None, cmd: O finally: proc.terminate() -def write_command(command: str, loc:Path) -> None: + +def write_command(command: str, loc: Path) -> None: with open(loc, "w") as f: f.write("#!/usr/bin/env bash\n") f.write(command) st = os.stat(loc) os.chmod(loc, st.st_mode | stat.S_IEXEC) -def spawn_process(func: Callable, **kwargs:Any) -> mp.Process: + +def spawn_process(func: Callable, **kwargs: Any) -> mp.Process: if mp.get_start_method(allow_none=True) is None: mp.set_start_method(method="spawn") @@ -57,7 +63,7 @@ def dump_env(env: Dict[str, str], loc: Path) -> None: with open(loc, "w") as f: f.write("#!/usr/bin/env bash\n") for k, v in cenv.items(): - if v.count('\n') > 0 or v.count("\"") > 0 or v.count("'") > 0: + if v.count("\n") > 0 or v.count('"') > 0 or v.count("'") > 0: continue f.write(f"export {k}='{v}'\n") st = os.stat(loc) diff --git a/pkgs/clan-cli/clan_cli/flakes/__init__.py b/pkgs/clan-cli/clan_cli/flakes/__init__.py index 78b80093f..628586cfc 100644 --- a/pkgs/clan-cli/clan_cli/flakes/__init__.py +++ b/pkgs/clan-cli/clan_cli/flakes/__init__.py @@ -4,6 +4,7 @@ import argparse from .create import register_create_parser from .list import register_list_parser + # takes a (sub)parser and configures it def register_parser(parser: argparse.ArgumentParser) -> None: subparser = parser.add_subparsers( diff --git a/pkgs/clan-cli/clan_cli/secrets/sops_generate.py b/pkgs/clan-cli/clan_cli/secrets/sops_generate.py index a87328fc5..adb1b792b 100644 --- a/pkgs/clan-cli/clan_cli/secrets/sops_generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/sops_generate.py @@ -1,3 +1,4 @@ +import logging import os import shlex import shutil @@ -6,7 +7,6 @@ import sys from pathlib import Path from tempfile import TemporaryDirectory from typing import Any -import logging from clan_cli.nix import nix_shell @@ -20,6 +20,7 @@ from .sops import generate_private_key log = logging.getLogger(__name__) + def generate_host_key(flake_name: FlakeName, machine_name: str) -> None: if has_machine(flake_name, machine_name): return @@ -97,7 +98,9 @@ def generate_secrets_from_nix( ) -> None: generate_host_key(flake_name, machine_name) errors = {} - log.debug("Generating secrets for machine %s and flake %s", machine_name, flake_name) + log.debug( + "Generating secrets for machine %s and flake %s", machine_name, flake_name + ) with TemporaryDirectory() as d: # if any of the secrets are missing, we regenerate all connected facts/secrets for secret_group, secret_options in secret_submodules.items(): diff --git a/pkgs/clan-cli/clan_cli/vms/create.py b/pkgs/clan-cli/clan_cli/vms/create.py index ddbc706c3..31852dc7c 100644 --- a/pkgs/clan-cli/clan_cli/vms/create.py +++ b/pkgs/clan-cli/clan_cli/vms/create.py @@ -2,20 +2,19 @@ import argparse import asyncio import json import os +import re import shlex import sys -import re from pathlib import Path -from typing import Iterator, Dict +from typing import Iterator from uuid import UUID from ..dirs import clan_flakes_dir, specific_flake_dir +from ..errors import ClanError from ..nix import nix_build, nix_config, nix_eval, nix_shell from ..task_manager import BaseTask, Command, create_task from ..types import validate_path from .inspect import VmConfig, inspect_vm -from ..errors import ClanError -from ..debug import repro_env_break def is_path_or_url(s: str) -> str | None: @@ -29,6 +28,7 @@ def is_path_or_url(s: str) -> str | None: else: return None + class BuildVmTask(BaseTask): def __init__(self, uuid: UUID, vm: VmConfig) -> None: super().__init__(uuid, num_cmds=7) @@ -95,7 +95,7 @@ class BuildVmTask(BaseTask): raise ClanError( f"flake_url must be a valid path or URL, got {self.vm.flake_url}" ) - elif res == "path": # Only generate secrets for local clans + elif res == "path": # Only generate secrets for local clans cmd = next(cmds) if Path(self.vm.flake_url).is_dir(): cmd.run( @@ -105,7 +105,6 @@ class BuildVmTask(BaseTask): else: self.log.warning("won't generate secrets for non local clan") - cmd = next(cmds) cmd.run( [vm_config["uploadSecrets"], clan_name], diff --git a/pkgs/clan-cli/clan_cli/webui/routers/machines.py b/pkgs/clan-cli/clan_cli/webui/routers/machines.py index e7bf2711e..f7aa419cf 100644 --- a/pkgs/clan-cli/clan_cli/webui/routers/machines.py +++ b/pkgs/clan-cli/clan_cli/webui/routers/machines.py @@ -3,7 +3,7 @@ import logging from typing import Annotated from fastapi import APIRouter, Body -from clan_cli.debug import repro_env_break + from ...config.machine import ( config_for_machine, schema_for_machine, diff --git a/pkgs/clan-cli/clan_cli/webui/server.py b/pkgs/clan-cli/clan_cli/webui/server.py index 057d2caa3..66b0f39e3 100644 --- a/pkgs/clan-cli/clan_cli/webui/server.py +++ b/pkgs/clan-cli/clan_cli/webui/server.py @@ -13,6 +13,7 @@ from typing import Iterator import uvicorn from pydantic import AnyUrl, IPvAnyAddress from pydantic.tools import parse_obj_as + from clan_cli.errors import ClanError log = logging.getLogger(__name__) @@ -25,8 +26,7 @@ def open_browser(base_url: AnyUrl, sub_url: str) -> None: break except OSError: time.sleep(i) - url = parse_obj_as( - AnyUrl,f"{base_url}/{sub_url.removeprefix('/')}") + url = parse_obj_as(AnyUrl, f"{base_url}/{sub_url.removeprefix('/')}") _open_browser(url) diff --git a/pkgs/clan-cli/pyproject.toml b/pkgs/clan-cli/pyproject.toml index 92d58cee9..d1e60f618 100644 --- a/pkgs/clan-cli/pyproject.toml +++ b/pkgs/clan-cli/pyproject.toml @@ -41,6 +41,10 @@ ignore_missing_imports = true module = "jsonschema.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "ipdb.*" +ignore_missing_imports = true + [[tool.mypy.overrides]] module = "pytest.*" ignore_missing_imports = true @@ -52,7 +56,7 @@ ignore_missing_imports = true [tool.ruff] line-length = 88 -select = [ "E", "F", "I", "U", "N"] +select = [ "E", "F", "I", "N"] ignore = [ "E501" ] [tool.black] diff --git a/pkgs/clan-cli/shell.nix b/pkgs/clan-cli/shell.nix index 0bb530c18..4c3d9cfdc 100644 --- a/pkgs/clan-cli/shell.nix +++ b/pkgs/clan-cli/shell.nix @@ -22,41 +22,41 @@ mkShell { ]; shellHook = '' - tmp_path=$(realpath ./.direnv) + tmp_path=$(realpath ./.direnv) - repo_root=$(realpath .) - mkdir -p "$tmp_path/python/${pythonWithDeps.sitePackages}" + repo_root=$(realpath .) + mkdir -p "$tmp_path/python/${pythonWithDeps.sitePackages}" - # Install the package in editable mode - # This allows executing `clan` from within the dev-shell using the current - # version of the code and its dependencies. - ${pythonWithDeps.interpreter} -m pip install \ - --quiet \ - --disable-pip-version-check \ - --no-index \ - --no-build-isolation \ - --prefix "$tmp_path/python" \ - --editable $repo_root + # Install the package in editable mode + # This allows executing `clan` from within the dev-shell using the current + # version of the code and its dependencies. + ${pythonWithDeps.interpreter} -m pip install \ + --quiet \ + --disable-pip-version-check \ + --no-index \ + --no-build-isolation \ + --prefix "$tmp_path/python" \ + --editable $repo_root - rm -f clan_cli/nixpkgs clan_cli/webui/assets - ln -sf ${clan-cli.nixpkgs} clan_cli/nixpkgs - ln -sf ${ui-assets} clan_cli/webui/assets + rm -f clan_cli/nixpkgs clan_cli/webui/assets + ln -sf ${clan-cli.nixpkgs} clan_cli/nixpkgs + ln -sf ${ui-assets} clan_cli/webui/assets - export PATH="$tmp_path/python/bin:${checkScript}/bin:$PATH" - export PYTHONPATH="$repo_root:$tmp_path/python/${pythonWithDeps.sitePackages}:" - export PYTHONBREAKPOINT=ipdb.set_trace + export PATH="$tmp_path/python/bin:${checkScript}/bin:$PATH" + export PYTHONPATH="$repo_root:$tmp_path/python/${pythonWithDeps.sitePackages}:" + export PYTHONBREAKPOINT=ipdb.set_trace - export XDG_DATA_DIRS="$tmp_path/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}" - export fish_complete_path="$tmp_path/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}" - mkdir -p \ - $tmp_path/share/fish/vendor_completions.d \ - $tmp_path/share/bash-completion/completions \ - $tmp_path/share/zsh/site-functions - register-python-argcomplete --shell fish clan > $tmp_path/share/fish/vendor_completions.d/clan.fish - register-python-argcomplete --shell bash clan > $tmp_path/share/bash-completion/completions/clan + export XDG_DATA_DIRS="$tmp_path/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}" + export fish_complete_path="$tmp_path/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}" + mkdir -p \ + $tmp_path/share/fish/vendor_completions.d \ + $tmp_path/share/bash-completion/completions \ + $tmp_path/share/zsh/site-functions + register-python-argcomplete --shell fish clan > $tmp_path/share/fish/vendor_completions.d/clan.fish + register-python-argcomplete --shell bash clan > $tmp_path/share/bash-completion/completions/clan - ./bin/clan flakes create example_clan - ./bin/clan machines create example_machine example_clan + ./bin/clan flakes create example_clan + ./bin/clan machines create example_machine example_clan ''; } diff --git a/pkgs/clan-cli/tests/fixtures_flakes.py b/pkgs/clan-cli/tests/fixtures_flakes.py index 9db5a37d8..ebedf19d4 100644 --- a/pkgs/clan-cli/tests/fixtures_flakes.py +++ b/pkgs/clan-cli/tests/fixtures_flakes.py @@ -112,5 +112,4 @@ def test_flake_with_core_and_pass( temporary_home, FlakeName("test_flake_with_core_and_pass"), CLAN_CORE, - ) diff --git a/pkgs/clan-cli/tests/test_config.py b/pkgs/clan-cli/tests/test_config.py index 329214497..4266c7121 100644 --- a/pkgs/clan-cli/tests/test_config.py +++ b/pkgs/clan-cli/tests/test_config.py @@ -5,11 +5,11 @@ from typing import Any, Optional import pytest from cli import Cli +from fixtures_flakes import FlakeForTest from clan_cli import config from clan_cli.config import parsing from clan_cli.errors import ClanError -from fixtures_flakes import FlakeForTest example_options = f"{Path(config.__file__).parent}/jsonschema/options.json" diff --git a/pkgs/clan-cli/tests/test_flake_api.py b/pkgs/clan-cli/tests/test_flake_api.py index 6f0414cda..6b1d6f08d 100644 --- a/pkgs/clan-cli/tests/test_flake_api.py +++ b/pkgs/clan-cli/tests/test_flake_api.py @@ -1,8 +1,8 @@ import json -from pathlib import Path -from fixtures_flakes import FlakeForTest + import pytest from api import TestClient +from fixtures_flakes import FlakeForTest @pytest.mark.impure diff --git a/pkgs/clan-cli/tests/test_import_sops_cli.py b/pkgs/clan-cli/tests/test_import_sops_cli.py index 872f3afbb..58d8e3f7e 100644 --- a/pkgs/clan-cli/tests/test_import_sops_cli.py +++ b/pkgs/clan-cli/tests/test_import_sops_cli.py @@ -1,10 +1,11 @@ from pathlib import Path from typing import TYPE_CHECKING -from fixtures_flakes import FlakeForTest -from clan_cli.debug import repro_env_break import pytest from cli import Cli +from fixtures_flakes import FlakeForTest + +from clan_cli.debug import repro_env_break if TYPE_CHECKING: from age_keys import KeyPair @@ -20,7 +21,9 @@ def test_import_sops( cli = Cli() monkeypatch.setenv("SOPS_AGE_KEY", age_keys[1].privkey) - cli.run(["secrets", "machines", "add", "machine1", age_keys[0].pubkey, test_flake.name]) + cli.run( + ["secrets", "machines", "add", "machine1", age_keys[0].pubkey, test_flake.name] + ) cli.run(["secrets", "users", "add", "user1", age_keys[1].pubkey, test_flake.name]) cli.run(["secrets", "users", "add", "user2", age_keys[2].pubkey, test_flake.name]) cli.run(["secrets", "groups", "add-user", "group1", "user1", test_flake.name]) @@ -29,19 +32,17 @@ def test_import_sops( # To edit: # SOPS_AGE_KEY=AGE-SECRET-KEY-1U5ENXZQAY62NC78Y2WC0SEGRRMAEEKH79EYY5TH4GPFWJKEAY0USZ6X7YQ sops --age age14tva0txcrl0zes05x7gkx56qd6wd9q3nwecjac74xxzz4l47r44sv3fz62 ./data/secrets.yaml cmd = [ - "secrets", - "import-sops", - "--group", - "group1", - "--machine", - "machine1", - str(test_root.joinpath("data", "secrets.yaml")), - test_flake.name - ] + "secrets", + "import-sops", + "--group", + "group1", + "--machine", + "machine1", + str(test_root.joinpath("data", "secrets.yaml")), + test_flake.name, + ] repro_env_break(work_dir=test_flake.path, cmd=cmd) - cli.run( - cmd - ) + cli.run(cmd) capsys.readouterr() cli.run(["secrets", "users", "list", test_flake.name]) users = sorted(capsys.readouterr().out.rstrip().split()) diff --git a/pkgs/clan-cli/tests/test_machines_api.py b/pkgs/clan-cli/tests/test_machines_api.py index 7cd71d43f..7287f18a4 100644 --- a/pkgs/clan-cli/tests/test_machines_api.py +++ b/pkgs/clan-cli/tests/test_machines_api.py @@ -1,14 +1,13 @@ -from pathlib import Path - from api import TestClient from fixtures_flakes import FlakeForTest -from clan_cli.debug import repro_env_break + def test_machines(api: TestClient, test_flake: FlakeForTest) -> None: response = api.get(f"/api/{test_flake.name}/machines") assert response.status_code == 200 assert response.json() == {"machines": []} + # TODO: Fails because the test_flake fixture needs to init a git repo, which it currently does not response = api.post(f"/api/{test_flake.name}/machines", json={"name": "test"}) assert response.status_code == 201 assert response.json() == {"machine": {"name": "test", "status": "unknown"}} diff --git a/pkgs/clan-cli/tests/test_secrets_generate.py b/pkgs/clan-cli/tests/test_secrets_generate.py index 350805bad..54b0311ec 100644 --- a/pkgs/clan-cli/tests/test_secrets_generate.py +++ b/pkgs/clan-cli/tests/test_secrets_generate.py @@ -21,7 +21,16 @@ def test_generate_secret( monkeypatch.chdir(test_flake_with_core.path) monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey) cli = Cli() - cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey, test_flake_with_core.name]) + cli.run( + [ + "secrets", + "users", + "add", + "user1", + age_keys[0].pubkey, + test_flake_with_core.name, + ] + ) cli.run(["secrets", "generate", "vm1", test_flake_with_core.name]) has_secret(test_flake_with_core.name, "vm1-age.key") has_secret(test_flake_with_core.name, "vm1-zerotier-identity-secret") diff --git a/pkgs/clan-cli/tests/test_secrets_upload.py b/pkgs/clan-cli/tests/test_secrets_upload.py index e68e8baf5..ec64d2c5d 100644 --- a/pkgs/clan-cli/tests/test_secrets_upload.py +++ b/pkgs/clan-cli/tests/test_secrets_upload.py @@ -1,9 +1,9 @@ -from pathlib import Path from typing import TYPE_CHECKING import pytest from cli import Cli from fixtures_flakes import FlakeForTest + from clan_cli.ssh import HostGroup if TYPE_CHECKING: @@ -21,9 +21,27 @@ def test_secrets_upload( monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey) cli = Cli() - cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey, test_flake_with_core.name]) + cli.run( + [ + "secrets", + "users", + "add", + "user1", + age_keys[0].pubkey, + test_flake_with_core.name, + ] + ) - cli.run(["secrets", "machines", "add", "vm1", age_keys[1].pubkey, test_flake_with_core.name]) + cli.run( + [ + "secrets", + "machines", + "add", + "vm1", + age_keys[1].pubkey, + test_flake_with_core.name, + ] + ) monkeypatch.setenv("SOPS_NIX_SECRET", age_keys[0].privkey) cli.run(["secrets", "set", "vm1-age.key", test_flake_with_core.name]) diff --git a/pkgs/clan-cli/tests/test_vms_api.py b/pkgs/clan-cli/tests/test_vms_api.py index fdc1963c3..daa6456d2 100644 --- a/pkgs/clan-cli/tests/test_vms_api.py +++ b/pkgs/clan-cli/tests/test_vms_api.py @@ -1,9 +1,8 @@ -from pathlib import Path - import pytest from api import TestClient from fixtures_flakes import FlakeForTest + @pytest.mark.impure def test_inspect(api: TestClient, test_flake_with_core: FlakeForTest) -> None: response = api.post( diff --git a/pkgs/clan-cli/tests/test_vms_api_create.py b/pkgs/clan-cli/tests/test_vms_api_create.py index df598f261..71b7b81be 100644 --- a/pkgs/clan-cli/tests/test_vms_api_create.py +++ b/pkgs/clan-cli/tests/test_vms_api_create.py @@ -9,7 +9,6 @@ from fixtures_flakes import FlakeForTest, create_flake from httpx import SyncByteStream from root import CLAN_CORE -from clan_cli.debug import repro_env_break from clan_cli.types import FlakeName if TYPE_CHECKING: @@ -18,8 +17,7 @@ if TYPE_CHECKING: @pytest.fixture def flake_with_vm_with_secrets( - monkeypatch: pytest.MonkeyPatch, - temporary_home: Path + monkeypatch: pytest.MonkeyPatch, temporary_home: Path ) -> Iterator[FlakeForTest]: yield from create_flake( monkeypatch, @@ -44,8 +42,6 @@ def remote_flake_with_vm_without_secrets( ) - - def generic_create_vm_test(api: TestClient, flake: Path, vm: str) -> None: print(f"flake_url: {flake} ") response = api.post( @@ -94,7 +90,14 @@ def test_create_local( ) -> None: monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey) cli = Cli() - cmd = ["secrets", "users", "add", "user1", age_keys[0].pubkey, flake_with_vm_with_secrets.name] + cmd = [ + "secrets", + "users", + "add", + "user1", + age_keys[0].pubkey, + flake_with_vm_with_secrets.name, + ] cli.run(cmd) generic_create_vm_test(api, flake_with_vm_with_secrets.path, "vm_with_secrets") diff --git a/pkgs/clan-cli/tests/test_vms_cli.py b/pkgs/clan-cli/tests/test_vms_cli.py index 75d7e8e72..d5a51e638 100644 --- a/pkgs/clan-cli/tests/test_vms_cli.py +++ b/pkgs/clan-cli/tests/test_vms_cli.py @@ -1,9 +1,9 @@ import os -from pathlib import Path from typing import TYPE_CHECKING -from fixtures_flakes import FlakeForTest + import pytest from cli import Cli +from fixtures_flakes import FlakeForTest if TYPE_CHECKING: from age_keys import KeyPair @@ -12,7 +12,9 @@ no_kvm = not os.path.exists("/dev/kvm") @pytest.mark.impure -def test_inspect(test_flake_with_core: FlakeForTest, capsys: pytest.CaptureFixture) -> None: +def test_inspect( + test_flake_with_core: FlakeForTest, capsys: pytest.CaptureFixture +) -> None: cli = Cli() cli.run(["vms", "inspect", "vm1", test_flake_with_core.name]) out = capsys.readouterr() # empty the buffer @@ -29,5 +31,14 @@ def test_create( monkeypatch.chdir(test_flake_with_core.path) monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey) cli = Cli() - cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey, test_flake_with_core.name]) + cli.run( + [ + "secrets", + "users", + "add", + "user1", + age_keys[0].pubkey, + test_flake_with_core.name, + ] + ) cli.run(["vms", "create", "vm1", test_flake_with_core.name])