From a65413c98e8f5f5a62b20b164ca62db95a482f9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 16:22:05 +0200 Subject: [PATCH 1/5] use experimental flags everywhere --- pkgs/clan-cli/clan_cli/admin.py | 20 ++++--- pkgs/clan-cli/clan_cli/config/parsing.py | 11 ++-- pkgs/clan-cli/clan_cli/machines/update.py | 4 +- pkgs/clan-cli/clan_cli/nix.py | 69 ++++++++++++----------- pkgs/clan-cli/tests/test_ssh_cli.py | 6 +- 5 files changed, 58 insertions(+), 52 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/admin.py b/pkgs/clan-cli/clan_cli/admin.py index 2efe0816d..9b47aaa2d 100644 --- a/pkgs/clan-cli/clan_cli/admin.py +++ b/pkgs/clan-cli/clan_cli/admin.py @@ -3,18 +3,22 @@ import argparse import os import subprocess +from .nix import nix_command + def create(args: argparse.Namespace) -> None: os.makedirs(args.folder, exist_ok=True) # TODO create clan template in flake subprocess.run( - [ - "nix", - "flake", - "init", - "-t", - "git+https://git.clan.lol/clan/clan-core#new-clan", - ] + nix_command( + [ + "flake", + "init", + "-t", + "git+https://git.clan.lol/clan/clan-core#new-clan", + ] + ), + check=True, ) @@ -24,7 +28,7 @@ def register_parser(parser: argparse.ArgumentParser) -> None: "-f", "--folder", help="the folder where the clan is defined, default to the current folder", - default=os.environ["PWD"], + default=os.getcwd(), ) subparser = parser.add_subparsers( title="command", diff --git a/pkgs/clan-cli/clan_cli/config/parsing.py b/pkgs/clan-cli/clan_cli/config/parsing.py index 92b476b99..3178224c5 100644 --- a/pkgs/clan-cli/clan_cli/config/parsing.py +++ b/pkgs/clan-cli/clan_cli/config/parsing.py @@ -3,7 +3,8 @@ import subprocess from pathlib import Path from typing import Any, Optional, Type, Union -from clan_cli.errors import ClanError +from ..errors import ClanError +from ..nix import nix_eval script_dir = Path(__file__).parent @@ -30,11 +31,9 @@ def schema_from_module_file( slib.parseModule {absolute_path} """ # run the nix expression and parse the output as json - return json.loads( - subprocess.check_output( - ["nix", "eval", "--impure", "--json", "--expr", nix_expr] - ) - ) + cmd = nix_eval(["--expr", nix_expr]) + proc = subprocess.run(cmd, stdout=subprocess.PIPE, check=True) + return json.loads(proc.stdout) def subtype_from_schema(schema: dict[str, Any]) -> Type: diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 963b0a15c..875f7fdde 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -5,7 +5,7 @@ import subprocess from typing import Optional from ..dirs import get_clan_flake_toplevel -from ..nix import nix_eval +from ..nix import nix_command, nix_eval from ..secrets.generate import generate_secrets from ..secrets.upload import upload_secrets from ..ssh import Host, HostGroup, HostKeyCheck @@ -22,7 +22,7 @@ def deploy_nixos(hosts: HostGroup) -> None: env = os.environ.copy() env["NIX_SSHOPTS"] = ssh_arg res = h.run_local( - ["nix", "flake", "archive", "--to", f"ssh://{target}", "--json"], + nix_command(["flake", "archive", "--to", f"ssh://{target}", "--json"]), check=True, stdout=subprocess.PIPE, extra_env=env, diff --git a/pkgs/clan-cli/clan_cli/nix.py b/pkgs/clan-cli/clan_cli/nix.py index e64790377..72dbced10 100644 --- a/pkgs/clan-cli/clan_cli/nix.py +++ b/pkgs/clan-cli/clan_cli/nix.py @@ -4,28 +4,35 @@ import tempfile from .dirs import nixpkgs_flake, nixpkgs_source, unfree_nixpkgs +def nix_command(flags: list[str]) -> list[str]: + return ["nix", "--experimental-features", "nix-command flakes"] + flags + + def nix_build( flags: list[str], ) -> list[str]: - return [ - "nix", - "build", - "--no-link", - "--print-out-paths", - "--extra-experimental-features", - "nix-command flakes", - ] + flags + return ( + nix_command( + [ + "build", + "--no-link", + "--print-out-paths", + "--extra-experimental-features", + "nix-command flakes", + ] + ) + + flags + ) def nix_eval(flags: list[str]) -> list[str]: - default_flags = [ - "nix", - "eval", - "--show-trace", - "--json", - "--extra-experimental-features", - "nix-command flakes", - ] + default_flags = nix_command( + [ + "eval", + "--show-trace", + "--json", + ] + ) if os.environ.get("IN_NIX_SANDBOX"): with tempfile.TemporaryDirectory() as nix_store: return ( @@ -51,14 +58,13 @@ def nix_shell(packages: list[str], cmd: list[str]) -> list[str]: return cmd wrapped_packages = [f"nixpkgs#{p}" for p in packages] return ( - [ - "nix", - "shell", - "--extra-experimental-features", - "nix-command flakes", - "--inputs-from", - f"{str(nixpkgs_flake())}", - ] + nix_command( + [ + "shell", + "--inputs-from", + f"{str(nixpkgs_flake())}", + ] + ) + wrapped_packages + ["-c"] + cmd @@ -69,14 +75,13 @@ def unfree_nix_shell(packages: list[str], cmd: list[str]) -> list[str]: if os.environ.get("IN_NIX_SANDBOX"): return cmd return ( - [ - "nix", - "shell", - "--extra-experimental-features", - "nix-command flakes", - "-f", - str(unfree_nixpkgs()), - ] + nix_command( + [ + "shell", + "-f", + str(unfree_nixpkgs()), + ] + ) + packages + ["-c"] + cmd diff --git a/pkgs/clan-cli/tests/test_ssh_cli.py b/pkgs/clan-cli/tests/test_ssh_cli.py index 11b839fe3..8a7f43d7b 100644 --- a/pkgs/clan-cli/tests/test_ssh_cli.py +++ b/pkgs/clan-cli/tests/test_ssh_cli.py @@ -30,9 +30,8 @@ def test_ssh_no_pass( monkeypatch.delenv("IN_NIX_SANDBOX") cmd: list[Union[str, utils.Any]] = [ "nix", + fp.any(), "shell", - "--extra-experimental-features", - "nix-command flakes", fp.any(), "-c", "torify", @@ -61,9 +60,8 @@ def test_ssh_with_pass( monkeypatch.delenv("IN_NIX_SANDBOX") cmd: list[Union[str, utils.Any]] = [ "nix", + fp.any(), "shell", - "--extra-experimental-features", - "nix-command flakes", fp.any(), "-c", "torify", From d41cca2e2874c828fbf92464307f34188722615b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 16:34:16 +0200 Subject: [PATCH 2/5] add clan_core path helper function --- pkgs/clan-cli/tests/root.py | 11 ++++++++++- pkgs/clan-cli/tests/test_flake.py | 7 +++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pkgs/clan-cli/tests/root.py b/pkgs/clan-cli/tests/root.py index c881ce026..0105e82a8 100644 --- a/pkgs/clan-cli/tests/root.py +++ b/pkgs/clan-cli/tests/root.py @@ -4,12 +4,13 @@ import pytest TEST_ROOT = Path(__file__).parent.resolve() PROJECT_ROOT = TEST_ROOT.parent +CLAN_CORE = PROJECT_ROOT.parent.parent @pytest.fixture(scope="session") def project_root() -> Path: """ - Root directory of the tests + Root directory the clan-cli """ return PROJECT_ROOT @@ -20,3 +21,11 @@ def test_root() -> Path: Root directory of the tests """ return TEST_ROOT + + +@pytest.fixture(scope="session") +def clan_core() -> Path: + """ + Directory of the clan-core flake + """ + return CLAN_CORE diff --git a/pkgs/clan-cli/tests/test_flake.py b/pkgs/clan-cli/tests/test_flake.py index cff8caf75..66083b19c 100644 --- a/pkgs/clan-cli/tests/test_flake.py +++ b/pkgs/clan-cli/tests/test_flake.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Iterator import pytest -from root import PROJECT_ROOT +from root import CLAN_CORE from clan_cli.dirs import nixpkgs_source @@ -46,9 +46,8 @@ def test_flake(monkeymodule: pytest.MonkeyPatch) -> Iterator[Path]: @pytest.fixture(scope="module") def test_flake_with_core(monkeymodule: pytest.MonkeyPatch) -> Iterator[Path]: - clan_core_flake = PROJECT_ROOT.parent.parent - if not (clan_core_flake / "flake.nix").exists(): + if not (CLAN_CORE / "flake.nix").exists(): raise Exception( "clan-core flake not found. This test requires the clan-core flake to be present" ) - yield from create_flake(monkeymodule, "test_flake_with_core", clan_core_flake) + yield from create_flake(monkeymodule, "test_flake_with_core", CLAN_CORE) From 164f95723dbbff40fe6bf41043cfbcc29e734b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 16:42:58 +0200 Subject: [PATCH 3/5] pytest: add impure marker --- pkgs/clan-cli/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/clan-cli/pyproject.toml b/pkgs/clan-cli/pyproject.toml index 127d6cfc6..1c16c93a1 100644 --- a/pkgs/clan-cli/pyproject.toml +++ b/pkgs/clan-cli/pyproject.toml @@ -18,6 +18,7 @@ clan_cli = [ "config/jsonschema/*", "webui/assets/**/*"] faulthandler_timeout = 30 addopts = "--cov . --cov-report term --cov-report html:.reports/html --no-cov-on-fail --workers auto --durations 5" norecursedirs = "tests/helpers" +markers = [ "impure" ] [tool.mypy] python_version = "3.10" From f6a7e42e38c4bd2e0c776a0fcbf0771c084517b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 16:36:09 +0200 Subject: [PATCH 4/5] cli: rename admin create to create --- pkgs/clan-cli/clan_cli/__init__.py | 6 +++--- pkgs/clan-cli/clan_cli/{admin.py => create.py} | 18 +----------------- pkgs/clan-cli/tests/test_admin_cli.py | 14 -------------- pkgs/clan-cli/tests/test_clan_template.py | 12 ++++++++++++ 4 files changed, 16 insertions(+), 34 deletions(-) rename pkgs/clan-cli/clan_cli/{admin.py => create.py} (50%) delete mode 100644 pkgs/clan-cli/tests/test_admin_cli.py create mode 100644 pkgs/clan-cli/tests/test_clan_template.py diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index ff67eeaf6..0eb6dd527 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -4,7 +4,7 @@ import sys from types import ModuleType from typing import Optional -from . import admin, config, machines, secrets, webui, zerotier +from . import config, create, machines, secrets, webui, zerotier from .errors import ClanError from .ssh import cli as ssh_cli @@ -19,8 +19,8 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog=prog, description="cLAN tool") subparsers = parser.add_subparsers() - parser_admin = subparsers.add_parser("admin", help="administrate a clan") - admin.register_parser(parser_admin) + parser_create = subparsers.add_parser("create", help="create a clan flake") + create.register_parser(parser_create) # DISABLED: this currently crashes if a flake does not define .#clanOptions if os.environ.get("CLAN_OPTIONS_FILE") is not None: diff --git a/pkgs/clan-cli/clan_cli/admin.py b/pkgs/clan-cli/clan_cli/create.py similarity index 50% rename from pkgs/clan-cli/clan_cli/admin.py rename to pkgs/clan-cli/clan_cli/create.py index 9b47aaa2d..2a6b3ce80 100644 --- a/pkgs/clan-cli/clan_cli/admin.py +++ b/pkgs/clan-cli/clan_cli/create.py @@ -1,13 +1,11 @@ # !/usr/bin/env python3 import argparse -import os import subprocess from .nix import nix_command def create(args: argparse.Namespace) -> None: - os.makedirs(args.folder, exist_ok=True) # TODO create clan template in flake subprocess.run( nix_command( @@ -24,18 +22,4 @@ def create(args: argparse.Namespace) -> None: # takes a (sub)parser and configures it def register_parser(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "-f", - "--folder", - help="the folder where the clan is defined, default to the current folder", - default=os.getcwd(), - ) - subparser = parser.add_subparsers( - title="command", - description="the command to run", - help="the command to run", - required=True, - ) - - parser_create = subparser.add_parser("create", help="create a new clan") - parser_create.set_defaults(func=create) + parser.set_defaults(func=create) diff --git a/pkgs/clan-cli/tests/test_admin_cli.py b/pkgs/clan-cli/tests/test_admin_cli.py deleted file mode 100644 index dba43d147..000000000 --- a/pkgs/clan-cli/tests/test_admin_cli.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import Union - -import pytest_subprocess.fake_process -from cli import Cli -from pytest_subprocess import utils - - -# using fp fixture from pytest-subprocess -def test_create(fp: pytest_subprocess.fake_process.FakeProcess) -> None: - cmd: list[Union[str, utils.Any]] = ["nix", "flake", "init", "-t", fp.any()] - fp.register(cmd) - cli = Cli() - cli.run(["admin", "--folder", "./my-clan", "create"]) - assert fp.call_count(cmd) == 1 diff --git a/pkgs/clan-cli/tests/test_clan_template.py b/pkgs/clan-cli/tests/test_clan_template.py new file mode 100644 index 000000000..10b4a9b3a --- /dev/null +++ b/pkgs/clan-cli/tests/test_clan_template.py @@ -0,0 +1,12 @@ +from pathlib import Path + +import pytest +from cli import Cli + + +@pytest.mark.impure +def test_template(monkeypatch: pytest.MonkeyPatch, temporary_dir: Path) -> None: + monkeypatch.chdir(temporary_dir) + cli = Cli() + cli.run(["create"]) + assert (temporary_dir / ".clan-flake").exists() From e753b9a87f2d381ae18468560ac4b38156891e21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 19 Sep 2023 13:02:11 +0200 Subject: [PATCH 5/5] try to fix ci by pinning nix-fast-build --- .gitea/workflows/checks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/checks.yaml b/.gitea/workflows/checks.yaml index 7f1fda2fb..6705af0d1 100644 --- a/.gitea/workflows/checks.yaml +++ b/.gitea/workflows/checks.yaml @@ -8,4 +8,4 @@ jobs: runs-on: nix steps: - uses: actions/checkout@v3 - - run: nix run --refresh github:Mic92/nix-ci-build + - run: nix run --refresh github:Mic92/nix-fast-build/ae50c356c2f9e790f3d9d8e00bfa9f4b54f49bdd