diff --git a/formatter.nix b/formatter.nix index 5dbbc7754..e576411a4 100644 --- a/formatter.nix +++ b/formatter.nix @@ -108,13 +108,6 @@ extraPythonPackages = (self'.packages.clan-app.devshellPyDeps pkgs.python3Packages); extraPythonPaths = [ "../../clan-cli" ]; }; - "generate-test-vars" = { - directory = "pkgs/generate-test-vars"; - extraPythonPackages = [ - (pkgs.python3.withPackages (ps: self'.packages.clan-cli.devshellPyDeps ps)) - ]; - extraPythonPaths = [ "../clan-cli" ]; - }; } // ( if pkgs.stdenv.isLinux then diff --git a/lib/clanTest/flake-module.nix b/lib/clanTest/flake-module.nix index 503e348ab..2fede3e21 100644 --- a/lib/clanTest/flake-module.nix +++ b/lib/clanTest/flake-module.nix @@ -77,9 +77,7 @@ in ) machineModules ); - update-vars-script = "${ - self.packages.${hostPkgs.system}.generate-test-vars - }/bin/generate-test-vars"; + update-vars-script = "${self.packages.${hostPkgs.system}.clan-cli}/bin/clan-generate-test-vars"; relativeDir = removePrefix "${self}/" (toString config.clan.directory); diff --git a/pkgs/clan-cli/bin/clan-generate-test-vars b/pkgs/clan-cli/bin/clan-generate-test-vars new file mode 100755 index 000000000..b9bd295ca --- /dev/null +++ b/pkgs/clan-cli/bin/clan-generate-test-vars @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +import os +import sys + +sys.path.insert( + 0, os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) +) + +from clan_cli.generate_test_vars.cli import main # NOQA + +if __name__ == "__main__": + main() diff --git a/pkgs/generate-test-vars/generate_test_vars/__init_.py b/pkgs/clan-cli/clan_cli/generate_test_vars/__init_.py similarity index 100% rename from pkgs/generate-test-vars/generate_test_vars/__init_.py rename to pkgs/clan-cli/clan_cli/generate_test_vars/__init_.py diff --git a/pkgs/generate-test-vars/generate_test_vars/cli.py b/pkgs/clan-cli/clan_cli/generate_test_vars/cli.py similarity index 82% rename from pkgs/generate-test-vars/generate_test_vars/cli.py rename to pkgs/clan-cli/clan_cli/generate_test_vars/cli.py index e5dc44b4f..7ddf0856b 100755 --- a/pkgs/generate-test-vars/generate_test_vars/cli.py +++ b/pkgs/clan-cli/clan_cli/generate_test_vars/cli.py @@ -9,7 +9,7 @@ import subprocess from dataclasses import dataclass from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any, override +from typing import TYPE_CHECKING, Any, override from clan_cli.vars.generator import Generator from clan_cli.vars.prompt import PromptType @@ -20,6 +20,12 @@ from clan_lib.machines.machines import Machine from clan_lib.nix import nix_config, nix_eval, nix_test_store from clan_lib.vars.generate import run_generators +if TYPE_CHECKING: + from clan_lib.machines.actions import ( + ListOptions, + MachineResponse, + ) + log = logging.getLogger(__name__) sops_priv_key = ( @@ -87,6 +93,26 @@ class TestFlake(Flake): full_selector = f'checks."{test_system}".{self.check_attr}.machinesCross.{system}."{machine_name}".{selector}' return self.select(full_selector) + # we don't want to evaluate all machines of the flake. Only the ones defined in the test + def set_machine_names(self, machine_names: list[str]) -> None: + """Set the machine names for this flake instance to fake the machines defined by the test""" + self._machine_names = machine_names + + def list_machines( + self, + opts: "ListOptions | None" = None, # noqa: ARG002 + ) -> "dict[str, MachineResponse]": + """List machines of a clan""" + from clan_lib.machines.actions import ( # noqa: PLC0415 + InventoryMachine, + MachineResponse, + ) + + res = {} + for name in self._machine_names: + res[name] = MachineResponse(data=InventoryMachine()) + return res + class TestMachine(Machine): """Machine class which is able to deal with not having an actual flake. @@ -180,13 +206,13 @@ def parse_args() -> Options: ) -def main() -> None: - logging.basicConfig(level=logging.DEBUG) - os.environ["CLAN_NO_COMMIT"] = "1" - opts = parse_args() - test_dir = opts.test_dir - - if opts.clean: +def generate_test_vars( + clean: bool, + repo_root: Path, + test_dir: Path, + check_attr: str, +) -> None: + if clean: shutil.rmtree(test_dir / "vars", ignore_errors=True) shutil.rmtree(test_dir / "sops", ignore_errors=True) @@ -196,25 +222,27 @@ def main() -> None: if system.endswith("-darwin"): test_system = system.rstrip("darwin") + "linux" - flake = TestFlake(opts.check_attr, test_dir, str(opts.repo_root)) + flake = TestFlake(check_attr, test_dir, str(repo_root)) machine_names = get_machine_names( - opts.repo_root, - opts.check_attr, + repo_root, + check_attr, test_system, ) + flake.set_machine_names(machine_names) + flake.precache( [ - f"checks.{test_system}.{opts.check_attr}.machinesCross.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.validationHash", + f"checks.{test_system}.{check_attr}.machinesCross.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.validationHash", ], ) # This hack is necessary because the sops store uses flake.path to find the machine keys # This hack does not work because flake.invalidate_cache resets _path - flake._path = opts.test_dir # noqa: SLF001 + flake._path = test_dir # noqa: SLF001 machines = [ - TestMachine(name, flake, test_dir, opts.check_attr) for name in machine_names + TestMachine(name, flake, test_dir, check_attr) for name in machine_names ] user = "admin" admin_key_path = Path(test_dir.resolve() / "sops" / "users" / user / "key.json") @@ -259,5 +287,17 @@ def main() -> None: run_generators(list(machines), prompt_values=mocked_prompts) +def main() -> None: + logging.basicConfig(level=logging.DEBUG) + os.environ["CLAN_NO_COMMIT"] = "1" + args = parse_args() + generate_test_vars( + clean=args.clean, + repo_root=args.repo_root, + test_dir=args.repo_root / args.test_dir, + check_attr=args.check_attr, + ) + + if __name__ == "__main__": main() diff --git a/pkgs/clan-cli/clan_cli/generate_test_vars/test_generate_test_vars.py b/pkgs/clan-cli/clan_cli/generate_test_vars/test_generate_test_vars.py new file mode 100644 index 000000000..130b603d8 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/generate_test_vars/test_generate_test_vars.py @@ -0,0 +1,26 @@ +import shutil +import subprocess +from pathlib import Path + +import pytest +from clan_cli.generate_test_vars.cli import generate_test_vars + + +@pytest.mark.with_core +def test_generate_test_vars( + clan_core: Path, + temp_dir: Path, +) -> None: + test_dir_original = clan_core / "checks/service-dummy-test" + service_dir = temp_dir / "service-dummy-test" + shutil.copytree(test_dir_original, service_dir) + + # Make the copied tree writable + subprocess.run(["chmod", "-R", "+w", str(service_dir)], check=True) + + generate_test_vars( + clean=True, + repo_root=clan_core, + test_dir=service_dir, + check_attr="service-dummy-test", + ) diff --git a/pkgs/clan-cli/clan_lib/flake/flake.py b/pkgs/clan-cli/clan_lib/flake/flake.py index 1b008dfe7..6814f9104 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake.py +++ b/pkgs/clan-cli/clan_lib/flake/flake.py @@ -9,7 +9,7 @@ from functools import cache from hashlib import sha1 from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any +from typing import TYPE_CHECKING, Any from clan_lib.cmd import Log, RunOpts, run from clan_lib.dirs import select_source, user_cache_dir @@ -22,6 +22,12 @@ from clan_lib.nix import ( nix_test_store, ) +if TYPE_CHECKING: + from clan_lib.machines.actions import ( + ListOptions, + MachineResponse, + ) + log = logging.getLogger(__name__) @@ -1102,6 +1108,15 @@ class Flake: full_selector = f'clanInternals.machines."{system}"."{machine_name}".{selector}' return self.select(full_selector) + def list_machines( + self, + opts: "ListOptions | None" = None, + ) -> "dict[str, MachineResponse]": + """List machines of a clan""" + from clan_lib.machines.actions import list_machines # noqa: PLC0415 + + return list_machines(self, opts) + def require_flake(flake: Flake | None) -> Flake: """Require that a flake argument is provided, if not in a clan flake. diff --git a/pkgs/clan-cli/clan_lib/vars/generate.py b/pkgs/clan-cli/clan_lib/vars/generate.py index 8f7d12725..6c52558c3 100644 --- a/pkgs/clan-cli/clan_lib/vars/generate.py +++ b/pkgs/clan-cli/clan_lib/vars/generate.py @@ -8,7 +8,6 @@ from clan_cli.vars.migration import check_can_migrate, migrate_files from clan_lib.api import API from clan_lib.errors import ClanError -from clan_lib.machines.actions import list_machines from clan_lib.machines.machines import Machine log = logging.getLogger(__name__) @@ -39,7 +38,7 @@ def get_generators( msg = "At least one machine must be provided" raise ClanError(msg) - all_machines = list_machines(machines[0].flake).keys() + all_machines = machines[0].flake.list_machines().keys() requested_machines = [machine.name for machine in machines] all_generators_list = Generator.get_machine_generators( diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index c831ec063..a4c6cf6b5 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -26,7 +26,7 @@ let ps.argcomplete # Enables shell completions # uncomment web-pdb for debugging: - # (pkgs.callPackage ./python-deps.nix {}).web-pdb + # (pkgs.callPackage ./python-deps.nix { }).web-pdb ]; devDeps = ps: [ ps.ipython diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index bab9fe618..4e48ff24a 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -28,6 +28,10 @@ "pkgs/zerotierone" "pkgs/minifakeroot" "pkgs/clan-cli/clan_cli/tests/flake-module.nix" + + # needed for test_generate_test_vars.py + "checks/service-dummy-test" + "checks/flake-module.nix" ]; }; }; diff --git a/pkgs/clan-cli/pyproject.toml b/pkgs/clan-cli/pyproject.toml index 077bac8c4..efbdde9b6 100644 --- a/pkgs/clan-cli/pyproject.toml +++ b/pkgs/clan-cli/pyproject.toml @@ -6,9 +6,12 @@ build-backend = "setuptools.build_meta" name = "clan" description = "clan cli tool" dynamic = ["version"] -scripts = { clan = "clan_cli.cli:main" } license = { text = "MIT" } +[project.scripts] +clan = "clan_cli.cli:main" +clan-generate-test-vars = "clan_cli.generate_test_vars.cli:main" + [project.urls] Homepage = "https://clan.lol/" Documentation = "https://docs.clan.lol/" @@ -55,4 +58,4 @@ warn_redundant_casts = true disallow_untyped_calls = true disallow_untyped_defs = true no_implicit_optional = true -exclude = "clan_lib.nixpkgs" \ No newline at end of file +exclude = "clan_lib.nixpkgs" diff --git a/pkgs/clan-cli/clan_cli/python-deps.nix b/pkgs/clan-cli/python-deps.nix similarity index 100% rename from pkgs/clan-cli/clan_cli/python-deps.nix rename to pkgs/clan-cli/python-deps.nix diff --git a/pkgs/flake-module.nix b/pkgs/flake-module.nix index d0a6e375a..bcdacba07 100644 --- a/pkgs/flake-module.nix +++ b/pkgs/flake-module.nix @@ -6,7 +6,6 @@ ./clan-vm-manager/flake-module.nix ./installer/flake-module.nix ./icon-update/flake-module.nix - ./generate-test-vars/flake-module.nix ./clan-core-flake/flake-module.nix ./clan-app/flake-module.nix ./testing/flake-module.nix diff --git a/pkgs/generate-test-vars/default.nix b/pkgs/generate-test-vars/default.nix deleted file mode 100644 index 783ac7a96..000000000 --- a/pkgs/generate-test-vars/default.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - buildPythonApplication, - python, - clan-cli, -}: -buildPythonApplication { - name = "generate-test-vars"; - src = ./.; - format = "pyproject"; - dependencies = [ (python.pkgs.toPythonModule clan-cli) ]; - nativeBuildInputs = [ - (python.withPackages (ps: [ ps.setuptools ])) - ]; - checkPhase = '' - runHook preCheck - $out/bin/generate-test-vars --help - runHook preCheck - ''; -} diff --git a/pkgs/generate-test-vars/flake-module.nix b/pkgs/generate-test-vars/flake-module.nix deleted file mode 100644 index d8178940e..000000000 --- a/pkgs/generate-test-vars/flake-module.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ ... }: -{ - perSystem = - { - config, - pkgs, - self', - ... - }: - { - devShells.vars-generator = pkgs.callPackage ./shell.nix { - inherit (self'.packages) generate-test-vars; - }; - - packages.generate-test-vars = pkgs.python3.pkgs.callPackage ./default.nix { - inherit (config.packages) clan-cli; - }; - }; -} diff --git a/pkgs/generate-test-vars/pyproject.toml b/pkgs/generate-test-vars/pyproject.toml deleted file mode 100644 index e3d0472e2..000000000 --- a/pkgs/generate-test-vars/pyproject.toml +++ /dev/null @@ -1,34 +0,0 @@ -[build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" - - -[project] -name = "generate-test-vars" -description = "vars generate" -dynamic = ["version"] -scripts = { generate-test-vars = "generate_test_vars.cli:main" } - -[project.urls] -Homepage = "https://clan.lol/" -Documentation = "https://docs.clan.lol/" -Repository = "https://git.clan.lol/clan/clan-core" - -[tool.setuptools.packages.find] -exclude = ["result", "**/__pycache__"] - -[tool.pytest.ini_options] -testpaths = "tests" -faulthandler_timeout = 60 -log_level = "DEBUG" -log_format = "%(levelname)s: %(message)s\n %(pathname)s:%(lineno)d::%(funcName)s" -addopts = "--durations 5 --color=yes --new-first" # Add --pdb for debugging -norecursedirs = "tests/helpers" - -[tool.mypy] -python_version = "3.13" -warn_redundant_casts = true -disallow_untyped_calls = true -disallow_untyped_defs = true -no_implicit_optional = true - diff --git a/pkgs/generate-test-vars/shell.nix b/pkgs/generate-test-vars/shell.nix deleted file mode 100644 index 903dd3ed5..000000000 --- a/pkgs/generate-test-vars/shell.nix +++ /dev/null @@ -1,7 +0,0 @@ -{ pkgs, generate-test-vars }: -pkgs.mkShell { - inputsFrom = [ - generate-test-vars - ]; - # packages = with pkgs; [ python3 ]; -} diff --git a/pkgs/generate-test-vars/test/vars.nix b/pkgs/generate-test-vars/test/vars.nix deleted file mode 100644 index 48d1d063f..000000000 --- a/pkgs/generate-test-vars/test/vars.nix +++ /dev/null @@ -1,21 +0,0 @@ -# Test that we can generate vars -{ - vars.generators = { - test_generator_1 = { - files.hello = { - secret = false; - }; - script = '' - echo "hello world 1" > $out/hello - ''; - }; - test_generator_2 = { - files.hello = { - secret = false; - }; - script = '' - echo "hello world 2" > $out/hello - ''; - }; - }; -}