332 lines
11 KiB
Python
332 lines
11 KiB
Python
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import subprocess as sp
|
|
import tempfile
|
|
from collections import defaultdict
|
|
from collections.abc import Callable, Iterator
|
|
from pathlib import Path
|
|
from typing import Any, NamedTuple
|
|
|
|
import pytest
|
|
from clan_cli.dirs import TemplateType, clan_templates, nixpkgs_source
|
|
from clan_cli.locked_open import locked_open
|
|
from fixture_error import FixtureError
|
|
from root import CLAN_CORE
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
lock_nix = os.environ.get("LOCK_NIX", "")
|
|
if not lock_nix:
|
|
lock_nix = tempfile.NamedTemporaryFile().name # NOQA: SIM115
|
|
|
|
|
|
# allows defining nested dictionary in a single line
|
|
def def_value() -> defaultdict:
|
|
return defaultdict(def_value)
|
|
|
|
|
|
nested_dict: Callable[[], dict[str, Any]] = lambda: defaultdict(def_value)
|
|
|
|
|
|
# Substitutes strings in a file.
|
|
# This can be used on the flake.nix or default.nix of a machine
|
|
def substitute(
|
|
file: Path,
|
|
clan_core_flake: Path | None = None,
|
|
flake: Path = Path(__file__).parent,
|
|
) -> None:
|
|
sops_key = str(flake.joinpath("sops.key"))
|
|
buf = ""
|
|
with file.open() as f:
|
|
for line in f:
|
|
line = line.replace("__NIXPKGS__", str(nixpkgs_source()))
|
|
if clan_core_flake:
|
|
line = line.replace("__CLAN_CORE__", str(clan_core_flake))
|
|
line = line.replace(
|
|
"git+https://git.clan.lol/clan/clan-core", str(clan_core_flake)
|
|
)
|
|
line = line.replace(
|
|
"https://git.clan.lol/clan/clan-core/archive/main.tar.gz",
|
|
str(clan_core_flake),
|
|
)
|
|
line = line.replace("__CLAN_SOPS_KEY_PATH__", sops_key)
|
|
line = line.replace("__CLAN_SOPS_KEY_DIR__", str(flake / "facts"))
|
|
buf += line
|
|
print(f"file: {file}")
|
|
print(f"clan_core: {clan_core_flake}")
|
|
print(f"flake: {flake}")
|
|
file.write_text(buf)
|
|
|
|
|
|
class FlakeForTest(NamedTuple):
|
|
path: Path
|
|
|
|
|
|
def set_machine_settings(
|
|
flake: Path,
|
|
machine_name: str,
|
|
machine_settings: dict,
|
|
) -> None:
|
|
config_path = flake / "machines" / machine_name / "configuration.json"
|
|
config_path.write_text(json.dumps(machine_settings, indent=2))
|
|
|
|
|
|
def set_git_credentials(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setenv("GIT_AUTHOR_NAME", "clan-tool")
|
|
monkeypatch.setenv("GIT_AUTHOR_EMAIL", "clan@example.com")
|
|
monkeypatch.setenv("GIT_COMMITTER_NAME", "clan-tool")
|
|
monkeypatch.setenv("GIT_COMMITTER_EMAIL", "clan@example.com")
|
|
|
|
|
|
def init_git(monkeypatch: pytest.MonkeyPatch, flake: Path) -> None:
|
|
set_git_credentials(monkeypatch)
|
|
sp.run(["git", "init", "-b", "main"], cwd=flake, check=True)
|
|
# TODO: Find out why test_vms_api.py fails in nix build
|
|
# but works in pytest when this bottom line is commented out
|
|
sp.run(["git", "add", "."], cwd=flake, check=True)
|
|
sp.run(["git", "commit", "-a", "-m", "Initial commit"], cwd=flake, check=True)
|
|
|
|
|
|
class ClanFlake:
|
|
"""
|
|
This class holds all attributes for generating a clan flake.
|
|
For example, inventory and machine configs can be set via self.inventory and self.machines["my_machine"] = {...}.
|
|
Whenever a flake's configuration is changed, it needs to be re-generated by calling refresh().
|
|
|
|
This class can also be used for managing templates.
|
|
Once initialized, all its settings including all generated files, if any, can be copied using the copy() method.
|
|
This avoids expensive re-computation, like for example creating the flake.lock over and over again.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
temporary_home: Path,
|
|
flake_template: Path,
|
|
suppress_tmp_home_warning: bool = False,
|
|
) -> None:
|
|
self._flake_template = flake_template
|
|
self.inventory = nested_dict()
|
|
self.machines = nested_dict()
|
|
self.substitutions: dict[str, str] = {
|
|
"git+https://git.clan.lol/clan/clan-core": "path://" + str(CLAN_CORE),
|
|
"https://git.clan.lol/clan/clan-core/archive/main.tar.gz": "path://"
|
|
+ str(CLAN_CORE),
|
|
}
|
|
self.clan_modules: list[str] = []
|
|
self.temporary_home = temporary_home
|
|
self.path = temporary_home / "flake"
|
|
if not suppress_tmp_home_warning:
|
|
if "/tmp" not in str(os.environ.get("HOME")):
|
|
log.warning(
|
|
f"!! $HOME does not point to a temp directory!! HOME={os.environ['HOME']}"
|
|
)
|
|
|
|
def copy(
|
|
self,
|
|
temporary_home: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> "ClanFlake":
|
|
# copy the files to the new location
|
|
shutil.copytree(self.path, temporary_home / "flake")
|
|
set_git_credentials(monkeypatch)
|
|
return ClanFlake(
|
|
temporary_home=temporary_home,
|
|
flake_template=self._flake_template,
|
|
)
|
|
|
|
def substitute(self) -> None:
|
|
for file in self.path.rglob("*"):
|
|
if ".git" in file.parts:
|
|
continue
|
|
if file.is_file():
|
|
buf = ""
|
|
with file.open() as f:
|
|
for line in f:
|
|
for key, value in self.substitutions.items():
|
|
line = line.replace(key, value)
|
|
buf += line
|
|
file.write_text(buf)
|
|
|
|
def init_from_template(self) -> None:
|
|
shutil.copytree(self._flake_template, self.path)
|
|
sp.run(["chmod", "+w", "-R", str(self.path)], check=True)
|
|
self.substitute()
|
|
if not (self.path / ".git").exists():
|
|
with locked_open(Path(lock_nix), "w"):
|
|
sp.run(
|
|
[
|
|
"nix",
|
|
"flake",
|
|
"lock",
|
|
"--extra-experimental-features",
|
|
"flakes nix-command",
|
|
],
|
|
cwd=self.path,
|
|
check=True,
|
|
)
|
|
with pytest.MonkeyPatch.context() as mp:
|
|
init_git(mp, self.path)
|
|
|
|
def refresh(self) -> None:
|
|
if not self.path.exists():
|
|
self.init_from_template()
|
|
self.substitute()
|
|
if self.inventory:
|
|
inventory_path = self.path / "inventory.json"
|
|
inventory_path.write_text(json.dumps(self.inventory, indent=2))
|
|
imports = "\n".join(
|
|
[f"clan-core.clanModules.{module}" for module in self.clan_modules]
|
|
)
|
|
for machine_name, machine_config in self.machines.items():
|
|
configuration_nix = (
|
|
self.path / "machines" / machine_name / "configuration.nix"
|
|
)
|
|
configuration_nix.parent.mkdir(parents=True, exist_ok=True)
|
|
configuration_nix.write_text(f"""
|
|
{{clan-core, ...}}:
|
|
{{
|
|
imports = [
|
|
(builtins.fromJSON (builtins.readFile ./configuration.json))
|
|
{imports}
|
|
];
|
|
}}
|
|
""")
|
|
set_machine_settings(self.path, machine_name, machine_config)
|
|
sp.run(["git", "add", "."], cwd=self.path, check=True)
|
|
sp.run(
|
|
["git", "commit", "-a", "-m", "Update by flake generator"],
|
|
cwd=self.path,
|
|
check=True,
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def minimal_flake_template() -> Iterator[ClanFlake]:
|
|
with (
|
|
tempfile.TemporaryDirectory(prefix="flake-") as home,
|
|
pytest.MonkeyPatch.context() as mp,
|
|
):
|
|
mp.setenv("HOME", home)
|
|
flake = ClanFlake(
|
|
temporary_home=Path(home),
|
|
flake_template=clan_templates(TemplateType.CLAN) / "minimal",
|
|
)
|
|
flake.init_from_template()
|
|
yield flake
|
|
|
|
|
|
@pytest.fixture
|
|
def flake(
|
|
temporary_home: Path,
|
|
minimal_flake_template: ClanFlake,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> ClanFlake:
|
|
return minimal_flake_template.copy(temporary_home, monkeypatch)
|
|
|
|
|
|
def create_flake(
|
|
temporary_home: Path,
|
|
flake_template: str | Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
clan_core_flake: Path | None = None,
|
|
# names referring to pre-defined machines from ../machines
|
|
machines: list[str] | None = None,
|
|
# alternatively specify the machines directly including their config
|
|
machine_configs: dict[str, dict] | None = None,
|
|
remote: bool = False,
|
|
) -> Iterator[FlakeForTest]:
|
|
"""
|
|
Creates a flake with the given name and machines.
|
|
The machine names map to the machines in ./test_machines
|
|
"""
|
|
if machine_configs is None:
|
|
machine_configs = {}
|
|
if machines is None:
|
|
machines = []
|
|
if isinstance(flake_template, Path):
|
|
template_path = flake_template
|
|
else:
|
|
template_path = Path(__file__).parent / flake_template
|
|
|
|
flake_template_name = template_path.name
|
|
|
|
# copy the template to a new temporary location
|
|
flake = temporary_home / flake_template_name
|
|
shutil.copytree(template_path, flake)
|
|
sp.run(["chmod", "+w", "-R", str(flake)], check=True)
|
|
|
|
# add the requested machines to the flake
|
|
if machines:
|
|
(flake / "machines").mkdir(parents=True, exist_ok=True)
|
|
for machine_name in machines:
|
|
machine_path = Path(__file__).parent / "machines" / machine_name
|
|
shutil.copytree(machine_path, flake / "machines" / machine_name)
|
|
substitute(flake / "machines" / machine_name / "default.nix", flake)
|
|
|
|
# generate machines from machineConfigs
|
|
for machine_name, machine_config in machine_configs.items():
|
|
settings_path = flake / "machines" / machine_name / "settings.json"
|
|
settings_path.parent.mkdir(parents=True, exist_ok=True)
|
|
settings_path.write_text(json.dumps(machine_config, indent=2))
|
|
|
|
# in the flake.nix file replace the string __CLAN_URL__ with the the clan flake
|
|
# provided by get_test_flake_toplevel
|
|
flake_nix = flake / "flake.nix"
|
|
# this is where we would install the sops key to, when updating
|
|
substitute(flake_nix, clan_core_flake, flake)
|
|
|
|
if "/tmp" not in str(os.environ.get("HOME")):
|
|
log.warning(
|
|
f"!! $HOME does not point to a temp directory!! HOME={os.environ['HOME']}"
|
|
)
|
|
|
|
init_git(monkeypatch, flake)
|
|
|
|
if remote:
|
|
with tempfile.TemporaryDirectory(prefix="flake-"):
|
|
yield FlakeForTest(flake)
|
|
else:
|
|
yield FlakeForTest(flake)
|
|
|
|
|
|
@pytest.fixture
|
|
def test_flake(
|
|
monkeypatch: pytest.MonkeyPatch, temporary_home: Path
|
|
) -> Iterator[FlakeForTest]:
|
|
yield from create_flake(
|
|
temporary_home=temporary_home,
|
|
flake_template="test_flake",
|
|
monkeypatch=monkeypatch,
|
|
)
|
|
# check that git diff on ./sops is empty
|
|
if (temporary_home / "test_flake" / "sops").exists():
|
|
git_proc = sp.run(
|
|
["git", "diff", "--exit-code", "./sops"],
|
|
cwd=temporary_home / "test_flake",
|
|
stderr=sp.PIPE,
|
|
check=False,
|
|
)
|
|
if git_proc.returncode != 0:
|
|
log.error(git_proc.stderr.decode())
|
|
msg = "git diff on ./sops is not empty. This should not happen as all changes should be committed"
|
|
raise FixtureError(msg)
|
|
|
|
|
|
@pytest.fixture
|
|
def test_flake_with_core(
|
|
monkeypatch: pytest.MonkeyPatch, temporary_home: Path
|
|
) -> Iterator[FlakeForTest]:
|
|
if not (CLAN_CORE / "flake.nix").exists():
|
|
msg = "clan-core flake not found. This test requires the clan-core flake to be present"
|
|
raise FixtureError(msg)
|
|
yield from create_flake(
|
|
temporary_home=temporary_home,
|
|
flake_template="test_flake_with_core",
|
|
clan_core_flake=CLAN_CORE,
|
|
monkeypatch=monkeypatch,
|
|
)
|