From 7f34ba5d5e7ffadea901ae56babe255edab30d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 27 Mar 2025 15:13:01 +0100 Subject: [PATCH] use recursive-nix for integration tests --- pkgs/clan-cli/clan_cli/flake.py | 7 ---- pkgs/clan-cli/clan_cli/machines/inventory.py | 5 +-- pkgs/clan-cli/clan_cli/machines/machines.py | 4 +-- pkgs/clan-cli/clan_cli/nix/__init__.py | 19 ----------- pkgs/clan-cli/clan_cli/vars/generate.py | 5 +-- pkgs/clan-cli/default.nix | 36 ++++++++------------ pkgs/clan-cli/tests/fixtures_flakes.py | 5 --- 7 files changed, 17 insertions(+), 64 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/flake.py b/pkgs/clan-cli/clan_cli/flake.py index 39614c378..af1428249 100644 --- a/pkgs/clan-cli/clan_cli/flake.py +++ b/pkgs/clan-cli/clan_cli/flake.py @@ -15,7 +15,6 @@ from clan_cli.nix import ( nix_command, nix_config, nix_metadata, - nix_test_store, ) log = logging.getLogger(__name__) @@ -476,18 +475,12 @@ class Flake: builtins.toJSON [ ({" ".join([f"flake.clanInternals.lib.select ''{attr}'' flake" for attr in selectors])}) ] ) """ - if tmp_store := nix_test_store(): - nix_options += ["--store", str(tmp_store)] - nix_options.append("--impure") - build_output = Path( run( nix_build(["--expr", nix_code, *nix_options]), RunOpts(log=Log.NONE) ).stdout.strip() ) - if tmp_store: - build_output = tmp_store.joinpath(*build_output.parts[1:]) outputs = json.loads(build_output.read_text()) if len(outputs) != len(selectors): msg = f"flake_prepare_cache: Expected {len(outputs)} outputs, got {len(outputs)}" diff --git a/pkgs/clan-cli/clan_cli/machines/inventory.py b/pkgs/clan-cli/clan_cli/machines/inventory.py index 28b1b5fe3..0771b2b0f 100644 --- a/pkgs/clan-cli/clan_cli/machines/inventory.py +++ b/pkgs/clan-cli/clan_cli/machines/inventory.py @@ -3,7 +3,7 @@ from pathlib import Path from clan_cli.cmd import run from clan_cli.flake import Flake -from clan_cli.nix import nix_build, nix_config, nix_test_store +from clan_cli.nix import nix_build, nix_config from .machines import Machine @@ -18,9 +18,6 @@ def get_all_machines(flake: Flake, nix_options: list[str]) -> list[Machine]: ).stdout.rstrip() ) - if test_store := nix_test_store(): - json_path = test_store.joinpath(*json_path.parts[1:]) - machines_json = json.loads(json_path.read_text()) machines = [] diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index cc5638555..0516fa115 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -10,7 +10,7 @@ from clan_cli.errors import ClanError from clan_cli.facts import public_modules as facts_public_modules from clan_cli.facts import secret_modules as facts_secret_modules from clan_cli.flake import Flake -from clan_cli.nix import nix_config, nix_test_store +from clan_cli.nix import nix_config from clan_cli.ssh.host import Host from clan_cli.ssh.host_key import HostKeyCheck from clan_cli.ssh.parse import parse_deployment_address @@ -243,8 +243,6 @@ class Machine: output = self.nix("build", attr, nix_options) output = Path(output) - if tmp_store := nix_test_store(): - output = tmp_store.joinpath(*output.parts[1:]) assert output.exists(), f"The output {output} doesn't exist" if isinstance(output, Path): return output diff --git a/pkgs/clan-cli/clan_cli/nix/__init__.py b/pkgs/clan-cli/clan_cli/nix/__init__.py index ad32e5d38..ad90ed928 100644 --- a/pkgs/clan-cli/clan_cli/nix/__init__.py +++ b/pkgs/clan-cli/clan_cli/nix/__init__.py @@ -1,22 +1,18 @@ import json import logging import os -import tempfile from pathlib import Path from typing import Any from clan_cli.cmd import run, run_no_stdout from clan_cli.dirs import nixpkgs_flake, nixpkgs_source from clan_cli.errors import ClanError -from clan_cli.locked_open import locked_open log = logging.getLogger(__name__) def nix_command(flags: list[str]) -> list[str]: args = ["nix", "--extra-experimental-features", "nix-command flakes", *flags] - if store := nix_test_store(): - args += ["--store", str(store)] return args @@ -61,21 +57,6 @@ def nix_config() -> dict[str, Any]: return config -def nix_test_store() -> Path | None: - store = os.environ.get("CLAN_TEST_STORE", None) - lock_nix = os.environ.get("LOCK_NIX", "") - - if not lock_nix: - lock_nix = tempfile.NamedTemporaryFile().name # NOQA: SIM115 - if not os.environ.get("IN_NIX_SANDBOX"): - return None - if store: - Path.mkdir(Path(store), exist_ok=True) - with locked_open(Path(lock_nix), "w"): - return Path(store) - return None - - def nix_eval(flags: list[str]) -> list[str]: default_flags = nix_command( [ diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 57ea6607d..f4f448bbc 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -17,7 +17,7 @@ from clan_cli.completions import ( from clan_cli.errors import ClanError from clan_cli.git import commit_files from clan_cli.machines.inventory import get_all_machines, get_selected_machines -from clan_cli.nix import nix_shell, nix_test_store +from clan_cli.nix import nix_shell from clan_cli.vars._types import StoreBase from .check import check_vars @@ -78,8 +78,6 @@ class Generator: def bubblewrap_cmd(generator: str, tmpdir: Path) -> list[str]: - test_store = nix_test_store() - # fmt: off return nix_shell( [ @@ -91,7 +89,6 @@ def bubblewrap_cmd(generator: str, tmpdir: Path) -> list[str]: "--unshare-all", "--tmpfs", "/", "--ro-bind", "/nix/store", "/nix/store", - *(["--ro-bind", str(test_store), str(test_store)] if test_store else []), "--dev", "/dev", # not allowed to bind procfs in some sandboxes "--bind", str(tmpdir), str(tmpdir), diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index a3214cb2d..42f9b6031 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -133,12 +133,13 @@ pythonRuntime.pkgs.buildPythonApplication { runCommand "clan-pytest-without-core" { nativeBuildInputs = testDependencies; + requiredSystemFeatures = [ "recursive-nix" ]; + closureInfo = pkgs.closureInfo { rootPaths = [ templateDerivation ]; }; - } '' set -u -o pipefail @@ -146,12 +147,10 @@ pythonRuntime.pkgs.buildPythonApplication { chmod +w -R ./src cd ./src - export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 PYTHONWARNINGS=error + export IN_NIX_SANDBOX=1 PYTHONWARNINGS=error # required to prevent concurrent 'nix flake lock' operations - export CLAN_TEST_STORE=$TMPDIR/store export LOCK_NIX=$TMPDIR/nix_lock - mkdir -p "$CLAN_TEST_STORE/nix/store" # limit build cores to 16 jobs="$((NIX_BUILD_CORES>16 ? 16 : NIX_BUILD_CORES))" @@ -159,15 +158,24 @@ pythonRuntime.pkgs.buildPythonApplication { python -m pytest -m "not impure and not with_core" -n $jobs ./tests touch $out ''; - } - // lib.optionalAttrs (!stdenv.isDarwin) { clan-pytest-with-core = runCommand "clan-pytest-with-core" { nativeBuildInputs = testDependencies; + requiredSystemFeatures = [ "recursive-nix" ]; + + closureInfo = pkgs.closureInfo { + rootPaths = [ + templateDerivation + ]; + }; + buildInputs = [ pkgs.bash pkgs.coreutils + pkgs.jq.dev + pkgs.stdenv + pkgs.stdenvNoCC # looks like Nix 2.26 fixes the profile creation race condition we were running into on Nix 2.24 # Switch this back to `pkgs.nix` when `pkgs.nix` is Nix 2.26+ ( @@ -177,16 +185,6 @@ pythonRuntime.pkgs.buildPythonApplication { pkgs.nixVersions.latest ) ]; - closureInfo = pkgs.closureInfo { - rootPaths = [ - templateDerivation - pkgs.bash - pkgs.coreutils - pkgs.jq.dev - pkgs.stdenv - pkgs.stdenvNoCC - ]; - }; } '' set -u -o pipefail @@ -195,16 +193,10 @@ pythonRuntime.pkgs.buildPythonApplication { cd ./src export CLAN_CORE_PATH=${clan-core-path} - export NIX_STATE_DIR=$TMPDIR/nix export IN_NIX_SANDBOX=1 export PYTHONWARNINGS=error - export CLAN_TEST_STORE=$TMPDIR/store # required to prevent concurrent 'nix flake lock' operations export LOCK_NIX=$TMPDIR/nix_lock - mkdir -p "$CLAN_TEST_STORE/nix/store" - mkdir -p "$CLAN_TEST_STORE/nix/var/nix/gcroots" - xargs cp --recursive --target "$CLAN_TEST_STORE/nix/store" < "$closureInfo/store-paths" - nix-store --load-db --store "$CLAN_TEST_STORE" < "$closureInfo/registration" # limit build cores to 16 jobs="$((NIX_BUILD_CORES>16 ? 16 : NIX_BUILD_CORES))" diff --git a/pkgs/clan-cli/tests/fixtures_flakes.py b/pkgs/clan-cli/tests/fixtures_flakes.py index aba71fff0..461cc5dba 100644 --- a/pkgs/clan-cli/tests/fixtures_flakes.py +++ b/pkgs/clan-cli/tests/fixtures_flakes.py @@ -13,7 +13,6 @@ import age_keys import pytest from clan_cli.dirs import TemplateType, clan_templates, nixpkgs_source from clan_cli.locked_open import locked_open -from clan_cli.nix import nix_test_store from fixture_error import FixtureError from root import CLAN_CORE from temporary_dir import TEMPDIR @@ -290,9 +289,6 @@ def create_flake( flake_nix = flake / "flake.nix" # this is where we would install the sops key to, when updating substitute(flake_nix, clan_core_flake, flake) - nix_options = [] - if tmp_store := nix_test_store(): - nix_options += ["--store", str(tmp_store)] with locked_open(Path(lock_nix), "w"): sp.run( @@ -303,7 +299,6 @@ def create_flake( flake, "--extra-experimental-features", "nix-command flakes", - *nix_options, ], check=True, )