yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

use recursive-nix for integration tests

Browse Source
This commit is contained in:
Jörg Thalheim
2025-03-27 15:13:01 +01:00
parent 1d60f94cc5
commit 7f34ba5d5e
7 changed files with 17 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
pkgs/clan-cli/clan_cli/flake.py
View File

@@ -15,7 +15,6 @@ from clan_cli.nix import (
nix_command,
nix_config,
nix_metadata,
nix_test_store,
)
log = logging.getLogger(__name__)
@@ -476,18 +475,12 @@ class Flake:
builtins.toJSON [ ({" ".join([f"flake.clanInternals.lib.select ''{attr}'' flake" for attr in selectors])}) ]
)
"""
if tmp_store := nix_test_store():
nix_options += ["--store", str(tmp_store)]
nix_options.append("--impure")
build_output = Path(
run(
nix_build(["--expr", nix_code, *nix_options]), RunOpts(log=Log.NONE)
).stdout.strip()
)
if tmp_store:
build_output = tmp_store.joinpath(*build_output.parts[1:])
outputs = json.loads(build_output.read_text())
if len(outputs) != len(selectors):
msg = f"flake_prepare_cache: Expected {len(outputs)} outputs, got {len(outputs)}"

5
pkgs/clan-cli/clan_cli/machines/inventory.py
View File

@@ -3,7 +3,7 @@ from pathlib import Path
from clan_cli.cmd import run
from clan_cli.flake import Flake
from clan_cli.nix import nix_build, nix_config, nix_test_store
from clan_cli.nix import nix_build, nix_config
from .machines import Machine
@@ -18,9 +18,6 @@ def get_all_machines(flake: Flake, nix_options: list[str]) -> list[Machine]:
).stdout.rstrip()
)
if test_store := nix_test_store():
json_path = test_store.joinpath(*json_path.parts[1:])
machines_json = json.loads(json_path.read_text())
machines = []

4
pkgs/clan-cli/clan_cli/machines/machines.py
View File

@@ -10,7 +10,7 @@ from clan_cli.errors import ClanError
from clan_cli.facts import public_modules as facts_public_modules
from clan_cli.facts import secret_modules as facts_secret_modules
from clan_cli.flake import Flake
from clan_cli.nix import nix_config, nix_test_store
from clan_cli.nix import nix_config
from clan_cli.ssh.host import Host
from clan_cli.ssh.host_key import HostKeyCheck
from clan_cli.ssh.parse import parse_deployment_address
@@ -243,8 +243,6 @@ class Machine:
output = self.nix("build", attr, nix_options)
output = Path(output)
if tmp_store := nix_test_store():
output = tmp_store.joinpath(*output.parts[1:])
assert output.exists(), f"The output {output} doesn't exist"
if isinstance(output, Path):
return output

19
pkgs/clan-cli/clan_cli/nix/__init__.py
View File

@@ -1,22 +1,18 @@
import json
import logging
import os
import tempfile
from pathlib import Path
from typing import Any
from clan_cli.cmd import run, run_no_stdout
from clan_cli.dirs import nixpkgs_flake, nixpkgs_source
from clan_cli.errors import ClanError
from clan_cli.locked_open import locked_open
log = logging.getLogger(__name__)
def nix_command(flags: list[str]) -> list[str]:
args = ["nix", "--extra-experimental-features", "nix-command flakes", *flags]
if store := nix_test_store():
args += ["--store", str(store)]
return args
@@ -61,21 +57,6 @@ def nix_config() -> dict[str, Any]:
return config
def nix_test_store() -> Path | None:
store = os.environ.get("CLAN_TEST_STORE", None)
lock_nix = os.environ.get("LOCK_NIX", "")
if not lock_nix:
lock_nix = tempfile.NamedTemporaryFile().name # NOQA: SIM115
if not os.environ.get("IN_NIX_SANDBOX"):
return None
if store:
Path.mkdir(Path(store), exist_ok=True)
with locked_open(Path(lock_nix), "w"):
return Path(store)
return None
def nix_eval(flags: list[str]) -> list[str]:
default_flags = nix_command(
[

5
pkgs/clan-cli/clan_cli/vars/generate.py
View File

@@ -17,7 +17,7 @@ from clan_cli.completions import (
from clan_cli.errors import ClanError
from clan_cli.git import commit_files
from clan_cli.machines.inventory import get_all_machines, get_selected_machines
from clan_cli.nix import nix_shell, nix_test_store
from clan_cli.nix import nix_shell
from clan_cli.vars._types import StoreBase
from .check import check_vars
@@ -78,8 +78,6 @@ class Generator:
def bubblewrap_cmd(generator: str, tmpdir: Path) -> list[str]:
test_store = nix_test_store()
# fmt: off
return nix_shell(
[
@@ -91,7 +89,6 @@ def bubblewrap_cmd(generator: str, tmpdir: Path) -> list[str]:
"--unshare-all",
"--tmpfs", "/",
"--ro-bind", "/nix/store", "/nix/store",
*(["--ro-bind", str(test_store), str(test_store)] if test_store else []),
"--dev", "/dev",
# not allowed to bind procfs in some sandboxes
"--bind", str(tmpdir), str(tmpdir),

36
pkgs/clan-cli/default.nix
View File

@@ -133,12 +133,13 @@ pythonRuntime.pkgs.buildPythonApplication {
runCommand "clan-pytest-without-core"
{
nativeBuildInputs = testDependencies;
requiredSystemFeatures = [ "recursive-nix" ];
closureInfo = pkgs.closureInfo {
rootPaths = [
templateDerivation
];
};
}
''
set -u -o pipefail
@@ -146,12 +147,10 @@ pythonRuntime.pkgs.buildPythonApplication {
chmod +w -R ./src
cd ./src
export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 PYTHONWARNINGS=error
export IN_NIX_SANDBOX=1 PYTHONWARNINGS=error
# required to prevent concurrent 'nix flake lock' operations
export CLAN_TEST_STORE=$TMPDIR/store
export LOCK_NIX=$TMPDIR/nix_lock
mkdir -p "$CLAN_TEST_STORE/nix/store"
# limit build cores to 16
jobs="$((NIX_BUILD_CORES>16 ? 16 : NIX_BUILD_CORES))"
@@ -159,15 +158,24 @@ pythonRuntime.pkgs.buildPythonApplication {
python -m pytest -m "not impure and not with_core" -n $jobs ./tests
touch $out
'';
}
// lib.optionalAttrs (!stdenv.isDarwin) {
clan-pytest-with-core =
runCommand "clan-pytest-with-core"
{
nativeBuildInputs = testDependencies;
requiredSystemFeatures = [ "recursive-nix" ];
closureInfo = pkgs.closureInfo {
rootPaths = [
templateDerivation
];
};
buildInputs = [
pkgs.bash
pkgs.coreutils
pkgs.jq.dev
pkgs.stdenv
pkgs.stdenvNoCC
# looks like Nix 2.26 fixes the profile creation race condition we were running into on Nix 2.24
# Switch this back to `pkgs.nix` when `pkgs.nix` is Nix 2.26+
(
@@ -177,16 +185,6 @@ pythonRuntime.pkgs.buildPythonApplication {
pkgs.nixVersions.latest
)
];
closureInfo = pkgs.closureInfo {
rootPaths = [
templateDerivation
pkgs.bash
pkgs.coreutils
pkgs.jq.dev
pkgs.stdenv
pkgs.stdenvNoCC
];
};
}
''
set -u -o pipefail
@@ -195,16 +193,10 @@ pythonRuntime.pkgs.buildPythonApplication {
cd ./src
export CLAN_CORE_PATH=${clan-core-path}
export NIX_STATE_DIR=$TMPDIR/nix
export IN_NIX_SANDBOX=1
export PYTHONWARNINGS=error
export CLAN_TEST_STORE=$TMPDIR/store
# required to prevent concurrent 'nix flake lock' operations
export LOCK_NIX=$TMPDIR/nix_lock
mkdir -p "$CLAN_TEST_STORE/nix/store"
mkdir -p "$CLAN_TEST_STORE/nix/var/nix/gcroots"
xargs cp --recursive --target "$CLAN_TEST_STORE/nix/store" < "$closureInfo/store-paths"
nix-store --load-db --store "$CLAN_TEST_STORE" < "$closureInfo/registration"
# limit build cores to 16
jobs="$((NIX_BUILD_CORES>16 ? 16 : NIX_BUILD_CORES))"

5
pkgs/clan-cli/tests/fixtures_flakes.py
View File

@@ -13,7 +13,6 @@ import age_keys
import pytest
from clan_cli.dirs import TemplateType, clan_templates, nixpkgs_source
from clan_cli.locked_open import locked_open
from clan_cli.nix import nix_test_store
from fixture_error import FixtureError
from root import CLAN_CORE
from temporary_dir import TEMPDIR
@@ -290,9 +289,6 @@ def create_flake(
flake_nix = flake / "flake.nix"
# this is where we would install the sops key to, when updating
substitute(flake_nix, clan_core_flake, flake)
nix_options = []
if tmp_store := nix_test_store():
nix_options += ["--store", str(tmp_store)]
with locked_open(Path(lock_nix), "w"):
sp.run(
@@ -303,7 +299,6 @@ def create_flake(
flake,
"--extra-experimental-features",
"nix-command flakes",
*nix_options,
],
check=True,
)