diff --git a/lib/clanTest/flake-module.nix b/lib/clanTest/flake-module.nix index 4e140a251..214615981 100644 --- a/lib/clanTest/flake-module.nix +++ b/lib/clanTest/flake-module.nix @@ -158,14 +158,26 @@ in ''; # the test's flake.nix with locked clan-core input - flakeForSandbox = hostPkgs.runCommand "offline-flake-for-test-${config.name}" { } '' - cp -r ${config.clan.directory} $out - chmod +w -R $out - substituteInPlace $out/flake.nix \ - --replace-fail \ - "https://git.clan.lol/clan/clan-core/archive/main.tar.gz" \ - "${clan-core.packages.${hostPkgs.system}.clan-core-flake}" - ''; + flakeForSandbox = + hostPkgs.runCommand "offline-flake-for-test-${config.name}" + { + nativeBuildInputs = [ hostPkgs.nix ]; + } + '' + cp -r ${config.clan.directory} $out + chmod +w -R $out + substituteInPlace $out/flake.nix \ + --replace-fail \ + "https://git.clan.lol/clan/clan-core/archive/main.tar.gz" \ + "${clan-core.packages.${hostPkgs.system}.clan-core-flake}" + + # Create a proper lock file for the test flake + export HOME=$(mktemp -d) + nix flake lock $out \ + --extra-experimental-features 'nix-command flakes' \ + --override-input clan-core ${clan-core.packages.${hostPkgs.system}.clan-core-flake} \ + --override-input nixpkgs ${clan-core.inputs.nixpkgs} + ''; in { imports = [ diff --git a/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py b/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py index 62b77f5ea..fefd8a262 100644 --- a/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py +++ b/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py @@ -14,6 +14,7 @@ from clan_cli.tests import age_keys from clan_cli.tests.fixture_error import FixtureError from clan_cli.tests.root import CLAN_CORE from clan_cli.tests.temporary_dir import TEMPDIR +from clan_lib import cmd from clan_lib.dirs import ( TemplateType, clan_templates, @@ -23,7 +24,7 @@ from clan_lib.dirs import ( from clan_lib.flake import Flake from clan_lib.locked_open import locked_open from clan_lib.machines.machines import Machine -from clan_lib.nix import nix_test_store +from clan_lib.nix import nix_command, nix_test_store log = logging.getLogger(__name__) @@ -86,6 +87,11 @@ def substitute( print(f"flake: {flake}") file.write_text(buf) + # Lock the flake after substitution if clan_core was replaced + if clan_core_flake: + flake_dir = file.parent + cmd.run(nix_command(["flake", "lock"]), cmd.RunOpts(cwd=flake_dir)) + class FlakeForTest(NamedTuple): path: Path diff --git a/pkgs/clan-cli/clan_lib/flake/flake.py b/pkgs/clan-cli/clan_lib/flake/flake.py index b84cd07e4..ed3b33375 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake.py +++ b/pkgs/clan-cli/clan_lib/flake/flake.py @@ -720,7 +720,7 @@ class Flake: AssertionError: If the cache or flake cache path is not properly initialized. """ from clan_lib.cmd import Log, RunOpts, run - from clan_lib.dirs import nixpkgs_source, select_source + from clan_lib.dirs import select_source from clan_lib.nix import ( nix_build, nix_config, @@ -731,7 +731,7 @@ class Flake: self.invalidate_cache() assert self._cache is not None - nix_options = self.nix_options if self.nix_options is not None else [] + nix_options = self.nix_options[:] if self.nix_options is not None else [] str_selectors: list[str] = [] for selector in selectors: @@ -739,23 +739,9 @@ class Flake: config = nix_config() - # these hashes should be filled in by `nix build` - # if we run this Python code directly then we use a fallback - # method to getting the NAR hash - fallback_nixpkgs_hash = "@fallback_nixpkgs_hash@" - if not fallback_nixpkgs_hash.startswith("sha256-"): - fallback_nixpkgs = Flake( - str(nixpkgs_source()), nix_options=self.nix_options - ) - fallback_nixpkgs.invalidate_cache() - assert fallback_nixpkgs.hash is not None, ( - "this should be impossible as invalidate_cache() should always set `hash`" - ) - fallback_nixpkgs_hash = fallback_nixpkgs.hash - select_hash = "@select_hash@" if not select_hash.startswith("sha256-"): - select_flake = Flake(str(select_source()), nix_options=self.nix_options) + select_flake = Flake(str(select_source()), nix_options=nix_options) select_flake.invalidate_cache() assert select_flake.hash is not None, ( "this should be impossible as invalidate_cache() should always set `hash`" @@ -800,12 +786,6 @@ class Flake: }} """ # fmt: on - if tmp_store := nix_test_store(): - nix_options.append("--impure") - - # build_output = Path( - # run(nix_build(["--expr", nix_code, *nix_options])).stdout.strip() - # ) build_output = Path( run( @@ -813,7 +793,7 @@ class Flake: ).stdout.strip() ) - if tmp_store: + if tmp_store := nix_test_store(): build_output = tmp_store.joinpath(*build_output.parts[1:]) outputs = json.loads(build_output.read_bytes()) if len(outputs) != len(selectors): diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index d35dd7a6c..81820c858 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -43,7 +43,10 @@ let deps: lib.filterAttrs (_: pkg: !pkg.meta.unsupported or false) (lib.genAttrs deps (name: pkgs.${name})); testRuntimeDependenciesMap = generateRuntimeDependenciesMap allDependencies; - testRuntimeDependencies = (lib.attrValues testRuntimeDependenciesMap); + # Filter out virt-viewer from test dependencies since it pulls quiet a lot of other packages, we don't run virt-viewer in tests. + testRuntimeDependencies = lib.filter (pkg: pkg.pname or "" != "virt-viewer") ( + lib.attrValues testRuntimeDependenciesMap + ); bundledRuntimeDependenciesMap = generateRuntimeDependenciesMap includedRuntimeDeps; bundledRuntimeDependencies = lib.attrValues bundledRuntimeDependenciesMap; @@ -70,7 +73,6 @@ let rm -f $out/clan_lib/select substituteInPlace $out/clan_lib/flake/flake.py \ - --replace-fail '@fallback_nixpkgs_hash@' "$(jq -r '.nodes.nixpkgs.locked.narHash' ${nixpkgs'}/flake.lock)" \ --replace-fail '@select_hash@' "$(jq -r '.nodes."nix-select".locked.narHash' ${../../flake.lock})" ln -sf ${nixpkgs'} $out/clan_lib/nixpkgs ln -sf ${nix-select} $out/clan_lib/select @@ -193,48 +195,6 @@ pythonRuntime.pkgs.buildPythonApplication { pkgs.coreutils pkgs.nix ]; - closureInfo = pkgs.closureInfo { - rootPaths = [ - templateDerivation - pkgs.bash - pkgs.coreutils - pkgs.jq.dev - pkgs.stdenv - pkgs.stdenvNoCC - pkgs.shellcheck-minimal - ]; - }; - } - '' - set -euo pipefail - cp -r ${sourceWithTests} ./src - chmod +w -R ./src - cd ./src - - ${setupNixInNix} - - export CLAN_CORE_PATH=${clan-core-path} - export PYTHONWARNINGS=error - - # limit build cores to 16 - jobs="$((NIX_BUILD_CORES>16 ? 16 : NIX_BUILD_CORES))" - - python -m pytest -m "not impure and with_core" ./clan_cli -n $jobs - touch $out - ''; - } - // lib.optionalAttrs (!stdenv.isDarwin) { - # disabled on macOS until we fix all remaining issues - clan-lib-pytest = - runCommand "clan-lib-pytest" - { - nativeBuildInputs = testDependencies; - buildInputs = [ - pkgs.bash - pkgs.coreutils - pkgs.nix - - ]; closureInfo = pkgs.closureInfo { rootPaths = [ templateDerivation @@ -266,10 +226,11 @@ pythonRuntime.pkgs.buildPythonApplication { export NIXPKGS=${nixpkgs} export NIX_SELECT=${nix-select} - # limit build cores to 4 - jobs="$((NIX_BUILD_CORES>4 ? 4 : NIX_BUILD_CORES))" + # limit build cores to 16 + jobs="$((NIX_BUILD_CORES>16 ? 16 : NIX_BUILD_CORES))" - python -m pytest -m "with_core" ./clan_lib -n $jobs + # Run all tests with core marker + python -m pytest -m "not impure and with_core" -n $jobs ./clan_cli ./clan_lib touch $out ''; }; diff --git a/pkgs/generate-test-vars/pyproject.toml b/pkgs/generate-test-vars/pyproject.toml index 3347a9efa..aede95ecb 100644 --- a/pkgs/generate-test-vars/pyproject.toml +++ b/pkgs/generate-test-vars/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" name = "generate-test-vars" description = "vars generate" dynamic = ["version"] -scripts = { generate-test-vars = "generate_test_vars:cli.main" } +scripts = { generate-test-vars = "generate_test_vars.cli:main" } [project.urls] Homepage = "https://clan.lol/"