Merge pull request 'clan_lib flake: fix handling garbage collected store paths as cached values' (#3699) from select-path-fix into main
Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3699
This commit is contained in:
lassulus
@@ -1,5 +1,6 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from enum import Enum
|
||||
from hashlib import sha1
|
||||
@@ -318,9 +319,10 @@ class FlakeCacheEntry:
|
||||
# strings need to be checked if they are store paths
|
||||
# if they are, we store them as a dict with the outPath key
|
||||
# this is to mirror nix behavior, where the outPath of an attrset is used if no further key is specified
|
||||
elif isinstance(value, str) and value.startswith("/nix/store/"):
|
||||
elif isinstance(value, str) and value.startswith(
|
||||
os.environ.get("NIX_STORE_DIR", "/nix/store")
|
||||
):
|
||||
assert selectors == []
|
||||
if value.startswith("/nix/store/"):
|
||||
self.value = {"outPath": FlakeCacheEntry(value)}
|
||||
|
||||
# if we have a normal scalar, we check if it conflicts with a maybe already store value
|
||||
@@ -337,7 +339,9 @@ class FlakeCacheEntry:
|
||||
selector: Selector
|
||||
|
||||
# for store paths we have to check if they still exist, otherwise they have to be rebuild and are thus not cached
|
||||
if isinstance(self.value, str) and self.value.startswith("/nix/store/"):
|
||||
if isinstance(self.value, str) and self.value.startswith(
|
||||
os.environ.get("NIX_STORE_DIR", "/nix/store")
|
||||
):
|
||||
return Path(self.value).exists()
|
||||
|
||||
# if self.value is not dict but we request more selectors, we assume we are cached and an error will be thrown in the select function
|
||||
@@ -345,7 +349,8 @@ class FlakeCacheEntry:
|
||||
return True
|
||||
|
||||
if selectors == []:
|
||||
return self.fetched_all
|
||||
selector = Selector(type=SelectorType.ALL)
|
||||
else:
|
||||
selector = selectors[0]
|
||||
|
||||
# we just fetch all subkeys, so we need to check of we inserted all keys at this level before
|
||||
@@ -739,18 +744,36 @@ class Flake:
|
||||
)
|
||||
select_hash = select_flake.hash
|
||||
|
||||
# fmt: off
|
||||
nix_code = f"""
|
||||
let
|
||||
flake = builtins.getFlake "path:{self.store_path}?narHash={self.hash}";
|
||||
selectLib = (builtins.getFlake "path:{select_source()}?narHash={select_hash}").lib;
|
||||
nixpkgs = flake.inputs.nixpkgs or (builtins.getFlake "path:{nixpkgs_source()}?narHash={fallback_nixpkgs_hash}");
|
||||
selectLib = (
|
||||
builtins.getFlake
|
||||
"path:{select_source()}?narHash={select_hash}"
|
||||
).lib;
|
||||
in
|
||||
nixpkgs.legacyPackages.{config["system"]}.writeText "clan-flake-select" (
|
||||
builtins.toJSON [ {" ".join([f"(selectLib.applySelectors (builtins.fromJSON ''{attr}'') flake)" for attr in str_selectors])} ]
|
||||
)
|
||||
derivation {{
|
||||
name = "clan-flake-select";
|
||||
system = "{config["system"]}";
|
||||
builder = "/bin/sh";
|
||||
args = [
|
||||
"-c"
|
||||
''
|
||||
printf %s '${{builtins.toJSON [
|
||||
{" ".join(
|
||||
[
|
||||
f"(selectLib.applySelectors (builtins.fromJSON ''{attr}'') flake)"
|
||||
for attr in str_selectors
|
||||
]
|
||||
)}
|
||||
]}}' > $out
|
||||
''
|
||||
];
|
||||
}}
|
||||
"""
|
||||
# fmt: on
|
||||
if tmp_store := nix_test_store():
|
||||
nix_options += ["--store", str(tmp_store)]
|
||||
nix_options.append("--impure")
|
||||
|
||||
build_output = Path(
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import logging
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from clan_cli.tests.fixtures_flakes import ClanFlake
|
||||
@@ -347,10 +351,6 @@ def test_conditional_all_selector(flake: ClanFlake) -> None:
|
||||
# Test that the caching works
|
||||
@pytest.mark.with_core
|
||||
def test_caching_works(flake: ClanFlake) -> None:
|
||||
from unittest.mock import patch
|
||||
|
||||
from clan_lib.flake import Flake
|
||||
|
||||
my_flake = Flake(str(flake.path))
|
||||
|
||||
with patch.object(
|
||||
@@ -363,6 +363,42 @@ def test_caching_works(flake: ClanFlake) -> None:
|
||||
assert tracked_build.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.with_core
|
||||
def test_cache_gc(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
with TemporaryDirectory() as tempdir_:
|
||||
tempdir = Path(tempdir_)
|
||||
|
||||
monkeypatch.setenv("NIX_STATE_DIR", str(tempdir / "var"))
|
||||
monkeypatch.setenv("NIX_LOG_DIR", str(tempdir / "var" / "log"))
|
||||
monkeypatch.setenv("NIX_STORE_DIR", str(tempdir / "store"))
|
||||
monkeypatch.setenv("NIX_CACHE_HOME", str(tempdir / "cache"))
|
||||
monkeypatch.setenv("HOME", str(tempdir / "home"))
|
||||
monkeypatch.delenv("CLAN_TEST_STORE")
|
||||
monkeypatch.setenv("NIX_BUILD_TOP", str(tempdir / "build"))
|
||||
|
||||
test_file = tempdir / "flake" / "testfile"
|
||||
test_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
test_file.write_text("test")
|
||||
|
||||
test_flake = tempdir / "flake" / "flake.nix"
|
||||
test_flake.write_text("""
|
||||
{
|
||||
outputs = _: {
|
||||
testfile = ./testfile;
|
||||
};
|
||||
}
|
||||
""")
|
||||
|
||||
my_flake = Flake(str(tempdir / "flake"))
|
||||
my_flake.select(
|
||||
"testfile", nix_options=["--sandbox-build-dir", str(tempdir / "build")]
|
||||
)
|
||||
assert my_flake._cache is not None # noqa: SLF001
|
||||
assert my_flake._cache.is_cached("testfile") # noqa: SLF001
|
||||
subprocess.run(["nix-collect-garbage"], check=True)
|
||||
assert not my_flake._cache.is_cached("testfile") # noqa: SLF001
|
||||
|
||||
|
||||
# This test fails because the CI sandbox does not have the required packages to run the generators
|
||||
# maybe @DavHau or @Qubasa can fix this at some point :)
|
||||
# @pytest.mark.with_core
|
||||
|
||||
Reference in New Issue
Block a user