From 102fa21b9c7819bc70cdf9fd41a5b87ec952cdba Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 19 May 2025 12:00:41 +0200 Subject: [PATCH 1/5] clan_lib flake: check the whole tree even if we fetched all keys before This is needed because nix garbage collection can remove store paths. So the cache can become invalid because a path needs to be rebuild. --- pkgs/clan-cli/clan_lib/flake/flake.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/flake/flake.py b/pkgs/clan-cli/clan_lib/flake/flake.py index 5d7e9b54d..eaad6aaf9 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake.py +++ b/pkgs/clan-cli/clan_lib/flake/flake.py @@ -345,8 +345,9 @@ class FlakeCacheEntry: return True if selectors == []: - return self.fetched_all - selector = selectors[0] + selector = Selector(type=SelectorType.ALL) + else: + selector = selectors[0] # we just fetch all subkeys, so we need to check of we inserted all keys at this level before if selector.type == SelectorType.ALL: From 8218bd3539943ff6950ceaed4ec2a6d763c3bedc Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 19 May 2025 15:28:05 +0200 Subject: [PATCH 2/5] clan_lib flake: get store path from NIX_STORE_DIR --- pkgs/clan-cli/clan_lib/flake/flake.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/flake/flake.py b/pkgs/clan-cli/clan_lib/flake/flake.py index eaad6aaf9..ead91889d 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake.py +++ b/pkgs/clan-cli/clan_lib/flake/flake.py @@ -1,5 +1,6 @@ import json import logging +import os from dataclasses import asdict, dataclass, field from enum import Enum from hashlib import sha1 @@ -318,10 +319,11 @@ class FlakeCacheEntry: # strings need to be checked if they are store paths # if they are, we store them as a dict with the outPath key # this is to mirror nix behavior, where the outPath of an attrset is used if no further key is specified - elif isinstance(value, str) and value.startswith("/nix/store/"): + elif isinstance(value, str) and value.startswith( + os.environ.get("NIX_STORE_DIR", "/nix/store") + ): assert selectors == [] - if value.startswith("/nix/store/"): - self.value = {"outPath": FlakeCacheEntry(value)} + self.value = {"outPath": FlakeCacheEntry(value)} # if we have a normal scalar, we check if it conflicts with a maybe already store value # since an empty attrset is the default value, we cannot check that, so we just set it to the value @@ -337,7 +339,9 @@ class FlakeCacheEntry: selector: Selector # for store paths we have to check if they still exist, otherwise they have to be rebuild and are thus not cached - if isinstance(self.value, str) and self.value.startswith("/nix/store/"): + if isinstance(self.value, str) and self.value.startswith( + os.environ.get("NIX_STORE_DIR", "/nix/store") + ): return Path(self.value).exists() # if self.value is not dict but we request more selectors, we assume we are cached and an error will be thrown in the select function From 9685e001223cf3cf413e98c1e80b913b9494552e Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 19 May 2025 15:28:42 +0200 Subject: [PATCH 3/5] clan_lib flake: get select output without nixpkgs --- pkgs/clan-cli/clan_lib/flake/flake.py | 29 ++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/flake/flake.py b/pkgs/clan-cli/clan_lib/flake/flake.py index ead91889d..839688198 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake.py +++ b/pkgs/clan-cli/clan_lib/flake/flake.py @@ -744,16 +744,35 @@ class Flake: ) select_hash = select_flake.hash + # fmt: off nix_code = f""" let flake = builtins.getFlake "path:{self.store_path}?narHash={self.hash}"; - selectLib = (builtins.getFlake "path:{select_source()}?narHash={select_hash}").lib; - nixpkgs = flake.inputs.nixpkgs or (builtins.getFlake "path:{nixpkgs_source()}?narHash={fallback_nixpkgs_hash}"); + selectLib = ( + builtins.getFlake + "path:{select_source()}?narHash={select_hash}" + ).lib; in - nixpkgs.legacyPackages.{config["system"]}.writeText "clan-flake-select" ( - builtins.toJSON [ {" ".join([f"(selectLib.applySelectors (builtins.fromJSON ''{attr}'') flake)" for attr in str_selectors])} ] - ) + derivation {{ + name = "clan-flake-select"; + system = "{config["system"]}"; + builder = "/bin/sh"; + args = [ + "-c" + '' + printf %s '${{builtins.toJSON [ + {" ".join( + [ + f"(selectLib.applySelectors (builtins.fromJSON ''{attr}'') flake)" + for attr in str_selectors + ] + )} + ]}}' > $out + '' + ]; + }} """ + # fmt: on if tmp_store := nix_test_store(): nix_options += ["--store", str(tmp_store)] nix_options.append("--impure") From 107bc91fa1c9ffb4cc089ccb8fdc78ea944c7087 Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 19 May 2025 15:29:08 +0200 Subject: [PATCH 4/5] clan_lib flake: test if cache gets invalidated with nix gc --- pkgs/clan-cli/clan_lib/flake/flake_test.py | 44 ++++++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/flake/flake_test.py b/pkgs/clan-cli/clan_lib/flake/flake_test.py index bdeee2583..0c665b0c8 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake_test.py +++ b/pkgs/clan-cli/clan_lib/flake/flake_test.py @@ -1,4 +1,8 @@ import logging +import subprocess +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch import pytest from clan_cli.tests.fixtures_flakes import ClanFlake @@ -347,10 +351,6 @@ def test_conditional_all_selector(flake: ClanFlake) -> None: # Test that the caching works @pytest.mark.with_core def test_caching_works(flake: ClanFlake) -> None: - from unittest.mock import patch - - from clan_lib.flake import Flake - my_flake = Flake(str(flake.path)) with patch.object( @@ -363,6 +363,42 @@ def test_caching_works(flake: ClanFlake) -> None: assert tracked_build.call_count == 1 +@pytest.mark.with_core +def test_cache_gc(monkeypatch: pytest.MonkeyPatch) -> None: + with TemporaryDirectory() as tempdir_: + tempdir = Path(tempdir_) + + monkeypatch.setenv("NIX_STATE_DIR", str(tempdir / "var")) + monkeypatch.setenv("NIX_LOG_DIR", str(tempdir / "var" / "log")) + monkeypatch.setenv("NIX_STORE_DIR", str(tempdir / "store")) + monkeypatch.setenv("NIX_CACHE_HOME", str(tempdir / "cache")) + monkeypatch.setenv("HOME", str(tempdir / "home")) + monkeypatch.delenv("CLAN_TEST_STORE") + monkeypatch.setenv("NIX_BUILD_TOP", str(tempdir / "build")) + + test_file = tempdir / "flake" / "testfile" + test_file.parent.mkdir(parents=True, exist_ok=True) + test_file.write_text("test") + + test_flake = tempdir / "flake" / "flake.nix" + test_flake.write_text(""" + { + outputs = _: { + testfile = ./testfile; + }; + } + """) + + my_flake = Flake(str(tempdir / "flake")) + my_flake.select( + "testfile", nix_options=["--sandbox-build-dir", str(tempdir / "build")] + ) + assert my_flake._cache is not None # noqa: SLF001 + assert my_flake._cache.is_cached("testfile") # noqa: SLF001 + subprocess.run(["nix-collect-garbage"], check=True) + assert not my_flake._cache.is_cached("testfile") # noqa: SLF001 + + # This test fails because the CI sandbox does not have the required packages to run the generators # maybe @DavHau or @Qubasa can fix this at some point :) # @pytest.mark.with_core From 1861c4447fc5e82b6773621465dd195185d04941 Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 19 May 2025 17:49:12 +0200 Subject: [PATCH 5/5] clan_lib flake: remove redundant store definition --- pkgs/clan-cli/clan_lib/flake/flake.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pkgs/clan-cli/clan_lib/flake/flake.py b/pkgs/clan-cli/clan_lib/flake/flake.py index 839688198..794dc57eb 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake.py +++ b/pkgs/clan-cli/clan_lib/flake/flake.py @@ -774,7 +774,6 @@ class Flake: """ # fmt: on if tmp_store := nix_test_store(): - nix_options += ["--store", str(tmp_store)] nix_options.append("--impure") build_output = Path(