clan-cli: use new flake caching for machines

checks/backups/flake-module.nix

@@ -154,6 +154,7 @@
           "inventory.json"
           "lib/build-clan"
           "lib/default.nix"
+          "lib/select.nix"
           "lib/flake-module.nix"
           "lib/frontmatter"
           "lib/inventory"

pkgs/clan-cli/clan_cli/backups/create.py

@@ -1,5 +1,4 @@
 import argparse
-import json
 import logging
 
 from clan_cli.completions import (
@@ -15,7 +14,7 @@ log = logging.getLogger(__name__)
 
 def create_backup(machine: Machine, provider: str | None = None) -> None:
     machine.info(f"creating backup for {machine.name}")
-    backup_scripts = json.loads(machine.eval_nix("config.clan.core.backups"))
+    backup_scripts = machine.eval_nix("config.clan.core.backups")
     if provider is None:
         if not backup_scripts["providers"]:
             msg = "No providers specified"

pkgs/clan-cli/clan_cli/backups/list.py

@@ -20,7 +20,7 @@ class Backup:
 
 def list_provider(machine: Machine, provider: str) -> list[Backup]:
     results = []
-    backup_metadata = json.loads(machine.eval_nix("config.clan.core.backups"))
+    backup_metadata = machine.eval_nix("config.clan.core.backups")
     list_command = backup_metadata["providers"][provider]["list"]
     proc = machine.target_host.run(
         [list_command],
@@ -46,7 +46,7 @@ def list_provider(machine: Machine, provider: str) -> list[Backup]:
 
 
 def list_backups(machine: Machine, provider: str | None = None) -> list[Backup]:
-    backup_metadata = json.loads(machine.eval_nix("config.clan.core.backups"))
+    backup_metadata = machine.eval_nix("config.clan.core.backups")
     results = []
     if provider is None:
         for _provider in backup_metadata["providers"]:

pkgs/clan-cli/clan_cli/backups/restore.py

@@ -1,5 +1,4 @@
 import argparse
-import json
 
 from clan_cli.cmd import Log, RunOpts
 from clan_cli.completions import (
@@ -12,14 +11,17 @@ from clan_cli.machines.machines import Machine
 
 
 def restore_service(machine: Machine, name: str, provider: str, service: str) -> None:
-    backup_metadata = json.loads(machine.eval_nix("config.clan.core.backups"))
+    backup_metadata = machine.eval_nix("config.clan.core.backups")
-    backup_folders = json.loads(machine.eval_nix("config.clan.core.state"))
+    backup_folders = machine.eval_nix("config.clan.core.state")
 
     if service not in backup_folders:
         msg = f"Service {service} not found in configuration. Available services are: {', '.join(backup_folders.keys())}"
         raise ClanError(msg)
 
-    folders = backup_folders[service]["folders"]
+    folders = backup_folders[service]["folders"].values()
+    assert all(isinstance(f, str) for f in folders), (
+        f"folders must be a list of strings instead of {folders}"
+    )
     env = {}
     env["NAME"] = name
     # FIXME: If we have too many folder this might overflow the stack.
@@ -63,7 +65,7 @@ def restore_backup(
 ) -> None:
     errors = []
     if service is None:
-        backup_folders = json.loads(machine.eval_nix("config.clan.core.state"))
+        backup_folders = machine.eval_nix("config.clan.core.state")
         for _service in backup_folders:
             try:
                 restore_service(machine, name, provider, _service)

pkgs/clan-cli/clan_cli/flake.py

@@ -63,11 +63,11 @@ class FlakeCacheEntry:
 
     def __init__(
         self,
-        value: str | float | dict[str, Any] | list[Any],
+        value: str | float | dict[str, Any] | list[Any] | None,
         selectors: list[Selector],
         is_out_path: bool = False,
     ) -> None:
-        self.value: str | float | int | dict[str | int, FlakeCacheEntry]
+        self.value: str | float | int | None | dict[str | int, FlakeCacheEntry]
         self.selector: set[int] | set[str] | AllSelector
         selector: Selector = AllSelector()
 
@@ -144,7 +144,7 @@ class FlakeCacheEntry:
                 value, selectors[1:], is_out_path=True
             )
 
-        elif isinstance(value, (str | float | int)):
+        elif isinstance(value, (str | float | int | None)):
             self.value = value
 
     def insert(
@@ -249,7 +249,7 @@ class FlakeCacheEntry:
         else:
             selector = selectors[0]
 
-        if isinstance(self.value, str | float | int):
+        if isinstance(self.value, str | float | int | None):
             return selectors == []
         if isinstance(selector, AllSelector):
             if isinstance(self.selector, AllSelector):
@@ -282,7 +282,7 @@ class FlakeCacheEntry:
         if selectors == [] and isinstance(self.value, dict) and "outPath" in self.value:
             return self.value["outPath"].value
 
-        if isinstance(self.value, str | float | int):
+        if isinstance(self.value, str | float | int | None):
             return self.value
         if isinstance(self.value, dict):
             if isinstance(selector, AllSelector):
@@ -389,6 +389,9 @@ class Flake:
         return self._path
 
     def prefetch(self) -> None:
+        """
+        Run prefetch to flush the cache as well as initializing it.
+        """
         flake_prefetch = run(
             nix_command(
                 [
@@ -424,20 +427,36 @@ class Flake:
             self._is_local = False
             self._path = Path(self.store_path)
 
-    def get_from_nix(self, selectors: list[str]) -> None:
+    def get_from_nix(
+        self,
+        selectors: list[str],
+        nix_options: list[str] | None = None,
+    ) -> None:
         if self._cache is None:
             self.prefetch()
         assert self._cache is not None
 
+        if nix_options is None:
+            nix_options = []
+
         config = nix_config()
         nix_code = f"""
             let
               flake = builtins.getFlake("path:{self.store_path}?narHash={self.hash}");
             in
-              flake.inputs.nixpkgs.legacyPackages.{config["system"]}.writeText "clan-flake-select" (builtins.toJSON [ ({" ".join([f'flake.clanInternals.lib.select "{attr}" flake' for attr in selectors])}) ])
+              flake.inputs.nixpkgs.legacyPackages.{config["system"]}.writeText "clan-flake-select" (
+                builtins.toJSON [ ({" ".join([f"flake.clanInternals.lib.select ''{attr}'' flake" for attr in selectors])}) ]
+              )
         """
-        build_output = Path(run(nix_build(["--expr", nix_code])).stdout.strip())
         if tmp_store := nix_test_store():
+            nix_options += ["--store", str(tmp_store)]
+            nix_options.append("--impure")
+
+        build_output = Path(
+            run(nix_build(["--expr", nix_code, *nix_options])).stdout.strip()
+        )
+
+        if tmp_store:
             build_output = tmp_store.joinpath(*build_output.parts[1:])
         outputs = json.loads(build_output.read_text())
         if len(outputs) != len(selectors):
@@ -448,7 +467,11 @@ class Flake:
             self._cache.insert(outputs[i], selector)
         self._cache.save_to_file(self.flake_cache_path)
 
-    def select(self, selector: str) -> Any:
+    def select(
+        self,
+        selector: str,
+        nix_options: list[str] | None = None,
+    ) -> Any:
         if self._cache is None:
             self.prefetch()
         assert self._cache is not None
@@ -456,5 +479,5 @@ class Flake:
         self._cache.load_from_file(self.flake_cache_path)
         if not self._cache.is_cached(selector):
             log.info(f"Cache miss for {selector}")
-            self.get_from_nix([selector])
+            self.get_from_nix([selector], nix_options)
         return self._cache.select(selector)

pkgs/clan-cli/clan_cli/machines/machines.py

@@ -4,7 +4,6 @@ import logging
 from dataclasses import dataclass, field
 from functools import cached_property
 from pathlib import Path
-from tempfile import NamedTemporaryFile
 from time import time
 from typing import TYPE_CHECKING, Any, Literal
 
@@ -13,7 +12,7 @@ from clan_cli.errors import ClanError
 from clan_cli.facts import public_modules as facts_public_modules
 from clan_cli.facts import secret_modules as facts_secret_modules
 from clan_cli.flake import Flake
-from clan_cli.nix import nix_build, nix_config, nix_eval, nix_metadata, nix_test_store
+from clan_cli.nix import nix_build, nix_config, nix_eval, nix_test_store
 from clan_cli.ssh.host import Host
 from clan_cli.ssh.host_key import HostKeyCheck
 from clan_cli.ssh.parse import parse_deployment_address
@@ -35,16 +34,11 @@ class Machine:
     override_build_host: None | str = None
     host_key_check: HostKeyCheck = HostKeyCheck.STRICT
 
-    _eval_cache: dict[str, str] = field(default_factory=dict)
-    _build_cache: dict[str, Path] = field(default_factory=dict)
-
     def get_id(self) -> str:
         return f"{self.flake}#{self.name}"
 
     def flush_caches(self) -> None:
-        self.cached_deployment = None
+        self.flake.prefetch()
-        self._build_cache.clear()
-        self._eval_cache.clear()
 
     def __str__(self) -> str:
         return f"Machine(name={self.name}, flake={self.flake})"
@@ -66,16 +60,9 @@ class Machine:
 
     @property
     def system(self) -> str:
-        # We filter out function attributes because they are not serializable.
+        return self.flake.select(
-        attr = f'(builtins.getFlake "{self.flake}").nixosConfigurations.{self.name}.pkgs.hostPlatform.system'
+            f"nixosConfigurations.{self.name}.pkgs.hostPlatform.system"
-        output = self._eval_cache.get(attr)
+        )
-        if output is None:
-            output = run_no_stdout(
-                nix_eval(["--impure", "--expr", attr]),
-                opts=RunOpts(prefix=self.name),
-            ).stdout.strip()
-            self._eval_cache[attr] = output
-        return json.loads(output)
 
     @property
     def can_build_locally(self) -> bool:
@@ -83,13 +70,19 @@ class Machine:
         if self.system == config["system"] or self.system in config["extra-platforms"]:
             return True
 
+        nix_code = f"""
+            let
+              flake = builtins.getFlake("path:{self.flake.store_path}?narHash={self.flake.hash}");
+            in
+              flake.inputs.nixpkgs.legacyPackages.{self.system}.runCommandNoCC "clan-can-build-{int(time())}" {{ }} "touch $out").drvPath
+        """
+
         unsubstitutable_drv = json.loads(
             run_no_stdout(
                 nix_eval(
                     [
-                        "--impure",
                         "--expr",
-                        f'((builtins.getFlake "{self.flake}").inputs.nixpkgs.legacyPackages.{self.system}.runCommandNoCC "clan-can-build-{int(time())}" {{ }} "touch $out").drvPath',
+                        nix_code,
                     ]
                 ),
                 opts=RunOpts(prefix=self.name),
@@ -234,81 +227,21 @@ class Machine:
         self,
         method: Literal["eval", "build"],
         attr: str,
-        extra_config: None | dict = None,
         nix_options: list[str] | None = None,
-    ) -> str | Path:
+    ) -> Any:
         """
         Build the machine and return the path to the result
         accepts a secret store and a facts store # TODO
         """
         if nix_options is None:
             nix_options = []
+
         config = nix_config()
         system = config["system"]
 
-        file_info = {}
+        return self.flake.select(
-        with NamedTemporaryFile(mode="w") as config_json:
+            f'clanInternals.machines."{system}".{self.name}.{attr}',
-            if extra_config is not None:
+            nix_options=nix_options,
-                json.dump(extra_config, config_json, indent=2)
-            else:
-                json.dump({}, config_json)
-            config_json.flush()
-
-            file_info = json.loads(
-                run_no_stdout(
-                    nix_eval(
-                        [
-                            "--impure",
-                            "--expr",
-                            f'let x = (builtins.fetchTree {{ type = "file"; url = "file://{config_json.name}"; }}); in {{ narHash = x.narHash; path = x.outPath; }}',
-                        ]
-                    ),
-                    opts=RunOpts(prefix=self.name),
-                ).stdout.strip()
-            )
-
-        args = []
-
-        # get git commit from flake
-        if extra_config is not None:
-            metadata = nix_metadata(self.flake_dir)
-            url = metadata["url"]
-            if (
-                "dirtyRevision" in metadata
-                or "dirtyRev" in metadata["locks"]["nodes"]["clan-core"]["locked"]
-            ):
-                args += ["--impure"]
-
-            args += [
-                "--expr",
-                f"""
-                    ((builtins.getFlake "{url}").clanInternals.machinesFunc."{system}"."{self.name}" {{
-                      extraConfig = builtins.fromJSON (builtins.readFile (builtins.fetchTree {{
-                        type = "file";
-                        url = if (builtins.compareVersions builtins.nixVersion "2.19") == -1 then "{file_info["path"]}" else "file:{file_info["path"]}";
-                        narHash = "{file_info["narHash"]}";
-                      }}));
-                    }}).{attr}
-                """,
-            ]
-        else:
-            if (self.flake_dir / ".git").exists():
-                flake = f"git+file://{self.flake_dir}"
-            else:
-                flake = f"path:{self.flake_dir}"
-
-            args += [f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}']
-        args += nix_options + self.nix_options
-
-        if method == "eval":
-            output = run_no_stdout(
-                nix_eval(args), opts=RunOpts(prefix=self.name)
-            ).stdout.strip()
-            return output
-        return Path(
-            run_no_stdout(
-                nix_build(args), opts=RunOpts(prefix=self.name)
-            ).stdout.strip()
         )
 
     def eval_nix(
@@ -317,27 +250,23 @@ class Machine:
         refresh: bool = False,
         extra_config: None | dict = None,
         nix_options: list[str] | None = None,
-    ) -> str:
+    ) -> Any:
         """
         eval a nix attribute of the machine
         @attr: the attribute to get
         """
+
+        if extra_config:
+            log.warning("extra_config in eval_nix is no longer supported")
+
         if nix_options is None:
             nix_options = []
-        if attr in self._eval_cache and not refresh and extra_config is None:
-            return self._eval_cache[attr]
 
-        output = self.nix("eval", attr, extra_config, nix_options)
+        return self.nix("eval", attr, nix_options)
-        if isinstance(output, str):
-            self._eval_cache[attr] = output
-            return output
-        msg = "eval_nix returned not a string"
-        raise ClanError(msg)
 
     def build_nix(
         self,
         attr: str,
-        refresh: bool = False,
         extra_config: None | dict = None,
         nix_options: list[str] | None = None,
     ) -> Path:
@@ -346,17 +275,18 @@ class Machine:
         @attr: the attribute to get
         """
 
+        if extra_config:
+            log.warning("extra_config in build_nix is no longer supported")
+
         if nix_options is None:
             nix_options = []
-        if attr in self._build_cache and not refresh and extra_config is None:
-            return self._build_cache[attr]
 
-        output = self.nix("build", attr, extra_config, nix_options)
+        output = self.nix("build", attr, nix_options)
-        assert isinstance(output, Path), "Nix build did not result in a single path"
+        output = Path(output)
         if tmp_store := nix_test_store():
             output = tmp_store.joinpath(*output.parts[1:])
+        assert output.exists(), f"The output {output} doesn't exist"
         if isinstance(output, Path):
-            self._build_cache[attr] = output
             return output
         msg = "build_nix returned not a Path"
         raise ClanError(msg)

pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py

@@ -1,5 +1,4 @@
 import io
-import json
 import logging
 import os
 import tarfile
@@ -32,9 +31,7 @@ class SecretStore(StoreBase):
 
     @property
     def _store_backend(self) -> str:
-        backend = json.loads(
+        backend = self.machine.eval_nix("config.clan.core.vars.settings.passBackend")
-            self.machine.eval_nix("config.clan.core.vars.settings.passBackend")
-        )
         return backend
 
     @property

pkgs/clan-cli/clan_cli/vms/inspect.py

@@ -1,6 +1,5 @@
 import argparse
 import dataclasses
-import json
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
@@ -55,7 +54,7 @@ class VmConfig:
 
 
 def inspect_vm(machine: Machine) -> VmConfig:
-    data = json.loads(machine.eval_nix("config.clan.core.vm.inspect"))
+    data = machine.eval_nix("config.clan.core.vm.inspect")
     # HACK!
     data["flake_url"] = dataclasses.asdict(machine.flake)
     return VmConfig.from_json(data)

pkgs/clan-cli/tests/fixtures_flakes.py

@@ -12,6 +12,7 @@ from typing import Any, NamedTuple
 import pytest
 from clan_cli.dirs import TemplateType, clan_templates, nixpkgs_source
 from clan_cli.locked_open import locked_open
+from clan_cli.nix import nix_test_store
 from fixture_error import FixtureError
 from root import CLAN_CORE
 
@@ -43,13 +44,13 @@ def substitute(
         for line in f:
             line = line.replace("__NIXPKGS__", str(nixpkgs_source()))
             if clan_core_flake:
-                line = line.replace("__CLAN_CORE__", str(clan_core_flake))
+                line = line.replace("__CLAN_CORE__", f"path:{clan_core_flake}")
                 line = line.replace(
-                    "git+https://git.clan.lol/clan/clan-core", str(clan_core_flake)
+                    "git+https://git.clan.lol/clan/clan-core", f"path:{clan_core_flake}"
                 )
                 line = line.replace(
                     "https://git.clan.lol/clan/clan-core/archive/main.tar.gz",
-                    str(clan_core_flake),
+                    f"path:{clan_core_flake}",
                 )
             line = line.replace("__CLAN_SOPS_KEY_PATH__", sops_key)
             line = line.replace("__CLAN_SOPS_KEY_DIR__", str(flake / "facts"))
@@ -278,6 +279,22 @@ def create_flake(
     flake_nix = flake / "flake.nix"
     # this is where we would install the sops key to, when updating
     substitute(flake_nix, clan_core_flake, flake)
+    nix_options = []
+    if tmp_store := nix_test_store():
+        nix_options += ["--store", str(tmp_store)]
+
+    sp.run(
+        [
+            "nix",
+            "flake",
+            "lock",
+            flake,
+            "--extra-experimental-features",
+            "nix-command flakes",
+            *nix_options,
+        ],
+        check=True,
+    )
 
     if "/tmp" not in str(os.environ.get("HOME")):
         log.warning(

pkgs/clan-cli/tests/test_secrets_upload.py

@@ -2,21 +2,28 @@ from typing import TYPE_CHECKING
 
 import pytest
 from clan_cli.ssh.host import Host
-from fixtures_flakes import FlakeForTest
+from fixtures_flakes import ClanFlake
 from helpers import cli
 
 if TYPE_CHECKING:
     from age_keys import KeyPair
 
 
-@pytest.mark.impure
+@pytest.mark.with_core
 def test_secrets_upload(
     monkeypatch: pytest.MonkeyPatch,
-    test_flake_with_core: FlakeForTest,
+    flake: ClanFlake,
     hosts: list[Host],
     age_keys: list["KeyPair"],
 ) -> None:
-    monkeypatch.chdir(test_flake_with_core.path)
+    config = flake.machines["vm1"]
+    config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
+    host = hosts[0]
+    addr = f"{host.user}@{host.host}:{host.port}?StrictHostKeyChecking=no&UserKnownHostsFile=/dev/null&IdentityFile={host.key}"
+    config["clan"]["networking"]["targetHost"] = addr
+    config["clan"]["core"]["facts"]["secretUploadDirectory"] = str(flake.path / "facts")
+    flake.refresh()
+    monkeypatch.chdir(str(flake.path))
     monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
 
     cli.run(
@@ -25,7 +32,7 @@ def test_secrets_upload(
             "users",
             "add",
             "--flake",
-            str(test_flake_with_core.path),
+            str(flake.path),
             "user1",
             age_keys[0].pubkey,
         ]
@@ -37,27 +44,20 @@ def test_secrets_upload(
             "machines",
             "add",
             "--flake",
-            str(test_flake_with_core.path),
+            str(flake.path),
             "vm1",
             age_keys[1].pubkey,
         ]
     )
     monkeypatch.setenv("SOPS_NIX_SECRET", age_keys[0].privkey)
-    cli.run(
+    cli.run(["secrets", "set", "--flake", str(flake.path), "vm1-age.key"])
-        ["secrets", "set", "--flake", str(test_flake_with_core.path), "vm1-age.key"]
-    )
 
-    flake = test_flake_with_core.path.joinpath("flake.nix")
+    flake_path = flake.path.joinpath("flake.nix")
-    host = hosts[0]
-    addr = f"{host.user}@{host.host}:{host.port}?StrictHostKeyChecking=no&UserKnownHostsFile=/dev/null&IdentityFile={host.key}"
-    new_text = flake.read_text().replace("__CLAN_TARGET_ADDRESS__", addr)
 
-    flake.write_text(new_text)
+    cli.run(["facts", "upload", "--flake", str(flake_path), "vm1"])
-
-    cli.run(["facts", "upload", "--flake", str(test_flake_with_core.path), "vm1"])
 
     # the flake defines this path as the location where the sops key should be installed
-    sops_key = test_flake_with_core.path / "facts" / "key.txt"
+    sops_key = flake.path / "facts" / "key.txt"
 
     assert sops_key.exists()
     assert sops_key.read_text() == age_keys[0].privkey