diff --git a/checks/backups/flake-module.nix b/checks/backups/flake-module.nix index 46a8f48ee..ff50c0b8b 100644 --- a/checks/backups/flake-module.nix +++ b/checks/backups/flake-module.nix @@ -154,6 +154,7 @@ "inventory.json" "lib/build-clan" "lib/default.nix" + "lib/select.nix" "lib/flake-module.nix" "lib/frontmatter" "lib/inventory" diff --git a/pkgs/clan-cli/clan_cli/backups/create.py b/pkgs/clan-cli/clan_cli/backups/create.py index 424f7e141..612d3cf45 100644 --- a/pkgs/clan-cli/clan_cli/backups/create.py +++ b/pkgs/clan-cli/clan_cli/backups/create.py @@ -1,5 +1,4 @@ import argparse -import json import logging from clan_cli.completions import ( @@ -15,7 +14,7 @@ log = logging.getLogger(__name__) def create_backup(machine: Machine, provider: str | None = None) -> None: machine.info(f"creating backup for {machine.name}") - backup_scripts = json.loads(machine.eval_nix("config.clan.core.backups")) + backup_scripts = machine.eval_nix("config.clan.core.backups") if provider is None: if not backup_scripts["providers"]: msg = "No providers specified" diff --git a/pkgs/clan-cli/clan_cli/backups/list.py b/pkgs/clan-cli/clan_cli/backups/list.py index cfe5cc575..af2c4ba46 100644 --- a/pkgs/clan-cli/clan_cli/backups/list.py +++ b/pkgs/clan-cli/clan_cli/backups/list.py @@ -20,7 +20,7 @@ class Backup: def list_provider(machine: Machine, provider: str) -> list[Backup]: results = [] - backup_metadata = json.loads(machine.eval_nix("config.clan.core.backups")) + backup_metadata = machine.eval_nix("config.clan.core.backups") list_command = backup_metadata["providers"][provider]["list"] proc = machine.target_host.run( [list_command], @@ -46,7 +46,7 @@ def list_provider(machine: Machine, provider: str) -> list[Backup]: def list_backups(machine: Machine, provider: str | None = None) -> list[Backup]: - backup_metadata = json.loads(machine.eval_nix("config.clan.core.backups")) + backup_metadata = machine.eval_nix("config.clan.core.backups") results = [] if provider is None: for _provider in backup_metadata["providers"]: diff --git a/pkgs/clan-cli/clan_cli/backups/restore.py b/pkgs/clan-cli/clan_cli/backups/restore.py index 306adab02..49902dc4b 100644 --- a/pkgs/clan-cli/clan_cli/backups/restore.py +++ b/pkgs/clan-cli/clan_cli/backups/restore.py @@ -1,5 +1,4 @@ import argparse -import json from clan_cli.cmd import Log, RunOpts from clan_cli.completions import ( @@ -12,14 +11,17 @@ from clan_cli.machines.machines import Machine def restore_service(machine: Machine, name: str, provider: str, service: str) -> None: - backup_metadata = json.loads(machine.eval_nix("config.clan.core.backups")) - backup_folders = json.loads(machine.eval_nix("config.clan.core.state")) + backup_metadata = machine.eval_nix("config.clan.core.backups") + backup_folders = machine.eval_nix("config.clan.core.state") if service not in backup_folders: msg = f"Service {service} not found in configuration. Available services are: {', '.join(backup_folders.keys())}" raise ClanError(msg) - folders = backup_folders[service]["folders"] + folders = backup_folders[service]["folders"].values() + assert all(isinstance(f, str) for f in folders), ( + f"folders must be a list of strings instead of {folders}" + ) env = {} env["NAME"] = name # FIXME: If we have too many folder this might overflow the stack. @@ -63,7 +65,7 @@ def restore_backup( ) -> None: errors = [] if service is None: - backup_folders = json.loads(machine.eval_nix("config.clan.core.state")) + backup_folders = machine.eval_nix("config.clan.core.state") for _service in backup_folders: try: restore_service(machine, name, provider, _service) diff --git a/pkgs/clan-cli/clan_cli/flake.py b/pkgs/clan-cli/clan_cli/flake.py index 2434c52c4..7a9aff8b6 100644 --- a/pkgs/clan-cli/clan_cli/flake.py +++ b/pkgs/clan-cli/clan_cli/flake.py @@ -63,11 +63,11 @@ class FlakeCacheEntry: def __init__( self, - value: str | float | dict[str, Any] | list[Any], + value: str | float | dict[str, Any] | list[Any] | None, selectors: list[Selector], is_out_path: bool = False, ) -> None: - self.value: str | float | int | dict[str | int, FlakeCacheEntry] + self.value: str | float | int | None | dict[str | int, FlakeCacheEntry] self.selector: set[int] | set[str] | AllSelector selector: Selector = AllSelector() @@ -144,7 +144,7 @@ class FlakeCacheEntry: value, selectors[1:], is_out_path=True ) - elif isinstance(value, (str | float | int)): + elif isinstance(value, (str | float | int | None)): self.value = value def insert( @@ -249,7 +249,7 @@ class FlakeCacheEntry: else: selector = selectors[0] - if isinstance(self.value, str | float | int): + if isinstance(self.value, str | float | int | None): return selectors == [] if isinstance(selector, AllSelector): if isinstance(self.selector, AllSelector): @@ -282,7 +282,7 @@ class FlakeCacheEntry: if selectors == [] and isinstance(self.value, dict) and "outPath" in self.value: return self.value["outPath"].value - if isinstance(self.value, str | float | int): + if isinstance(self.value, str | float | int | None): return self.value if isinstance(self.value, dict): if isinstance(selector, AllSelector): @@ -389,6 +389,9 @@ class Flake: return self._path def prefetch(self) -> None: + """ + Run prefetch to flush the cache as well as initializing it. + """ flake_prefetch = run( nix_command( [ @@ -424,20 +427,36 @@ class Flake: self._is_local = False self._path = Path(self.store_path) - def get_from_nix(self, selectors: list[str]) -> None: + def get_from_nix( + self, + selectors: list[str], + nix_options: list[str] | None = None, + ) -> None: if self._cache is None: self.prefetch() assert self._cache is not None + if nix_options is None: + nix_options = [] + config = nix_config() nix_code = f""" let flake = builtins.getFlake("path:{self.store_path}?narHash={self.hash}"); in - flake.inputs.nixpkgs.legacyPackages.{config["system"]}.writeText "clan-flake-select" (builtins.toJSON [ ({" ".join([f'flake.clanInternals.lib.select "{attr}" flake' for attr in selectors])}) ]) + flake.inputs.nixpkgs.legacyPackages.{config["system"]}.writeText "clan-flake-select" ( + builtins.toJSON [ ({" ".join([f"flake.clanInternals.lib.select ''{attr}'' flake" for attr in selectors])}) ] + ) """ - build_output = Path(run(nix_build(["--expr", nix_code])).stdout.strip()) if tmp_store := nix_test_store(): + nix_options += ["--store", str(tmp_store)] + nix_options.append("--impure") + + build_output = Path( + run(nix_build(["--expr", nix_code, *nix_options])).stdout.strip() + ) + + if tmp_store: build_output = tmp_store.joinpath(*build_output.parts[1:]) outputs = json.loads(build_output.read_text()) if len(outputs) != len(selectors): @@ -448,7 +467,11 @@ class Flake: self._cache.insert(outputs[i], selector) self._cache.save_to_file(self.flake_cache_path) - def select(self, selector: str) -> Any: + def select( + self, + selector: str, + nix_options: list[str] | None = None, + ) -> Any: if self._cache is None: self.prefetch() assert self._cache is not None @@ -456,5 +479,5 @@ class Flake: self._cache.load_from_file(self.flake_cache_path) if not self._cache.is_cached(selector): log.info(f"Cache miss for {selector}") - self.get_from_nix([selector]) + self.get_from_nix([selector], nix_options) return self._cache.select(selector) diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index ce8b78c98..ad07c7a00 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -4,7 +4,6 @@ import logging from dataclasses import dataclass, field from functools import cached_property from pathlib import Path -from tempfile import NamedTemporaryFile from time import time from typing import TYPE_CHECKING, Any, Literal @@ -13,7 +12,7 @@ from clan_cli.errors import ClanError from clan_cli.facts import public_modules as facts_public_modules from clan_cli.facts import secret_modules as facts_secret_modules from clan_cli.flake import Flake -from clan_cli.nix import nix_build, nix_config, nix_eval, nix_metadata, nix_test_store +from clan_cli.nix import nix_build, nix_config, nix_eval, nix_test_store from clan_cli.ssh.host import Host from clan_cli.ssh.host_key import HostKeyCheck from clan_cli.ssh.parse import parse_deployment_address @@ -35,16 +34,11 @@ class Machine: override_build_host: None | str = None host_key_check: HostKeyCheck = HostKeyCheck.STRICT - _eval_cache: dict[str, str] = field(default_factory=dict) - _build_cache: dict[str, Path] = field(default_factory=dict) - def get_id(self) -> str: return f"{self.flake}#{self.name}" def flush_caches(self) -> None: - self.cached_deployment = None - self._build_cache.clear() - self._eval_cache.clear() + self.flake.prefetch() def __str__(self) -> str: return f"Machine(name={self.name}, flake={self.flake})" @@ -66,16 +60,9 @@ class Machine: @property def system(self) -> str: - # We filter out function attributes because they are not serializable. - attr = f'(builtins.getFlake "{self.flake}").nixosConfigurations.{self.name}.pkgs.hostPlatform.system' - output = self._eval_cache.get(attr) - if output is None: - output = run_no_stdout( - nix_eval(["--impure", "--expr", attr]), - opts=RunOpts(prefix=self.name), - ).stdout.strip() - self._eval_cache[attr] = output - return json.loads(output) + return self.flake.select( + f"nixosConfigurations.{self.name}.pkgs.hostPlatform.system" + ) @property def can_build_locally(self) -> bool: @@ -83,13 +70,19 @@ class Machine: if self.system == config["system"] or self.system in config["extra-platforms"]: return True + nix_code = f""" + let + flake = builtins.getFlake("path:{self.flake.store_path}?narHash={self.flake.hash}"); + in + flake.inputs.nixpkgs.legacyPackages.{self.system}.runCommandNoCC "clan-can-build-{int(time())}" {{ }} "touch $out").drvPath + """ + unsubstitutable_drv = json.loads( run_no_stdout( nix_eval( [ - "--impure", "--expr", - f'((builtins.getFlake "{self.flake}").inputs.nixpkgs.legacyPackages.{self.system}.runCommandNoCC "clan-can-build-{int(time())}" {{ }} "touch $out").drvPath', + nix_code, ] ), opts=RunOpts(prefix=self.name), @@ -234,81 +227,21 @@ class Machine: self, method: Literal["eval", "build"], attr: str, - extra_config: None | dict = None, nix_options: list[str] | None = None, - ) -> str | Path: + ) -> Any: """ Build the machine and return the path to the result accepts a secret store and a facts store # TODO """ if nix_options is None: nix_options = [] + config = nix_config() system = config["system"] - file_info = {} - with NamedTemporaryFile(mode="w") as config_json: - if extra_config is not None: - json.dump(extra_config, config_json, indent=2) - else: - json.dump({}, config_json) - config_json.flush() - - file_info = json.loads( - run_no_stdout( - nix_eval( - [ - "--impure", - "--expr", - f'let x = (builtins.fetchTree {{ type = "file"; url = "file://{config_json.name}"; }}); in {{ narHash = x.narHash; path = x.outPath; }}', - ] - ), - opts=RunOpts(prefix=self.name), - ).stdout.strip() - ) - - args = [] - - # get git commit from flake - if extra_config is not None: - metadata = nix_metadata(self.flake_dir) - url = metadata["url"] - if ( - "dirtyRevision" in metadata - or "dirtyRev" in metadata["locks"]["nodes"]["clan-core"]["locked"] - ): - args += ["--impure"] - - args += [ - "--expr", - f""" - ((builtins.getFlake "{url}").clanInternals.machinesFunc."{system}"."{self.name}" {{ - extraConfig = builtins.fromJSON (builtins.readFile (builtins.fetchTree {{ - type = "file"; - url = if (builtins.compareVersions builtins.nixVersion "2.19") == -1 then "{file_info["path"]}" else "file:{file_info["path"]}"; - narHash = "{file_info["narHash"]}"; - }})); - }}).{attr} - """, - ] - else: - if (self.flake_dir / ".git").exists(): - flake = f"git+file://{self.flake_dir}" - else: - flake = f"path:{self.flake_dir}" - - args += [f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}'] - args += nix_options + self.nix_options - - if method == "eval": - output = run_no_stdout( - nix_eval(args), opts=RunOpts(prefix=self.name) - ).stdout.strip() - return output - return Path( - run_no_stdout( - nix_build(args), opts=RunOpts(prefix=self.name) - ).stdout.strip() + return self.flake.select( + f'clanInternals.machines."{system}".{self.name}.{attr}', + nix_options=nix_options, ) def eval_nix( @@ -317,27 +250,23 @@ class Machine: refresh: bool = False, extra_config: None | dict = None, nix_options: list[str] | None = None, - ) -> str: + ) -> Any: """ eval a nix attribute of the machine @attr: the attribute to get """ + + if extra_config: + log.warning("extra_config in eval_nix is no longer supported") + if nix_options is None: nix_options = [] - if attr in self._eval_cache and not refresh and extra_config is None: - return self._eval_cache[attr] - output = self.nix("eval", attr, extra_config, nix_options) - if isinstance(output, str): - self._eval_cache[attr] = output - return output - msg = "eval_nix returned not a string" - raise ClanError(msg) + return self.nix("eval", attr, nix_options) def build_nix( self, attr: str, - refresh: bool = False, extra_config: None | dict = None, nix_options: list[str] | None = None, ) -> Path: @@ -346,17 +275,18 @@ class Machine: @attr: the attribute to get """ + if extra_config: + log.warning("extra_config in build_nix is no longer supported") + if nix_options is None: nix_options = [] - if attr in self._build_cache and not refresh and extra_config is None: - return self._build_cache[attr] - output = self.nix("build", attr, extra_config, nix_options) - assert isinstance(output, Path), "Nix build did not result in a single path" + output = self.nix("build", attr, nix_options) + output = Path(output) if tmp_store := nix_test_store(): output = tmp_store.joinpath(*output.parts[1:]) + assert output.exists(), f"The output {output} doesn't exist" if isinstance(output, Path): - self._build_cache[attr] = output return output msg = "build_nix returned not a Path" raise ClanError(msg) diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py index 31ee8d789..e583c69a8 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py @@ -1,5 +1,4 @@ import io -import json import logging import os import tarfile @@ -32,9 +31,7 @@ class SecretStore(StoreBase): @property def _store_backend(self) -> str: - backend = json.loads( - self.machine.eval_nix("config.clan.core.vars.settings.passBackend") - ) + backend = self.machine.eval_nix("config.clan.core.vars.settings.passBackend") return backend @property diff --git a/pkgs/clan-cli/clan_cli/vms/inspect.py b/pkgs/clan-cli/clan_cli/vms/inspect.py index b86d97c3d..898bda1ae 100644 --- a/pkgs/clan-cli/clan_cli/vms/inspect.py +++ b/pkgs/clan-cli/clan_cli/vms/inspect.py @@ -1,6 +1,5 @@ import argparse import dataclasses -import json from dataclasses import dataclass from pathlib import Path from typing import Any @@ -55,7 +54,7 @@ class VmConfig: def inspect_vm(machine: Machine) -> VmConfig: - data = json.loads(machine.eval_nix("config.clan.core.vm.inspect")) + data = machine.eval_nix("config.clan.core.vm.inspect") # HACK! data["flake_url"] = dataclasses.asdict(machine.flake) return VmConfig.from_json(data) diff --git a/pkgs/clan-cli/tests/fixtures_flakes.py b/pkgs/clan-cli/tests/fixtures_flakes.py index b7e6f41a8..114536718 100644 --- a/pkgs/clan-cli/tests/fixtures_flakes.py +++ b/pkgs/clan-cli/tests/fixtures_flakes.py @@ -12,6 +12,7 @@ from typing import Any, NamedTuple import pytest from clan_cli.dirs import TemplateType, clan_templates, nixpkgs_source from clan_cli.locked_open import locked_open +from clan_cli.nix import nix_test_store from fixture_error import FixtureError from root import CLAN_CORE @@ -43,13 +44,13 @@ def substitute( for line in f: line = line.replace("__NIXPKGS__", str(nixpkgs_source())) if clan_core_flake: - line = line.replace("__CLAN_CORE__", str(clan_core_flake)) + line = line.replace("__CLAN_CORE__", f"path:{clan_core_flake}") line = line.replace( - "git+https://git.clan.lol/clan/clan-core", str(clan_core_flake) + "git+https://git.clan.lol/clan/clan-core", f"path:{clan_core_flake}" ) line = line.replace( "https://git.clan.lol/clan/clan-core/archive/main.tar.gz", - str(clan_core_flake), + f"path:{clan_core_flake}", ) line = line.replace("__CLAN_SOPS_KEY_PATH__", sops_key) line = line.replace("__CLAN_SOPS_KEY_DIR__", str(flake / "facts")) @@ -278,6 +279,22 @@ def create_flake( flake_nix = flake / "flake.nix" # this is where we would install the sops key to, when updating substitute(flake_nix, clan_core_flake, flake) + nix_options = [] + if tmp_store := nix_test_store(): + nix_options += ["--store", str(tmp_store)] + + sp.run( + [ + "nix", + "flake", + "lock", + flake, + "--extra-experimental-features", + "nix-command flakes", + *nix_options, + ], + check=True, + ) if "/tmp" not in str(os.environ.get("HOME")): log.warning( diff --git a/pkgs/clan-cli/tests/test_secrets_upload.py b/pkgs/clan-cli/tests/test_secrets_upload.py index 058bd0b7a..2556fb863 100644 --- a/pkgs/clan-cli/tests/test_secrets_upload.py +++ b/pkgs/clan-cli/tests/test_secrets_upload.py @@ -2,21 +2,28 @@ from typing import TYPE_CHECKING import pytest from clan_cli.ssh.host import Host -from fixtures_flakes import FlakeForTest +from fixtures_flakes import ClanFlake from helpers import cli if TYPE_CHECKING: from age_keys import KeyPair -@pytest.mark.impure +@pytest.mark.with_core def test_secrets_upload( monkeypatch: pytest.MonkeyPatch, - test_flake_with_core: FlakeForTest, + flake: ClanFlake, hosts: list[Host], age_keys: list["KeyPair"], ) -> None: - monkeypatch.chdir(test_flake_with_core.path) + config = flake.machines["vm1"] + config["nixpkgs"]["hostPlatform"] = "x86_64-linux" + host = hosts[0] + addr = f"{host.user}@{host.host}:{host.port}?StrictHostKeyChecking=no&UserKnownHostsFile=/dev/null&IdentityFile={host.key}" + config["clan"]["networking"]["targetHost"] = addr + config["clan"]["core"]["facts"]["secretUploadDirectory"] = str(flake.path / "facts") + flake.refresh() + monkeypatch.chdir(str(flake.path)) monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey) cli.run( @@ -25,7 +32,7 @@ def test_secrets_upload( "users", "add", "--flake", - str(test_flake_with_core.path), + str(flake.path), "user1", age_keys[0].pubkey, ] @@ -37,27 +44,20 @@ def test_secrets_upload( "machines", "add", "--flake", - str(test_flake_with_core.path), + str(flake.path), "vm1", age_keys[1].pubkey, ] ) monkeypatch.setenv("SOPS_NIX_SECRET", age_keys[0].privkey) - cli.run( - ["secrets", "set", "--flake", str(test_flake_with_core.path), "vm1-age.key"] - ) + cli.run(["secrets", "set", "--flake", str(flake.path), "vm1-age.key"]) - flake = test_flake_with_core.path.joinpath("flake.nix") - host = hosts[0] - addr = f"{host.user}@{host.host}:{host.port}?StrictHostKeyChecking=no&UserKnownHostsFile=/dev/null&IdentityFile={host.key}" - new_text = flake.read_text().replace("__CLAN_TARGET_ADDRESS__", addr) + flake_path = flake.path.joinpath("flake.nix") - flake.write_text(new_text) - - cli.run(["facts", "upload", "--flake", str(test_flake_with_core.path), "vm1"]) + cli.run(["facts", "upload", "--flake", str(flake_path), "vm1"]) # the flake defines this path as the location where the sops key should be installed - sops_key = test_flake_with_core.path / "facts" / "key.txt" + sops_key = flake.path / "facts" / "key.txt" assert sops_key.exists() assert sops_key.read_text() == age_keys[0].privkey