yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

clan-cli: use new flake caching for machines

Browse Source
This commit is contained in:
lassulus
2025-02-23 14:33:59 +01:00
parent 35eaaea68e
commit 300aaa48e7
10 changed files with 113 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
checks/backups/flake-module.nix
View File

@@ -154,6 +154,7 @@
"inventory.json"
"lib/build-clan"
"lib/default.nix"
"lib/select.nix"
"lib/flake-module.nix"
"lib/frontmatter"
"lib/inventory"

3
pkgs/clan-cli/clan_cli/backups/create.py
View File

@@ -1,5 +1,4 @@
import argparse
import json
import logging
from clan_cli.completions import (
@@ -15,7 +14,7 @@ log = logging.getLogger(__name__)
def create_backup(machine: Machine, provider: str | None = None) -> None:
machine.info(f"creating backup for {machine.name}")
backup_scripts = json.loads(machine.eval_nix("config.clan.core.backups"))
backup_scripts = machine.eval_nix("config.clan.core.backups")
if provider is None:
if not backup_scripts["providers"]:
msg = "No providers specified"

4
pkgs/clan-cli/clan_cli/backups/list.py
View File

@@ -20,7 +20,7 @@ class Backup:
def list_provider(machine: Machine, provider: str) -> list[Backup]:
results = []
backup_metadata = json.loads(machine.eval_nix("config.clan.core.backups"))
backup_metadata = machine.eval_nix("config.clan.core.backups")
list_command = backup_metadata["providers"][provider]["list"]
proc = machine.target_host.run(
[list_command],
@@ -46,7 +46,7 @@ def list_provider(machine: Machine, provider: str) -> list[Backup]:
def list_backups(machine: Machine, provider: str | None = None) -> list[Backup]:
backup_metadata = json.loads(machine.eval_nix("config.clan.core.backups"))
backup_metadata = machine.eval_nix("config.clan.core.backups")
results = []
if provider is None:
for _provider in backup_metadata["providers"]:

12
pkgs/clan-cli/clan_cli/backups/restore.py
View File

@@ -1,5 +1,4 @@
import argparse
import json
from clan_cli.cmd import Log, RunOpts
from clan_cli.completions import (
@@ -12,14 +11,17 @@ from clan_cli.machines.machines import Machine
def restore_service(machine: Machine, name: str, provider: str, service: str) -> None:
backup_metadata = json.loads(machine.eval_nix("config.clan.core.backups"))
backup_folders = json.loads(machine.eval_nix("config.clan.core.state"))
backup_metadata = machine.eval_nix("config.clan.core.backups")
backup_folders = machine.eval_nix("config.clan.core.state")
if service not in backup_folders:
msg = f"Service {service} not found in configuration. Available services are: {', '.join(backup_folders.keys())}"
raise ClanError(msg)
folders = backup_folders[service]["folders"]
folders = backup_folders[service]["folders"].values()
assert all(isinstance(f, str) for f in folders), (
f"folders must be a list of strings instead of {folders}"
)
env = {}
env["NAME"] = name
# FIXME: If we have too many folder this might overflow the stack.
@@ -63,7 +65,7 @@ def restore_backup(
) -> None:
errors = []
if service is None:
backup_folders = json.loads(machine.eval_nix("config.clan.core.state"))
backup_folders = machine.eval_nix("config.clan.core.state")
for _service in backup_folders:
try:
restore_service(machine, name, provider, _service)

43
pkgs/clan-cli/clan_cli/flake.py
View File

@@ -63,11 +63,11 @@ class FlakeCacheEntry:
def __init__(
self,
value: str | float | dict[str, Any] | list[Any],
value: str | float | dict[str, Any] | list[Any] | None,
selectors: list[Selector],
is_out_path: bool = False,
) -> None:
self.value: str | float | int | dict[str | int, FlakeCacheEntry]
self.value: str | float | int | None | dict[str | int, FlakeCacheEntry]
self.selector: set[int] | set[str] | AllSelector
selector: Selector = AllSelector()
@@ -144,7 +144,7 @@ class FlakeCacheEntry:
value, selectors[1:], is_out_path=True
)
elif isinstance(value, (str | float | int)):
elif isinstance(value, (str | float | int | None)):
self.value = value
def insert(
@@ -249,7 +249,7 @@ class FlakeCacheEntry:
else:
selector = selectors[0]
if isinstance(self.value, str | float | int):
if isinstance(self.value, str | float | int | None):
return selectors == []
if isinstance(selector, AllSelector):
if isinstance(self.selector, AllSelector):
@@ -282,7 +282,7 @@ class FlakeCacheEntry:
if selectors == [] and isinstance(self.value, dict) and "outPath" in self.value:
return self.value["outPath"].value
if isinstance(self.value, str | float | int):
if isinstance(self.value, str | float | int | None):
return self.value
if isinstance(self.value, dict):
if isinstance(selector, AllSelector):
@@ -389,6 +389,9 @@ class Flake:
return self._path
def prefetch(self) -> None:
"""
Run prefetch to flush the cache as well as initializing it.
"""
flake_prefetch = run(
nix_command(
[
@@ -424,20 +427,36 @@ class Flake:
self._is_local = False
self._path = Path(self.store_path)
def get_from_nix(self, selectors: list[str]) -> None:
def get_from_nix(
self,
selectors: list[str],
nix_options: list[str] | None = None,
) -> None:
if self._cache is None:
self.prefetch()
assert self._cache is not None
if nix_options is None:
nix_options = []
config = nix_config()
nix_code = f"""
let
flake = builtins.getFlake("path:{self.store_path}?narHash={self.hash}");
in
flake.inputs.nixpkgs.legacyPackages.{config["system"]}.writeText "clan-flake-select" (builtins.toJSON [ ({" ".join([f'flake.clanInternals.lib.select "{attr}" flake' for attr in selectors])}) ])
flake.inputs.nixpkgs.legacyPackages.{config["system"]}.writeText "clan-flake-select" (
builtins.toJSON [ ({" ".join([f"flake.clanInternals.lib.select ''{attr}'' flake" for attr in selectors])}) ]
)
"""
build_output = Path(run(nix_build(["--expr", nix_code])).stdout.strip())
if tmp_store := nix_test_store():
nix_options += ["--store", str(tmp_store)]
nix_options.append("--impure")
build_output = Path(
run(nix_build(["--expr", nix_code, *nix_options])).stdout.strip()
)
if tmp_store:
build_output = tmp_store.joinpath(*build_output.parts[1:])
outputs = json.loads(build_output.read_text())
if len(outputs) != len(selectors):
@@ -448,7 +467,11 @@ class Flake:
self._cache.insert(outputs[i], selector)
self._cache.save_to_file(self.flake_cache_path)
def select(self, selector: str) -> Any:
def select(
self,
selector: str,
nix_options: list[str] | None = None,
) -> Any:
if self._cache is None:
self.prefetch()
assert self._cache is not None
@@ -456,5 +479,5 @@ class Flake:
self._cache.load_from_file(self.flake_cache_path)
if not self._cache.is_cached(selector):
log.info(f"Cache miss for {selector}")
self.get_from_nix([selector])
self.get_from_nix([selector], nix_options)
return self._cache.select(selector)

130
pkgs/clan-cli/clan_cli/machines/machines.py
View File

@@ -4,7 +4,6 @@ import logging
from dataclasses import dataclass, field
from functools import cached_property
from pathlib import Path
from tempfile import NamedTemporaryFile
from time import time
from typing import TYPE_CHECKING, Any, Literal
@@ -13,7 +12,7 @@ from clan_cli.errors import ClanError
from clan_cli.facts import public_modules as facts_public_modules
from clan_cli.facts import secret_modules as facts_secret_modules
from clan_cli.flake import Flake
from clan_cli.nix import nix_build, nix_config, nix_eval, nix_metadata, nix_test_store
from clan_cli.nix import nix_build, nix_config, nix_eval, nix_test_store
from clan_cli.ssh.host import Host
from clan_cli.ssh.host_key import HostKeyCheck
from clan_cli.ssh.parse import parse_deployment_address
@@ -35,16 +34,11 @@ class Machine:
override_build_host: None | str = None
host_key_check: HostKeyCheck = HostKeyCheck.STRICT
_eval_cache: dict[str, str] = field(default_factory=dict)
_build_cache: dict[str, Path] = field(default_factory=dict)
def get_id(self) -> str:
return f"{self.flake}#{self.name}"
def flush_caches(self) -> None:
self.cached_deployment = None
self._build_cache.clear()
self._eval_cache.clear()
self.flake.prefetch()
def __str__(self) -> str:
return f"Machine(name={self.name}, flake={self.flake})"
@@ -66,16 +60,9 @@ class Machine:
@property
def system(self) -> str:
# We filter out function attributes because they are not serializable.
attr = f'(builtins.getFlake "{self.flake}").nixosConfigurations.{self.name}.pkgs.hostPlatform.system'
output = self._eval_cache.get(attr)
if output is None:
output = run_no_stdout(
nix_eval(["--impure", "--expr", attr]),
opts=RunOpts(prefix=self.name),
).stdout.strip()
self._eval_cache[attr] = output
return json.loads(output)
return self.flake.select(
f"nixosConfigurations.{self.name}.pkgs.hostPlatform.system"
)
@property
def can_build_locally(self) -> bool:
@@ -83,13 +70,19 @@ class Machine:
if self.system == config["system"] or self.system in config["extra-platforms"]:
return True
nix_code = f"""
let
flake = builtins.getFlake("path:{self.flake.store_path}?narHash={self.flake.hash}");
in
flake.inputs.nixpkgs.legacyPackages.{self.system}.runCommandNoCC "clan-can-build-{int(time())}" {{ }} "touch $out").drvPath
"""
unsubstitutable_drv = json.loads(
run_no_stdout(
nix_eval(
[
"--impure",
"--expr",
f'((builtins.getFlake "{self.flake}").inputs.nixpkgs.legacyPackages.{self.system}.runCommandNoCC "clan-can-build-{int(time())}" {{ }} "touch $out").drvPath',
nix_code,
]
),
opts=RunOpts(prefix=self.name),
@@ -234,81 +227,21 @@ class Machine:
self,
method: Literal["eval", "build"],
attr: str,
extra_config: None | dict = None,
nix_options: list[str] | None = None,
) -> str | Path:
) -> Any:
"""
Build the machine and return the path to the result
accepts a secret store and a facts store # TODO
"""
if nix_options is None:
nix_options = []
config = nix_config()
system = config["system"]
file_info = {}
with NamedTemporaryFile(mode="w") as config_json:
if extra_config is not None:
json.dump(extra_config, config_json, indent=2)
else:
json.dump({}, config_json)
config_json.flush()
file_info = json.loads(
run_no_stdout(
nix_eval(
[
"--impure",
"--expr",
f'let x = (builtins.fetchTree {{ type = "file"; url = "file://{config_json.name}"; }}); in {{ narHash = x.narHash; path = x.outPath; }}',
]
),
opts=RunOpts(prefix=self.name),
).stdout.strip()
)
args = []
# get git commit from flake
if extra_config is not None:
metadata = nix_metadata(self.flake_dir)
url = metadata["url"]
if (
"dirtyRevision" in metadata
or "dirtyRev" in metadata["locks"]["nodes"]["clan-core"]["locked"]
):
args += ["--impure"]
args += [
"--expr",
f"""
((builtins.getFlake "{url}").clanInternals.machinesFunc."{system}"."{self.name}" {{
extraConfig = builtins.fromJSON (builtins.readFile (builtins.fetchTree {{
type = "file";
url = if (builtins.compareVersions builtins.nixVersion "2.19") == -1 then "{file_info["path"]}" else "file:{file_info["path"]}";
narHash = "{file_info["narHash"]}";
}}));
}}).{attr}
""",
]
else:
if (self.flake_dir / ".git").exists():
flake = f"git+file://{self.flake_dir}"
else:
flake = f"path:{self.flake_dir}"
args += [f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}']
args += nix_options + self.nix_options
if method == "eval":
output = run_no_stdout(
nix_eval(args), opts=RunOpts(prefix=self.name)
).stdout.strip()
return output
return Path(
run_no_stdout(
nix_build(args), opts=RunOpts(prefix=self.name)
).stdout.strip()
return self.flake.select(
f'clanInternals.machines."{system}".{self.name}.{attr}',
nix_options=nix_options,
)
def eval_nix(
@@ -317,27 +250,23 @@ class Machine:
refresh: bool = False,
extra_config: None | dict = None,
nix_options: list[str] | None = None,
) -> str:
) -> Any:
"""
eval a nix attribute of the machine
@attr: the attribute to get
"""
if extra_config:
log.warning("extra_config in eval_nix is no longer supported")
if nix_options is None:
nix_options = []
if attr in self._eval_cache and not refresh and extra_config is None:
return self._eval_cache[attr]
output = self.nix("eval", attr, extra_config, nix_options)
if isinstance(output, str):
self._eval_cache[attr] = output
return output
msg = "eval_nix returned not a string"
raise ClanError(msg)
return self.nix("eval", attr, nix_options)
def build_nix(
self,
attr: str,
refresh: bool = False,
extra_config: None | dict = None,
nix_options: list[str] | None = None,
) -> Path:
@@ -346,17 +275,18 @@ class Machine:
@attr: the attribute to get
"""
if extra_config:
log.warning("extra_config in build_nix is no longer supported")
if nix_options is None:
nix_options = []
if attr in self._build_cache and not refresh and extra_config is None:
return self._build_cache[attr]
output = self.nix("build", attr, extra_config, nix_options)
assert isinstance(output, Path), "Nix build did not result in a single path"
output = self.nix("build", attr, nix_options)
output = Path(output)
if tmp_store := nix_test_store():
output = tmp_store.joinpath(*output.parts[1:])
assert output.exists(), f"The output {output} doesn't exist"
if isinstance(output, Path):
self._build_cache[attr] = output
return output
msg = "build_nix returned not a Path"
raise ClanError(msg)

5
pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py
View File

@@ -1,5 +1,4 @@
import io
import json
import logging
import os
import tarfile
@@ -32,9 +31,7 @@ class SecretStore(StoreBase):
@property
def _store_backend(self) -> str:
backend = json.loads(
self.machine.eval_nix("config.clan.core.vars.settings.passBackend")
)
backend = self.machine.eval_nix("config.clan.core.vars.settings.passBackend")
return backend
@property

3
pkgs/clan-cli/clan_cli/vms/inspect.py
View File

@@ -1,6 +1,5 @@
import argparse
import dataclasses
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any
@@ -55,7 +54,7 @@ class VmConfig:
def inspect_vm(machine: Machine) -> VmConfig:
data = json.loads(machine.eval_nix("config.clan.core.vm.inspect"))
data = machine.eval_nix("config.clan.core.vm.inspect")
# HACK!
data["flake_url"] = dataclasses.asdict(machine.flake)
return VmConfig.from_json(data)

23
pkgs/clan-cli/tests/fixtures_flakes.py
View File

@@ -12,6 +12,7 @@ from typing import Any, NamedTuple
import pytest
from clan_cli.dirs import TemplateType, clan_templates, nixpkgs_source
from clan_cli.locked_open import locked_open
from clan_cli.nix import nix_test_store
from fixture_error import FixtureError
from root import CLAN_CORE
@@ -43,13 +44,13 @@ def substitute(
for line in f:
line = line.replace("__NIXPKGS__", str(nixpkgs_source()))
if clan_core_flake:
line = line.replace("__CLAN_CORE__", str(clan_core_flake))
line = line.replace("__CLAN_CORE__", f"path:{clan_core_flake}")
line = line.replace(
"git+https://git.clan.lol/clan/clan-core", str(clan_core_flake)
"git+https://git.clan.lol/clan/clan-core", f"path:{clan_core_flake}"
)
line = line.replace(
"https://git.clan.lol/clan/clan-core/archive/main.tar.gz",
str(clan_core_flake),
f"path:{clan_core_flake}",
)
line = line.replace("__CLAN_SOPS_KEY_PATH__", sops_key)
line = line.replace("__CLAN_SOPS_KEY_DIR__", str(flake / "facts"))
@@ -278,6 +279,22 @@ def create_flake(
flake_nix = flake / "flake.nix"
# this is where we would install the sops key to, when updating
substitute(flake_nix, clan_core_flake, flake)
nix_options = []
if tmp_store := nix_test_store():
nix_options += ["--store", str(tmp_store)]
sp.run(
[
"nix",
"flake",
"lock",
flake,
"--extra-experimental-features",
"nix-command flakes",
*nix_options,
],
check=True,
)
if "/tmp" not in str(os.environ.get("HOME")):
log.warning(

34
pkgs/clan-cli/tests/test_secrets_upload.py
View File

@@ -2,21 +2,28 @@ from typing import TYPE_CHECKING
import pytest
from clan_cli.ssh.host import Host
from fixtures_flakes import FlakeForTest
from fixtures_flakes import ClanFlake
from helpers import cli
if TYPE_CHECKING:
from age_keys import KeyPair
@pytest.mark.impure
@pytest.mark.with_core
def test_secrets_upload(
monkeypatch: pytest.MonkeyPatch,
test_flake_with_core: FlakeForTest,
flake: ClanFlake,
hosts: list[Host],
age_keys: list["KeyPair"],
) -> None:
monkeypatch.chdir(test_flake_with_core.path)
config = flake.machines["vm1"]
config["nixpkgs"]["hostPlatform"] = "x86_64-linux"
host = hosts[0]
addr = f"{host.user}@{host.host}:{host.port}?StrictHostKeyChecking=no&UserKnownHostsFile=/dev/null&IdentityFile={host.key}"
config["clan"]["networking"]["targetHost"] = addr
config["clan"]["core"]["facts"]["secretUploadDirectory"] = str(flake.path / "facts")
flake.refresh()
monkeypatch.chdir(str(flake.path))
monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
cli.run(
@@ -25,7 +32,7 @@ def test_secrets_upload(
"users",
"add",
"--flake",
str(test_flake_with_core.path),
str(flake.path),
"user1",
age_keys[0].pubkey,
]
@@ -37,27 +44,20 @@ def test_secrets_upload(
"machines",
"add",
"--flake",
str(test_flake_with_core.path),
str(flake.path),
"vm1",
age_keys[1].pubkey,
]
)
monkeypatch.setenv("SOPS_NIX_SECRET", age_keys[0].privkey)
cli.run(
["secrets", "set", "--flake", str(test_flake_with_core.path), "vm1-age.key"]
)
cli.run(["secrets", "set", "--flake", str(flake.path), "vm1-age.key"])
flake = test_flake_with_core.path.joinpath("flake.nix")
host = hosts[0]
addr = f"{host.user}@{host.host}:{host.port}?StrictHostKeyChecking=no&UserKnownHostsFile=/dev/null&IdentityFile={host.key}"
new_text = flake.read_text().replace("__CLAN_TARGET_ADDRESS__", addr)
flake_path = flake.path.joinpath("flake.nix")
flake.write_text(new_text)
cli.run(["facts", "upload", "--flake", str(test_flake_with_core.path), "vm1"])
cli.run(["facts", "upload", "--flake", str(flake_path), "vm1"])
# the flake defines this path as the location where the sops key should be installed
sops_key = test_flake_with_core.path / "facts" / "key.txt"
sops_key = flake.path / "facts" / "key.txt"
assert sops_key.exists()
assert sops_key.read_text() == age_keys[0].privkey