From 723d72255cebc64be9261107b4cd95bc97ebd4fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 21 Apr 2025 13:03:25 +0200 Subject: [PATCH 1/7] Reapply "remove nix_shell_legacy" This reverts commit c5001f19fc23a6a8f8d001f73a8b6bf729198321. --- docs/CONTRIBUTING.md | 2 +- pkgs/clan-app/tests/conftest.py | 6 ++--- pkgs/clan-cli/clan_cli/api/directory.py | 6 ++--- pkgs/clan-cli/clan_cli/api/mdns_discovery.py | 6 ++--- pkgs/clan-cli/clan_cli/bwrap/__init__.py | 8 +++--- pkgs/clan-cli/clan_cli/clan/create.py | 4 +-- pkgs/clan-cli/clan_cli/facts/generate.py | 8 +++--- .../facts/secret_modules/password_store.py | 18 ++++++------- pkgs/clan-cli/clan_cli/flash/flash.py | 6 ++--- pkgs/clan-cli/clan_cli/machines/install.py | 27 ++++++++----------- pkgs/clan-cli/clan_cli/machines/list.py | 6 ++--- pkgs/clan-cli/clan_cli/nix/__init__.py | 19 ------------- .../clan_cli/nix/allowed-packages.json | 4 +++ pkgs/clan-cli/clan_cli/secrets/import_sops.py | 4 +-- pkgs/clan-cli/clan_cli/secrets/sops.py | 8 +++--- pkgs/clan-cli/clan_cli/ssh/deploy_info.py | 6 ++--- pkgs/clan-cli/clan_cli/ssh/tor.py | 6 ++--- pkgs/clan-cli/clan_cli/tests/git_repo.py | 10 +++---- .../tests/test_secrets_password_store.py | 10 +++---- pkgs/clan-cli/clan_cli/vars/generate.py | 8 +++--- .../vars/secret_modules/password_store.py | 14 +++++----- pkgs/clan-cli/clan_cli/vms/run.py | 12 ++++----- pkgs/clan-cli/clan_cli/vms/virtiofsd.py | 6 ++--- pkgs/clan-cli/clan_cli/vms/waypipe.py | 6 ++--- 24 files changed, 91 insertions(+), 119 deletions(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 070b9b6ae..09cfc3f66 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -82,7 +82,7 @@ For instance, if you need to update `nixos-anywhere` in clan-cli, find its usage ```python run( nix_shell( - ["nixpkgs#nixos-anywhere"], + ["nixos-anywhere"], cmd, ), RunOpts(log=Log.BOTH, prefix=machine.name, needs_user_terminal=True), diff --git a/pkgs/clan-app/tests/conftest.py b/pkgs/clan-app/tests/conftest.py index 530f4ad1c..047f8724a 100644 --- a/pkgs/clan-app/tests/conftest.py +++ b/pkgs/clan-app/tests/conftest.py @@ -29,12 +29,12 @@ def pytest_sessionstart(session: pytest.Session) -> None: @pytest.fixture def git_repo(tmp_path: Path) -> Path: # initialize a git repository - cmd = nix_shell(["nixpkgs#git"], ["git", "init"]) + cmd = nix_shell(["git"], ["git", "init"]) subprocess.run(cmd, cwd=tmp_path, check=True) # set user.name and user.email - cmd = nix_shell(["nixpkgs#git"], ["git", "config", "user.name", "test"]) + cmd = nix_shell(["git"], ["git", "config", "user.name", "test"]) subprocess.run(cmd, cwd=tmp_path, check=True) - cmd = nix_shell(["nixpkgs#git"], ["git", "config", "user.email", "test@test.test"]) + cmd = nix_shell(["git"], ["git", "config", "user.email", "test@test.test"]) subprocess.run(cmd, cwd=tmp_path, check=True) # return the path to the git repository return tmp_path diff --git a/pkgs/clan-cli/clan_cli/api/directory.py b/pkgs/clan-cli/clan_cli/api/directory.py index cdd297ba6..907705686 100644 --- a/pkgs/clan-cli/clan_cli/api/directory.py +++ b/pkgs/clan-cli/clan_cli/api/directory.py @@ -6,7 +6,7 @@ from typing import Any, Literal from clan_cli.cmd import RunOpts from clan_cli.errors import ClanError -from clan_cli.nix import nix_shell_legacy, run_no_stdout +from clan_cli.nix import nix_shell, run_no_stdout from . import API @@ -126,8 +126,8 @@ def show_block_devices() -> Blockdevices: It must return a list of block devices. """ - cmd = nix_shell_legacy( - ["nixpkgs#util-linux"], + cmd = nix_shell( + ["util-linux"], [ "lsblk", "--json", diff --git a/pkgs/clan-cli/clan_cli/api/mdns_discovery.py b/pkgs/clan-cli/clan_cli/api/mdns_discovery.py index 6f45734e0..8dea84f75 100644 --- a/pkgs/clan-cli/clan_cli/api/mdns_discovery.py +++ b/pkgs/clan-cli/clan_cli/api/mdns_discovery.py @@ -3,7 +3,7 @@ import re from dataclasses import dataclass from clan_cli.cmd import run_no_stdout -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell from . import API @@ -89,8 +89,8 @@ def parse_avahi_output(output: str) -> DNSInfo: @API.register def show_mdns() -> DNSInfo: - cmd = nix_shell_legacy( - ["nixpkgs#avahi"], + cmd = nix_shell( + ["avahi"], [ "avahi-browse", "--all", diff --git a/pkgs/clan-cli/clan_cli/bwrap/__init__.py b/pkgs/clan-cli/clan_cli/bwrap/__init__.py index f733ba3d6..4d855e767 100644 --- a/pkgs/clan-cli/clan_cli/bwrap/__init__.py +++ b/pkgs/clan-cli/clan_cli/bwrap/__init__.py @@ -1,5 +1,5 @@ from clan_cli.cmd import run -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell _works: bool | None = None @@ -13,10 +13,10 @@ def bubblewrap_works() -> bool: def _bubblewrap_works() -> bool: # fmt: off - cmd = nix_shell_legacy( + cmd = nix_shell( [ - "nixpkgs#bash", - "nixpkgs#bubblewrap", + "bash", + "bubblewrap", ], [ "bwrap", diff --git a/pkgs/clan-cli/clan_cli/clan/create.py b/pkgs/clan-cli/clan_cli/clan/create.py index f311e38b3..fbd4e0bc5 100644 --- a/pkgs/clan-cli/clan_cli/clan/create.py +++ b/pkgs/clan-cli/clan_cli/clan/create.py @@ -9,7 +9,7 @@ from clan_cli.cmd import CmdOut, RunOpts, run from clan_cli.errors import ClanError from clan_cli.flake import Flake from clan_cli.inventory import Inventory, init_inventory -from clan_cli.nix import nix_command, nix_metadata, nix_shell_legacy +from clan_cli.nix import nix_command, nix_metadata, nix_shell from clan_cli.templates import ( InputPrio, TemplateName, @@ -41,7 +41,7 @@ class CreateOptions: def git_command(directory: Path, *args: str) -> list[str]: - return nix_shell_legacy(["nixpkgs#git"], ["git", "-C", str(directory), *args]) + return nix_shell(["git"], ["git", "-C", str(directory), *args]) @API.register diff --git a/pkgs/clan-cli/clan_cli/facts/generate.py b/pkgs/clan-cli/clan_cli/facts/generate.py index 72511fe05..090e578fa 100644 --- a/pkgs/clan-cli/clan_cli/facts/generate.py +++ b/pkgs/clan-cli/clan_cli/facts/generate.py @@ -18,7 +18,7 @@ from clan_cli.errors import ClanError from clan_cli.git import commit_files from clan_cli.machines.inventory import get_all_machines, get_selected_machines from clan_cli.machines.machines import Machine -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell from .check import check_secrets from .public_modules import FactStoreBase @@ -39,10 +39,10 @@ def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str: def bubblewrap_cmd(generator: str, facts_dir: Path, secrets_dir: Path) -> list[str]: # fmt: off - return nix_shell_legacy( + return nix_shell( [ - "nixpkgs#bash", - "nixpkgs#bubblewrap", + "bash", + "bubblewrap", ], [ "bwrap", diff --git a/pkgs/clan-cli/clan_cli/facts/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/facts/secret_modules/password_store.py index 47cf48a77..ca7caaadf 100644 --- a/pkgs/clan-cli/clan_cli/facts/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/facts/secret_modules/password_store.py @@ -5,7 +5,7 @@ from typing import override from clan_cli.cmd import Log, RunOpts from clan_cli.machines.machines import Machine -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell from . import SecretStoreBase @@ -18,8 +18,8 @@ class SecretStore(SecretStoreBase): self, service: str, name: str, value: bytes, groups: list[str] ) -> Path | None: subprocess.run( - nix_shell_legacy( - ["nixpkgs#pass"], + nix_shell( + ["pass"], ["pass", "insert", "-m", f"machines/{self.machine.name}/{name}"], ), input=value, @@ -29,8 +29,8 @@ class SecretStore(SecretStoreBase): def get(self, service: str, name: str) -> bytes: return subprocess.run( - nix_shell_legacy( - ["nixpkgs#pass"], + nix_shell( + ["pass"], ["pass", "show", f"machines/{self.machine.name}/{name}"], ), check=True, @@ -51,8 +51,8 @@ class SecretStore(SecretStoreBase): hashes = [] hashes.append( subprocess.run( - nix_shell_legacy( - ["nixpkgs#git"], + nix_shell( + ["git"], [ "git", "-C", @@ -71,8 +71,8 @@ class SecretStore(SecretStoreBase): if symlink.is_symlink(): hashes.append( subprocess.run( - nix_shell_legacy( - ["nixpkgs#git"], + nix_shell( + ["git"], [ "git", "-C", diff --git a/pkgs/clan-cli/clan_cli/flash/flash.py b/pkgs/clan-cli/clan_cli/flash/flash.py index 51d2fc61c..0c43435ef 100644 --- a/pkgs/clan-cli/clan_cli/flash/flash.py +++ b/pkgs/clan-cli/clan_cli/flash/flash.py @@ -13,7 +13,7 @@ from clan_cli.errors import ClanError from clan_cli.facts.generate import generate_facts from clan_cli.facts.secret_modules import SecretStoreBase from clan_cli.machines.machines import Machine -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell from clan_cli.vars.generate import generate_vars from .automount import pause_automounting @@ -147,8 +147,8 @@ def flash_machine( disko_install.extend(["--option", "dry-run", "true"]) disko_install.extend(extra_args) - cmd = nix_shell_legacy( - ["nixpkgs#disko"], + cmd = nix_shell( + ["disko"], disko_install, ) run( diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index 462986510..93e7f8d68 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -18,7 +18,7 @@ from clan_cli.errors import ClanError from clan_cli.facts.generate import generate_facts from clan_cli.machines.hardware import HardwareConfig from clan_cli.machines.machines import Machine -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell from clan_cli.ssh.deploy_info import DeployInfo, find_reachable_host, ssh_command_parse from clan_cli.ssh.host_key import HostKeyCheck from clan_cli.vars.generate import generate_vars @@ -145,24 +145,19 @@ def install_machine(opts: InstallOptions) -> None: # nix copy does not support tor socks proxy # cmd.append("--ssh-option") # cmd.append("ProxyCommand=nc -x 127.0.0.1:9050 -X 5 %h %p") - run( - nix_shell_legacy( - [ - "nixpkgs#nixos-anywhere", - "nixpkgs#tor", - ], - ["torify", *cmd], - ), - RunOpts(log=Log.BOTH, prefix=machine.name, needs_user_terminal=True), + cmd = nix_shell( + [ + "nixos-anywhere", + "tor", + ], + ["torify", *cmd], ) else: - run( - nix_shell_legacy( - ["nixpkgs#nixos-anywhere"], - cmd, - ), - RunOpts(log=Log.BOTH, prefix=machine.name, needs_user_terminal=True), + cmd = nix_shell( + ["nixos-anywhere"], + cmd, ) + run(cmd, RunOpts(log=Log.BOTH, prefix=machine.name, needs_user_terminal=True)) def install_command(args: argparse.Namespace) -> None: diff --git a/pkgs/clan-cli/clan_cli/machines/list.py b/pkgs/clan-cli/clan_cli/machines/list.py index c0ae9b875..30360c364 100644 --- a/pkgs/clan-cli/clan_cli/machines/list.py +++ b/pkgs/clan-cli/clan_cli/machines/list.py @@ -20,7 +20,7 @@ from clan_cli.inventory import ( patch_inventory_with, ) from clan_cli.machines.hardware import HardwareConfig -from clan_cli.nix import nix_eval, nix_shell_legacy +from clan_cli.nix import nix_eval, nix_shell from clan_cli.tags import list_nixos_machines_by_tags log = logging.getLogger(__name__) @@ -126,8 +126,8 @@ def check_machine_online( timeout = opts.timeout if opts and opts.timeout else 20 - cmd = nix_shell_legacy( - ["nixpkgs#util-linux", *(["nixpkgs#openssh"] if hostname else [])], + cmd = nix_shell( + ["util-linux", *(["openssh"] if hostname else [])], [ "ssh", *(["-i", f"{opts.keyfile}"] if opts and opts.keyfile else []), diff --git a/pkgs/clan-cli/clan_cli/nix/__init__.py b/pkgs/clan-cli/clan_cli/nix/__init__.py index 72e63b4d4..05b5d4367 100644 --- a/pkgs/clan-cli/clan_cli/nix/__init__.py +++ b/pkgs/clan-cli/clan_cli/nix/__init__.py @@ -105,24 +105,6 @@ def nix_metadata(flake_url: str | Path) -> dict[str, Any]: return data -# Deprecated: use nix_shell() instead -def nix_shell_legacy(packages: list[str], cmd: list[str]) -> list[str]: - # we cannot use nix-shell inside the nix sandbox - # in our tests we just make sure we have all the packages - if ( - os.environ.get("IN_NIX_SANDBOX") - or os.environ.get("CLAN_NO_DYNAMIC_DEPS") - or len(packages) == 0 - ): - return cmd - return [ - *nix_command(["shell", "--inputs-from", f"{nixpkgs_flake()!s}"]), - *packages, - "-c", - *cmd, - ] - - # lazy loads list of allowed and static programs class Packages: allowed_packages: set[str] | None = None @@ -152,7 +134,6 @@ class Packages: return program in cls.static_packages -# Alternative implementation of nix_shell() to replace nix_shell_legacy() at some point # Features: # - allow list for programs (need to be specified in allowed-packages.json) # - be abe to compute a closure of all deps for testing diff --git a/pkgs/clan-cli/clan_cli/nix/allowed-packages.json b/pkgs/clan-cli/clan_cli/nix/allowed-packages.json index b03599487..ebf9efd71 100644 --- a/pkgs/clan-cli/clan_cli/nix/allowed-packages.json +++ b/pkgs/clan-cli/clan_cli/nix/allowed-packages.json @@ -3,12 +3,14 @@ "avahi", "bash", "bubblewrap", + "disko", "e2fsprogs", "git", "gnupg", "mypy", "netcat", "nix", + "nixos-anywhere", "openssh", "pass", "qemu", @@ -18,6 +20,8 @@ "sshpass", "tor", "util-linux", + "virt-viewer", "virtiofsd", + "waypipe", "zbar" ] diff --git a/pkgs/clan-cli/clan_cli/secrets/import_sops.py b/pkgs/clan-cli/clan_cli/secrets/import_sops.py index 30fd110ee..6a9218bc3 100644 --- a/pkgs/clan-cli/clan_cli/secrets/import_sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/import_sops.py @@ -11,7 +11,7 @@ from clan_cli.completions import ( complete_users, ) from clan_cli.errors import ClanError -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell from .secrets import encrypt_secret, sops_secrets_folder @@ -30,7 +30,7 @@ def import_sops(args: argparse.Namespace) -> None: if args.input_type: cmd += ["--input-type", args.input_type] cmd += ["--output-type", "json", "--decrypt", args.sops_file] - cmd = nix_shell_legacy(["nixpkgs#sops", "nixpkgs#gnupg"], cmd) + cmd = nix_shell(["sops", "gnupg"], cmd) res = run(cmd, RunOpts(error_msg=f"Could not import sops file {file}")) secrets = json.loads(res.stdout) diff --git a/pkgs/clan-cli/clan_cli/secrets/sops.py b/pkgs/clan-cli/clan_cli/secrets/sops.py index 6f0eefa10..e2e9ac315 100644 --- a/pkgs/clan-cli/clan_cli/secrets/sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/sops.py @@ -16,7 +16,7 @@ from clan_cli.api import API from clan_cli.cmd import Log, RunOpts, run from clan_cli.dirs import user_config_dir from clan_cli.errors import ClanError -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell from .folders import sops_machines_folder, sops_users_folder @@ -233,7 +233,7 @@ def sops_run( raise ClanError(msg) sops_cmd.append(str(secret_path)) - cmd = nix_shell_legacy(["nixpkgs#sops", "nixpkgs#gnupg"], sops_cmd) + cmd = nix_shell(["sops", "gnupg"], sops_cmd) opts = ( dataclasses.replace(run_opts, env=environ) if run_opts @@ -249,7 +249,7 @@ def sops_run( def get_public_age_key(privkey: str) -> str: - cmd = nix_shell_legacy(["nixpkgs#age"], ["age-keygen", "-y"]) + cmd = nix_shell(["age"], ["age-keygen", "-y"]) error_msg = "Failed to get public key for age private key. Is the key malformed?" res = run(cmd, RunOpts(input=privkey.encode(), error_msg=error_msg)) @@ -257,7 +257,7 @@ def get_public_age_key(privkey: str) -> str: def generate_private_key(out_file: Path | None = None) -> tuple[str, str]: - cmd = nix_shell_legacy(["nixpkgs#age"], ["age-keygen"]) + cmd = nix_shell(["age"], ["age-keygen"]) try: proc = run(cmd) res = proc.stdout.strip() diff --git a/pkgs/clan-cli/clan_cli/ssh/deploy_info.py b/pkgs/clan-cli/clan_cli/ssh/deploy_info.py index 338118d97..45915b71c 100644 --- a/pkgs/clan-cli/clan_cli/ssh/deploy_info.py +++ b/pkgs/clan-cli/clan_cli/ssh/deploy_info.py @@ -14,7 +14,7 @@ from clan_cli.completions import ( ) from clan_cli.errors import ClanError from clan_cli.machines.machines import Machine -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell from clan_cli.ssh.host import Host, is_ssh_reachable from clan_cli.ssh.host_key import HostKeyCheck from clan_cli.ssh.parse import parse_deployment_address @@ -65,8 +65,8 @@ def find_reachable_host( def qrcode_scan(picture_file: Path) -> str: - cmd = nix_shell_legacy( - ["nixpkgs#zbar"], + cmd = nix_shell( + ["zbar"], [ "zbarimg", "--quiet", diff --git a/pkgs/clan-cli/clan_cli/ssh/tor.py b/pkgs/clan-cli/clan_cli/ssh/tor.py index a91fac6f1..1a7554164 100755 --- a/pkgs/clan-cli/clan_cli/ssh/tor.py +++ b/pkgs/clan-cli/clan_cli/ssh/tor.py @@ -10,7 +10,7 @@ from dataclasses import dataclass from clan_cli.async_run import AsyncRuntime from clan_cli.cmd import Log, RunOpts, run from clan_cli.errors import TorConnectionError, TorSocksError -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell log = logging.getLogger(__name__) @@ -116,8 +116,8 @@ def spawn_tor(runtime: AsyncRuntime) -> None: def start_tor() -> None: """Starts Tor process using nix-shell.""" cmd_args = ["tor", "--HardwareAccel", "1"] - packages = ["nixpkgs#tor"] - cmd = nix_shell_legacy(packages, cmd_args) + packages = ["tor"] + cmd = nix_shell(packages, cmd_args) runtime.async_run(None, run, cmd, RunOpts(log=Log.BOTH)) log.debug("Attempting to start Tor") diff --git a/pkgs/clan-cli/clan_cli/tests/git_repo.py b/pkgs/clan-cli/clan_cli/tests/git_repo.py index fa7b7d08f..073ce5aa4 100644 --- a/pkgs/clan-cli/clan_cli/tests/git_repo.py +++ b/pkgs/clan-cli/clan_cli/tests/git_repo.py @@ -2,21 +2,19 @@ import subprocess from pathlib import Path import pytest -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell # fixture for git_repo @pytest.fixture def git_repo(temp_dir: Path) -> Path: # initialize a git repository - cmd = nix_shell_legacy(["nixpkgs#git"], ["git", "init"]) + cmd = nix_shell(["git"], ["git", "init"]) subprocess.run(cmd, cwd=temp_dir, check=True) # set user.name and user.email - cmd = nix_shell_legacy(["nixpkgs#git"], ["git", "config", "user.name", "test"]) + cmd = nix_shell(["git"], ["git", "config", "user.name", "test"]) subprocess.run(cmd, cwd=temp_dir, check=True) - cmd = nix_shell_legacy( - ["nixpkgs#git"], ["git", "config", "user.email", "test@test.test"] - ) + cmd = nix_shell(["git"], ["git", "config", "user.email", "test@test.test"]) subprocess.run(cmd, cwd=temp_dir, check=True) # return the path to the git repository return temp_dir diff --git a/pkgs/clan-cli/clan_cli/tests/test_secrets_password_store.py b/pkgs/clan-cli/clan_cli/tests/test_secrets_password_store.py index 44ff26821..92c8b6443 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_secrets_password_store.py +++ b/pkgs/clan-cli/clan_cli/tests/test_secrets_password_store.py @@ -6,7 +6,7 @@ from clan_cli.facts.secret_modules.password_store import SecretStore from clan_cli.flake import Flake from clan_cli.machines.facts import machine_get_fact from clan_cli.machines.machines import Machine -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell from clan_cli.ssh.host import Host from clan_cli.tests.fixtures_flakes import ClanFlake from clan_cli.tests.helpers import cli @@ -58,14 +58,10 @@ def test_upload_secret( """ ) subprocess.run( - nix_shell_legacy( - ["nixpkgs#gnupg"], ["gpg", "--batch", "--gen-key", str(gpg_key_spec)] - ), + nix_shell(["gnupg"], ["gpg", "--batch", "--gen-key", str(gpg_key_spec)]), check=True, ) - subprocess.run( - nix_shell_legacy(["nixpkgs#pass"], ["pass", "init", "test@local"]), check=True - ) + subprocess.run(nix_shell(["pass"], ["pass", "init", "test@local"]), check=True) cli.run(["facts", "generate", "vm1", "--flake", str(flake.path)]) store = SecretStore(Machine(name="vm1", flake=Flake(str(flake.path)))) diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 6a82a541e..feaf3832e 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -17,7 +17,7 @@ from clan_cli.completions import ( from clan_cli.errors import ClanError from clan_cli.git import commit_files from clan_cli.machines.inventory import get_all_machines, get_selected_machines -from clan_cli.nix import nix_config, nix_shell_legacy, nix_test_store +from clan_cli.nix import nix_config, nix_shell, nix_test_store from clan_cli.vars._types import StoreBase from .check import check_vars @@ -84,10 +84,10 @@ def bubblewrap_cmd(generator: str, tmpdir: Path) -> list[str]: test_store = nix_test_store() # fmt: off - return nix_shell_legacy( + return nix_shell( [ - "nixpkgs#bash", - "nixpkgs#bubblewrap", + "bash", + "bubblewrap", ], [ "bwrap", diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py index e38d55c5c..67c90a0ec 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py @@ -9,7 +9,7 @@ from tempfile import TemporaryDirectory from clan_cli.cmd import CmdOut, Log, RunOpts, run from clan_cli.machines.machines import Machine -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell from clan_cli.ssh.upload import upload from clan_cli.vars._types import StoreBase from clan_cli.vars.generate import Generator, Var @@ -49,9 +49,7 @@ class SecretStore(StoreBase): return Path(self.entry_prefix) / self.rel_dir(generator, name) def _run_pass(self, *args: str, options: RunOpts | None = None) -> CmdOut: - cmd = nix_shell_legacy( - packages=["nixpkgs#pass"], cmd=[self._store_backend, *args] - ) + cmd = nix_shell(packages=["pass"], cmd=[self._store_backend, *args]) return run(cmd, options) def _set( @@ -92,8 +90,8 @@ class SecretStore(StoreBase): hashes = [] hashes.append( run( - nix_shell_legacy( - ["nixpkgs#git"], + nix_shell( + ["git"], [ "git", "-C", @@ -120,8 +118,8 @@ class SecretStore(StoreBase): if symlink.is_symlink(): hashes.append( run( - nix_shell_legacy( - ["nixpkgs#git"], + nix_shell( + ["git"], [ "git", "-C", diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index 4f69a4c60..80301d81f 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -19,7 +19,7 @@ from clan_cli.dirs import module_root, user_cache_dir, vm_state_dir from clan_cli.errors import ClanCmdError, ClanError from clan_cli.facts.generate import generate_facts from clan_cli.machines.machines import Machine -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell from clan_cli.qemu.qga import QgaSession from clan_cli.qemu.qmp import QEMUMonitorProtocol @@ -96,8 +96,8 @@ def prepare_disk( file_name: str = "disk.img", ) -> Path: disk_img = directory / file_name - cmd = nix_shell_legacy( - ["nixpkgs#qemu"], + cmd = nix_shell( + ["qemu"], [ "qemu-img", "create", @@ -127,7 +127,7 @@ def start_vm( ) -> Iterator[subprocess.Popen]: env = os.environ.copy() env.update(extra_env) - cmd = nix_shell_legacy(packages, args) + cmd = nix_shell(packages, args) machine.debug(f"Starting VM with command: {cmd}") with subprocess.Popen( @@ -280,11 +280,11 @@ def spawn_vm( interactive=stdin is None, ) - packages = ["nixpkgs#qemu"] + packages = ["qemu"] extra_env = {} if vm.graphics and not vm.waypipe.enable: - packages.append("nixpkgs#virt-viewer") + packages.append("virt-viewer") remote_viewer_mimetypes = module_root() / "vms" / "mimetypes" extra_env["XDG_DATA_DIRS"] = ( f"{remote_viewer_mimetypes}:{os.environ.get('XDG_DATA_DIRS', '')}" diff --git a/pkgs/clan-cli/clan_cli/vms/virtiofsd.py b/pkgs/clan-cli/clan_cli/vms/virtiofsd.py index 38b185521..158a9954d 100644 --- a/pkgs/clan-cli/clan_cli/vms/virtiofsd.py +++ b/pkgs/clan-cli/clan_cli/vms/virtiofsd.py @@ -6,7 +6,7 @@ from collections.abc import Iterator from pathlib import Path from clan_cli.errors import ClanError -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell @contextlib.contextmanager @@ -14,8 +14,8 @@ def start_virtiofsd(socket_path: Path) -> Iterator[None]: sandbox = "namespace" if shutil.which("newuidmap") is None: sandbox = "none" - virtiofsd = nix_shell_legacy( - ["nixpkgs#virtiofsd"], + virtiofsd = nix_shell( + ["virtiofsd"], [ "virtiofsd", "--socket-path", diff --git a/pkgs/clan-cli/clan_cli/vms/waypipe.py b/pkgs/clan-cli/clan_cli/vms/waypipe.py index 4adf2597b..3c0186ba5 100644 --- a/pkgs/clan-cli/clan_cli/vms/waypipe.py +++ b/pkgs/clan-cli/clan_cli/vms/waypipe.py @@ -6,7 +6,7 @@ import time from collections.abc import Iterator from clan_cli.errors import ClanError -from clan_cli.nix import nix_shell_legacy +from clan_cli.nix import nix_shell VMADDR_CID_HYPERVISOR = 2 @@ -29,8 +29,8 @@ def start_waypipe(cid: int | None, title_prefix: str) -> Iterator[None]: if cid is None: yield return - waypipe = nix_shell_legacy( - ["nixpkgs#waypipe"], + waypipe = nix_shell( + ["waypipe"], [ "waypipe", "--vsock", From 59125a8f42fc9279759e52d76a581eb8984dd714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 21 Apr 2025 13:23:03 +0200 Subject: [PATCH 2/7] devshell/clan-cli: fix CLAN_PROVIDED_PACKAGES having too many packages --- pkgs/clan-cli/flake-module.nix | 2 +- pkgs/clan-cli/shell.nix | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index f7bd869a9..ce736765c 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -127,7 +127,7 @@ in { devShells.clan-cli = pkgs.callPackage ./shell.nix { - inherit (self'.packages) clan-cli clan-cli-full; + inherit (self'.packages) clan-cli; inherit self'; }; packages = { diff --git a/pkgs/clan-cli/shell.nix b/pkgs/clan-cli/shell.nix index 4f3a5dc82..563246064 100644 --- a/pkgs/clan-cli/shell.nix +++ b/pkgs/clan-cli/shell.nix @@ -2,7 +2,6 @@ lib, nix-unit, clan-cli, - clan-cli-full, mkShell, ruff, self', @@ -26,7 +25,7 @@ mkShell { inputsFrom = [ self'.devShells.default ]; CLAN_PROVIDED_PACKAGES = lib.concatStringsSep ":" ( - lib.attrNames clan-cli-full.passthru.runtimeDependenciesMap + lib.attrNames clan-cli.passthru.runtimeDependenciesMap ); shellHook = '' From 180ee83e1d46a3ce64e30dd1742dcb9ce98910b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 21 Apr 2025 13:37:11 +0200 Subject: [PATCH 3/7] rename vars-generate to generate-test-vars --- pkgs/flake-module.nix | 2 +- pkgs/{vars-generate => generate-test-vars}/default.nix | 4 ++-- pkgs/{vars-generate => generate-test-vars}/flake-module.nix | 2 +- .../cli => generate-test-vars/generate_test_vars}/__init_.py | 0 .../main.py => generate-test-vars/generate_test_vars/cli.py} | 0 pkgs/{vars-generate => generate-test-vars}/pyproject.toml | 4 ++-- pkgs/{vars-generate => generate-test-vars}/test/vars.nix | 0 7 files changed, 6 insertions(+), 6 deletions(-) rename pkgs/{vars-generate => generate-test-vars}/default.nix (68%) rename pkgs/{vars-generate => generate-test-vars}/flake-module.nix (72%) rename pkgs/{vars-generate/cli => generate-test-vars/generate_test_vars}/__init_.py (100%) rename pkgs/{vars-generate/cli/main.py => generate-test-vars/generate_test_vars/cli.py} (100%) rename pkgs/{vars-generate => generate-test-vars}/pyproject.toml (90%) rename pkgs/{vars-generate => generate-test-vars}/test/vars.nix (100%) diff --git a/pkgs/flake-module.nix b/pkgs/flake-module.nix index 027ca14a1..fd2e59fd2 100644 --- a/pkgs/flake-module.nix +++ b/pkgs/flake-module.nix @@ -9,7 +9,7 @@ ./webview-ui/flake-module.nix ./distro-packages/flake-module.nix ./icon-update/flake-module.nix - ./vars-generate/flake-module.nix + ./generate-test-vars/flake-module.nix ]; flake.packages.x86_64-linux = diff --git a/pkgs/vars-generate/default.nix b/pkgs/generate-test-vars/default.nix similarity index 68% rename from pkgs/vars-generate/default.nix rename to pkgs/generate-test-vars/default.nix index 620f9a44c..22349b9ac 100644 --- a/pkgs/vars-generate/default.nix +++ b/pkgs/generate-test-vars/default.nix @@ -4,10 +4,10 @@ clan-cli, }: buildPythonApplication { - name = "vars-generate"; + name = "generate-test-vars"; src = ./.; format = "pyproject"; - buildInputs = [ (python.pkgs.toPythonModule clan-cli) ]; + dependencies = [ (python.pkgs.toPythonModule clan-cli) ]; nativeBuildInputs = [ (python.withPackages (ps: [ ps.setuptools ])) ]; diff --git a/pkgs/vars-generate/flake-module.nix b/pkgs/generate-test-vars/flake-module.nix similarity index 72% rename from pkgs/vars-generate/flake-module.nix rename to pkgs/generate-test-vars/flake-module.nix index 206275352..a81b03ca7 100644 --- a/pkgs/vars-generate/flake-module.nix +++ b/pkgs/generate-test-vars/flake-module.nix @@ -10,7 +10,7 @@ # devShells.vars-generator = pkgs.callPackage ./shell.nix { # }; - packages.vars-generator = pkgs.python3.pkgs.callPackage ./default.nix { + packages.generate-test-vars = pkgs.python3.pkgs.callPackage ./default.nix { inherit (config.packages) clan-cli; }; }; diff --git a/pkgs/vars-generate/cli/__init_.py b/pkgs/generate-test-vars/generate_test_vars/__init_.py similarity index 100% rename from pkgs/vars-generate/cli/__init_.py rename to pkgs/generate-test-vars/generate_test_vars/__init_.py diff --git a/pkgs/vars-generate/cli/main.py b/pkgs/generate-test-vars/generate_test_vars/cli.py similarity index 100% rename from pkgs/vars-generate/cli/main.py rename to pkgs/generate-test-vars/generate_test_vars/cli.py diff --git a/pkgs/vars-generate/pyproject.toml b/pkgs/generate-test-vars/pyproject.toml similarity index 90% rename from pkgs/vars-generate/pyproject.toml rename to pkgs/generate-test-vars/pyproject.toml index c0cb62f6c..d6052d9d2 100644 --- a/pkgs/vars-generate/pyproject.toml +++ b/pkgs/generate-test-vars/pyproject.toml @@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta" [project] -name = "vars-generate" +name = "generate-test-vars" description = "vars generate" dynamic = ["version"] -scripts = { vars-generate = "cli:main.main" } +scripts = { generate-test-vars = "generate_test_vars:cli.main" } [project.urls] Homepage = "https://clan.lol/" diff --git a/pkgs/vars-generate/test/vars.nix b/pkgs/generate-test-vars/test/vars.nix similarity index 100% rename from pkgs/vars-generate/test/vars.nix rename to pkgs/generate-test-vars/test/vars.nix From d12f7de9b4a728f3e8b07039f4ada23009db4b25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 21 Apr 2025 13:38:43 +0200 Subject: [PATCH 4/7] generate-test-vars: use absolute paths in flake urls flakes don't work with releative paths --- pkgs/generate-test-vars/generate_test_vars/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkgs/generate-test-vars/generate_test_vars/cli.py b/pkgs/generate-test-vars/generate_test_vars/cli.py index 85747a834..04053adc3 100755 --- a/pkgs/generate-test-vars/generate_test_vars/cli.py +++ b/pkgs/generate-test-vars/generate_test_vars/cli.py @@ -52,7 +52,7 @@ class TestMachine(Machine): return self._deployment cmd = nix_build( [ - f"{self.flake.path}#checks.{nix_config()['system']}.{self.check_attr}.nodes.{self.name}.system.clan.deployment.file" + f"{self.flake.path.resolve()}#checks.{nix_config()['system']}.{self.check_attr}.nodes.{self.name}.system.clan.deployment.file" ] ) out = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE) @@ -77,7 +77,7 @@ class TestMachine(Machine): # return self.nix("eval", attr, nix_options) cmd = nix_eval( [ - f"{self.flake.path}#checks.{nix_config()['system']}.{self.check_attr}.nodes.{self.name}.{attr}" + f"{self.flake.path.resolve()}#checks.{nix_config()['system']}.{self.check_attr}.nodes.{self.name}.{attr}" ] ) out = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE) @@ -99,7 +99,7 @@ class TestMachine(Machine): cmd = nix_build( [ - f"{self.flake.path}#checks.{nix_config()['system']}.{self.check_attr}.nodes.{self.name}.{attr}" + f"{self.flake.path.resolve()}#checks.{nix_config()['system']}.{self.check_attr}.nodes.{self.name}.{attr}" ] ) out = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE) @@ -164,7 +164,7 @@ def main() -> None: ) ] user = "admin" - admin_key_path = Path(flake.path / "sops" / "users" / user / "key.json") + admin_key_path = Path(flake.path.resolve() / "sops" / "users" / user / "key.json") admin_key_path.parent.mkdir(parents=True, exist_ok=True) admin_key_path.write_text( json.dumps( From f69bfa7ea951798c2e2d9dd2d694fb8ef9fef119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 21 Apr 2025 14:56:44 +0200 Subject: [PATCH 5/7] Machines: remove unused method argument from self.nix() --- pkgs/clan-cli/clan_cli/machines/machines.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 6727cb719..d8bb280f3 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -198,7 +198,6 @@ class Machine: def nix( self, - method: Literal["eval", "build"], attr: str, nix_options: list[str] | None = None, ) -> Any: @@ -235,7 +234,7 @@ class Machine: if nix_options is None: nix_options = [] - return self.nix("eval", attr, nix_options) + return self.nix(attr, nix_options) def build_nix( self, @@ -254,7 +253,7 @@ class Machine: if nix_options is None: nix_options = [] - output = self.nix("build", attr, nix_options) + output = self.nix(attr, nix_options) output = Path(output) if tmp_store := nix_test_store(): output = tmp_store.joinpath(*output.parts[1:]) From 177dea9dffc15643f289418976103b646064e7d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 21 Apr 2025 14:58:03 +0200 Subject: [PATCH 6/7] generate-test-vars: improve caching and reduce number of overrides in base class --- .../generate_test_vars/cli.py | 109 +++++++----------- 1 file changed, 41 insertions(+), 68 deletions(-) diff --git a/pkgs/generate-test-vars/generate_test_vars/cli.py b/pkgs/generate-test-vars/generate_test_vars/cli.py index 04053adc3..bd896873d 100755 --- a/pkgs/generate-test-vars/generate_test_vars/cli.py +++ b/pkgs/generate-test-vars/generate_test_vars/cli.py @@ -4,14 +4,15 @@ import argparse import json import os import subprocess +from dataclasses import dataclass from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any +from typing import Any, override from clan_cli.dirs import find_git_repo_root from clan_cli.flake import Flake from clan_cli.machines.machines import Machine -from clan_cli.nix import nix_build, nix_config, nix_eval +from clan_cli.nix import nix_config, nix_eval from clan_cli.vars.generate import generate_vars sops_priv_key = ( @@ -26,7 +27,7 @@ def machine_names(repo_root: Path, check_attr: str) -> list[str]: """ cmd = nix_eval( [ - f"{repo_root}#checks.{nix_config()['system']}.{check_attr}.nodes", + f"git+file://{repo_root}#checks.{nix_config()['system']}.{check_attr}.nodes", "--apply", "builtins.attrNames", ] @@ -42,79 +43,48 @@ class TestMachine(Machine): clan-core#checks...nodes.. """ - def __init__(self, name: str, flake: Flake, check_attr: str) -> None: + @override + def __init__( + self, name: str, flake: Flake, test_dir: Path, check_attr: str + ) -> None: super().__init__(name, flake) self.check_attr = check_attr + self.test_dir = test_dir @property - def deployment(self) -> dict: - if getattr(self, "_deployment", None): - return self._deployment - cmd = nix_build( - [ - f"{self.flake.path.resolve()}#checks.{nix_config()['system']}.{self.check_attr}.nodes.{self.name}.system.clan.deployment.file" - ] - ) - out = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE) - self._deployment = json.loads(Path(out.stdout.strip()).read_text()) - return self._deployment + def flake_dir(self) -> Path: + return self.test_dir - def eval_nix( + @override + def nix( self, attr: str, - refresh: bool = False, - extra_config: None | dict = None, nix_options: list[str] | None = None, ) -> Any: """ - eval a nix attribute of the machine - @attr: the attribute to get + Build the machine and return the path to the result + accepts a secret store and a facts store # TODO """ - if nix_options is None: nix_options = [] - # return self.nix("eval", attr, nix_options) - cmd = nix_eval( - [ - f"{self.flake.path.resolve()}#checks.{nix_config()['system']}.{self.check_attr}.nodes.{self.name}.{attr}" - ] + config = nix_config() + system = config["system"] + + return self.flake.select( + f'checks."{system}".{self.check_attr}.nodes.{self.name}.{attr}', + nix_options=nix_options, ) - out = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE) - return json.loads(out.stdout.strip()) - - def build_nix( - self, - attr: str, - extra_config: None | dict = None, - nix_options: list[str] | None = None, - ) -> Path: - """ - build a nix attribute of the machine - @attr: the attribute to get - """ - - if nix_options is None: - nix_options = [] - - cmd = nix_build( - [ - f"{self.flake.path.resolve()}#checks.{nix_config()['system']}.{self.check_attr}.nodes.{self.name}.{attr}" - ] - ) - out = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE) - return Path(out.stdout.strip()) - - def flush_caches(self) -> None: - """ - Disable flush, because it calls prefetch() which resets the overridden Flake._path - """ - return -def parse_args() -> argparse.Namespace: - import argparse +@dataclass +class Options: + repo_root: Path + test_dir: Path + check_attr: str + +def parse_args() -> Options: parser = argparse.ArgumentParser( description=""" Update the vars of a 'makeTestClan' integration test. @@ -145,26 +115,29 @@ def parse_args() -> argparse.Namespace: type=str, help="The attribute name of the flake#checks to update", ) - return parser.parse_args() + args = parser.parse_args() + return Options( + repo_root=args.repo_root, + test_dir=args.test_dir, + check_attr=args.check_attr, + ) def main() -> None: os.environ["CLAN_NO_COMMIT"] = "1" - args = parse_args() - test_dir = args.repo_root / args.test_dir + opts = parse_args() + test_dir = opts.repo_root / opts.test_dir subprocess.run(["rm", "-rf", f"{test_dir}/vars", f"{test_dir}/sops"]) - flake = Flake(str(test_dir)) - flake._path = test_dir # noqa SLF001 - flake._is_local = True # noqa SLF001 + flake = Flake(str(opts.repo_root)) machines = [ - TestMachine(name, flake, args.check_attr) + TestMachine(name, flake, test_dir, opts.check_attr) for name in machine_names( - args.repo_root, - args.check_attr, + opts.repo_root, + opts.check_attr, ) ] user = "admin" - admin_key_path = Path(flake.path.resolve() / "sops" / "users" / user / "key.json") + admin_key_path = Path(test_dir.resolve() / "sops" / "users" / user / "key.json") admin_key_path.parent.mkdir(parents=True, exist_ok=True) admin_key_path.write_text( json.dumps( From 8a47cc4e78cf46d1c0168c970f13348f0522e176 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 21 Apr 2025 15:03:42 +0200 Subject: [PATCH 7/7] generate-test-vars: use shutil.rmtree instead of rm More portable. --- pkgs/generate-test-vars/generate_test_vars/cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkgs/generate-test-vars/generate_test_vars/cli.py b/pkgs/generate-test-vars/generate_test_vars/cli.py index bd896873d..46f0f7ef1 100755 --- a/pkgs/generate-test-vars/generate_test_vars/cli.py +++ b/pkgs/generate-test-vars/generate_test_vars/cli.py @@ -3,6 +3,7 @@ import argparse import json import os +import shutil import subprocess from dataclasses import dataclass from pathlib import Path @@ -127,7 +128,10 @@ def main() -> None: os.environ["CLAN_NO_COMMIT"] = "1" opts = parse_args() test_dir = opts.repo_root / opts.test_dir - subprocess.run(["rm", "-rf", f"{test_dir}/vars", f"{test_dir}/sops"]) + + shutil.rmtree(test_dir / "vars", ignore_errors=True) + shutil.rmtree(test_dir / "sops", ignore_errors=True) + flake = Flake(str(opts.repo_root)) machines = [ TestMachine(name, flake, test_dir, opts.check_attr)