diff --git a/pkgs/clan-cli/clan_cli/cmd.py b/pkgs/clan-cli/clan_cli/cmd.py index 16dcc255f..d9534c164 100644 --- a/pkgs/clan-cli/clan_cli/cmd.py +++ b/pkgs/clan-cli/clan_cli/cmd.py @@ -5,6 +5,7 @@ import shlex import subprocess import sys from collections.abc import Callable +from enum import Enum from pathlib import Path from typing import IO, Any, NamedTuple @@ -16,10 +17,16 @@ log = logging.getLogger(__name__) class CmdOut(NamedTuple): stdout: str stderr: str - cwd: Path | None = None + cwd: Path -def handle_output(process: subprocess.Popen) -> tuple[str, str]: +class Log(Enum): + STDERR = 1 + STDOUT = 2 + BOTH = 3 + + +def handle_output(process: subprocess.Popen, log: Log) -> tuple[str, str]: rlist = [process.stdout, process.stderr] stdout_buf = b"" stderr_buf = b"" @@ -36,21 +43,25 @@ def handle_output(process: subprocess.Popen) -> tuple[str, str]: return b"" ret = handle_fd(process.stdout) - sys.stdout.buffer.write(ret) + if log in [Log.STDOUT, Log.BOTH]: + sys.stdout.buffer.write(ret) + stdout_buf += ret ret = handle_fd(process.stderr) - sys.stderr.buffer.write(ret) + + if log in [Log.STDERR, Log.BOTH]: + sys.stderr.buffer.write(ret) stderr_buf += ret return stdout_buf.decode("utf-8"), stderr_buf.decode("utf-8") -def run(cmd: list[str], cwd: Path = Path.cwd()) -> CmdOut: +def run(cmd: list[str], cwd: Path = Path.cwd(), log: Log = Log.STDERR) -> CmdOut: # Start the subprocess process = subprocess.Popen( cmd, cwd=str(cwd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) - stdout_buf, stderr_buf = handle_output(process) + stdout_buf, stderr_buf = handle_output(process, log) # Wait for the subprocess to finish rc = process.wait() diff --git a/pkgs/clan-cli/clan_cli/config/parsing.py b/pkgs/clan-cli/clan_cli/config/parsing.py index b128d62bf..df8961387 100644 --- a/pkgs/clan-cli/clan_cli/config/parsing.py +++ b/pkgs/clan-cli/clan_cli/config/parsing.py @@ -1,8 +1,8 @@ import json -import subprocess from pathlib import Path from typing import Any +from ..cmd import run from ..errors import ClanError from ..nix import nix_eval @@ -32,7 +32,7 @@ def schema_from_module_file( """ # run the nix expression and parse the output as json cmd = nix_eval(["--expr", nix_expr]) - proc = subprocess.run(cmd, stdout=subprocess.PIPE, check=True) + proc = run(cmd) return json.loads(proc.stdout) diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index b871c679a..45a7fcf89 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -1,9 +1,9 @@ import argparse -import subprocess from dataclasses import dataclass from pathlib import Path from tempfile import TemporaryDirectory +from ..cmd import Log, run from ..machines.machines import Machine from ..nix import nix_shell from ..secrets.generate import generate_secrets @@ -40,12 +40,12 @@ def install_nixos(machine: Machine, kexec: str | None = None) -> None: cmd += ["--kexec", kexec] cmd.append(target_host) - subprocess.run( + run( nix_shell( ["nixpkgs#nixos-anywhere"], cmd, ), - check=True, + log=Log.BOTH, ) diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 22aef38ed..88f63bb8e 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -4,7 +4,7 @@ import subprocess import sys from pathlib import Path -from ..errors import ClanError +from ..cmd import run from ..nix import nix_build, nix_config, nix_eval from ..ssh import Host, parse_deployment_address @@ -13,21 +13,14 @@ def build_machine_data(machine_name: str, clan_dir: Path) -> dict: config = nix_config() system = config["system"] - proc = subprocess.run( + proc = run( nix_build( [ f'{clan_dir}#clanInternals.machines."{system}"."{machine_name}".config.system.clan.deployment.file' ] ), - stdout=subprocess.PIPE, - check=True, - text=True, ) - if proc.returncode != 0: - ClanError("failed to build machine data") - exit(1) - return json.loads(Path(proc.stdout.strip()).read_text()) @@ -99,11 +92,8 @@ class Machine: if attr in self.eval_cache and not refresh: return self.eval_cache[attr] - output = subprocess.run( + output = run( nix_eval([f"path:{self.flake_dir}#{attr}"]), - stdout=subprocess.PIPE, - check=True, - text=True, ).stdout.strip() self.eval_cache[attr] = output return output @@ -115,11 +105,8 @@ class Machine: """ if attr in self.build_cache and not refresh: return self.build_cache[attr] - outpath = subprocess.run( + outpath = run( nix_build([f"path:{self.flake_dir}#{attr}"]), - stdout=subprocess.PIPE, - check=True, - text=True, ).stdout.strip() self.build_cache[attr] = Path(outpath) return Path(outpath) diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index a4fa25e01..9710e8cb5 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -4,6 +4,7 @@ import os import subprocess from pathlib import Path +from ..cmd import run from ..errors import ClanError from ..machines.machines import Machine from ..nix import nix_build, nix_command, nix_config @@ -79,11 +80,8 @@ def deploy_nixos(hosts: HostGroup, clan_dir: Path) -> None: def get_all_machines(clan_dir: Path) -> HostGroup: config = nix_config() system = config["system"] - machines_json = subprocess.run( - nix_build([f'{clan_dir}#clanInternals.all-machines-json."{system}"']), - stdout=subprocess.PIPE, - check=True, - text=True, + machines_json = run( + nix_build([f'{clan_dir}#clanInternals.all-machines-json."{system}"']) ).stdout machines = json.loads(Path(machines_json.rstrip()).read_text()) diff --git a/pkgs/clan-cli/clan_cli/nix.py b/pkgs/clan-cli/clan_cli/nix.py index b9b0acaaf..a8672c409 100644 --- a/pkgs/clan-cli/clan_cli/nix.py +++ b/pkgs/clan-cli/clan_cli/nix.py @@ -1,10 +1,10 @@ import json import os -import subprocess import tempfile from pathlib import Path from typing import Any +from .cmd import run from .dirs import nixpkgs_flake, nixpkgs_source @@ -55,7 +55,7 @@ def nix_build(flags: list[str], gcroot: Path | None = None) -> list[str]: def nix_config() -> dict[str, Any]: cmd = nix_command(["show-config", "--json"]) - proc = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE) + proc = run(cmd) data = json.loads(proc.stdout) config = {} for key, value in data.items(): @@ -90,7 +90,7 @@ def nix_eval(flags: list[str]) -> list[str]: def nix_metadata(flake_url: str | Path) -> dict[str, Any]: cmd = nix_command(["flake", "metadata", "--json", f"{flake_url}"]) - proc = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE) + proc = run(cmd) data = json.loads(proc.stdout) return data diff --git a/pkgs/clan-cli/tests/test_machines_cli.py b/pkgs/clan-cli/tests/test_machines_cli.py index 2b0832e7c..325f45e23 100644 --- a/pkgs/clan-cli/tests/test_machines_cli.py +++ b/pkgs/clan-cli/tests/test_machines_cli.py @@ -14,6 +14,7 @@ def test_machine_subcommands( capsys.readouterr() cli.run(["--flake", str(test_flake_with_core.path), "machines", "list"]) + out = capsys.readouterr() assert "machine1\nvm1\nvm2\n" == out.out diff --git a/pkgs/clan-vm-manager/clan_vm_manager/executor.py b/pkgs/clan-vm-manager/clan_vm_manager/executor.py index e5828eb82..3845c57c1 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/executor.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/executor.py @@ -111,8 +111,7 @@ def spawn( if not log_path.is_dir(): raise ClanError(f"Log path {log_path} is not a directory") - if not log_path.exists(): - log_path.mkdir(parents=True) + log_path.mkdir(parents=True, exist_ok=True) # Set names proc_name = f"MPExec:{func.__name__}" @@ -128,8 +127,6 @@ def spawn( proc.start() # Print some information - assert proc.pid is not None - cmd = f"tail -f {out_file}" print(f"Connect to stdout with: {cmd}")