diff --git a/nixosModules/clanCore/secrets/sops.nix b/nixosModules/clanCore/secrets/sops.nix index 99f6b84a5..ea508863b 100644 --- a/nixosModules/clanCore/secrets/sops.nix +++ b/nixosModules/clanCore/secrets/sops.nix @@ -30,8 +30,10 @@ in generateSecrets = pkgs.writeScript "generate-secrets" '' #!${pkgs.python3}/bin/python import json + import sys from clan_cli.secrets.sops_generate import generate_secrets_from_nix args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; secret_submodules = config.clanCore.secrets; })}) + args["flake_name"] = sys.argv[1] generate_secrets_from_nix(**args) ''; uploadSecrets = pkgs.writeScript "upload-secrets" '' @@ -40,6 +42,7 @@ in from clan_cli.secrets.sops_generate import upload_age_key_from_nix # the second toJSON is needed to escape the string for the python args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; })}) + args["flake_name"] = sys.argv[1] upload_age_key_from_nix(**args) ''; }; diff --git a/pkgs/clan-cli/.envrc b/pkgs/clan-cli/.envrc index 53d6aa325..0ded7613f 100644 --- a/pkgs/clan-cli/.envrc +++ b/pkgs/clan-cli/.envrc @@ -2,6 +2,7 @@ source_up + if type nix_direnv_watch_file &>/dev/null; then nix_direnv_watch_file flake-module.nix nix_direnv_watch_file default.nix diff --git a/pkgs/clan-cli/clan_cli/config/__init__.py b/pkgs/clan-cli/clan_cli/config/__init__.py index 4477803e4..34fd3ed4a 100644 --- a/pkgs/clan-cli/clan_cli/config/__init__.py +++ b/pkgs/clan-cli/clan_cli/config/__init__.py @@ -158,7 +158,11 @@ def read_machine_option_value( def get_or_set_option(args: argparse.Namespace) -> None: if args.value == []: - print(read_machine_option_value(args.machine, args.option, args.show_trace)) + print( + read_machine_option_value( + args.flake, args.machine, args.option, args.show_trace + ) + ) else: # load options if args.options_file is None: @@ -355,6 +359,7 @@ def register_parser( help="name of the flake to set machine options for", ) + def main(argv: Optional[list[str]] = None) -> None: if argv is None: argv = sys.argv diff --git a/pkgs/clan-cli/clan_cli/debug.py b/pkgs/clan-cli/clan_cli/debug.py new file mode 100644 index 000000000..72bdfc0cf --- /dev/null +++ b/pkgs/clan-cli/clan_cli/debug.py @@ -0,0 +1,66 @@ +from typing import Dict, Optional, Tuple, Callable, Any, Mapping, List +from pathlib import Path +import ipdb +import os +import stat +import subprocess +from .dirs import find_git_repo_root +import multiprocessing as mp +from .types import FlakeName +import logging +import sys +import shlex +import time + +log = logging.getLogger(__name__) + +def command_exec(cmd: List[str], work_dir:Path, env: Dict[str, str]) -> None: + subprocess.run(cmd, check=True, env=env, cwd=work_dir.resolve()) + +def repro_env_break(work_dir: Path, env: Optional[Dict[str, str]] = None, cmd: Optional[List[str]] = None) -> None: + if env is None: + env = os.environ.copy() + else: + env = env.copy() + + # Error checking + if "bash" in env["SHELL"]: + raise Exception("I assumed you use zsh, not bash") + + # Cmd appending + args = ["xterm", "-e", "zsh", "-df"] + if cmd is not None: + mycommand = shlex.join(cmd) + write_command(mycommand, work_dir / "cmd.sh") + print(f"Adding to zsh history the command: {mycommand}", file=sys.stderr) + proc = spawn_process(func=command_exec, cmd=args, work_dir=work_dir, env=env) + + try: + ipdb.set_trace() + finally: + proc.terminate() + +def write_command(command: str, loc:Path) -> None: + with open(loc, "w") as f: + f.write("#!/usr/bin/env bash\n") + f.write(command) + st = os.stat(loc) + os.chmod(loc, st.st_mode | stat.S_IEXEC) + +def spawn_process(func: Callable, **kwargs:Any) -> mp.Process: + mp.set_start_method(method="spawn") + proc = mp.Process(target=func, kwargs=kwargs) + proc.start() + return proc + + +def dump_env(env: Dict[str, str], loc: Path) -> None: + cenv = env.copy() + with open(loc, "w") as f: + f.write("#!/usr/bin/env bash\n") + for k, v in cenv.items(): + if v.count('\n') > 0 or v.count("\"") > 0 or v.count("'") > 0: + continue + f.write(f"export {k}='{v}'\n") + st = os.stat(loc) + os.chmod(loc, st.st_mode | stat.S_IEXEC) diff --git a/pkgs/clan-cli/clan_cli/flakes/create.py b/pkgs/clan-cli/clan_cli/flakes/create.py index 28e2bdc63..ebdaff8a6 100644 --- a/pkgs/clan-cli/clan_cli/flakes/create.py +++ b/pkgs/clan-cli/clan_cli/flakes/create.py @@ -12,7 +12,8 @@ from ..errors import ClanError from ..nix import nix_command, nix_shell DEFAULT_URL: AnyUrl = parse_obj_as( - AnyUrl, "git+https://git.clan.lol/clan/clan-core?ref=Qubasa-main#new-clan" # TODO: Change me back to main branch + AnyUrl, + "git+https://git.clan.lol/clan/clan-core?ref=Qubasa-main#new-clan", # TODO: Change me back to main branch ) diff --git a/pkgs/clan-cli/clan_cli/secrets/sops_generate.py b/pkgs/clan-cli/clan_cli/secrets/sops_generate.py index d067e7104..a87328fc5 100644 --- a/pkgs/clan-cli/clan_cli/secrets/sops_generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/sops_generate.py @@ -6,6 +6,7 @@ import sys from pathlib import Path from tempfile import TemporaryDirectory from typing import Any +import logging from clan_cli.nix import nix_shell @@ -17,6 +18,7 @@ from .machines import add_machine, has_machine from .secrets import decrypt_secret, encrypt_secret, has_secret from .sops import generate_private_key +log = logging.getLogger(__name__) def generate_host_key(flake_name: FlakeName, machine_name: str) -> None: if has_machine(flake_name, machine_name): @@ -95,7 +97,7 @@ def generate_secrets_from_nix( ) -> None: generate_host_key(flake_name, machine_name) errors = {} - + log.debug("Generating secrets for machine %s and flake %s", machine_name, flake_name) with TemporaryDirectory() as d: # if any of the secrets are missing, we regenerate all connected facts/secrets for secret_group, secret_options in secret_submodules.items(): @@ -117,6 +119,7 @@ def upload_age_key_from_nix( flake_name: FlakeName, machine_name: str, ) -> None: + log.debug("Uploading secrets for machine %s and flake %s", machine_name, flake_name) secret_name = f"{machine_name}-age.key" if not has_secret( flake_name, secret_name diff --git a/pkgs/clan-cli/clan_cli/task_manager.py b/pkgs/clan-cli/clan_cli/task_manager.py index c54a97481..3e659cbe6 100644 --- a/pkgs/clan-cli/clan_cli/task_manager.py +++ b/pkgs/clan-cli/clan_cli/task_manager.py @@ -63,7 +63,6 @@ class Command: os.set_blocking(self.p.stdout.fileno(), False) os.set_blocking(self.p.stderr.fileno(), False) - while self.p.poll() is None: # Check if stderr is ready to be read from rlist, _, _ = select.select([self.p.stderr, self.p.stdout], [], [], 0) diff --git a/pkgs/clan-cli/clan_cli/types.py b/pkgs/clan-cli/clan_cli/types.py index be54b3da3..a56c0519d 100644 --- a/pkgs/clan-cli/clan_cli/types.py +++ b/pkgs/clan-cli/clan_cli/types.py @@ -1,6 +1,6 @@ -from typing import NewType -from pathlib import Path import logging +from pathlib import Path +from typing import NewType log = logging.getLogger(__name__) @@ -20,4 +20,4 @@ def validate_path(base_dir: Path, value: Path) -> Path: log.warning( f"Detected pytest tmpdir. Skipping path validation for {user_path}" ) - return user_path \ No newline at end of file + return user_path diff --git a/pkgs/clan-cli/clan_cli/vms/create.py b/pkgs/clan-cli/clan_cli/vms/create.py index 158e243e2..8c9930d0f 100644 --- a/pkgs/clan-cli/clan_cli/vms/create.py +++ b/pkgs/clan-cli/clan_cli/vms/create.py @@ -4,17 +4,18 @@ import json import os import shlex import sys -import tempfile from pathlib import Path -from typing import Iterator +from typing import Iterator, Dict from uuid import UUID -from ..dirs import specific_flake_dir, clan_flakes_dir -from ..nix import nix_build, nix_config, nix_shell, nix_eval +from ..dirs import clan_flakes_dir, specific_flake_dir +from ..nix import nix_build, nix_config, nix_eval, nix_shell from ..task_manager import BaseTask, Command, create_task -from .inspect import VmConfig, inspect_vm -from ..flakes.create import create_flake from ..types import validate_path +from .inspect import VmConfig, inspect_vm +from ..errors import ClanError +from ..debug import repro_env_break + class BuildVmTask(BaseTask): def __init__(self, uuid: UUID, vm: VmConfig) -> None: @@ -43,14 +44,8 @@ class BuildVmTask(BaseTask): def get_clan_name(self, cmds: Iterator[Command]) -> str: clan_dir = self.vm.flake_url cmd = next(cmds) - cmd.run( - nix_eval( - [ - f'{clan_dir}#clanInternals.clanName' - ] - ) - ) - clan_name = "".join(cmd.stdout).strip() + cmd.run(nix_eval([f"{clan_dir}#clanInternals.clanName"])) + clan_name = cmd.stdout[0].strip().strip('"') return clan_name def run(self) -> None: @@ -63,9 +58,11 @@ class BuildVmTask(BaseTask): vm_config = self.get_vm_create_info(cmds) clan_name = self.get_clan_name(cmds) + self.log.debug(f"Building VM for clan name: {clan_name}") flake_dir = clan_flakes_dir() / clan_name validate_path(clan_flakes_dir(), flake_dir) + flake_dir.mkdir(exist_ok=True) xchg_dir = flake_dir / "xchg" xchg_dir.mkdir() @@ -82,9 +79,10 @@ class BuildVmTask(BaseTask): env["SECRETS_DIR"] = str(secrets_dir) cmd = next(cmds) + repro_env_break(work_dir=flake_dir, env=env, cmd=[vm_config["generateSecrets"], clan_name]) if Path(self.vm.flake_url).is_dir(): cmd.run( - [vm_config["generateSecrets"]], + [vm_config["generateSecrets"], clan_name], env=env, ) else: diff --git a/pkgs/clan-cli/clan_cli/webui/api_inputs.py b/pkgs/clan-cli/clan_cli/webui/api_inputs.py index c1e9fd891..94a27b84b 100644 --- a/pkgs/clan-cli/clan_cli/webui/api_inputs.py +++ b/pkgs/clan-cli/clan_cli/webui/api_inputs.py @@ -8,9 +8,9 @@ from ..dirs import clan_data_dir, clan_flakes_dir from ..flakes.create import DEFAULT_URL from ..types import validate_path - log = logging.getLogger(__name__) + class ClanDataPath(BaseModel): dest: Path diff --git a/pkgs/clan-cli/shell.nix b/pkgs/clan-cli/shell.nix index 62d28d2b6..0bb530c18 100644 --- a/pkgs/clan-cli/shell.nix +++ b/pkgs/clan-cli/shell.nix @@ -44,7 +44,7 @@ mkShell { export PATH="$tmp_path/python/bin:${checkScript}/bin:$PATH" export PYTHONPATH="$repo_root:$tmp_path/python/${pythonWithDeps.sitePackages}:" - export PYTHONBREAKPOINT=ipdb.set_trace + export PYTHONBREAKPOINT=ipdb.set_trace export XDG_DATA_DIRS="$tmp_path/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}" export fish_complete_path="$tmp_path/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}" @@ -55,6 +55,7 @@ mkShell { register-python-argcomplete --shell fish clan > $tmp_path/share/fish/vendor_completions.d/clan.fish register-python-argcomplete --shell bash clan > $tmp_path/share/bash-completion/completions/clan + ./bin/clan flakes create example_clan ./bin/clan machines create example_machine example_clan ''; diff --git a/pkgs/clan-cli/tests/temporary_dir.py b/pkgs/clan-cli/tests/temporary_dir.py index 4c9bfa55c..4d6ca1747 100644 --- a/pkgs/clan-cli/tests/temporary_dir.py +++ b/pkgs/clan-cli/tests/temporary_dir.py @@ -11,14 +11,14 @@ log = logging.getLogger(__name__) @pytest.fixture def temporary_home(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: - if os.getenv("TEST_KEEP_TEMPORARY_DIR") is not None: - temp_dir = tempfile.mkdtemp(prefix="pytest-") - path = Path(temp_dir) + env_dir = os.getenv("TEST_TEMPORARY_DIR") + if env_dir is not None: + path = Path(env_dir).resolve() log.debug("Temp HOME directory: %s", str(path)) - monkeypatch.setenv("HOME", str(temp_dir)) + monkeypatch.setenv("HOME", str(path)) yield path else: - log.debug("TEST_KEEP_TEMPORARY_DIR not set, using TemporaryDirectory") + log.debug("TEST_TEMPORARY_DIR not set, using TemporaryDirectory") with tempfile.TemporaryDirectory(prefix="pytest-") as dirpath: monkeypatch.setenv("HOME", str(dirpath)) log.debug("Temp HOME directory: %s", str(dirpath)) diff --git a/pkgs/clan-cli/tests/test_config.py b/pkgs/clan-cli/tests/test_config.py index 428921dae..329214497 100644 --- a/pkgs/clan-cli/tests/test_config.py +++ b/pkgs/clan-cli/tests/test_config.py @@ -9,6 +9,7 @@ from cli import Cli from clan_cli import config from clan_cli.config import parsing from clan_cli.errors import ClanError +from fixtures_flakes import FlakeForTest example_options = f"{Path(config.__file__).parent}/jsonschema/options.json" @@ -29,7 +30,7 @@ example_options = f"{Path(config.__file__).parent}/jsonschema/options.json" def test_set_some_option( args: list[str], expected: dict[str, Any], - test_flake: Path, + test_flake: FlakeForTest, ) -> None: # create temporary file for out_file with tempfile.NamedTemporaryFile() as out_file: @@ -46,24 +47,24 @@ def test_set_some_option( out_file.name, ] + args + + [test_flake.name] ) json_out = json.loads(open(out_file.name).read()) assert json_out == expected def test_configure_machine( - test_flake: Path, - temporary_dir: Path, + test_flake: FlakeForTest, + temporary_home: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, ) -> None: - cli = Cli() - cli.run(["config", "-m", "machine1", "clan.jitsi.enable", "true"]) + cli.run(["config", "-m", "machine1", "clan.jitsi.enable", "true", test_flake.name]) # clear the output buffer capsys.readouterr() # read a option value - cli.run(["config", "-m", "machine1", "clan.jitsi.enable"]) + cli.run(["config", "-m", "machine1", "clan.jitsi.enable", test_flake.name]) # read the output assert capsys.readouterr().out == "true\n" diff --git a/pkgs/clan-cli/tests/test_create_flake.py b/pkgs/clan-cli/tests/test_create_flake.py index 571bf73d4..0de4e177f 100644 --- a/pkgs/clan-cli/tests/test_create_flake.py +++ b/pkgs/clan-cli/tests/test_create_flake.py @@ -5,8 +5,10 @@ from pathlib import Path import pytest from api import TestClient from cli import Cli + +from clan_cli.dirs import clan_flakes_dir from clan_cli.flakes.create import DEFAULT_URL -from clan_cli.dirs import clan_flakes_dir, clan_data_dir + @pytest.fixture def cli() -> Cli: @@ -65,6 +67,17 @@ def test_create_flake( pytest.fail("nixosConfigurations.machine1 not found in flake outputs") # configure machine1 capsys.readouterr() - cli.run(["config", "--machine", "machine1", "services.openssh.enable", "", flake_name]) + cli.run( + ["config", "--machine", "machine1", "services.openssh.enable", "", flake_name] + ) capsys.readouterr() - cli.run(["config", "--machine", "machine1", "services.openssh.enable", "true", flake_name]) + cli.run( + [ + "config", + "--machine", + "machine1", + "services.openssh.enable", + "true", + flake_name, + ] + ) diff --git a/pkgs/clan-cli/tests/test_vms_api_create.py b/pkgs/clan-cli/tests/test_vms_api_create.py index fc2c7b3da..e2c0f94ce 100644 --- a/pkgs/clan-cli/tests/test_vms_api_create.py +++ b/pkgs/clan-cli/tests/test_vms_api_create.py @@ -17,11 +17,11 @@ if TYPE_CHECKING: @pytest.fixture def flake_with_vm_with_secrets( - monkeypatch: pytest.MonkeyPatch, temporary_dir: Path + monkeypatch: pytest.MonkeyPatch, temporary_home: Path ) -> Iterator[FlakeForTest]: yield from create_flake( monkeypatch, - temporary_dir, + temporary_home, FlakeName("test_flake_with_core_dynamic_machines"), CLAN_CORE, machines=["vm_with_secrets"], @@ -30,11 +30,11 @@ def flake_with_vm_with_secrets( @pytest.fixture def remote_flake_with_vm_without_secrets( - monkeypatch: pytest.MonkeyPatch, temporary_dir: Path + monkeypatch: pytest.MonkeyPatch, temporary_home: Path ) -> Iterator[FlakeForTest]: yield from create_flake( monkeypatch, - temporary_dir, + temporary_home, FlakeName("test_flake_with_core_dynamic_machines"), CLAN_CORE, machines=["vm_without_secrets"],