diff --git a/pkgs/clan-cli/clan_cli/flakes/create.py b/pkgs/clan-cli/clan_cli/flakes/create.py index b5a2154ab..f208c0dd0 100644 --- a/pkgs/clan-cli/clan_cli/flakes/create.py +++ b/pkgs/clan-cli/clan_cli/flakes/create.py @@ -26,28 +26,30 @@ async def create_flake(directory: Path, url: str) -> dict[str, CmdOut]: out = await run(command, cwd=directory) response["flake init"] = out - command = nix_shell(["git"], ["git", "init"]) + command = nix_shell(["nixpkgs#git"], ["git", "init"]) out = await run(command, cwd=directory) response["git init"] = out - command = nix_shell(["git"], ["git", "add", "."]) + command = nix_shell(["nixpkgs#git"], ["git", "add", "."]) out = await run(command, cwd=directory) response["git add"] = out - # command = nix_shell(["git"], ["git", "config", "init.defaultBranch", "main"]) + # command = nix_shell(["nixpkgs#git"], ["git", "config", "init.defaultBranch", "main"]) # out = await run(command, cwd=directory) # response["git config"] = out - command = nix_shell(["git"], ["git", "config", "user.name", "clan-tool"]) + command = nix_shell(["nixpkgs#git"], ["git", "config", "user.name", "clan-tool"]) out = await run(command, cwd=directory) response["git config"] = out - command = nix_shell(["git"], ["git", "config", "user.email", "clan@example.com"]) + command = nix_shell( + ["nixpkgs#git"], ["git", "config", "user.email", "clan@example.com"] + ) out = await run(command, cwd=directory) response["git config"] = out # TODO: Find out why this fails on Johannes machine - # command = nix_shell(["git"], ["git", "commit", "-a", "-m", "Initial commit"]) + # command = nix_shell(["nixpkgs#git"], ["git", "commit", "-a", "-m", "Initial commit"]) # out = await run(command, cwd=directory) # response["git commit"] = out diff --git a/pkgs/clan-cli/clan_cli/git.py b/pkgs/clan-cli/clan_cli/git.py index 60d8f5852..126f8587d 100644 --- a/pkgs/clan-cli/clan_cli/git.py +++ b/pkgs/clan-cli/clan_cli/git.py @@ -38,7 +38,7 @@ def _commit_file_to_git(repo_dir: Path, file_path: Path, commit_message: str) -> :raises ClanError: If the file is not in the git repository. """ cmd = nix_shell( - ["git"], + ["nixpkgs#git"], ["git", "-C", str(repo_dir), "add", str(file_path)], ) # add the file to the git index @@ -51,7 +51,7 @@ def _commit_file_to_git(repo_dir: Path, file_path: Path, commit_message: str) -> # check if there is a diff cmd = nix_shell( - ["git"], + ["nixpkgs#git"], ["git", "-C", str(repo_dir), "diff", "--cached", "--exit-code", str(file_path)], ) result = subprocess.run(cmd, cwd=repo_dir) @@ -61,7 +61,7 @@ def _commit_file_to_git(repo_dir: Path, file_path: Path, commit_message: str) -> # commit only that file cmd = nix_shell( - ["git"], + ["nixpkgs#git"], [ "git", "-C", diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index fc036fa99..75559618f 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -28,7 +28,7 @@ def install_nixos(machine: Machine) -> None: subprocess.run( nix_shell( - ["nixos-anywhere"], + ["nixpkgs#nixos-anywhere"], [ "nixos-anywhere", "-f", diff --git a/pkgs/clan-cli/clan_cli/nix.py b/pkgs/clan-cli/clan_cli/nix.py index c90611e21..3a7aca7ee 100644 --- a/pkgs/clan-cli/clan_cli/nix.py +++ b/pkgs/clan-cli/clan_cli/nix.py @@ -95,16 +95,15 @@ def nix_shell(packages: list[str], cmd: list[str]) -> list[str]: # in our tests we just make sure we have all the packages if os.environ.get("IN_NIX_SANDBOX"): return cmd - wrapped_packages = [f"nixpkgs#{p}" for p in packages] - return ( - nix_command( + return [ + *nix_command( [ "shell", "--inputs-from", f"{nixpkgs_flake()!s}", ] - ) - + wrapped_packages - + ["-c"] - + cmd - ) + ), + *packages, + "-c", + *cmd, + ] diff --git a/pkgs/clan-cli/clan_cli/secrets/import_sops.py b/pkgs/clan-cli/clan_cli/secrets/import_sops.py index 1d0d511ea..82ae7737e 100644 --- a/pkgs/clan-cli/clan_cli/secrets/import_sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/import_sops.py @@ -22,7 +22,7 @@ def import_sops(args: argparse.Namespace) -> None: if args.input_type: cmd += ["--input-type", args.input_type] cmd += ["--output-type", "json", "--decrypt", args.sops_file] - cmd = nix_shell(["sops"], cmd) + cmd = nix_shell(["nixpkgs#sops"], cmd) try: res = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE) except subprocess.CalledProcessError as e: diff --git a/pkgs/clan-cli/clan_cli/secrets/sops.py b/pkgs/clan-cli/clan_cli/secrets/sops.py index c23f08f2c..5f4493ac9 100644 --- a/pkgs/clan-cli/clan_cli/secrets/sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/sops.py @@ -21,7 +21,7 @@ class SopsKey: def get_public_key(privkey: str) -> str: - cmd = nix_shell(["age"], ["age-keygen", "-y"]) + cmd = nix_shell(["nixpkgs#age"], ["age-keygen", "-y"]) try: res = subprocess.run( cmd, input=privkey, stdout=subprocess.PIPE, text=True, check=True @@ -34,7 +34,7 @@ def get_public_key(privkey: str) -> str: def generate_private_key() -> tuple[str, str]: - cmd = nix_shell(["age"], ["age-keygen"]) + cmd = nix_shell(["nixpkgs#age"], ["age-keygen"]) try: proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, text=True) res = proc.stdout.strip() @@ -119,7 +119,7 @@ def sops_manifest(keys: list[str]) -> Iterator[Path]: def update_keys(secret_path: Path, keys: list[str]) -> None: with sops_manifest(keys) as manifest: cmd = nix_shell( - ["sops"], + ["nixpkgs#sops"], [ "sops", "--config", @@ -146,7 +146,7 @@ def encrypt_file( if not content: args = ["sops", "--config", str(manifest)] args.extend([str(secret_path)]) - cmd = nix_shell(["sops"], args) + cmd = nix_shell(["nixpkgs#sops"], args) p = subprocess.run(cmd) # returns 200 if the file is changed if p.returncode != 0 and p.returncode != 200: @@ -166,7 +166,7 @@ def encrypt_file( # we pass an empty manifest to pick up existing configuration of the user args = ["sops", "--config", str(manifest)] args.extend(["-i", "--encrypt", str(f.name)]) - cmd = nix_shell(["sops"], args) + cmd = nix_shell(["nixpkgs#sops"], args) subprocess.run(cmd, check=True) # atomic copy of the encrypted file with NamedTemporaryFile(dir=folder, delete=False) as f2: @@ -182,7 +182,8 @@ def encrypt_file( def decrypt_file(secret_path: Path) -> str: with sops_manifest([]) as manifest: cmd = nix_shell( - ["sops"], ["sops", "--config", str(manifest), "--decrypt", str(secret_path)] + ["nixpkgs#sops"], + ["sops", "--config", str(manifest), "--decrypt", str(secret_path)], ) res = subprocess.run(cmd, stdout=subprocess.PIPE, text=True) if res.returncode != 0: diff --git a/pkgs/clan-cli/clan_cli/secrets/sops_generate.py b/pkgs/clan-cli/clan_cli/secrets/sops_generate.py index c99a7a3b9..405ab994a 100644 --- a/pkgs/clan-cli/clan_cli/secrets/sops_generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/sops_generate.py @@ -61,7 +61,7 @@ export secrets={shlex.quote(str(secrets_dir))} {generator} """ try: - cmd = nix_shell(["bash"], ["bash", "-c", text]) + cmd = nix_shell(["nixpkgs#bash"], ["bash", "-c", text]) subprocess.run(cmd, check=True) except subprocess.CalledProcessError: msg = "failed to the following command:\n" diff --git a/pkgs/clan-cli/clan_cli/secrets/upload.py b/pkgs/clan-cli/clan_cli/secrets/upload.py index 8250d3e10..a4cacf138 100644 --- a/pkgs/clan-cli/clan_cli/secrets/upload.py +++ b/pkgs/clan-cli/clan_cli/secrets/upload.py @@ -21,7 +21,7 @@ def upload_secrets(machine: Machine) -> None: ssh_cmd = host.ssh_cmd() subprocess.run( nix_shell( - ["rsync"], + ["nixpkgs#rsync"], [ "rsync", "-e", diff --git a/pkgs/clan-cli/clan_cli/ssh/cli.py b/pkgs/clan-cli/clan_cli/ssh/cli.py index b46766fc9..fe97d6082 100644 --- a/pkgs/clan-cli/clan_cli/ssh/cli.py +++ b/pkgs/clan-cli/clan_cli/ssh/cli.py @@ -11,10 +11,10 @@ def ssh( password: str | None = None, ssh_args: list[str] = [], ) -> None: - packages = ["tor", "openssh"] + packages = ["nixpkgs#tor", "nixpkgs#openssh"] password_args = [] if password: - packages.append("sshpass") + packages.append("nixpkgs#sshpass") password_args = [ "sshpass", "-p", @@ -37,7 +37,7 @@ def qrcode_scan(picture_file: str) -> str: return ( subprocess.run( nix_shell( - ["zbar"], + ["nixpkgs#zbar"], [ "zbarimg", "--quiet", diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index f10e08daf..1fdbb526c 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -18,13 +18,47 @@ from .inspect import VmConfig, inspect_vm log = logging.getLogger(__name__) +def graphics_options(vm: VmConfig) -> list[str]: + if vm.wayland: + # fmt: off + return [ + "-nographic", + "-vga", "none", + "-device", "virtio-gpu-rutabaga,gfxstream-vulkan=on,cross-domain=on,hostmem=4G,wsi=headless", + ] + # fmt: on + else: + # fmt: off + return [ + "-audiodev", "spice,id=audio0", + "-device", "intel-hda", + "-device", "hda-duplex,audiodev=audio0", + "-display", "gtk,gl=on", + "-device", "virtio-gpu-gl", + "-display", "spice-app,gl=on", + "-device", "virtio-serial-pci", + "-chardev", "spicevmc,id=vdagent0,name=vdagent", + "-device", "virtserialport,chardev=vdagent0,name=com.redhat.spice.0", + "-device", "qemu-xhci,id=spicepass", + "-chardev", "spicevmc,id=usbredirchardev1,name=usbredir", + "-device", "usb-redir,chardev=usbredirchardev1,id=usbredirdev1", + "-chardev", "spicevmc,id=usbredirchardev2,name=usbredir", + "-device", "usb-redir,chardev=usbredirchardev2,id=usbredirdev2", + "-chardev", "spicevmc,id=usbredirchardev3,name=usbredir", + "-device", "usb-redir,chardev=usbredirchardev3,id=usbredirdev3", + "-device", "pci-ohci,id=smartpass", + "-device", "usb-ccid", + "-chardev", "spicevmc,id=ccid,name=smartcard", + ] + # fmt: on + + def qemu_command( vm: VmConfig, nixos_config: dict[str, str], xchg_dir: Path, secrets_dir: Path, disk_img: Path, - spice_socket: Path, ) -> list[str]: kernel_cmdline = [ (Path(nixos_config["toplevel"]) / "kernel-params").read_text(), @@ -40,6 +74,7 @@ def qemu_command( "-m", f'{nixos_config["memorySize"]}M', "-smp", str(nixos_config["cores"]), "-cpu", "max", + "-enable-kvm", "-device", "virtio-rng-pci", "-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0", @@ -50,9 +85,6 @@ def qemu_command( "-drive", f"cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report", "-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root", "-device", "virtio-keyboard", - # TODO: we also need to fixup timezone than... - # "-rtc", "base=localtime,clock=host,driftfix=slew", - "-vga", "virtio", "-usb", "-device", "usb-tablet,bus=usb-bus.0", "-kernel", f'{nixos_config["toplevel"]}/kernel', "-initrd", nixos_config["initrd"], @@ -60,31 +92,7 @@ def qemu_command( ] # fmt: on if vm.graphics: - # fmt: off - command.extend( - [ - "-audiodev", "spice,id=audio0", - "-device", "intel-hda", - "-device", "hda-duplex,audiodev=audio0", - "-vga", "none", - "-device", "virtio-gpu-gl", - "-display", "spice-app,gl=on", - "-device", "virtio-serial-pci", - "-chardev", "spicevmc,id=vdagent0,name=vdagent", - "-device", "virtserialport,chardev=vdagent0,name=com.redhat.spice.0", - "-device", "qemu-xhci,id=spicepass", - "-chardev", "spicevmc,id=usbredirchardev1,name=usbredir", - "-device", "usb-redir,chardev=usbredirchardev1,id=usbredirdev1", - "-chardev", "spicevmc,id=usbredirchardev2,name=usbredir", - "-device", "usb-redir,chardev=usbredirchardev2,id=usbredirdev2", - "-chardev", "spicevmc,id=usbredirchardev3,name=usbredir", - "-device", "usb-redir,chardev=usbredirchardev3,id=usbredirdev3", - "-device", "pci-ohci,id=smartpass", - "-device", "usb-ccid", - "-chardev", "spicevmc,id=ccid,name=smartcard", - ] - ) - # fmt: on + command.extend(graphics_options(vm)) else: command.append("-nographic") return command @@ -134,6 +142,96 @@ def get_clan_name(vm: VmConfig, nix_options: list[str]) -> str: return proc.stdout.strip().strip('"') +def generate_secrets( + vm: VmConfig, + clan_name: str, + nixos_config: dict[str, str], + tmpdir: Path, + log_fd: IO[str] | None, +) -> Path: + secrets_dir = tmpdir / "secrets" + secrets_dir.mkdir(exist_ok=True) + + env = os.environ.copy() + env["CLAN_DIR"] = str(vm.flake_url) + + env["PYTHONPATH"] = str(":".join(sys.path)) # TODO do this in the clanCore module + env["SECRETS_DIR"] = str(secrets_dir) + + # Only generate secrets for local clans + if isinstance(vm.flake_url, Path) and vm.flake_url.is_dir(): + if Path(vm.flake_url).is_dir(): + subprocess.run( + [nixos_config["generateSecrets"], clan_name], + env=env, + check=False, + stdout=log_fd, + stderr=log_fd, + ) + else: + log.warning("won't generate secrets for non local clan") + + cmd = [nixos_config["uploadSecrets"]] + res = subprocess.run( + cmd, + env=env, + check=False, + stdout=log_fd, + stderr=log_fd, + ) + if res.returncode != 0: + raise ClanError( + f"Failed to upload secrets: {shlex.join(cmd)} failed with {res.returncode}" + ) + return secrets_dir + + +def prepare_disk(tmpdir: Path, log_fd: IO[str] | None) -> Path: + disk_img = tmpdir / "disk.img" + cmd = nix_shell( + ["nixpkgs#qemu"], + [ + "qemu-img", + "create", + "-f", + "raw", + str(disk_img), + "1024M", + ], + ) + res = subprocess.run( + cmd, + check=False, + stdout=log_fd, + stderr=log_fd, + ) + if res.returncode != 0: + raise ClanError( + f"Failed to create disk image: {shlex.join(cmd)} failed with {res.returncode}" + ) + + cmd = nix_shell( + ["nixpkgs#e2fsprogs"], + [ + "mkfs.ext4", + "-L", + "nixos", + str(disk_img), + ], + ) + res = subprocess.run( + cmd, + check=False, + stdout=log_fd, + stderr=log_fd, + ) + if res.returncode != 0: + raise ClanError( + f"Failed to create ext4 filesystem: {shlex.join(cmd)} failed with {res.returncode}" + ) + return disk_img + + def run_vm( vm: VmConfig, nix_options: list[str] = [], log_fd: IO[str] | None = None ) -> None: @@ -156,86 +254,9 @@ def run_vm( tmpdir = Path(tmpdir_) xchg_dir = tmpdir / "xchg" xchg_dir.mkdir(exist_ok=True) - secrets_dir = tmpdir / "secrets" - secrets_dir.mkdir(exist_ok=True) - disk_img = tmpdir / "disk.img" - spice_socket = tmpdir / "spice.sock" - env = os.environ.copy() - env["CLAN_DIR"] = str(vm.flake_url) - - env["PYTHONPATH"] = str( - ":".join(sys.path) - ) # TODO do this in the clanCore module - env["SECRETS_DIR"] = str(secrets_dir) - - # Only generate secrets for local clans - if isinstance(vm.flake_url, Path) and vm.flake_url.is_dir(): - if Path(vm.flake_url).is_dir(): - subprocess.run( - [nixos_config["generateSecrets"], clan_name], - env=env, - check=False, - stdout=log_fd, - stderr=log_fd, - ) - else: - log.warning("won't generate secrets for non local clan") - - cmd = [nixos_config["uploadSecrets"]] - res = subprocess.run( - cmd, - env=env, - check=False, - stdout=log_fd, - stderr=log_fd, - ) - if res.returncode != 0: - raise ClanError( - f"Failed to upload secrets: {shlex.join(cmd)} failed with {res.returncode}" - ) - - cmd = nix_shell( - ["qemu"], - [ - "qemu-img", - "create", - "-f", - "raw", - str(disk_img), - "1024M", - ], - ) - res = subprocess.run( - cmd, - check=False, - stdout=log_fd, - stderr=log_fd, - ) - if res.returncode != 0: - raise ClanError( - f"Failed to create disk image: {shlex.join(cmd)} failed with {res.returncode}" - ) - - cmd = nix_shell( - ["e2fsprogs"], - [ - "mkfs.ext4", - "-L", - "nixos", - str(disk_img), - ], - ) - res = subprocess.run( - cmd, - check=False, - stdout=log_fd, - stderr=log_fd, - ) - if res.returncode != 0: - raise ClanError( - f"Failed to create ext4 filesystem: {shlex.join(cmd)} failed with {res.returncode}" - ) + secrets_dir = generate_secrets(vm, clan_name, nixos_config, tmpdir, log_fd) + disk_img = prepare_disk(tmpdir, log_fd) qemu_cmd = qemu_command( vm, @@ -243,20 +264,22 @@ def run_vm( xchg_dir=xchg_dir, secrets_dir=secrets_dir, disk_img=disk_img, - spice_socket=spice_socket, ) - print("$ " + shlex.join(qemu_cmd)) - packages = ["qemu"] - if vm.graphics: - packages.append("virt-viewer") + if vm.wayland: + packages = ["git+https://git.clan.lol/clan/clan-core.git#qemu-wayland"] + else: + packages = ["nixpkgs#qemu"] env = os.environ.copy() - remote_viewer_mimetypes = module_root() / "vms" / "mimetypes" - env[ - "XDG_DATA_DIRS" - ] = f"{remote_viewer_mimetypes}:{env.get('XDG_DATA_DIRS', '')}" - print(env["XDG_DATA_DIRS"]) + if vm.graphics and not vm.wayland: + packages.append("nixpkgs#virt-viewer") + remote_viewer_mimetypes = module_root() / "vms" / "mimetypes" + env[ + "XDG_DATA_DIRS" + ] = f"{remote_viewer_mimetypes}:{env.get('XDG_DATA_DIRS', '')}" + + print("$ " + shlex.join(qemu_cmd)) res = subprocess.run( nix_shell(packages, qemu_cmd), env=env, @@ -274,6 +297,7 @@ class RunOptions: flake_url: str | None flake: Path nix_options: list[str] = field(default_factory=list) + wayland: bool = False def run_command(args: argparse.Namespace) -> None: @@ -282,10 +306,13 @@ def run_command(args: argparse.Namespace) -> None: flake_url=args.flake_url, flake=args.flake or Path.cwd(), nix_options=args.option, + wayland=args.wayland, ) flake_url = run_options.flake_url or run_options.flake vm = inspect_vm(flake_url=flake_url, flake_attr=run_options.machine) + # TODO: allow to set this in the config + vm.wayland = run_options.wayland run_vm(vm, run_options.nix_options) @@ -293,4 +320,5 @@ def run_command(args: argparse.Namespace) -> None: def register_run_parser(parser: argparse.ArgumentParser) -> None: parser.add_argument("machine", type=str, help="machine in the flake to run") parser.add_argument("--flake-url", type=str, help="flake url") + parser.add_argument("--wayland", action="store_true", help="use wayland") parser.set_defaults(func=run_command) diff --git a/pkgs/clan-cli/tests/conftest.py b/pkgs/clan-cli/tests/conftest.py index c3c11e75e..8cfdde305 100644 --- a/pkgs/clan-cli/tests/conftest.py +++ b/pkgs/clan-cli/tests/conftest.py @@ -26,12 +26,12 @@ pytest_plugins = [ @pytest.fixture def git_repo(tmp_path: Path) -> Path: # initialize a git repository - cmd = nix_shell(["git"], ["git", "init"]) + cmd = nix_shell(["nixpkgs#git"], ["git", "init"]) subprocess.run(cmd, cwd=tmp_path, check=True) # set user.name and user.email - cmd = nix_shell(["git"], ["git", "config", "user.name", "test"]) + cmd = nix_shell(["nixpkgs#git"], ["git", "config", "user.name", "test"]) subprocess.run(cmd, cwd=tmp_path, check=True) - cmd = nix_shell(["git"], ["git", "config", "user.email", "test@test.test"]) + cmd = nix_shell(["nixpkgs#git"], ["git", "config", "user.email", "test@test.test"]) subprocess.run(cmd, cwd=tmp_path, check=True) # return the path to the git repository return tmp_path diff --git a/pkgs/clan-cli/tests/test_secrets_password_store.py b/pkgs/clan-cli/tests/test_secrets_password_store.py index 3e32278c5..ac9fc49ad 100644 --- a/pkgs/clan-cli/tests/test_secrets_password_store.py +++ b/pkgs/clan-cli/tests/test_secrets_password_store.py @@ -35,10 +35,14 @@ def test_upload_secret( ) cli = Cli() subprocess.run( - nix_shell(["gnupg"], ["gpg", "--batch", "--gen-key", str(gpg_key_spec)]), + nix_shell( + ["nixpkgs#gnupg"], ["gpg", "--batch", "--gen-key", str(gpg_key_spec)] + ), check=True, ) - subprocess.run(nix_shell(["pass"], ["pass", "init", "test@local"]), check=True) + subprocess.run( + nix_shell(["nixpkgs#pass"], ["pass", "init", "test@local"]), check=True + ) cli.run(["secrets", "generate", "vm1"]) network_id = machine_get_fact( test_flake_with_core_and_pass.path, "vm1", "zerotier-network-id"