From 1f261353819498356c368a532f7f14486fb2bd6d Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Fri, 4 Jul 2025 10:20:58 +0100 Subject: [PATCH 001/258] feat(ui): alert component --- .../ui/src/components/v2/Alert/Alert.css | 39 +++++ .../src/components/v2/Alert/Alert.stories.tsx | 138 ++++++++++++++++++ .../ui/src/components/v2/Alert/Alert.tsx | 43 ++++++ 3 files changed, 220 insertions(+) create mode 100644 pkgs/clan-app/ui/src/components/v2/Alert/Alert.css create mode 100644 pkgs/clan-app/ui/src/components/v2/Alert/Alert.stories.tsx create mode 100644 pkgs/clan-app/ui/src/components/v2/Alert/Alert.tsx diff --git a/pkgs/clan-app/ui/src/components/v2/Alert/Alert.css b/pkgs/clan-app/ui/src/components/v2/Alert/Alert.css new file mode 100644 index 000000000..0fd90cc0d --- /dev/null +++ b/pkgs/clan-app/ui/src/components/v2/Alert/Alert.css @@ -0,0 +1,39 @@ +div.alert { + @apply flex gap-2.5 px-6 py-4 size-full rounded-md items-start; + + &.has-icon { + @apply pl-4; + + svg.icon { + @apply relative top-0.5; + } + } + + &.has-dismiss { + @apply pr-4; + } + + & > div.content { + @apply flex flex-col gap-2 size-full; + } + + &.info { + @apply bg-semantic-info-1 border border-semantic-info-3 fg-semantic-info-3; + } + + &.error { + @apply bg-semantic-error-2 border border-semantic-error-3 fg-semantic-error-3; + } + + &.warning { + @apply bg-semantic-warning-2 border border-semantic-warning-3 fg-semantic-warning-3; + } + + &.success { + @apply bg-semantic-success-1 border border-semantic-success-3 fg-semantic-success-3; + } + + & > button.dismiss-trigger { + @apply relative top-0.5; + } +} diff --git a/pkgs/clan-app/ui/src/components/v2/Alert/Alert.stories.tsx b/pkgs/clan-app/ui/src/components/v2/Alert/Alert.stories.tsx new file mode 100644 index 000000000..5434becd7 --- /dev/null +++ b/pkgs/clan-app/ui/src/components/v2/Alert/Alert.stories.tsx @@ -0,0 +1,138 @@ +import type { Meta, StoryObj } from "@kachurun/storybook-solid"; +import { Alert, AlertProps } from "@/src/components/v2/Alert/Alert"; +import { expect, fn } from "storybook/test"; +import { StoryContext } from "@kachurun/storybook-solid-vite"; + +const meta: Meta = { + title: "Components/Alert", + component: Alert, + decorators: [ + (Story: StoryObj) => ( +
+ +
+ ), + ], +}; + +export default meta; + +type Story = StoryObj; + +export const Info: Story = { + args: { + type: "info", + title: "Headline", + description: + "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua.", + }, +}; + +export const Error: Story = { + args: { + ...Info.args, + type: "error", + }, +}; + +export const Warning: Story = { + args: { + ...Info.args, + type: "warning", + }, +}; + +export const Success: Story = { + args: { + ...Info.args, + type: "success", + }, +}; + +export const InfoIcon: Story = { + args: { + ...Info.args, + icon: "Info", + }, +}; + +export const ErrorIcon: Story = { + args: { + ...Error.args, + icon: "WarningFilled", + }, +}; + +export const WarningIcon: Story = { + args: { + ...Warning.args, + icon: "WarningFilled", + }, +}; + +export const SuccessIcon: Story = { + args: { + ...Success.args, + icon: "Checkmark", + }, +}; + +export const InfoDismiss: Story = { + args: { + ...Info.args, + onDismiss: fn(), + play: async ({ canvas, step, userEvent, args }: StoryContext) => { + await userEvent.click(canvas.getByRole("button")); + await expect(args.onDismiss).toHaveBeenCalled(); + }, + }, +}; + +export const ErrorDismiss: Story = { + args: { + ...InfoDismiss.args, + type: "error", + }, +}; + +export const WarningDismiss: Story = { + args: { + ...InfoDismiss.args, + type: "warning", + }, +}; + +export const SuccessDismiss: Story = { + args: { + ...InfoDismiss.args, + type: "success", + }, +}; + +export const InfoIconDismiss: Story = { + args: { + ...InfoDismiss.args, + icon: "Info", + }, +}; + +export const ErrorIconDismiss: Story = { + args: { + ...ErrorDismiss.args, + icon: "WarningFilled", + }, +}; + +export const WarningIconDismiss: Story = { + args: { + ...WarningDismiss.args, + icon: "WarningFilled", + }, +}; + +export const SuccessIconDismiss: Story = { + args: { + ...SuccessDismiss.args, + icon: "Checkmark", + }, +}; diff --git a/pkgs/clan-app/ui/src/components/v2/Alert/Alert.tsx b/pkgs/clan-app/ui/src/components/v2/Alert/Alert.tsx new file mode 100644 index 000000000..7dc156a2f --- /dev/null +++ b/pkgs/clan-app/ui/src/components/v2/Alert/Alert.tsx @@ -0,0 +1,43 @@ +import "./Alert.css"; +import cx from "classnames"; +import Icon, { IconVariant } from "@/src/components/v2/Icon/Icon"; +import { Typography } from "@/src/components/v2/Typography/Typography"; +import { Button } from "@kobalte/core/button"; +import { Alert as KAlert } from "@kobalte/core/alert"; + +export interface AlertProps { + type: "success" | "error" | "warning" | "info"; + title: string; + description: string; + icon?: IconVariant; + onDismiss?: () => void; +} + +export const Alert = (props: AlertProps) => ( + + {props.icon && } +
+ + {props.title} + + + {props.description} + +
+ {props.onDismiss && ( + + )} + +); From 6aab8ffd0c76e98fb1f996331d1d8f355bf9a7e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 9 Jun 2025 14:02:14 +0200 Subject: [PATCH 002/258] change install test to run clan outside of the VM --- checks/installation/flake-module.nix | 326 +++++++++++++++++++++------ 1 file changed, 259 insertions(+), 67 deletions(-) diff --git a/checks/installation/flake-module.nix b/checks/installation/flake-module.nix index 527890f53..d9313aae8 100644 --- a/checks/installation/flake-module.nix +++ b/checks/installation/flake-module.nix @@ -1,23 +1,12 @@ { self, lib, + ... }: let - installer = + target = { modulesPath, pkgs, ... }: - let - dependencies = [ - self.clan.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.toplevel - self.clan.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.diskoScript - pkgs.stdenv.drvPath - pkgs.bash.drvPath - pkgs.nixos-anywhere - pkgs.bubblewrap - pkgs.buildPackages.xorg.lndir - ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs); - closureInfo = pkgs.closureInfo { rootPaths = dependencies; }; - in { imports = [ (modulesPath + "/../tests/common/auto-format-root-device.nix") @@ -28,33 +17,30 @@ let services.openssh.settings.PasswordAuthentication = false; system.nixos.variant_id = "installer"; environment.systemPackages = [ - self.packages.${pkgs.system}.clan-cli-full pkgs.nixos-facter ]; - environment.etc."install-closure".source = "${closureInfo}/store-paths"; + # Disable cache.nixos.org to speed up tests + nix.settings.substituters = [ ]; + nix.settings.trusted-public-keys = [ ]; virtualisation.emptyDiskImages = [ 512 ]; virtualisation.diskSize = 8 * 1024; virtualisation.rootDevice = "/dev/vdb"; # both installer and target need to use the same diskImage virtualisation.diskImage = "./target.qcow2"; virtualisation.memorySize = 3048; - nix.settings = { - substituters = lib.mkForce [ ]; - hashed-mirrors = null; - connect-timeout = lib.mkForce 3; - flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}''; - experimental-features = [ - "nix-command" - "flakes" - ]; - }; users.users.nonrootuser = { isNormalUser = true; - openssh.authorizedKeys.keyFiles = [ ../assets/ssh/pubkey ]; + openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ]; extraGroups = [ "wheel" ]; }; + users.users.root.openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ]; + # Allow users to manage their own SSH keys + services.openssh.authorizedKeysFiles = [ + "/root/.ssh/authorized_keys" + "/home/%u/.ssh/authorized_keys" + "/etc/ssh/authorized_keys.d/%u" + ]; security.sudo.wheelNeedsPassword = false; - system.extraDependencies = dependencies; }; in { @@ -105,6 +91,25 @@ in environment.etc."install-successful".text = "ok"; + # Enable SSH and add authorized key for testing + services.openssh.enable = true; + services.openssh.settings.PasswordAuthentication = false; + users.users.nonrootuser = { + isNormalUser = true; + openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ]; + extraGroups = [ "wheel" ]; + home = "/home/nonrootuser"; + createHome = true; + }; + users.users.root.openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ]; + # Allow users to manage their own SSH keys + services.openssh.authorizedKeysFiles = [ + "/root/.ssh/authorized_keys" + "/home/%u/.ssh/authorized_keys" + "/etc/ssh/authorized_keys.d/%u" + ]; + security.sudo.wheelNeedsPassword = false; + boot.consoleLogLevel = lib.mkForce 100; boot.kernelParams = [ "boot.shell_on_fail" ]; @@ -181,55 +186,242 @@ in # vm-test-run-test-installation-> target: waiting for the VM to finish booting # vm-test-run-test-installation-> target: Guest root shell did not produce any data yet... # vm-test-run-test-installation-> target: To debug, enter the VM and run 'systemctl status backdoor.service'. - checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux && !pkgs.stdenv.isAarch64) { - nixos-test-installation = self.clanLib.test.baseTest { - name = "installation"; - nodes.target = { - services.openssh.enable = true; - virtualisation.diskImage = "./target.qcow2"; - virtualisation.useBootLoader = true; + checks = + let + # Custom Python package for port management utilities + portUtils = pkgs.python3Packages.buildPythonPackage { + pname = "port-utils"; + version = "1.0.0"; + format = "other"; + src = lib.fileset.toSource { + root = ./.; + fileset = ./port_utils.py; + }; + dontUnpack = true; + installPhase = '' + install -D $src/port_utils.py $out/${pkgs.python3.sitePackages}/port_utils.py + touch $out/${pkgs.python3.sitePackages}/py.typed + ''; + doCheck = false; }; - nodes.installer = installer; + closureInfo = pkgs.closureInfo { + rootPaths = [ + self.checks.x86_64-linux.clan-core-for-checks + self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.toplevel + self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.initialRamdisk + self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.diskoScript + pkgs.stdenv.drvPath + pkgs.bash.drvPath + pkgs.buildPackages.xorg.lndir + ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs); + }; + in + pkgs.lib.mkIf (pkgs.stdenv.isLinux && !pkgs.stdenv.isAarch64) { + nixos-test-installation = self.clanLib.test.baseTest { + name = "installation"; + nodes.target = target; + extraPythonPackages = _p: [ + portUtils + self.legacyPackages.${pkgs.system}.setupNixInNixPythonPackage + ]; - testScript = '' - installer.start() + testScript = '' + import tempfile + import os + import subprocess + from port_utils import find_free_port, setup_port_forwarding # type: ignore[import-untyped] + from setup_nix_in_nix import setup_nix_in_nix # type: ignore[import-untyped] - installer.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../assets/ssh/privkey} /root/.ssh/id_ed25519") + def create_test_machine(oldmachine=None, **kwargs): + """Create a new test machine from an installed disk image""" + start_command = [ + "${pkgs.qemu_test}/bin/qemu-kvm", + "-cpu", "max", + "-m", "3048", + "-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store", + "-drive", f"file={oldmachine.state_dir}/target.qcow2,id=drive1,if=none,index=1,werror=report", + "-device", "virtio-blk-pci,drive=drive1", + "-netdev", "user,id=net0", + "-device", "virtio-net-pci,netdev=net0", + ] + machine = create_machine(start_command=" ".join(start_command), **kwargs) + driver.machines.append(machine) + return machine - installer.wait_until_succeeds("timeout 2 ssh -o StrictHostKeyChecking=accept-new -v nonrootuser@localhost hostname") - installer.succeed("cp -r ${self.checks.x86_64-linux.clan-core-for-checks} test-flake && chmod -R +w test-flake") + if "NIX_REMOTE" in os.environ: + del os.environ["NIX_REMOTE"] # we don't have any nix daemon running + target.start() - installer.succeed("clan machines install --no-reboot --debug --flake test-flake --yes test-install-machine-without-system --target-host nonrootuser@localhost --update-hardware-config nixos-facter >&2") - installer.shutdown() + # Set up SSH key on host (test driver environment) + with tempfile.TemporaryDirectory() as temp_dir: - # We are missing the test instrumentation somehow. Test this later. - target.state_dir = installer.state_dir - target.start() - target.wait_for_unit("multi-user.target") - ''; - } { inherit pkgs self; }; + # Set up nix chroot store environment + os.environ["closureInfo"] = "${closureInfo}" + os.environ["TMPDIR"] = temp_dir + + # Run setup function + setup_nix_in_nix() - nixos-test-update-hardware-configuration = self.clanLib.test.baseTest { - name = "update-hardware-configuration"; - nodes.installer = installer; - testScript = '' - installer.start() - installer.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../assets/ssh/privkey} /root/.ssh/id_ed25519") - installer.wait_until_succeeds("timeout 2 ssh -o StrictHostKeyChecking=accept-new -v nonrootuser@localhost hostname") - installer.succeed("cp -r ${self.checks.x86_64-linux.clan-core-for-checks} test-flake && chmod -R +w test-flake") - installer.fail("test -f test-flake/machines/test-install-machine/hardware-configuration.nix") - installer.fail("test -f test-flake/machines/test-install-machine/facter.json") + host_port = find_free_port() + target.wait_for_unit("sshd.service") + target.wait_for_open_port(22) - installer.succeed("clan machines update-hardware-config --debug --flake test-flake test-install-machine-without-system nonrootuser@localhost >&2") - installer.succeed("test -f test-flake/machines/test-install-machine-without-system/facter.json") - installer.succeed("rm test-flake/machines/test-install-machine-without-system/facter.json") + setup_port_forwarding(target, host_port) - installer.succeed("clan machines update-hardware-config --debug --backend nixos-generate-config --flake test-flake test-install-machine-without-system nonrootuser@localhost >&2") - installer.succeed("test -f test-flake/machines/test-install-machine-without-system/hardware-configuration.nix") - installer.succeed("rm test-flake/machines/test-install-machine-without-system/hardware-configuration.nix") - ''; - } { inherit pkgs self; }; - }; + ssh_key = os.path.join(temp_dir, "id_ed25519") + with open(ssh_key, "w") as f: + with open("${../assets/ssh/privkey}", "r") as src: + f.write(src.read()) + os.chmod(ssh_key, 0o600) + + # Copy test flake to temp directory + flake_dir = os.path.join(temp_dir, "test-flake") + subprocess.run(["cp", "-r", "${self.checks.x86_64-linux.clan-core-for-checks}", flake_dir], check=True) + subprocess.run(["chmod", "-R", "+w", flake_dir], check=True) + + # Run clan install from host using port forwarding + clan_cmd = [ + "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan", + "machines", + "install", + "--phases", "disko,install", + "--debug", + "--flake", env.flake_dir, + "--yes", "test-install-machine-without-system", + "--target-host", f"nonrootuser@localhost:{host_port}", + "-i", ssh_key, + "--option", "store", os.environ['CLAN_TEST_STORE'], + "--update-hardware-config", "nixos-facter", + ] + + # Set NIX_CONFIG to disable cache.nixos.org for speed + env = os.environ.copy() + env["NIX_CONFIG"] = "substituters = \ntrusted-public-keys = " + subprocess.run(clan_cmd, check=True, env=env) + + # Shutdown the installer machine gracefully + try: + target.shutdown() + except BrokenPipeError: + # qemu has already exited + pass + + # Create a new machine instance that boots from the installed system + installed_machine = create_test_machine(oldmachine=target, name="after_install") + installed_machine.start() + installed_machine.wait_for_unit("multi-user.target") + installed_machine.succeed("test -f /etc/install-successful") + ''; + } { inherit pkgs self; }; + + nixos-test-update-hardware-configuration = self.clanLib.test.baseTest { + name = "update-hardware-configuration"; + nodes.target = target; + extraPythonPackages = _p: [ + portUtils + self.legacyPackages.${pkgs.system}.setupNixInNixPythonPackage + ]; + + testScript = '' + import tempfile + import os + import subprocess + from port_utils import find_free_port, setup_port_forwarding # type: ignore[import-untyped] + from setup_nix_in_nix import setup_nix_in_nix # type: ignore[import-untyped] + + # Keep reference to closureInfo: ${closureInfo} + + # Set up nix chroot store environment + os.environ["closureInfo"] = "${closureInfo}" + + # Run setup function + setup_nix_in_nix() + + host_port = find_free_port() + + target.start() + + setup_port_forwarding(target, host_port) + + # Set up SSH key and flake on host + with tempfile.TemporaryDirectory() as temp_dir: + ssh_key = os.path.join(temp_dir, "id_ed25519") + with open(ssh_key, "w") as f: + with open("${../assets/ssh/privkey}", "r") as src: + f.write(src.read()) + os.chmod(ssh_key, 0o600) + + flake_dir = os.path.join(temp_dir, "test-flake") + subprocess.run(["cp", "-r", "${self.checks.x86_64-linux.clan-core-for-checks}", flake_dir], check=True) + subprocess.run(["chmod", "-R", "+w", flake_dir], check=True) + + # Verify files don't exist initially + hw_config_file = os.path.join(flake_dir, "machines/test-install-machine/hardware-configuration.nix") + facter_file = os.path.join(flake_dir, "machines/test-install-machine/facter.json") + + assert not os.path.exists(hw_config_file), "hardware-configuration.nix should not exist initially" + assert not os.path.exists(facter_file), "facter.json should not exist initially" + + target.wait_for_unit("sshd.service") + target.wait_for_open_port(22) + + # Test facter backend + clan_cmd = [ + "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan", + "machines", + "update-hardware-config", + "--debug", + "--flake", ".", + "--host-key-check", "none", + "test-install-machine-without-system", + "-i", ssh_key, + "--option", "store", os.environ['CLAN_TEST_STORE'], + f"nonrootuser@localhost:{env.host_port}" + ] + + env = os.environ.copy() + env["CLAN_FLAKE"] = flake_dir + # Set NIX_CONFIG to disable cache.nixos.org for speed + env["NIX_CONFIG"] = "substituters = \ntrusted-public-keys = " + result = subprocess.run(clan_cmd, capture_output=True, cwd=flake_dir, env=env) + if result.returncode != 0: + print(f"Clan update-hardware-config failed: {result.stderr.decode()}") + raise Exception(f"Clan update-hardware-config failed with return code {result.returncode}") + + facter_without_system_file = os.path.join(env.flake_dir, "machines/test-install-machine-without-system/facter.json") + assert os.path.exists(facter_without_system_file), "facter.json should exist after update" + os.remove(facter_without_system_file) + + # Test nixos-generate-config backend + clan_cmd = [ + "${self.packages.${pkgs.system}.clan-cli-full}/bin/clan", + "machines", + "update-hardware-config", + "--debug", + "--backend", "nixos-generate-config", + "--host-key-check", "none", + "--flake", ".", + "test-install-machine-without-system", + "--option", "ssh-option", f"-i {ssh_key} -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null", + "--option", "store", os.environ['CLAN_TEST_STORE'], + f"nonrootuser@localhost:{env.host_port}" + ] + + env = os.environ.copy() + env["CLAN_FLAKE"] = flake_dir + # Set NIX_CONFIG to disable cache.nixos.org for speed + env["NIX_CONFIG"] = "substituters = \ntrusted-public-keys = " + result = subprocess.run(clan_cmd, capture_output=True, cwd=flake_dir, env=env) + if result.returncode != 0: + print(f"Clan update-hardware-config (nixos-generate-config) failed: {result.stderr.decode()}") + raise Exception(f"Clan update-hardware-config failed with return code {result.returncode}") + + hw_config_without_system_file = os.path.join(env.flake_dir, "machines/test-install-machine-without-system/hardware-configuration.nix") + assert os.path.exists(hw_config_without_system_file), "hardware-configuration.nix should exist after update" + ''; + } { inherit pkgs self; }; + + }; }; } From 1558a366de4edbf30cdd88c829723836dd8d86aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 17 Jun 2025 12:30:14 +0200 Subject: [PATCH 003/258] bump clan-core-for-checks --- checks/clan-core-for-checks.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checks/clan-core-for-checks.nix b/checks/clan-core-for-checks.nix index cdff657fa..12aa373aa 100644 --- a/checks/clan-core-for-checks.nix +++ b/checks/clan-core-for-checks.nix @@ -1,6 +1,6 @@ { fetchgit }: fetchgit { url = "https://git.clan.lol/clan/clan-core.git"; - rev = "28131afbbcd379a8ff04c79c66c670ef655ed889"; - sha256 = "1294cwjlnc341fl6zbggn4rgq8z33gqkcyggjfvk9cf7zdgygrf6"; + rev = "eea93ea22c9818da67e148ba586277bab9e73cea"; + sha256 = "sha256-PV0Z+97QuxQbkYSVuNIJwUNXMbHZG/vhsA9M4cDTCOE="; } From 541732462bdb4343a297197cb89fc1b5904a1974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 2 Jul 2025 14:26:43 +0200 Subject: [PATCH 004/258] add port_utils module for installation testions --- checks/installation/port_utils.py | 51 +++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 checks/installation/port_utils.py diff --git a/checks/installation/port_utils.py b/checks/installation/port_utils.py new file mode 100644 index 000000000..a836785c4 --- /dev/null +++ b/checks/installation/port_utils.py @@ -0,0 +1,51 @@ +"""Port management utilities for NixOS installation tests.""" + +import socket +import time +from typing import Any + + +class PortUtilsError(Exception): + """Port utils related errors.""" + + +def find_free_port() -> int: + """Find a free port on the host.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +def check_host_port_open(port: int) -> bool: + """Verify port forwarding is working by checking if the host port is listening.""" + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(1) + result = s.connect_ex(("localhost", port)) + return result == 0 + except Exception as e: + print(f"Port check failed: {e}") + return False + + +def setup_port_forwarding(target: Any, host_port: int) -> None: + """Set up port forwarding and wait for it to be ready.""" + print(f"Setting up port forwarding from host port {host_port} to guest port 22") + target.forward_port(host_port=host_port, guest_port=22) + + # Give the port forwarding time to establish + time.sleep(2) + + # Wait up to 30 seconds for the port to become available + port_ready = False + for i in range(30): + if check_host_port_open(host_port): + port_ready = True + print(f"Host port {host_port} is now listening") + break + print(f"Waiting for host port {host_port} to be ready... attempt {i + 1}/30") + time.sleep(1) + + if not port_ready: + msg = f"Host port {host_port} never became available for forwarding" + raise PortUtilsError(msg) From 76b0a9bf13f447514b3e9c73ef54f43d6df92750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 2 Jul 2025 15:24:44 +0200 Subject: [PATCH 005/258] add -i option to update-hardware-config --- pkgs/clan-cli/clan_cli/machines/hardware.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/machines/hardware.py b/pkgs/clan-cli/clan_cli/machines/hardware.py index 67f0ac923..b65cf7831 100644 --- a/pkgs/clan-cli/clan_cli/machines/hardware.py +++ b/pkgs/clan-cli/clan_cli/machines/hardware.py @@ -1,5 +1,6 @@ import argparse import logging +from pathlib import Path from clan_lib.machines.hardware import ( HardwareConfig, @@ -30,9 +31,11 @@ def update_hardware_config_command(args: argparse.Namespace) -> None: if args.target_host: target_host = Remote.from_ssh_uri( machine_name=machine.name, address=args.target_host - ).override(host_key_check=host_key_check) + ).override(host_key_check=host_key_check, private_key=args.identity_file) else: - target_host = machine.target_host().override(host_key_check=host_key_check) + target_host = machine.target_host().override( + host_key_check=host_key_check, private_key=args.identity_file + ) generate_machine_hardware_info(opts, target_host) @@ -69,3 +72,9 @@ def register_update_hardware_config(parser: argparse.ArgumentParser) -> None: choices=["nixos-generate-config", "nixos-facter"], default="nixos-facter", ) + parser.add_argument( + "-i", + dest="identity_file", + type=Path, + help="specify which SSH private key file to use", + ) From b5262427446ae00bd501312f664c3781538089c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 2 Jul 2025 15:25:57 +0200 Subject: [PATCH 006/258] share more code between installation and update test --- checks/installation/flake-module.nix | 186 ++++-------------- .../installation/nixos_test_lib/__init__.py | 3 + .../nixos_test_lib/environment.py | 13 ++ checks/installation/nixos_test_lib/machine.py | 25 +++ .../{port_utils.py => nixos_test_lib/port.py} | 0 checks/installation/nixos_test_lib/py.typed | 0 checks/installation/nixos_test_lib/ssh.py | 53 +++++ checks/installation/pyproject.toml | 44 +++++ checks/installation/test-helpers.nix | 173 ++++++++++++++++ 9 files changed, 346 insertions(+), 151 deletions(-) create mode 100644 checks/installation/nixos_test_lib/__init__.py create mode 100644 checks/installation/nixos_test_lib/environment.py create mode 100644 checks/installation/nixos_test_lib/machine.py rename checks/installation/{port_utils.py => nixos_test_lib/port.py} (100%) create mode 100644 checks/installation/nixos_test_lib/py.typed create mode 100644 checks/installation/nixos_test_lib/ssh.py create mode 100644 checks/installation/pyproject.toml create mode 100644 checks/installation/test-helpers.nix diff --git a/checks/installation/flake-module.nix b/checks/installation/flake-module.nix index d9313aae8..4d19bed2b 100644 --- a/checks/installation/flake-module.nix +++ b/checks/installation/flake-module.nix @@ -4,45 +4,6 @@ ... }: -let - target = - { modulesPath, pkgs, ... }: - { - imports = [ - (modulesPath + "/../tests/common/auto-format-root-device.nix") - ]; - networking.useNetworkd = true; - services.openssh.enable = true; - services.openssh.settings.UseDns = false; - services.openssh.settings.PasswordAuthentication = false; - system.nixos.variant_id = "installer"; - environment.systemPackages = [ - pkgs.nixos-facter - ]; - # Disable cache.nixos.org to speed up tests - nix.settings.substituters = [ ]; - nix.settings.trusted-public-keys = [ ]; - virtualisation.emptyDiskImages = [ 512 ]; - virtualisation.diskSize = 8 * 1024; - virtualisation.rootDevice = "/dev/vdb"; - # both installer and target need to use the same diskImage - virtualisation.diskImage = "./target.qcow2"; - virtualisation.memorySize = 3048; - users.users.nonrootuser = { - isNormalUser = true; - openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ]; - extraGroups = [ "wheel" ]; - }; - users.users.root.openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ]; - # Allow users to manage their own SSH keys - services.openssh.authorizedKeysFiles = [ - "/root/.ssh/authorized_keys" - "/home/%u/.ssh/authorized_keys" - "/etc/ssh/authorized_keys.d/%u" - ]; - security.sudo.wheelNeedsPassword = false; - }; -in { # The purpose of this test is to ensure `clan machines install` works @@ -189,21 +150,6 @@ in checks = let # Custom Python package for port management utilities - portUtils = pkgs.python3Packages.buildPythonPackage { - pname = "port-utils"; - version = "1.0.0"; - format = "other"; - src = lib.fileset.toSource { - root = ./.; - fileset = ./port_utils.py; - }; - dontUnpack = true; - installPhase = '' - install -D $src/port_utils.py $out/${pkgs.python3.sitePackages}/port_utils.py - touch $out/${pkgs.python3.sitePackages}/py.typed - ''; - doCheck = false; - }; closureInfo = pkgs.closureInfo { rootPaths = [ self.checks.x86_64-linux.clan-core-for-checks @@ -219,9 +165,9 @@ in pkgs.lib.mkIf (pkgs.stdenv.isLinux && !pkgs.stdenv.isAarch64) { nixos-test-installation = self.clanLib.test.baseTest { name = "installation"; - nodes.target = target; + nodes.target = (import ./test-helpers.nix { inherit lib pkgs self; }).target; extraPythonPackages = _p: [ - portUtils + (import ./test-helpers.nix { inherit lib pkgs self; }).nixosTestLib self.legacyPackages.${pkgs.system}.setupNixInNixPythonPackage ]; @@ -229,56 +175,20 @@ in import tempfile import os import subprocess - from port_utils import find_free_port, setup_port_forwarding # type: ignore[import-untyped] - from setup_nix_in_nix import setup_nix_in_nix # type: ignore[import-untyped] + from nixos_test_lib.ssh import setup_test_environment # type: ignore[import-untyped] + from nixos_test_lib.machine import create_test_machine # type: ignore[import-untyped] - def create_test_machine(oldmachine=None, **kwargs): - """Create a new test machine from an installed disk image""" - start_command = [ - "${pkgs.qemu_test}/bin/qemu-kvm", - "-cpu", "max", - "-m", "3048", - "-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store", - "-drive", f"file={oldmachine.state_dir}/target.qcow2,id=drive1,if=none,index=1,werror=report", - "-device", "virtio-blk-pci,drive=drive1", - "-netdev", "user,id=net0", - "-device", "virtio-net-pci,netdev=net0", - ] - machine = create_machine(start_command=" ".join(start_command), **kwargs) - driver.machines.append(machine) - return machine - - if "NIX_REMOTE" in os.environ: - del os.environ["NIX_REMOTE"] # we don't have any nix daemon running target.start() - # Set up SSH key on host (test driver environment) + # Set up test environment using common helper with tempfile.TemporaryDirectory() as temp_dir: - - # Set up nix chroot store environment - os.environ["closureInfo"] = "${closureInfo}" - os.environ["TMPDIR"] = temp_dir - - # Run setup function - setup_nix_in_nix() - - - host_port = find_free_port() - target.wait_for_unit("sshd.service") - target.wait_for_open_port(22) - - setup_port_forwarding(target, host_port) - - ssh_key = os.path.join(temp_dir, "id_ed25519") - with open(ssh_key, "w") as f: - with open("${../assets/ssh/privkey}", "r") as src: - f.write(src.read()) - os.chmod(ssh_key, 0o600) - - # Copy test flake to temp directory - flake_dir = os.path.join(temp_dir, "test-flake") - subprocess.run(["cp", "-r", "${self.checks.x86_64-linux.clan-core-for-checks}", flake_dir], check=True) - subprocess.run(["chmod", "-R", "+w", flake_dir], check=True) + env = setup_test_environment( + target, + temp_dir, + "${closureInfo}", + "${../assets/ssh/privkey}", + "${self.checks.x86_64-linux.clan-core-for-checks}" + ) # Run clan install from host using port forwarding clan_cmd = [ @@ -289,16 +199,13 @@ in "--debug", "--flake", env.flake_dir, "--yes", "test-install-machine-without-system", - "--target-host", f"nonrootuser@localhost:{host_port}", - "-i", ssh_key, + "--target-host", f"nonrootuser@localhost:{env.host_port}", + "-i", env.ssh_key, "--option", "store", os.environ['CLAN_TEST_STORE'], "--update-hardware-config", "nixos-facter", ] - # Set NIX_CONFIG to disable cache.nixos.org for speed - env = os.environ.copy() - env["NIX_CONFIG"] = "substituters = \ntrusted-public-keys = " - subprocess.run(clan_cmd, check=True, env=env) + subprocess.run(clan_cmd, check=True) # Shutdown the installer machine gracefully try: @@ -308,7 +215,7 @@ in pass # Create a new machine instance that boots from the installed system - installed_machine = create_test_machine(oldmachine=target, name="after_install") + installed_machine = create_test_machine(target, "${pkgs.qemu_test}", name="after_install") installed_machine.start() installed_machine.wait_for_unit("multi-user.target") installed_machine.succeed("test -f /etc/install-successful") @@ -317,9 +224,9 @@ in nixos-test-update-hardware-configuration = self.clanLib.test.baseTest { name = "update-hardware-configuration"; - nodes.target = target; + nodes.target = (import ./test-helpers.nix { inherit lib pkgs self; }).target; extraPythonPackages = _p: [ - portUtils + (import ./test-helpers.nix { inherit lib pkgs self; }).nixosTestLib self.legacyPackages.${pkgs.system}.setupNixInNixPythonPackage ]; @@ -327,44 +234,29 @@ in import tempfile import os import subprocess - from port_utils import find_free_port, setup_port_forwarding # type: ignore[import-untyped] - from setup_nix_in_nix import setup_nix_in_nix # type: ignore[import-untyped] - - # Keep reference to closureInfo: ${closureInfo} - - # Set up nix chroot store environment - os.environ["closureInfo"] = "${closureInfo}" - - # Run setup function - setup_nix_in_nix() - - host_port = find_free_port() + from nixos_test_lib.ssh import setup_test_environment # type: ignore[import-untyped] target.start() - setup_port_forwarding(target, host_port) - - # Set up SSH key and flake on host + # Set up test environment using common helper with tempfile.TemporaryDirectory() as temp_dir: - ssh_key = os.path.join(temp_dir, "id_ed25519") - with open(ssh_key, "w") as f: - with open("${../assets/ssh/privkey}", "r") as src: - f.write(src.read()) - os.chmod(ssh_key, 0o600) - - flake_dir = os.path.join(temp_dir, "test-flake") - subprocess.run(["cp", "-r", "${self.checks.x86_64-linux.clan-core-for-checks}", flake_dir], check=True) - subprocess.run(["chmod", "-R", "+w", flake_dir], check=True) + env = setup_test_environment( + target, + temp_dir, + "${closureInfo}", + "${../assets/ssh/privkey}", + "${self.checks.x86_64-linux.clan-core-for-checks}" + ) # Verify files don't exist initially - hw_config_file = os.path.join(flake_dir, "machines/test-install-machine/hardware-configuration.nix") - facter_file = os.path.join(flake_dir, "machines/test-install-machine/facter.json") + hw_config_file = os.path.join(env.flake_dir, "machines/test-install-machine/hardware-configuration.nix") + facter_file = os.path.join(env.flake_dir, "machines/test-install-machine/facter.json") assert not os.path.exists(hw_config_file), "hardware-configuration.nix should not exist initially" assert not os.path.exists(facter_file), "facter.json should not exist initially" - target.wait_for_unit("sshd.service") - target.wait_for_open_port(22) + # Set CLAN_FLAKE for the commands + os.environ["CLAN_FLAKE"] = env.flake_dir # Test facter backend clan_cmd = [ @@ -375,16 +267,12 @@ in "--flake", ".", "--host-key-check", "none", "test-install-machine-without-system", - "-i", ssh_key, + "-i", env.ssh_key, "--option", "store", os.environ['CLAN_TEST_STORE'], f"nonrootuser@localhost:{env.host_port}" ] - env = os.environ.copy() - env["CLAN_FLAKE"] = flake_dir - # Set NIX_CONFIG to disable cache.nixos.org for speed - env["NIX_CONFIG"] = "substituters = \ntrusted-public-keys = " - result = subprocess.run(clan_cmd, capture_output=True, cwd=flake_dir, env=env) + result = subprocess.run(clan_cmd, capture_output=True, cwd=env.flake_dir) if result.returncode != 0: print(f"Clan update-hardware-config failed: {result.stderr.decode()}") raise Exception(f"Clan update-hardware-config failed with return code {result.returncode}") @@ -403,16 +291,12 @@ in "--host-key-check", "none", "--flake", ".", "test-install-machine-without-system", - "--option", "ssh-option", f"-i {ssh_key} -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null", + "-i", env.ssh_key, "--option", "store", os.environ['CLAN_TEST_STORE'], f"nonrootuser@localhost:{env.host_port}" ] - env = os.environ.copy() - env["CLAN_FLAKE"] = flake_dir - # Set NIX_CONFIG to disable cache.nixos.org for speed - env["NIX_CONFIG"] = "substituters = \ntrusted-public-keys = " - result = subprocess.run(clan_cmd, capture_output=True, cwd=flake_dir, env=env) + result = subprocess.run(clan_cmd, capture_output=True, cwd=env.flake_dir) if result.returncode != 0: print(f"Clan update-hardware-config (nixos-generate-config) failed: {result.stderr.decode()}") raise Exception(f"Clan update-hardware-config failed with return code {result.returncode}") diff --git a/checks/installation/nixos_test_lib/__init__.py b/checks/installation/nixos_test_lib/__init__.py new file mode 100644 index 000000000..ded39654b --- /dev/null +++ b/checks/installation/nixos_test_lib/__init__.py @@ -0,0 +1,3 @@ +"""NixOS test library for clan VM testing""" + +__version__ = "1.0.0" diff --git a/checks/installation/nixos_test_lib/environment.py b/checks/installation/nixos_test_lib/environment.py new file mode 100644 index 000000000..c66be466e --- /dev/null +++ b/checks/installation/nixos_test_lib/environment.py @@ -0,0 +1,13 @@ +"""Environment setup utilities for VM tests""" + +import os + + +def setup_nix_environment(temp_dir: str, closure_info: str) -> None: + """Set up nix chroot store environment""" + if "NIX_REMOTE" in os.environ: + del os.environ["NIX_REMOTE"] # we don't have any nix daemon running + + os.environ["TMPDIR"] = temp_dir + # Set NIX_CONFIG globally to disable substituters for speed + os.environ["NIX_CONFIG"] = "substituters = \ntrusted-public-keys = " diff --git a/checks/installation/nixos_test_lib/machine.py b/checks/installation/nixos_test_lib/machine.py new file mode 100644 index 000000000..c20bad723 --- /dev/null +++ b/checks/installation/nixos_test_lib/machine.py @@ -0,0 +1,25 @@ +"""VM machine management utilities""" + + +def create_test_machine(oldmachine, qemu_test_bin: str, **kwargs): + """Create a new test machine from an installed disk image""" + start_command = [ + f"{qemu_test_bin}/bin/qemu-kvm", + "-cpu", + "max", + "-m", + "3048", + "-virtfs", + "local,path=/nix/store,security_model=none,mount_tag=nix-store", + "-drive", + f"file={oldmachine.state_dir}/target.qcow2,id=drive1,if=none,index=1,werror=report", + "-device", + "virtio-blk-pci,drive=drive1", + "-netdev", + "user,id=net0", + "-device", + "virtio-net-pci,netdev=net0", + ] + machine = create_machine(start_command=" ".join(start_command), **kwargs) + driver.machines.append(machine) + return machine diff --git a/checks/installation/port_utils.py b/checks/installation/nixos_test_lib/port.py similarity index 100% rename from checks/installation/port_utils.py rename to checks/installation/nixos_test_lib/port.py diff --git a/checks/installation/nixos_test_lib/py.typed b/checks/installation/nixos_test_lib/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/checks/installation/nixos_test_lib/ssh.py b/checks/installation/nixos_test_lib/ssh.py new file mode 100644 index 000000000..367fd4bb0 --- /dev/null +++ b/checks/installation/nixos_test_lib/ssh.py @@ -0,0 +1,53 @@ +"""SSH and test setup utilities""" + +import os +import subprocess +from typing import NamedTuple + +from .environment import setup_nix_environment +from .port import find_free_port, setup_port_forwarding + + +class TestEnvironment(NamedTuple): + host_port: int + ssh_key: str + flake_dir: str + + +def setup_test_environment( + target, + temp_dir: str, + closure_info: str, + assets_ssh_privkey: str, + clan_core_for_checks: str, +) -> TestEnvironment: + """Set up common test environment including SSH, port forwarding, and flake setup + + Returns: + TestEnvironment with host_port, ssh_key, and flake_dir + """ + from setup_nix_in_nix import setup_nix_in_nix # type: ignore[import-untyped] + + setup_nix_environment(temp_dir, closure_info) + + # Run setup function + setup_nix_in_nix() + + host_port = find_free_port() + target.wait_for_unit("sshd.service") + target.wait_for_open_port(22) + + setup_port_forwarding(target, host_port) + + ssh_key = os.path.join(temp_dir, "id_ed25519") + with open(ssh_key, "w") as f: + with open(assets_ssh_privkey) as src: + f.write(src.read()) + os.chmod(ssh_key, 0o600) + + # Copy test flake to temp directory + flake_dir = os.path.join(temp_dir, "test-flake") + subprocess.run(["cp", "-r", clan_core_for_checks, flake_dir], check=True) + subprocess.run(["chmod", "-R", "+w", flake_dir], check=True) + + return TestEnvironment(host_port, ssh_key, flake_dir) diff --git a/checks/installation/pyproject.toml b/checks/installation/pyproject.toml new file mode 100644 index 000000000..b82e1bc08 --- /dev/null +++ b/checks/installation/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "nixos-test-lib" +version = "1.0.0" +description = "NixOS test utilities for clan VM testing" +authors = [ + {name = "Clan Core Team"} +] +dependencies = [] + +[project.optional-dependencies] +dev = [ + "mypy", + "ruff" +] + +[tool.setuptools.packages.find] +where = ["."] +include = ["nixos_test_lib*"] + +[tool.setuptools.package-data] +"nixos_test_lib" = ["py.typed"] + +[tool.mypy] +python_version = "3.12" +strict = true +warn_return_any = true +warn_unused_configs = true + +[tool.ruff] +target-version = "py312" +line-length = 88 + +[tool.ruff.lint] +select = ["ALL"] +ignore = [ + "D", # docstrings + "ANN", # type annotations + "COM812", # trailing comma + "ISC001", # string concatenation +] \ No newline at end of file diff --git a/checks/installation/test-helpers.nix b/checks/installation/test-helpers.nix new file mode 100644 index 000000000..dddf81d4f --- /dev/null +++ b/checks/installation/test-helpers.nix @@ -0,0 +1,173 @@ +{ + lib, + pkgs, + self, + ... +}: +let + # Common target VM configuration used by both installation and update tests + target = + { modulesPath, pkgs, ... }: + { + imports = [ + (modulesPath + "/../tests/common/auto-format-root-device.nix") + ]; + networking.useNetworkd = true; + services.openssh.enable = true; + services.openssh.settings.UseDns = false; + services.openssh.settings.PasswordAuthentication = false; + system.nixos.variant_id = "installer"; + environment.systemPackages = [ + pkgs.nixos-facter + ]; + # Disable cache.nixos.org to speed up tests + nix.settings.substituters = [ ]; + nix.settings.trusted-public-keys = [ ]; + virtualisation.emptyDiskImages = [ 512 ]; + virtualisation.diskSize = 8 * 1024; + virtualisation.rootDevice = "/dev/vdb"; + # both installer and target need to use the same diskImage + virtualisation.diskImage = "./target.qcow2"; + virtualisation.memorySize = 3048; + users.users.nonrootuser = { + isNormalUser = true; + openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ]; + extraGroups = [ "wheel" ]; + }; + users.users.root.openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ]; + # Allow users to manage their own SSH keys + services.openssh.authorizedKeysFiles = [ + "/root/.ssh/authorized_keys" + "/home/%u/.ssh/authorized_keys" + "/etc/ssh/authorized_keys.d/%u" + ]; + security.sudo.wheelNeedsPassword = false; + }; + + # Common base test machine configuration + baseTestMachine = + { lib, modulesPath, ... }: + { + imports = [ + (modulesPath + "/testing/test-instrumentation.nix") + (modulesPath + "/profiles/qemu-guest.nix") + self.clanLib.test.minifyModule + ]; + + # Enable SSH and add authorized key for testing + services.openssh.enable = true; + services.openssh.settings.PasswordAuthentication = false; + users.users.nonrootuser = { + isNormalUser = true; + openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ]; + extraGroups = [ "wheel" ]; + home = "/home/nonrootuser"; + createHome = true; + }; + users.users.root.openssh.authorizedKeys.keys = [ (builtins.readFile ../assets/ssh/pubkey) ]; + # Allow users to manage their own SSH keys + services.openssh.authorizedKeysFiles = [ + "/root/.ssh/authorized_keys" + "/home/%u/.ssh/authorized_keys" + "/etc/ssh/authorized_keys.d/%u" + ]; + security.sudo.wheelNeedsPassword = false; + + boot.consoleLogLevel = lib.mkForce 100; + boot.kernelParams = [ "boot.shell_on_fail" ]; + + # disko config + boot.loader.grub.efiSupport = lib.mkDefault true; + boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true; + clan.core.vars.settings.secretStore = "vm"; + clan.core.vars.generators.test = { + files.test.neededFor = "partitioning"; + script = '' + echo "notok" > "$out"/test + ''; + }; + disko.devices = { + disk = { + main = { + type = "disk"; + device = "/dev/vda"; + + preCreateHook = '' + test -e /run/partitioning-secrets/test/test + ''; + + content = { + type = "gpt"; + partitions = { + boot = { + size = "1M"; + type = "EF02"; # for grub MBR + priority = 1; + }; + ESP = { + size = "512M"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + mountOptions = [ "umask=0077" ]; + }; + }; + root = { + size = "100%"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + }; + }; + }; + }; + }; + }; + }; + }; + + # NixOS test library combining port utils and clan VM test utilities + nixosTestLib = pkgs.python3Packages.buildPythonPackage { + pname = "nixos-test-lib"; + version = "1.0.0"; + format = "pyproject"; + src = lib.fileset.toSource { + root = ./.; + fileset = lib.fileset.unions [ + ./pyproject.toml + ./nixos_test_lib + ]; + }; + nativeBuildInputs = with pkgs.python3Packages; [ + setuptools + wheel + ]; + doCheck = false; + }; + + # Common closure info + closureInfo = pkgs.closureInfo { + rootPaths = [ + self.checks.x86_64-linux.clan-core-for-checks + self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.toplevel + self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.initialRamdisk + self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.diskoScript + self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.clan.deployment.file + pkgs.stdenv.drvPath + pkgs.bash.drvPath + pkgs.buildPackages.xorg.lndir + ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs); + }; + +in +{ + inherit + target + baseTestMachine + nixosTestLib + closureInfo + ; +} From c148ece02e8467ea47f8745b80dbe2a98f2d2956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 2 Jul 2025 16:01:28 +0200 Subject: [PATCH 007/258] move setup_nix_in_nix into nixos_test_lib --- checks/installation/flake-module.nix | 2 - .../installation/nixos_test_lib/nix_setup.py | 70 +++++++++++++++++++ checks/installation/nixos_test_lib/ssh.py | 8 +-- .../service-dummy-test-from-flake/default.nix | 4 +- pkgs/testing/flake-module.nix | 51 -------------- 5 files changed, 74 insertions(+), 61 deletions(-) create mode 100644 checks/installation/nixos_test_lib/nix_setup.py diff --git a/checks/installation/flake-module.nix b/checks/installation/flake-module.nix index 4d19bed2b..e297d88ad 100644 --- a/checks/installation/flake-module.nix +++ b/checks/installation/flake-module.nix @@ -168,7 +168,6 @@ nodes.target = (import ./test-helpers.nix { inherit lib pkgs self; }).target; extraPythonPackages = _p: [ (import ./test-helpers.nix { inherit lib pkgs self; }).nixosTestLib - self.legacyPackages.${pkgs.system}.setupNixInNixPythonPackage ]; testScript = '' @@ -227,7 +226,6 @@ nodes.target = (import ./test-helpers.nix { inherit lib pkgs self; }).target; extraPythonPackages = _p: [ (import ./test-helpers.nix { inherit lib pkgs self; }).nixosTestLib - self.legacyPackages.${pkgs.system}.setupNixInNixPythonPackage ]; testScript = '' diff --git a/checks/installation/nixos_test_lib/nix_setup.py b/checks/installation/nixos_test_lib/nix_setup.py new file mode 100644 index 000000000..5e518bcfd --- /dev/null +++ b/checks/installation/nixos_test_lib/nix_setup.py @@ -0,0 +1,70 @@ +"""Nix store setup utilities for VM tests""" + +import os +import shutil +import subprocess +from pathlib import Path + + +def setup_nix_in_nix(closure_info: str) -> None: + """Set up Nix store inside test environment + + Args: + closure_info: Path to closure info directory containing store-paths file + """ + tmpdir = os.environ.get("TMPDIR", "/tmp") + + # Remove NIX_REMOTE if present (we don't have any nix daemon running) + if "NIX_REMOTE" in os.environ: + del os.environ["NIX_REMOTE"] + + # Set NIX_CONFIG globally to disable substituters for speed + os.environ["NIX_CONFIG"] = "substituters = \ntrusted-public-keys = " + + # Set up environment variables for test environment + os.environ["HOME"] = tmpdir + os.environ["NIX_STATE_DIR"] = f"{tmpdir}/nix" + os.environ["NIX_CONF_DIR"] = f"{tmpdir}/etc" + os.environ["IN_NIX_SANDBOX"] = "1" + os.environ["CLAN_TEST_STORE"] = f"{tmpdir}/store" + os.environ["LOCK_NIX"] = f"{tmpdir}/nix_lock" + + # Create necessary directories + Path(f"{tmpdir}/nix").mkdir(parents=True, exist_ok=True) + Path(f"{tmpdir}/etc").mkdir(parents=True, exist_ok=True) + Path(f"{tmpdir}/store").mkdir(parents=True, exist_ok=True) + + # Set up Nix store if closure info is provided + if closure_info and os.path.exists(closure_info): + store_paths_file = os.path.join(closure_info, "store-paths") + if os.path.exists(store_paths_file): + with open(store_paths_file) as f: + store_paths = f.read().strip().split("\n") + + # Copy store paths to test store + for store_path in store_paths: + if store_path.strip(): + dest_path = f"{tmpdir}/store{store_path}" + if not os.path.exists(dest_path): + # Create parent directories + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + # Copy the store path + if os.path.isdir(store_path): + shutil.copytree(store_path, dest_path, dirs_exist_ok=True) + else: + shutil.copy2(store_path, dest_path) + + # Load Nix database + registration_file = os.path.join(closure_info, "registration") + if os.path.exists(registration_file): + env = os.environ.copy() + env["NIX_REMOTE"] = f"local?store={tmpdir}/store" + + with open(registration_file) as f: + subprocess.run( + ["nix-store", "--load-db"], + input=f.read(), + text=True, + env=env, + check=True, + ) diff --git a/checks/installation/nixos_test_lib/ssh.py b/checks/installation/nixos_test_lib/ssh.py index 367fd4bb0..645a0769a 100644 --- a/checks/installation/nixos_test_lib/ssh.py +++ b/checks/installation/nixos_test_lib/ssh.py @@ -4,7 +4,7 @@ import os import subprocess from typing import NamedTuple -from .environment import setup_nix_environment +from .nix_setup import setup_nix_in_nix from .port import find_free_port, setup_port_forwarding @@ -26,12 +26,8 @@ def setup_test_environment( Returns: TestEnvironment with host_port, ssh_key, and flake_dir """ - from setup_nix_in_nix import setup_nix_in_nix # type: ignore[import-untyped] - - setup_nix_environment(temp_dir, closure_info) - # Run setup function - setup_nix_in_nix() + setup_nix_in_nix(closure_info) host_port = find_free_port() target.wait_for_unit("sshd.service") diff --git a/checks/service-dummy-test-from-flake/default.nix b/checks/service-dummy-test-from-flake/default.nix index 0faa8f9b1..aaf2444b6 100644 --- a/checks/service-dummy-test-from-flake/default.nix +++ b/checks/service-dummy-test-from-flake/default.nix @@ -29,8 +29,8 @@ nixosLib.runTest ( testScript = { nodes, ... }: '' - from setup_nix_in_nix import setup_nix_in_nix # type: ignore[import-untyped] - setup_nix_in_nix() + from nixos_test_lib.nix_setup import setup_nix_in_nix # type: ignore[import-untyped] + setup_nix_in_nix(None) # No closure info for this test def run_clan(cmd: list[str], **kwargs) -> str: import subprocess diff --git a/pkgs/testing/flake-module.nix b/pkgs/testing/flake-module.nix index 51c39729a..11c81bfd9 100644 --- a/pkgs/testing/flake-module.nix +++ b/pkgs/testing/flake-module.nix @@ -21,57 +21,6 @@ fi ''; - setupNixInNixPythonPackage = pkgs.python3Packages.buildPythonPackage { - pname = "setup-nix-in-nix"; - version = "1.0.0"; - format = "other"; - - dontUnpack = true; - - installPhase = '' - mkdir -p $out/${pkgs.python3.sitePackages} - cat > $out/${pkgs.python3.sitePackages}/setup_nix_in_nix.py << 'EOF' - from os import environ - import subprocess - from pathlib import Path - - def setup_nix_in_nix(): - """Set up a Nix store inside the test environment.""" - environ['HOME'] = environ['TMPDIR'] - environ['NIX_STATE_DIR'] = environ['TMPDIR'] + '/nix' - environ['NIX_CONF_DIR'] = environ['TMPDIR'] + '/etc' - environ['IN_NIX_SANDBOX'] = '1' - environ['CLAN_TEST_STORE'] = environ['TMPDIR'] + '/store' - environ['LOCK_NIX'] = environ['TMPDIR'] + '/nix_lock' - - Path(environ['CLAN_TEST_STORE'] + '/nix/store').mkdir(parents=True, exist_ok=True) - Path(environ['CLAN_TEST_STORE'] + '/nix/var/nix/gcroots').mkdir(parents=True, exist_ok=True) - - if 'closureInfo' in environ: - # Read store paths from the closure info file - with open(environ['closureInfo'] + '/store-paths', 'r') as f: - store_paths = f.read().strip().split('\n') - - # Copy store paths using absolute path to cp - subprocess.run( - ['${pkgs.coreutils}/bin/cp', '--recursive', '--target', environ['CLAN_TEST_STORE'] + '/nix/store'] + store_paths, - check=True - ) - - # Load the nix database using absolute path to nix-store - with open(environ['closureInfo'] + '/registration', 'r') as f: - subprocess.run( - ['${pkgs.nix}/bin/nix-store', '--load-db', '--store', environ['CLAN_TEST_STORE']], - input=f.read(), - text=True, - check=True - ) - EOF - touch $out/${pkgs.python3.sitePackages}/py.typed - ''; - - doCheck = false; - }; }; }; } From 1e7453ab04673244a12e64ae1880e52c9f6d6547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 2 Jul 2025 16:03:01 +0200 Subject: [PATCH 008/258] move nixosTestLib to pkgs/testing --- checks/installation/flake-module.nix | 4 +- .../nixos_test_lib/environment.py | 13 ------ .../service-dummy-test-from-flake/default.nix | 2 +- pkgs/testing/flake-module.nix | 21 ++++++++- .../testing}/nixos_test_lib/__init__.py | 0 pkgs/testing/nixos_test_lib/machine.py | 25 +++++++++++ .../testing}/nixos_test_lib/nix_setup.py | 16 +++---- .../testing}/nixos_test_lib/port.py | 0 .../testing}/nixos_test_lib/py.typed | 0 .../testing}/nixos_test_lib/ssh.py | 1 - pkgs/testing/pyproject.toml | 44 +++++++++++++++++++ 11 files changed, 100 insertions(+), 26 deletions(-) delete mode 100644 checks/installation/nixos_test_lib/environment.py rename {checks/installation => pkgs/testing}/nixos_test_lib/__init__.py (100%) create mode 100644 pkgs/testing/nixos_test_lib/machine.py rename {checks/installation => pkgs/testing}/nixos_test_lib/nix_setup.py (84%) rename {checks/installation => pkgs/testing}/nixos_test_lib/port.py (100%) rename {checks/installation => pkgs/testing}/nixos_test_lib/py.typed (100%) rename {checks/installation => pkgs/testing}/nixos_test_lib/ssh.py (99%) create mode 100644 pkgs/testing/pyproject.toml diff --git a/checks/installation/flake-module.nix b/checks/installation/flake-module.nix index e297d88ad..6041d082e 100644 --- a/checks/installation/flake-module.nix +++ b/checks/installation/flake-module.nix @@ -167,7 +167,7 @@ name = "installation"; nodes.target = (import ./test-helpers.nix { inherit lib pkgs self; }).target; extraPythonPackages = _p: [ - (import ./test-helpers.nix { inherit lib pkgs self; }).nixosTestLib + self.legacyPackages.${pkgs.system}.nixosTestLib ]; testScript = '' @@ -225,7 +225,7 @@ name = "update-hardware-configuration"; nodes.target = (import ./test-helpers.nix { inherit lib pkgs self; }).target; extraPythonPackages = _p: [ - (import ./test-helpers.nix { inherit lib pkgs self; }).nixosTestLib + self.legacyPackages.${pkgs.system}.nixosTestLib ]; testScript = '' diff --git a/checks/installation/nixos_test_lib/environment.py b/checks/installation/nixos_test_lib/environment.py deleted file mode 100644 index c66be466e..000000000 --- a/checks/installation/nixos_test_lib/environment.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Environment setup utilities for VM tests""" - -import os - - -def setup_nix_environment(temp_dir: str, closure_info: str) -> None: - """Set up nix chroot store environment""" - if "NIX_REMOTE" in os.environ: - del os.environ["NIX_REMOTE"] # we don't have any nix daemon running - - os.environ["TMPDIR"] = temp_dir - # Set NIX_CONFIG globally to disable substituters for speed - os.environ["NIX_CONFIG"] = "substituters = \ntrusted-public-keys = " diff --git a/checks/service-dummy-test-from-flake/default.nix b/checks/service-dummy-test-from-flake/default.nix index aaf2444b6..324e2f087 100644 --- a/checks/service-dummy-test-from-flake/default.nix +++ b/checks/service-dummy-test-from-flake/default.nix @@ -23,7 +23,7 @@ nixosLib.runTest ( clan.test.fromFlake = ./.; extraPythonPackages = _p: [ - clan-core.legacyPackages.${hostPkgs.system}.setupNixInNixPythonPackage + clan-core.legacyPackages.${hostPkgs.system}.nixosTestLib ]; testScript = diff --git a/pkgs/testing/flake-module.nix b/pkgs/testing/flake-module.nix index 11c81bfd9..c30aaf383 100644 --- a/pkgs/testing/flake-module.nix +++ b/pkgs/testing/flake-module.nix @@ -1,6 +1,6 @@ { perSystem = - { pkgs, ... }: + { pkgs, lib, ... }: { legacyPackages = { setupNixInNix = '' @@ -21,6 +21,25 @@ fi ''; + # NixOS test library combining port utils and clan VM test utilities + nixosTestLib = pkgs.python3Packages.buildPythonPackage { + pname = "nixos-test-lib"; + version = "1.0.0"; + format = "pyproject"; + src = lib.fileset.toSource { + root = ./.; + fileset = lib.fileset.unions [ + ./pyproject.toml + ./nixos_test_lib + ]; + }; + nativeBuildInputs = with pkgs.python3Packages; [ + setuptools + wheel + ]; + doCheck = false; + }; + }; }; } diff --git a/checks/installation/nixos_test_lib/__init__.py b/pkgs/testing/nixos_test_lib/__init__.py similarity index 100% rename from checks/installation/nixos_test_lib/__init__.py rename to pkgs/testing/nixos_test_lib/__init__.py diff --git a/pkgs/testing/nixos_test_lib/machine.py b/pkgs/testing/nixos_test_lib/machine.py new file mode 100644 index 000000000..c20bad723 --- /dev/null +++ b/pkgs/testing/nixos_test_lib/machine.py @@ -0,0 +1,25 @@ +"""VM machine management utilities""" + + +def create_test_machine(oldmachine, qemu_test_bin: str, **kwargs): + """Create a new test machine from an installed disk image""" + start_command = [ + f"{qemu_test_bin}/bin/qemu-kvm", + "-cpu", + "max", + "-m", + "3048", + "-virtfs", + "local,path=/nix/store,security_model=none,mount_tag=nix-store", + "-drive", + f"file={oldmachine.state_dir}/target.qcow2,id=drive1,if=none,index=1,werror=report", + "-device", + "virtio-blk-pci,drive=drive1", + "-netdev", + "user,id=net0", + "-device", + "virtio-net-pci,netdev=net0", + ] + machine = create_machine(start_command=" ".join(start_command), **kwargs) + driver.machines.append(machine) + return machine diff --git a/checks/installation/nixos_test_lib/nix_setup.py b/pkgs/testing/nixos_test_lib/nix_setup.py similarity index 84% rename from checks/installation/nixos_test_lib/nix_setup.py rename to pkgs/testing/nixos_test_lib/nix_setup.py index 5e518bcfd..5fc9d1d69 100644 --- a/checks/installation/nixos_test_lib/nix_setup.py +++ b/pkgs/testing/nixos_test_lib/nix_setup.py @@ -12,7 +12,7 @@ def setup_nix_in_nix(closure_info: str) -> None: Args: closure_info: Path to closure info directory containing store-paths file """ - tmpdir = os.environ.get("TMPDIR", "/tmp") + tmpdir = Path(os.environ.get("TMPDIR", "/tmp")) # Remove NIX_REMOTE if present (we don't have any nix daemon running) if "NIX_REMOTE" in os.environ: @@ -35,10 +35,10 @@ def setup_nix_in_nix(closure_info: str) -> None: Path(f"{tmpdir}/store").mkdir(parents=True, exist_ok=True) # Set up Nix store if closure info is provided - if closure_info and os.path.exists(closure_info): - store_paths_file = os.path.join(closure_info, "store-paths") - if os.path.exists(store_paths_file): - with open(store_paths_file) as f: + if closure_info and Path(closure_info).exists(): + store_paths_file = Path(closure_info) / "store-paths" + if store_paths_file.exists(): + with store_paths_file.open() as f: store_paths = f.read().strip().split("\n") # Copy store paths to test store @@ -55,12 +55,12 @@ def setup_nix_in_nix(closure_info: str) -> None: shutil.copy2(store_path, dest_path) # Load Nix database - registration_file = os.path.join(closure_info, "registration") - if os.path.exists(registration_file): + registration_file = Path(closure_info) / "registration" + if registration_file.exists(): env = os.environ.copy() env["NIX_REMOTE"] = f"local?store={tmpdir}/store" - with open(registration_file) as f: + with registration_file.open() as f: subprocess.run( ["nix-store", "--load-db"], input=f.read(), diff --git a/checks/installation/nixos_test_lib/port.py b/pkgs/testing/nixos_test_lib/port.py similarity index 100% rename from checks/installation/nixos_test_lib/port.py rename to pkgs/testing/nixos_test_lib/port.py diff --git a/checks/installation/nixos_test_lib/py.typed b/pkgs/testing/nixos_test_lib/py.typed similarity index 100% rename from checks/installation/nixos_test_lib/py.typed rename to pkgs/testing/nixos_test_lib/py.typed diff --git a/checks/installation/nixos_test_lib/ssh.py b/pkgs/testing/nixos_test_lib/ssh.py similarity index 99% rename from checks/installation/nixos_test_lib/ssh.py rename to pkgs/testing/nixos_test_lib/ssh.py index 645a0769a..ae6a84f89 100644 --- a/checks/installation/nixos_test_lib/ssh.py +++ b/pkgs/testing/nixos_test_lib/ssh.py @@ -1,6 +1,5 @@ """SSH and test setup utilities""" -import os import subprocess from typing import NamedTuple diff --git a/pkgs/testing/pyproject.toml b/pkgs/testing/pyproject.toml new file mode 100644 index 000000000..b82e1bc08 --- /dev/null +++ b/pkgs/testing/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "nixos-test-lib" +version = "1.0.0" +description = "NixOS test utilities for clan VM testing" +authors = [ + {name = "Clan Core Team"} +] +dependencies = [] + +[project.optional-dependencies] +dev = [ + "mypy", + "ruff" +] + +[tool.setuptools.packages.find] +where = ["."] +include = ["nixos_test_lib*"] + +[tool.setuptools.package-data] +"nixos_test_lib" = ["py.typed"] + +[tool.mypy] +python_version = "3.12" +strict = true +warn_return_any = true +warn_unused_configs = true + +[tool.ruff] +target-version = "py312" +line-length = 88 + +[tool.ruff.lint] +select = ["ALL"] +ignore = [ + "D", # docstrings + "ANN", # type annotations + "COM812", # trailing comma + "ISC001", # string concatenation +] \ No newline at end of file From 68b2aaea8951bf08f397891a80de11a27931964c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 2 Jul 2025 16:10:24 +0200 Subject: [PATCH 009/258] setup_nix_in_nix: use cp intead of shutil it's faster and handles symlinks --- pkgs/testing/nixos_test_lib/nix_setup.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/pkgs/testing/nixos_test_lib/nix_setup.py b/pkgs/testing/nixos_test_lib/nix_setup.py index 5fc9d1d69..2bd3b0f1f 100644 --- a/pkgs/testing/nixos_test_lib/nix_setup.py +++ b/pkgs/testing/nixos_test_lib/nix_setup.py @@ -6,11 +6,11 @@ import subprocess from pathlib import Path -def setup_nix_in_nix(closure_info: str) -> None: +def setup_nix_in_nix(closure_info: str | None) -> None: """Set up Nix store inside test environment Args: - closure_info: Path to closure info directory containing store-paths file + closure_info: Path to closure info directory containing store-paths file, or None if no closure info """ tmpdir = Path(os.environ.get("TMPDIR", "/tmp")) @@ -41,18 +41,12 @@ def setup_nix_in_nix(closure_info: str) -> None: with store_paths_file.open() as f: store_paths = f.read().strip().split("\n") - # Copy store paths to test store - for store_path in store_paths: - if store_path.strip(): - dest_path = f"{tmpdir}/store{store_path}" - if not os.path.exists(dest_path): - # Create parent directories - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - # Copy the store path - if os.path.isdir(store_path): - shutil.copytree(store_path, dest_path, dirs_exist_ok=True) - else: - shutil.copy2(store_path, dest_path) + # Copy store paths to test store using external cp command (handles symlinks better) + subprocess.run( + ["cp", "--recursive", "--target", f"{tmpdir}/store"] + + [path.strip() for path in store_paths if path.strip()], + check=True + ) # Load Nix database registration_file = Path(closure_info) / "registration" From ea93d8fec7a3876b33144f3a1d6ed44f76d24c1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 2 Jul 2025 16:14:45 +0200 Subject: [PATCH 010/258] inline create_test_machine again --- checks/installation/flake-module.nix | 24 +++++++++++++++++- checks/installation/nixos_test_lib/machine.py | 25 ------------------- pkgs/testing/nixos_test_lib/machine.py | 25 ------------------- 3 files changed, 23 insertions(+), 51 deletions(-) delete mode 100644 checks/installation/nixos_test_lib/machine.py delete mode 100644 pkgs/testing/nixos_test_lib/machine.py diff --git a/checks/installation/flake-module.nix b/checks/installation/flake-module.nix index 6041d082e..163e6e653 100644 --- a/checks/installation/flake-module.nix +++ b/checks/installation/flake-module.nix @@ -175,7 +175,29 @@ import os import subprocess from nixos_test_lib.ssh import setup_test_environment # type: ignore[import-untyped] - from nixos_test_lib.machine import create_test_machine # type: ignore[import-untyped] + + def create_test_machine(oldmachine, qemu_test_bin: str, **kwargs): + """Create a new test machine from an installed disk image""" + start_command = [ + f"{qemu_test_bin}/bin/qemu-kvm", + "-cpu", + "max", + "-m", + "3048", + "-virtfs", + "local,path=/nix/store,security_model=none,mount_tag=nix-store", + "-drive", + f"file={oldmachine.state_dir}/target.qcow2,id=drive1,if=none,index=1,werror=report", + "-device", + "virtio-blk-pci,drive=drive1", + "-netdev", + "user,id=net0", + "-device", + "virtio-net-pci,netdev=net0", + ] + machine = create_machine(start_command=" ".join(start_command), **kwargs) + driver.machines.append(machine) + return machine target.start() diff --git a/checks/installation/nixos_test_lib/machine.py b/checks/installation/nixos_test_lib/machine.py deleted file mode 100644 index c20bad723..000000000 --- a/checks/installation/nixos_test_lib/machine.py +++ /dev/null @@ -1,25 +0,0 @@ -"""VM machine management utilities""" - - -def create_test_machine(oldmachine, qemu_test_bin: str, **kwargs): - """Create a new test machine from an installed disk image""" - start_command = [ - f"{qemu_test_bin}/bin/qemu-kvm", - "-cpu", - "max", - "-m", - "3048", - "-virtfs", - "local,path=/nix/store,security_model=none,mount_tag=nix-store", - "-drive", - f"file={oldmachine.state_dir}/target.qcow2,id=drive1,if=none,index=1,werror=report", - "-device", - "virtio-blk-pci,drive=drive1", - "-netdev", - "user,id=net0", - "-device", - "virtio-net-pci,netdev=net0", - ] - machine = create_machine(start_command=" ".join(start_command), **kwargs) - driver.machines.append(machine) - return machine diff --git a/pkgs/testing/nixos_test_lib/machine.py b/pkgs/testing/nixos_test_lib/machine.py deleted file mode 100644 index c20bad723..000000000 --- a/pkgs/testing/nixos_test_lib/machine.py +++ /dev/null @@ -1,25 +0,0 @@ -"""VM machine management utilities""" - - -def create_test_machine(oldmachine, qemu_test_bin: str, **kwargs): - """Create a new test machine from an installed disk image""" - start_command = [ - f"{qemu_test_bin}/bin/qemu-kvm", - "-cpu", - "max", - "-m", - "3048", - "-virtfs", - "local,path=/nix/store,security_model=none,mount_tag=nix-store", - "-drive", - f"file={oldmachine.state_dir}/target.qcow2,id=drive1,if=none,index=1,werror=report", - "-device", - "virtio-blk-pci,drive=drive1", - "-netdev", - "user,id=net0", - "-device", - "virtio-net-pci,netdev=net0", - ] - machine = create_machine(start_command=" ".join(start_command), **kwargs) - driver.machines.append(machine) - return machine From c509f333e4fd9e038e6fcd27e72b03dafd38626e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 2 Jul 2025 16:22:58 +0200 Subject: [PATCH 011/258] nixosTestLib: fix various linting issues --- pkgs/testing/nixos_test_lib/nix_setup.py | 15 ++++++++------- pkgs/testing/nixos_test_lib/port.py | 2 +- pkgs/testing/nixos_test_lib/ssh.py | 18 +++++++++--------- pkgs/testing/pyproject.toml | 1 + 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/pkgs/testing/nixos_test_lib/nix_setup.py b/pkgs/testing/nixos_test_lib/nix_setup.py index 2bd3b0f1f..b42126b44 100644 --- a/pkgs/testing/nixos_test_lib/nix_setup.py +++ b/pkgs/testing/nixos_test_lib/nix_setup.py @@ -1,7 +1,6 @@ """Nix store setup utilities for VM tests""" import os -import shutil import subprocess from pathlib import Path @@ -10,7 +9,8 @@ def setup_nix_in_nix(closure_info: str | None) -> None: """Set up Nix store inside test environment Args: - closure_info: Path to closure info directory containing store-paths file, or None if no closure info + closure_info: Path to closure info directory containing store-paths file, + or None if no closure info """ tmpdir = Path(os.environ.get("TMPDIR", "/tmp")) @@ -22,7 +22,7 @@ def setup_nix_in_nix(closure_info: str | None) -> None: os.environ["NIX_CONFIG"] = "substituters = \ntrusted-public-keys = " # Set up environment variables for test environment - os.environ["HOME"] = tmpdir + os.environ["HOME"] = str(tmpdir) os.environ["NIX_STATE_DIR"] = f"{tmpdir}/nix" os.environ["NIX_CONF_DIR"] = f"{tmpdir}/etc" os.environ["IN_NIX_SANDBOX"] = "1" @@ -41,11 +41,12 @@ def setup_nix_in_nix(closure_info: str | None) -> None: with store_paths_file.open() as f: store_paths = f.read().strip().split("\n") - # Copy store paths to test store using external cp command (handles symlinks better) + # Copy store paths to test store using external cp command + # (handles symlinks better) subprocess.run( - ["cp", "--recursive", "--target", f"{tmpdir}/store"] + - [path.strip() for path in store_paths if path.strip()], - check=True + ["cp", "--recursive", "--target", f"{tmpdir}/store"] + + [path.strip() for path in store_paths if path.strip()], + check=True, ) # Load Nix database diff --git a/pkgs/testing/nixos_test_lib/port.py b/pkgs/testing/nixos_test_lib/port.py index a836785c4..b81353fa8 100644 --- a/pkgs/testing/nixos_test_lib/port.py +++ b/pkgs/testing/nixos_test_lib/port.py @@ -23,7 +23,7 @@ def check_host_port_open(port: int) -> bool: s.settimeout(1) result = s.connect_ex(("localhost", port)) return result == 0 - except Exception as e: + except OSError as e: print(f"Port check failed: {e}") return False diff --git a/pkgs/testing/nixos_test_lib/ssh.py b/pkgs/testing/nixos_test_lib/ssh.py index ae6a84f89..97a52766c 100644 --- a/pkgs/testing/nixos_test_lib/ssh.py +++ b/pkgs/testing/nixos_test_lib/ssh.py @@ -1,6 +1,7 @@ """SSH and test setup utilities""" import subprocess +from pathlib import Path from typing import NamedTuple from .nix_setup import setup_nix_in_nix @@ -34,15 +35,14 @@ def setup_test_environment( setup_port_forwarding(target, host_port) - ssh_key = os.path.join(temp_dir, "id_ed25519") - with open(ssh_key, "w") as f: - with open(assets_ssh_privkey) as src: - f.write(src.read()) - os.chmod(ssh_key, 0o600) + ssh_key = Path(temp_dir) / "id_ed25519" + with ssh_key.open("w") as f, Path(assets_ssh_privkey).open() as src: + f.write(src.read()) + ssh_key.chmod(0o600) # Copy test flake to temp directory - flake_dir = os.path.join(temp_dir, "test-flake") - subprocess.run(["cp", "-r", clan_core_for_checks, flake_dir], check=True) - subprocess.run(["chmod", "-R", "+w", flake_dir], check=True) + flake_dir = Path(temp_dir) / "test-flake" + subprocess.run(["cp", "-r", clan_core_for_checks, flake_dir], check=True) # noqa: S603, S607 + subprocess.run(["chmod", "-R", "+w", flake_dir], check=True) # noqa: S603, S607 - return TestEnvironment(host_port, ssh_key, flake_dir) + return TestEnvironment(host_port, str(ssh_key), str(flake_dir)) diff --git a/pkgs/testing/pyproject.toml b/pkgs/testing/pyproject.toml index b82e1bc08..c29fd67f5 100644 --- a/pkgs/testing/pyproject.toml +++ b/pkgs/testing/pyproject.toml @@ -41,4 +41,5 @@ ignore = [ "ANN", # type annotations "COM812", # trailing comma "ISC001", # string concatenation + "T201", # print statements ] \ No newline at end of file From a53efb9386346e6a23f6b64c49222431918b51d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 2 Jul 2025 16:26:23 +0200 Subject: [PATCH 012/258] nixosTestLib: substitute dependencies on tools in --- pkgs/testing/flake-module.nix | 5 +++++ pkgs/testing/nixos_test_lib/nix_setup.py | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pkgs/testing/flake-module.nix b/pkgs/testing/flake-module.nix index c30aaf383..05f4c53cb 100644 --- a/pkgs/testing/flake-module.nix +++ b/pkgs/testing/flake-module.nix @@ -37,6 +37,11 @@ setuptools wheel ]; + postPatch = '' + substituteInPlace nixos_test_lib/nix_setup.py \ + --replace '@cp@' '${pkgs.coreutils}/bin/cp' \ + --replace '@nix-store@' '${pkgs.nix}/bin/nix-store' + ''; doCheck = false; }; diff --git a/pkgs/testing/nixos_test_lib/nix_setup.py b/pkgs/testing/nixos_test_lib/nix_setup.py index b42126b44..a8ed4d72d 100644 --- a/pkgs/testing/nixos_test_lib/nix_setup.py +++ b/pkgs/testing/nixos_test_lib/nix_setup.py @@ -4,6 +4,10 @@ import os import subprocess from pathlib import Path +# These paths will be substituted during package build +CP_BIN = "@cp@" +NIX_STORE_BIN = "@nix-store@" + def setup_nix_in_nix(closure_info: str | None) -> None: """Set up Nix store inside test environment @@ -44,7 +48,7 @@ def setup_nix_in_nix(closure_info: str | None) -> None: # Copy store paths to test store using external cp command # (handles symlinks better) subprocess.run( - ["cp", "--recursive", "--target", f"{tmpdir}/store"] + [CP_BIN, "--recursive", "--target", f"{tmpdir}/store"] + [path.strip() for path in store_paths if path.strip()], check=True, ) @@ -57,7 +61,7 @@ def setup_nix_in_nix(closure_info: str | None) -> None: with registration_file.open() as f: subprocess.run( - ["nix-store", "--load-db"], + [NIX_STORE_BIN, "--load-db"], input=f.read(), text=True, env=env, From 7f4f11751e2727e5586e43b1d95d56540d66d9bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 2 Jul 2025 16:38:20 +0200 Subject: [PATCH 013/258] nixosTestLib: use xargs for copying store inputs --- pkgs/testing/flake-module.nix | 3 +- pkgs/testing/nixos_test_lib/nix_setup.py | 36 +++++++++++++----------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/pkgs/testing/flake-module.nix b/pkgs/testing/flake-module.nix index 05f4c53cb..235514393 100644 --- a/pkgs/testing/flake-module.nix +++ b/pkgs/testing/flake-module.nix @@ -40,7 +40,8 @@ postPatch = '' substituteInPlace nixos_test_lib/nix_setup.py \ --replace '@cp@' '${pkgs.coreutils}/bin/cp' \ - --replace '@nix-store@' '${pkgs.nix}/bin/nix-store' + --replace '@nix-store@' '${pkgs.nix}/bin/nix-store' \ + --replace '@xargs@' '${pkgs.findutils}/bin/xargs' ''; doCheck = false; }; diff --git a/pkgs/testing/nixos_test_lib/nix_setup.py b/pkgs/testing/nixos_test_lib/nix_setup.py index a8ed4d72d..8ec2b76cc 100644 --- a/pkgs/testing/nixos_test_lib/nix_setup.py +++ b/pkgs/testing/nixos_test_lib/nix_setup.py @@ -7,6 +7,7 @@ from pathlib import Path # These paths will be substituted during package build CP_BIN = "@cp@" NIX_STORE_BIN = "@nix-store@" +XARGS_BIN = "@xargs@" def setup_nix_in_nix(closure_info: str | None) -> None: @@ -16,7 +17,7 @@ def setup_nix_in_nix(closure_info: str | None) -> None: closure_info: Path to closure info directory containing store-paths file, or None if no closure info """ - tmpdir = Path(os.environ.get("TMPDIR", "/tmp")) + tmpdir = Path(os.environ.get("TMPDIR", "/tmp")) # noqa: S108 # Remove NIX_REMOTE if present (we don't have any nix daemon running) if "NIX_REMOTE" in os.environ: @@ -37,33 +38,36 @@ def setup_nix_in_nix(closure_info: str | None) -> None: Path(f"{tmpdir}/nix").mkdir(parents=True, exist_ok=True) Path(f"{tmpdir}/etc").mkdir(parents=True, exist_ok=True) Path(f"{tmpdir}/store").mkdir(parents=True, exist_ok=True) + Path(f"{tmpdir}/store/nix/store").mkdir(parents=True, exist_ok=True) + Path(f"{tmpdir}/store/nix/var/nix/gcroots").mkdir(parents=True, exist_ok=True) # Set up Nix store if closure info is provided if closure_info and Path(closure_info).exists(): store_paths_file = Path(closure_info) / "store-paths" if store_paths_file.exists(): + # Use xargs to handle potentially long lists of store paths + # Equivalent to: xargs cp --recursive --target-directory + # "$CLAN_TEST_STORE/nix/store" < "$closureInfo/store-paths" with store_paths_file.open() as f: - store_paths = f.read().strip().split("\n") - - # Copy store paths to test store using external cp command - # (handles symlinks better) - subprocess.run( - [CP_BIN, "--recursive", "--target", f"{tmpdir}/store"] - + [path.strip() for path in store_paths if path.strip()], - check=True, - ) + subprocess.run( # noqa: S603 + [ + XARGS_BIN, + CP_BIN, + "--recursive", + "--target-directory", + f"{tmpdir}/store/nix/store", + ], + stdin=f, + check=True, + ) # Load Nix database registration_file = Path(closure_info) / "registration" if registration_file.exists(): - env = os.environ.copy() - env["NIX_REMOTE"] = f"local?store={tmpdir}/store" - with registration_file.open() as f: - subprocess.run( - [NIX_STORE_BIN, "--load-db"], + subprocess.run( # noqa: S603 + [NIX_STORE_BIN, "--load-db", "--store", f"{tmpdir}/store"], input=f.read(), text=True, - env=env, check=True, ) From 543c518ed0573a23883aa1f0f0be0163bb098add Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 2 Jul 2025 17:33:32 +0200 Subject: [PATCH 014/258] make host key check an enum instead of an literal type this is more typesafe at runtime. --- pkgs/clan-cli/clan_cli/host_key_check.py | 28 +++++++++++++++++++ pkgs/clan-cli/clan_cli/machines/hardware.py | 8 ++---- pkgs/clan-cli/clan_cli/machines/install.py | 8 ++---- pkgs/clan-cli/clan_cli/machines/update.py | 8 ++---- pkgs/clan-cli/clan_cli/ssh/deploy_info.py | 11 +++----- .../clan-cli/clan_cli/ssh/test_deploy_info.py | 9 ++++-- pkgs/clan-cli/clan_cli/tests/hosts.py | 3 +- pkgs/clan-cli/clan_lib/ssh/host_key.py | 22 +++++++-------- pkgs/clan-cli/clan_lib/ssh/remote.py | 2 +- pkgs/clan-cli/clan_lib/ssh/remote_test.py | 5 ++-- pkgs/clan-cli/clan_lib/tests/test_create.py | 3 +- 11 files changed, 63 insertions(+), 44 deletions(-) create mode 100644 pkgs/clan-cli/clan_cli/host_key_check.py diff --git a/pkgs/clan-cli/clan_cli/host_key_check.py b/pkgs/clan-cli/clan_cli/host_key_check.py new file mode 100644 index 000000000..df1331574 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/host_key_check.py @@ -0,0 +1,28 @@ +"""Common argument types and utilities for host key checking in clan CLI commands.""" + +import argparse + +from clan_lib.ssh.host_key import HostKeyCheck + + +def host_key_check_type(value: str) -> HostKeyCheck: + """ + Argparse type converter for HostKeyCheck enum. + """ + try: + return HostKeyCheck(value) + except ValueError: + valid_values = [e.value for e in HostKeyCheck] + msg = f"Invalid host key check mode: {value}. Valid options: {', '.join(valid_values)}" + raise argparse.ArgumentTypeError(msg) from None + + +def add_host_key_check_arg( + parser: argparse.ArgumentParser, default: HostKeyCheck = HostKeyCheck.ASK +) -> None: + parser.add_argument( + "--host-key-check", + type=host_key_check_type, + default=default, + help=f"Host key (.ssh/known_hosts) check mode. Options: {', '.join([e.value for e in HostKeyCheck])}", + ) diff --git a/pkgs/clan-cli/clan_cli/machines/hardware.py b/pkgs/clan-cli/clan_cli/machines/hardware.py index b65cf7831..2db751fd3 100644 --- a/pkgs/clan-cli/clan_cli/machines/hardware.py +++ b/pkgs/clan-cli/clan_cli/machines/hardware.py @@ -12,6 +12,7 @@ from clan_lib.machines.suggestions import validate_machine_names from clan_lib.ssh.remote import Remote from clan_cli.completions import add_dynamic_completer, complete_machines +from clan_cli.host_key_check import add_host_key_check_arg from .types import machine_name_type @@ -54,12 +55,7 @@ def register_update_hardware_config(parser: argparse.ArgumentParser) -> None: nargs="?", help="ssh address to install to in the form of user@host:2222", ) - parser.add_argument( - "--host-key-check", - choices=["strict", "ask", "tofu", "none"], - default="ask", - help="Host key (.ssh/known_hosts) check mode.", - ) + add_host_key_check_arg(parser) parser.add_argument( "--password", help="Pre-provided password the cli will prompt otherwise if needed.", diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index b93a7331d..a6222e88f 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -13,6 +13,7 @@ from clan_cli.completions import ( complete_machines, complete_target_host, ) +from clan_cli.host_key_check import add_host_key_check_arg from clan_cli.machines.hardware import HardwareConfig from clan_cli.ssh.deploy_info import DeployInfo, find_reachable_host, ssh_command_parse @@ -97,12 +98,7 @@ def register_install_parser(parser: argparse.ArgumentParser) -> None: help="do not reboot after installation (deprecated)", default=False, ) - parser.add_argument( - "--host-key-check", - choices=["strict", "ask", "tofu", "none"], - default="ask", - help="Host key (.ssh/known_hosts) check mode.", - ) + add_host_key_check_arg(parser) parser.add_argument( "--build-on", choices=[x.value for x in BuildOn], diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 19ed3412b..76f5afb38 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -16,6 +16,7 @@ from clan_cli.completions import ( complete_machines, complete_tags, ) +from clan_cli.host_key_check import add_host_key_check_arg log = logging.getLogger(__name__) @@ -163,12 +164,7 @@ def register_update_parser(parser: argparse.ArgumentParser) -> None: ) add_dynamic_completer(tag_parser, complete_tags) - parser.add_argument( - "--host-key-check", - choices=["strict", "ask", "tofu", "none"], - default="ask", - help="Host key (.ssh/known_hosts) check mode.", - ) + add_host_key_check_arg(parser) parser.add_argument( "--target-host", type=str, diff --git a/pkgs/clan-cli/clan_cli/ssh/deploy_info.py b/pkgs/clan-cli/clan_cli/ssh/deploy_info.py index 9ba609c95..3734b47a7 100644 --- a/pkgs/clan-cli/clan_cli/ssh/deploy_info.py +++ b/pkgs/clan-cli/clan_cli/ssh/deploy_info.py @@ -8,12 +8,14 @@ from typing import Any from clan_lib.cmd import run from clan_lib.errors import ClanError from clan_lib.nix import nix_shell -from clan_lib.ssh.remote import HostKeyCheck, Remote +from clan_lib.ssh.host_key import HostKeyCheck +from clan_lib.ssh.remote import Remote from clan_cli.completions import ( add_dynamic_completer, complete_machines, ) +from clan_cli.host_key_check import add_host_key_check_arg from clan_cli.ssh.tor import TorTarget, spawn_tor, ssh_tor_reachable log = logging.getLogger(__name__) @@ -181,10 +183,5 @@ def register_parser(parser: argparse.ArgumentParser) -> None: "--png", help="specify the json file for ssh data as the qrcode image (generated by starting the clan installer)", ) - parser.add_argument( - "--host-key-check", - choices=["strict", "ask", "tofu", "none"], - default="tofu", - help="Host key (.ssh/known_hosts) check mode.", - ) + add_host_key_check_arg(parser, default=HostKeyCheck.TOFU) parser.set_defaults(func=ssh_command) diff --git a/pkgs/clan-cli/clan_cli/ssh/test_deploy_info.py b/pkgs/clan-cli/clan_cli/ssh/test_deploy_info.py index 3a4996f7d..5efa7cbfb 100644 --- a/pkgs/clan-cli/clan_cli/ssh/test_deploy_info.py +++ b/pkgs/clan-cli/clan_cli/ssh/test_deploy_info.py @@ -4,6 +4,7 @@ from pathlib import Path import pytest from clan_lib.cmd import RunOpts, run from clan_lib.nix import nix_shell +from clan_lib.ssh.host_key import HostKeyCheck from clan_lib.ssh.remote import Remote from clan_cli.ssh.deploy_info import DeployInfo, find_reachable_host @@ -23,7 +24,7 @@ def test_qrcode_scan(temp_dir: Path) -> None: run(cmd, RunOpts(input=data.encode())) # Call the qrcode_scan function - deploy_info = DeployInfo.from_qr_code(img_path, "none") + deploy_info = DeployInfo.from_qr_code(img_path, HostKeyCheck.NONE) host = deploy_info.addrs[0] assert host.address == "192.168.122.86" @@ -46,7 +47,7 @@ def test_qrcode_scan(temp_dir: Path) -> None: def test_from_json() -> None: data = '{"pass":"scabbed-defender-headlock","tor":"qjeerm4r6t55hcfum4pinnvscn5njlw2g3k7ilqfuu7cdt3ahaxhsbid.onion","addrs":["192.168.122.86"]}' - deploy_info = DeployInfo.from_json(json.loads(data), "none") + deploy_info = DeployInfo.from_json(json.loads(data), HostKeyCheck.NONE) host = deploy_info.addrs[0] assert host.password == "scabbed-defender-headlock" @@ -69,7 +70,9 @@ def test_from_json() -> None: @pytest.mark.with_core def test_find_reachable_host(hosts: list[Remote]) -> None: host = hosts[0] - deploy_info = DeployInfo.from_hostnames(["172.19.1.2", host.ssh_url()], "none") + deploy_info = DeployInfo.from_hostnames( + ["172.19.1.2", host.ssh_url()], HostKeyCheck.NONE + ) assert deploy_info.addrs[0].address == "172.19.1.2" diff --git a/pkgs/clan-cli/clan_cli/tests/hosts.py b/pkgs/clan-cli/clan_cli/tests/hosts.py index 84c32e720..f40c79e63 100644 --- a/pkgs/clan-cli/clan_cli/tests/hosts.py +++ b/pkgs/clan-cli/clan_cli/tests/hosts.py @@ -4,6 +4,7 @@ from pathlib import Path import pytest from clan_cli.tests.sshd import Sshd +from clan_lib.ssh.host_key import HostKeyCheck from clan_lib.ssh.remote import Remote @@ -16,7 +17,7 @@ def hosts(sshd: Sshd) -> list[Remote]: port=sshd.port, user=login, private_key=Path(sshd.key), - host_key_check="none", + host_key_check=HostKeyCheck.NONE, command_prefix="local_test", ) ] diff --git a/pkgs/clan-cli/clan_lib/ssh/host_key.py b/pkgs/clan-cli/clan_lib/ssh/host_key.py index 3f2e6968c..e97932ef3 100644 --- a/pkgs/clan-cli/clan_lib/ssh/host_key.py +++ b/pkgs/clan-cli/clan_lib/ssh/host_key.py @@ -1,15 +1,15 @@ # Adapted from https://github.com/numtide/deploykit -from typing import Literal +from enum import Enum from clan_lib.errors import ClanError -HostKeyCheck = Literal[ - "strict", # Strictly check ssh host keys, prompt for unknown ones - "ask", # Ask for confirmation on first use - "tofu", # Trust on ssh keys on first use - "none", # Do not check ssh host keys -] + +class HostKeyCheck(Enum): + STRICT = "strict" # Strictly check ssh host keys, prompt for unknown ones + ASK = "ask" # Ask for confirmation on first use + TOFU = "tofu" # Trust on ssh keys on first use + NONE = "none" # Do not check ssh host keys def hostkey_to_ssh_opts(host_key_check: HostKeyCheck) -> list[str]: @@ -17,13 +17,13 @@ def hostkey_to_ssh_opts(host_key_check: HostKeyCheck) -> list[str]: Convert a HostKeyCheck value to SSH options. """ match host_key_check: - case "strict": + case HostKeyCheck.STRICT: return ["-o", "StrictHostKeyChecking=yes"] - case "ask": + case HostKeyCheck.ASK: return [] - case "tofu": + case HostKeyCheck.TOFU: return ["-o", "StrictHostKeyChecking=accept-new"] - case "none": + case HostKeyCheck.NONE: return [ "-o", "StrictHostKeyChecking=no", diff --git a/pkgs/clan-cli/clan_lib/ssh/remote.py b/pkgs/clan-cli/clan_lib/ssh/remote.py index de486d80f..41db26ad5 100644 --- a/pkgs/clan-cli/clan_lib/ssh/remote.py +++ b/pkgs/clan-cli/clan_lib/ssh/remote.py @@ -39,7 +39,7 @@ class Remote: private_key: Path | None = None password: str | None = None forward_agent: bool = True - host_key_check: HostKeyCheck = "ask" + host_key_check: HostKeyCheck = HostKeyCheck.ASK verbose_ssh: bool = False ssh_options: dict[str, str] = field(default_factory=dict) tor_socks: bool = False diff --git a/pkgs/clan-cli/clan_lib/ssh/remote_test.py b/pkgs/clan-cli/clan_lib/ssh/remote_test.py index 6d8a094b7..00b6ca53c 100644 --- a/pkgs/clan-cli/clan_lib/ssh/remote_test.py +++ b/pkgs/clan-cli/clan_lib/ssh/remote_test.py @@ -9,6 +9,7 @@ from clan_lib.async_run import AsyncRuntime from clan_lib.cmd import ClanCmdTimeoutError, Log, RunOpts from clan_lib.errors import ClanError, CmdOut from clan_lib.ssh.remote import Remote +from clan_lib.ssh.host_key import HostKeyCheck from clan_lib.ssh.sudo_askpass_proxy import SudoAskpassProxy if sys.platform == "darwin": @@ -113,7 +114,7 @@ def test_parse_deployment_address( result = Remote.from_ssh_uri( machine_name=machine_name, address=test_addr, - ).override(host_key_check="strict") + ).override(host_key_check=HostKeyCheck.STRICT) if expected_exception: return @@ -131,7 +132,7 @@ def test_parse_deployment_address( def test_parse_ssh_options() -> None: addr = "root@example.com:2222?IdentityFile=/path/to/private/key&StrictRemoteKeyChecking=yes" host = Remote.from_ssh_uri(machine_name="foo", address=addr).override( - host_key_check="strict" + host_key_check=HostKeyCheck.STRICT ) assert host.address == "example.com" assert host.port == 2222 diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index 7090e8b85..ccbdce0ec 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -33,6 +33,7 @@ from clan_lib.nix_models.clan import ( ) from clan_lib.nix_models.clan import InventoryMachineDeploy as MachineDeploy from clan_lib.persist.util import set_value_by_path +from clan_lib.ssh.host_key import HostKeyCheck from clan_lib.ssh.remote import Remote, can_ssh_login log = logging.getLogger(__name__) @@ -198,7 +199,7 @@ def test_clan_create_api( clan_dir_flake.invalidate_cache() target_host = machine.target_host().override( - private_key=private_key, host_key_check="none" + private_key=private_key, host_key_check=HostKeyCheck.NONE ) result = can_ssh_login(target_host) assert result == "Online", f"Machine {machine.name} is not online" From 247151e93f892eb962809899a6aea01a01df562b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 2 Jul 2025 17:43:04 +0200 Subject: [PATCH 015/258] only override identify/host_key_check in a single place --- pkgs/clan-cli/clan_cli/machines/hardware.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/machines/hardware.py b/pkgs/clan-cli/clan_cli/machines/hardware.py index 2db751fd3..cc36cdf0d 100644 --- a/pkgs/clan-cli/clan_cli/machines/hardware.py +++ b/pkgs/clan-cli/clan_cli/machines/hardware.py @@ -21,7 +21,6 @@ log = logging.getLogger(__name__) def update_hardware_config_command(args: argparse.Namespace) -> None: validate_machine_names([args.machine], args.flake) - host_key_check = args.host_key_check machine = Machine(flake=args.flake, name=args.machine) opts = HardwareGenerateOptions( machine=machine, @@ -32,11 +31,13 @@ def update_hardware_config_command(args: argparse.Namespace) -> None: if args.target_host: target_host = Remote.from_ssh_uri( machine_name=machine.name, address=args.target_host - ).override(host_key_check=host_key_check, private_key=args.identity_file) - else: - target_host = machine.target_host().override( - host_key_check=host_key_check, private_key=args.identity_file ) + else: + target_host = machine.target_host() + + target_host = target_host.override( + host_key_check=args.host_key_check, private_key=args.identity_file + ) generate_machine_hardware_info(opts, target_host) From eb54fdc7417daee1edf938b719476b5e59c54b9d Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Fri, 4 Jul 2025 23:56:52 +1000 Subject: [PATCH 016/258] clanServices/wifi: fix `autoConnect` setting not doing anything --- clanServices/wifi/default.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/clanServices/wifi/default.nix b/clanServices/wifi/default.nix index f7a75e70a..e6226b476 100644 --- a/clanServices/wifi/default.nix +++ b/clanServices/wifi/default.nix @@ -73,9 +73,10 @@ in ]; networking.networkmanager.ensureProfiles.profiles = flip mapAttrs settings.networks ( - name: _network: { + name: networkCfg: { connection.id = "$ssid_${name}"; connection.type = "wifi"; + connection.autoconnect = networkCfg.autoConnect; wifi.mode = "infrastructure"; wifi.ssid = "$ssid_${name}"; wifi-security.psk = "$pw_${name}"; @@ -102,7 +103,7 @@ in # Generate the secrets file echo "Generating wifi secrets file: $env_file" ${flip (concatMapAttrsStringSep "\n") settings.networks ( - name: _network: '' + name: _networkCfg: '' echo "ssid_${name}=\"$(cat "${ssid_path name}")\"" >> /run/secrets/NetworkManager/wifi-secrets echo "pw_${name}=\"$(cat "${password_path name}")\"" >> /run/secrets/NetworkManager/wifi-secrets '' From 76e653f37f49e2a3ad1f3580f798786962d5755b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 4 Jul 2025 16:56:42 +0200 Subject: [PATCH 017/258] nixoTestLib: split setting up function for port-forwarding and setuping up flake --- checks/installation/flake-module.nix | 74 ++++++++++++++--------- pkgs/clan-cli/clan_lib/ssh/remote_test.py | 2 +- pkgs/testing/nixos_test_lib/nix_setup.py | 23 +++++++ pkgs/testing/nixos_test_lib/ssh.py | 30 ++++----- 4 files changed, 79 insertions(+), 50 deletions(-) diff --git a/checks/installation/flake-module.nix b/checks/installation/flake-module.nix index 163e6e653..473dde6c1 100644 --- a/checks/installation/flake-module.nix +++ b/checks/installation/flake-module.nix @@ -174,7 +174,8 @@ import tempfile import os import subprocess - from nixos_test_lib.ssh import setup_test_environment # type: ignore[import-untyped] + from nixos_test_lib.ssh import setup_ssh_connection # type: ignore[import-untyped] + from nixos_test_lib.nix_setup import prepare_test_flake # type: ignore[import-untyped] def create_test_machine(oldmachine, qemu_test_bin: str, **kwargs): """Create a new test machine from an installed disk image""" @@ -201,14 +202,20 @@ target.start() - # Set up test environment using common helper + # Set up test environment with tempfile.TemporaryDirectory() as temp_dir: - env = setup_test_environment( - target, - temp_dir, - "${closureInfo}", - "${../assets/ssh/privkey}", - "${self.checks.x86_64-linux.clan-core-for-checks}" + # Prepare test flake and Nix store + flake_dir = prepare_test_flake( + temp_dir, + "${self.checks.x86_64-linux.clan-core-for-checks}", + "${closureInfo}" + ) + + # Set up SSH connection + ssh_conn = setup_ssh_connection( + target, + temp_dir, + "${../assets/ssh/privkey}" ) # Run clan install from host using port forwarding @@ -218,10 +225,10 @@ "install", "--phases", "disko,install", "--debug", - "--flake", env.flake_dir, + "--flake", flake_dir, "--yes", "test-install-machine-without-system", - "--target-host", f"nonrootuser@localhost:{env.host_port}", - "-i", env.ssh_key, + "--target-host", f"nonrootuser@localhost:{ssh_conn.host_port}", + "-i", ssh_conn.ssh_key, "--option", "store", os.environ['CLAN_TEST_STORE'], "--update-hardware-config", "nixos-facter", ] @@ -254,29 +261,36 @@ import tempfile import os import subprocess - from nixos_test_lib.ssh import setup_test_environment # type: ignore[import-untyped] + from nixos_test_lib.ssh import setup_ssh_connection # type: ignore[import-untyped] + from nixos_test_lib.nix_setup import prepare_test_flake # type: ignore[import-untyped] target.start() - # Set up test environment using common helper + # Set up test environment with tempfile.TemporaryDirectory() as temp_dir: - env = setup_test_environment( - target, - temp_dir, - "${closureInfo}", - "${../assets/ssh/privkey}", - "${self.checks.x86_64-linux.clan-core-for-checks}" + # Prepare test flake and Nix store + flake_dir = prepare_test_flake( + temp_dir, + "${self.checks.x86_64-linux.clan-core-for-checks}", + "${closureInfo}" + ) + + # Set up SSH connection + ssh_conn = setup_ssh_connection( + target, + temp_dir, + "${../assets/ssh/privkey}" ) # Verify files don't exist initially - hw_config_file = os.path.join(env.flake_dir, "machines/test-install-machine/hardware-configuration.nix") - facter_file = os.path.join(env.flake_dir, "machines/test-install-machine/facter.json") + hw_config_file = os.path.join(flake_dir, "machines/test-install-machine/hardware-configuration.nix") + facter_file = os.path.join(flake_dir, "machines/test-install-machine/facter.json") assert not os.path.exists(hw_config_file), "hardware-configuration.nix should not exist initially" assert not os.path.exists(facter_file), "facter.json should not exist initially" # Set CLAN_FLAKE for the commands - os.environ["CLAN_FLAKE"] = env.flake_dir + os.environ["CLAN_FLAKE"] = flake_dir # Test facter backend clan_cmd = [ @@ -287,17 +301,17 @@ "--flake", ".", "--host-key-check", "none", "test-install-machine-without-system", - "-i", env.ssh_key, + "-i", ssh_conn.ssh_key, "--option", "store", os.environ['CLAN_TEST_STORE'], - f"nonrootuser@localhost:{env.host_port}" + f"nonrootuser@localhost:{ssh_conn.host_port}" ] - result = subprocess.run(clan_cmd, capture_output=True, cwd=env.flake_dir) + result = subprocess.run(clan_cmd, capture_output=True, cwd=flake_dir) if result.returncode != 0: print(f"Clan update-hardware-config failed: {result.stderr.decode()}") raise Exception(f"Clan update-hardware-config failed with return code {result.returncode}") - facter_without_system_file = os.path.join(env.flake_dir, "machines/test-install-machine-without-system/facter.json") + facter_without_system_file = os.path.join(flake_dir, "machines/test-install-machine-without-system/facter.json") assert os.path.exists(facter_without_system_file), "facter.json should exist after update" os.remove(facter_without_system_file) @@ -311,17 +325,17 @@ "--host-key-check", "none", "--flake", ".", "test-install-machine-without-system", - "-i", env.ssh_key, + "-i", ssh_conn.ssh_key, "--option", "store", os.environ['CLAN_TEST_STORE'], - f"nonrootuser@localhost:{env.host_port}" + f"nonrootuser@localhost:{ssh_conn.host_port}" ] - result = subprocess.run(clan_cmd, capture_output=True, cwd=env.flake_dir) + result = subprocess.run(clan_cmd, capture_output=True, cwd=flake_dir) if result.returncode != 0: print(f"Clan update-hardware-config (nixos-generate-config) failed: {result.stderr.decode()}") raise Exception(f"Clan update-hardware-config failed with return code {result.returncode}") - hw_config_without_system_file = os.path.join(env.flake_dir, "machines/test-install-machine-without-system/hardware-configuration.nix") + hw_config_without_system_file = os.path.join(flake_dir, "machines/test-install-machine-without-system/hardware-configuration.nix") assert os.path.exists(hw_config_without_system_file), "hardware-configuration.nix should exist after update" ''; } { inherit pkgs self; }; diff --git a/pkgs/clan-cli/clan_lib/ssh/remote_test.py b/pkgs/clan-cli/clan_lib/ssh/remote_test.py index 00b6ca53c..7eeb5feac 100644 --- a/pkgs/clan-cli/clan_lib/ssh/remote_test.py +++ b/pkgs/clan-cli/clan_lib/ssh/remote_test.py @@ -8,8 +8,8 @@ import pytest from clan_lib.async_run import AsyncRuntime from clan_lib.cmd import ClanCmdTimeoutError, Log, RunOpts from clan_lib.errors import ClanError, CmdOut -from clan_lib.ssh.remote import Remote from clan_lib.ssh.host_key import HostKeyCheck +from clan_lib.ssh.remote import Remote from clan_lib.ssh.sudo_askpass_proxy import SudoAskpassProxy if sys.platform == "darwin": diff --git a/pkgs/testing/nixos_test_lib/nix_setup.py b/pkgs/testing/nixos_test_lib/nix_setup.py index 8ec2b76cc..aeb4e35b7 100644 --- a/pkgs/testing/nixos_test_lib/nix_setup.py +++ b/pkgs/testing/nixos_test_lib/nix_setup.py @@ -71,3 +71,26 @@ def setup_nix_in_nix(closure_info: str | None) -> None: text=True, check=True, ) + + +def prepare_test_flake( + temp_dir: str, clan_core_for_checks: str, closure_info: str +) -> str: + """Set up Nix store and copy test flake to temporary directory + + Args: + temp_dir: Temporary directory + clan_core_for_checks: Path to clan-core-for-checks + closure_info: Path to closure info for Nix store setup + + Returns: + Path to the test flake directory + """ + # Set up Nix store + setup_nix_in_nix(closure_info) + + # Copy test flake + flake_dir = Path(temp_dir) / "test-flake" + subprocess.run(["cp", "-r", clan_core_for_checks, flake_dir], check=True) # noqa: S603, S607 + subprocess.run(["chmod", "-R", "+w", flake_dir], check=True) # noqa: S603, S607 + return str(flake_dir) diff --git a/pkgs/testing/nixos_test_lib/ssh.py b/pkgs/testing/nixos_test_lib/ssh.py index 97a52766c..dc51213ac 100644 --- a/pkgs/testing/nixos_test_lib/ssh.py +++ b/pkgs/testing/nixos_test_lib/ssh.py @@ -1,34 +1,31 @@ """SSH and test setup utilities""" -import subprocess from pathlib import Path from typing import NamedTuple -from .nix_setup import setup_nix_in_nix from .port import find_free_port, setup_port_forwarding -class TestEnvironment(NamedTuple): +class SSHConnection(NamedTuple): host_port: int ssh_key: str - flake_dir: str -def setup_test_environment( +def setup_ssh_connection( target, temp_dir: str, - closure_info: str, assets_ssh_privkey: str, - clan_core_for_checks: str, -) -> TestEnvironment: - """Set up common test environment including SSH, port forwarding, and flake setup +) -> SSHConnection: + """Set up SSH connection with port forwarding to test VM + + Args: + target: Test VM target + temp_dir: Temporary directory for SSH key + assets_ssh_privkey: Path to SSH private key asset Returns: - TestEnvironment with host_port, ssh_key, and flake_dir + SSHConnection with host_port and ssh_key path """ - # Run setup function - setup_nix_in_nix(closure_info) - host_port = find_free_port() target.wait_for_unit("sshd.service") target.wait_for_open_port(22) @@ -40,9 +37,4 @@ def setup_test_environment( f.write(src.read()) ssh_key.chmod(0o600) - # Copy test flake to temp directory - flake_dir = Path(temp_dir) / "test-flake" - subprocess.run(["cp", "-r", clan_core_for_checks, flake_dir], check=True) # noqa: S603, S607 - subprocess.run(["chmod", "-R", "+w", flake_dir], check=True) # noqa: S603, S607 - - return TestEnvironment(host_port, str(ssh_key), str(flake_dir)) + return SSHConnection(host_port, str(ssh_key)) From b35ca4f1a868fc6e661403c7a94c348519482dee Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 30 Jun 2025 09:21:45 +0200 Subject: [PATCH 018/258] Chore: bump nixpkgs --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 8460b729d..997733c48 100644 --- a/flake.lock +++ b/flake.lock @@ -164,10 +164,10 @@ "nixpkgs": { "locked": { "lastModified": 315532800, - "narHash": "sha256-VgDAFPxHNhCfC7rI5I5wFqdiVJBH43zUefVo8hwo7cI=", - "rev": "41da1e3ea8e23e094e5e3eeb1e6b830468a7399e", + "narHash": "sha256-0HRxGUoOMtOYnwlMWY0AkuU88WHaI3Q5GEILmsWpI8U=", + "rev": "a48741b083d4f36dd79abd9f760c84da6b4dc0e5", "type": "tarball", - "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre814815.41da1e3ea8e2/nixexprs.tar.xz" + "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre823094.a48741b083d4/nixexprs.tar.xz" }, "original": { "type": "tarball", From 3f1fdc0aae57ec8fbc11ec6ad0f95741dddf32b8 Mon Sep 17 00:00:00 2001 From: Qubasa Date: Mon, 30 Jun 2025 15:28:15 +0700 Subject: [PATCH 019/258] treefmt/ruff: Set python lint version to 3.13. Fix all new lints coming up. --- pkgs/clan-app/clan_app/api/file_gtk.py | 1 - pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py | 11 ++++++++--- pkgs/clan-cli/clan_cli/tests/helpers/nixos_config.py | 11 ----------- pkgs/clan-cli/clan_cli/vars/check.py | 5 ++--- pkgs/clan-cli/clan_lib/api/__init__.py | 4 ++-- pkgs/clan-cli/clan_lib/machines/delete.py | 5 ++++- pkgs/clan-cli/clan_lib/ssh/remote.py | 1 - pkgs/clan-cli/docs.py | 4 +--- pkgs/clan-cli/pyproject.toml | 5 ++++- pkgs/classgen/main.py | 1 - 10 files changed, 21 insertions(+), 27 deletions(-) delete mode 100644 pkgs/clan-cli/clan_cli/tests/helpers/nixos_config.py diff --git a/pkgs/clan-app/clan_app/api/file_gtk.py b/pkgs/clan-app/clan_app/api/file_gtk.py index aef9a06d7..d3deef616 100644 --- a/pkgs/clan-app/clan_app/api/file_gtk.py +++ b/pkgs/clan-app/clan_app/api/file_gtk.py @@ -1,4 +1,3 @@ -# ruff: noqa: N801 import gi gi.require_version("Gtk", "4.0") diff --git a/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py b/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py index fefd8a262..59667d2f3 100644 --- a/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py +++ b/pkgs/clan-cli/clan_cli/tests/fixtures_flakes.py @@ -5,9 +5,9 @@ import shutil import subprocess as sp import tempfile from collections import defaultdict -from collections.abc import Callable, Iterator +from collections.abc import Iterator from pathlib import Path -from typing import Any, NamedTuple +from typing import NamedTuple import pytest from clan_cli.tests import age_keys @@ -38,7 +38,12 @@ def def_value() -> defaultdict: return defaultdict(def_value) -nested_dict: Callable[[], dict[str, Any]] = lambda: defaultdict(def_value) +def nested_dict() -> defaultdict: + """ + Creates a defaultdict that allows for arbitrary levels of nesting. + For example: d['a']['b']['c'] = value + """ + return defaultdict(def_value) # Substitutes strings in a file. diff --git a/pkgs/clan-cli/clan_cli/tests/helpers/nixos_config.py b/pkgs/clan-cli/clan_cli/tests/helpers/nixos_config.py deleted file mode 100644 index b922c6bf9..000000000 --- a/pkgs/clan-cli/clan_cli/tests/helpers/nixos_config.py +++ /dev/null @@ -1,11 +0,0 @@ -from collections import defaultdict -from collections.abc import Callable -from typing import Any - - -def def_value() -> defaultdict: - return defaultdict(def_value) - - -# allows defining nested dictionary in a single line -nested_dict: Callable[[], dict[str, Any]] = lambda: defaultdict(def_value) diff --git a/pkgs/clan-cli/clan_cli/vars/check.py b/pkgs/clan-cli/clan_cli/vars/check.py index e7087f73d..09f0dd0c9 100644 --- a/pkgs/clan-cli/clan_cli/vars/check.py +++ b/pkgs/clan-cli/clan_cli/vars/check.py @@ -4,14 +4,13 @@ import logging from clan_cli.completions import add_dynamic_completer, complete_machines from clan_lib.errors import ClanError from clan_lib.machines.machines import Machine - -log = logging.getLogger(__name__) - from typing import TYPE_CHECKING if TYPE_CHECKING: from .generate import Var +log = logging.getLogger(__name__) + class VarStatus: def __init__( diff --git a/pkgs/clan-cli/clan_lib/api/__init__.py b/pkgs/clan-cli/clan_lib/api/__init__.py index 789101f15..76c813936 100644 --- a/pkgs/clan-cli/clan_lib/api/__init__.py +++ b/pkgs/clan-cli/clan_lib/api/__init__.py @@ -16,14 +16,14 @@ from typing import ( ) from clan_lib.api.util import JSchemaTypeError +from clan_lib.errors import ClanError +from .serde import dataclass_to_dict, from_dict, sanitize_string log = logging.getLogger(__name__) -from .serde import dataclass_to_dict, from_dict, sanitize_string __all__ = ["dataclass_to_dict", "from_dict", "sanitize_string"] -from clan_lib.errors import ClanError T = TypeVar("T") diff --git a/pkgs/clan-cli/clan_lib/machines/delete.py b/pkgs/clan-cli/clan_lib/machines/delete.py index 29bf5c473..68f210cb8 100644 --- a/pkgs/clan-cli/clan_lib/machines/delete.py +++ b/pkgs/clan-cli/clan_lib/machines/delete.py @@ -42,7 +42,10 @@ def delete_machine(machine: Machine) -> None: # louis@(2025-02-04): clean-up legacy (pre-vars) secrets: sops_folder = sops_secrets_folder(machine.flake.path) - filter_fn = lambda secret_name: secret_name.startswith(f"{machine.name}-") + + def filter_fn(secret_name: str) -> bool: + return secret_name.startswith(f"{machine.name}-") + for secret_name in list_secrets(machine.flake.path, filter_fn): secret_path = sops_folder / secret_name changed_paths.append(secret_path) diff --git a/pkgs/clan-cli/clan_lib/ssh/remote.py b/pkgs/clan-cli/clan_lib/ssh/remote.py index de486d80f..8f4d4f8af 100644 --- a/pkgs/clan-cli/clan_lib/ssh/remote.py +++ b/pkgs/clan-cli/clan_lib/ssh/remote.py @@ -1,4 +1,3 @@ -# ruff: noqa: SLF001 import ipaddress import logging import os diff --git a/pkgs/clan-cli/docs.py b/pkgs/clan-cli/docs.py index 746fa9ed7..abd6aeadc 100644 --- a/pkgs/clan-cli/docs.py +++ b/pkgs/clan-cli/docs.py @@ -1,5 +1,6 @@ import argparse import sys +import re from dataclasses import dataclass from pathlib import Path @@ -118,9 +119,6 @@ def epilog_to_md(text: str) -> str: return md -import re - - def contains_https_link(line: str) -> bool: pattern = r"https://\S+" return re.search(pattern, line) is not None diff --git a/pkgs/clan-cli/pyproject.toml b/pkgs/clan-cli/pyproject.toml index 905caa974..c203368e0 100644 --- a/pkgs/clan-cli/pyproject.toml +++ b/pkgs/clan-cli/pyproject.toml @@ -49,9 +49,12 @@ filterwarnings = "default::ResourceWarning" python_files = ["test_*.py", "*_test.py"] [tool.mypy] -python_version = "3.12" +python_version = "3.13" warn_redundant_casts = true disallow_untyped_calls = true disallow_untyped_defs = true no_implicit_optional = true exclude = "clan_lib.nixpkgs" + +[tool.ruff] +target-version = "py313" \ No newline at end of file diff --git a/pkgs/classgen/main.py b/pkgs/classgen/main.py index 6eeab97b1..46cc22880 100644 --- a/pkgs/classgen/main.py +++ b/pkgs/classgen/main.py @@ -1,4 +1,3 @@ -# ruff: noqa: RUF001 import argparse import json import logging From d5aa917ee78eeed65908f173a6370753234ef6a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 4 Jul 2025 17:18:13 +0200 Subject: [PATCH 020/258] migrate all projects to python 3.13 linting --- lib/test/container-test-driver/pyproject.toml | 2 +- pkgs/clan-app/pyproject.toml | 2 +- pkgs/clan-app/tests/wayland.py | 4 ++-- pkgs/clan-vm-manager/clan_vm_manager/components/gkvstore.py | 4 ++-- pkgs/clan-vm-manager/clan_vm_manager/components/vmobj.py | 2 +- pkgs/clan-vm-manager/clan_vm_manager/views/details.py | 2 +- pkgs/clan-vm-manager/clan_vm_manager/views/list.py | 2 +- pkgs/clan-vm-manager/pyproject.toml | 2 +- pkgs/clan-vm-manager/tests/wayland.py | 4 ++-- pkgs/generate-test-vars/pyproject.toml | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/test/container-test-driver/pyproject.toml b/lib/test/container-test-driver/pyproject.toml index c9c3abf54..4a4c3e062 100644 --- a/lib/test/container-test-driver/pyproject.toml +++ b/lib/test/container-test-driver/pyproject.toml @@ -15,7 +15,7 @@ find = {} [tool.setuptools.package-data] test_driver = ["py.typed"] [tool.mypy] -python_version = "3.12" +python_version = "3.13" warn_redundant_casts = true disallow_untyped_calls = true disallow_untyped_defs = true diff --git a/pkgs/clan-app/pyproject.toml b/pkgs/clan-app/pyproject.toml index 60083f92a..25147331f 100644 --- a/pkgs/clan-app/pyproject.toml +++ b/pkgs/clan-app/pyproject.toml @@ -30,7 +30,7 @@ norecursedirs = "tests/helpers" markers = ["impure"] [tool.mypy] -python_version = "3.12" +python_version = "3.13" warn_redundant_casts = true disallow_untyped_calls = true disallow_untyped_defs = true diff --git a/pkgs/clan-app/tests/wayland.py b/pkgs/clan-app/tests/wayland.py index 9fa73961c..2c4ee1ee9 100644 --- a/pkgs/clan-app/tests/wayland.py +++ b/pkgs/clan-app/tests/wayland.py @@ -7,7 +7,7 @@ import pytest @pytest.fixture(scope="session") -def wayland_compositor() -> Generator[Popen, None, None]: +def wayland_compositor() -> Generator[Popen]: # Start the Wayland compositor (e.g., Weston) # compositor = Popen(["weston", "--backend=headless-backend.so"]) compositor = Popen(["weston"]) @@ -20,7 +20,7 @@ GtkProc = NewType("GtkProc", Popen) @pytest.fixture -def app() -> Generator[GtkProc, None, None]: +def app() -> Generator[GtkProc]: cmd = [sys.executable, "-m", "clan_app"] print(f"Running: {cmd}") rapp = Popen( diff --git a/pkgs/clan-vm-manager/clan_vm_manager/components/gkvstore.py b/pkgs/clan-vm-manager/clan_vm_manager/components/gkvstore.py index 9d4d0abab..f830b1073 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/components/gkvstore.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/components/gkvstore.py @@ -1,6 +1,6 @@ import logging from collections.abc import Callable -from typing import Any, Generic, TypeVar +from typing import Any, TypeVar import gi @@ -22,7 +22,7 @@ V = TypeVar( # clan_vm_manager/components/gkvstore.py:21: error: Definition of "newv" in base class "Object" is incompatible with definition in base class "GInterface" [misc] # clan_vm_manager/components/gkvstore.py:21: error: Definition of "install_properties" in base class "Object" is incompatible with definition in base class "GInterface" [misc] # clan_vm_manager/components/gkvstore.py:21: error: Definition of "getv" in base class "Object" is incompatible with definition in base class "GInterface" [misc] -class GKVStore(GObject.GObject, Gio.ListModel, Generic[K, V]): # type: ignore[misc] +class GKVStore[K, V: GObject.Object](GObject.GObject, Gio.ListModel): # type: ignore[misc] """ A simple key-value store that implements the Gio.ListModel interface, with generic types for keys and values. Only use self[key] and del self[key] for accessing the items for better performance. diff --git a/pkgs/clan-vm-manager/clan_vm_manager/components/vmobj.py b/pkgs/clan-vm-manager/clan_vm_manager/components/vmobj.py index 0be941739..1c4c60f75 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/components/vmobj.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/components/vmobj.py @@ -143,7 +143,7 @@ class VMObject(GObject.Object): # We use a context manager to create the machine object # and make sure it is destroyed when the context is exited @contextmanager - def _create_machine(self) -> Generator[Machine, None, None]: + def _create_machine(self) -> Generator[Machine]: uri = ClanURI.from_str( url=str(self.data.flake.flake_url), machine_name=self.data.flake.flake_attr ) diff --git a/pkgs/clan-vm-manager/clan_vm_manager/views/details.py b/pkgs/clan-vm-manager/clan_vm_manager/views/details.py index c9ec2f93f..df2d8fa17 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/views/details.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/views/details.py @@ -12,7 +12,7 @@ from gi.repository import Adw, Gio, GObject, Gtk ListItem = TypeVar("ListItem", bound=GObject.Object) -def create_details_list( +def create_details_list[ListItem: GObject.Object]( model: Gio.ListStore, render_row: Callable[[Gtk.ListBox, ListItem], Gtk.Widget] ) -> Gtk.ListBox: boxed_list = Gtk.ListBox() diff --git a/pkgs/clan-vm-manager/clan_vm_manager/views/list.py b/pkgs/clan-vm-manager/clan_vm_manager/views/list.py index b0ff077c5..5f104989b 100644 --- a/pkgs/clan-vm-manager/clan_vm_manager/views/list.py +++ b/pkgs/clan-vm-manager/clan_vm_manager/views/list.py @@ -32,7 +32,7 @@ ListItem = TypeVar("ListItem", bound=GObject.Object) CustomStore = TypeVar("CustomStore", bound=Gio.ListModel) -def create_boxed_list( +def create_boxed_list[CustomStore: Gio.ListModel, ListItem: GObject.Object]( model: CustomStore, render_row: Callable[[Gtk.ListBox, ListItem], Gtk.Widget], ) -> Gtk.ListBox: diff --git a/pkgs/clan-vm-manager/pyproject.toml b/pkgs/clan-vm-manager/pyproject.toml index 6e39b3fec..4de277719 100644 --- a/pkgs/clan-vm-manager/pyproject.toml +++ b/pkgs/clan-vm-manager/pyproject.toml @@ -30,7 +30,7 @@ norecursedirs = "tests/helpers" markers = ["impure"] [tool.mypy] -python_version = "3.12" +python_version = "3.13" warn_redundant_casts = true disallow_untyped_calls = true disallow_untyped_defs = true diff --git a/pkgs/clan-vm-manager/tests/wayland.py b/pkgs/clan-vm-manager/tests/wayland.py index d2515ba86..c9563cd6e 100644 --- a/pkgs/clan-vm-manager/tests/wayland.py +++ b/pkgs/clan-vm-manager/tests/wayland.py @@ -7,7 +7,7 @@ import pytest @pytest.fixture(scope="session") -def wayland_compositor() -> Generator[Popen, None, None]: +def wayland_compositor() -> Generator[Popen]: # Start the Wayland compositor (e.g., Weston) # compositor = Popen(["weston", "--backend=headless-backend.so"]) compositor = Popen(["weston"]) @@ -20,7 +20,7 @@ GtkProc = NewType("GtkProc", Popen) @pytest.fixture -def app() -> Generator[GtkProc, None, None]: +def app() -> Generator[GtkProc]: rapp = Popen([sys.executable, "-m", "clan_vm_manager"], text=True) yield GtkProc(rapp) # Cleanup: Terminate your application diff --git a/pkgs/generate-test-vars/pyproject.toml b/pkgs/generate-test-vars/pyproject.toml index aede95ecb..e3d0472e2 100644 --- a/pkgs/generate-test-vars/pyproject.toml +++ b/pkgs/generate-test-vars/pyproject.toml @@ -26,7 +26,7 @@ addopts = "--durations 5 --color=yes --new-first" # Add --pdb for debugging norecursedirs = "tests/helpers" [tool.mypy] -python_version = "3.12" +python_version = "3.13" warn_redundant_casts = true disallow_untyped_calls = true disallow_untyped_defs = true From 65904d8d8eae77b6d120df5c81e64132a75b1768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 4 Jul 2025 17:50:49 +0200 Subject: [PATCH 021/258] clan-cli: handle None in union types to prevent TypeError Add comprehensive test coverage for union types with None to prevent regression of the issubclass() TypeError that was occurring when checking if None is in a union type. --- pkgs/clan-cli/clan_lib/api/serde.py | 27 ++++++++++++- .../clan_lib/api/serde_deserialize_test.py | 39 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/api/serde.py b/pkgs/clan-cli/clan_lib/api/serde.py index d85700dee..f5940e065 100644 --- a/pkgs/clan-cli/clan_lib/api/serde.py +++ b/pkgs/clan-cli/clan_lib/api/serde.py @@ -146,8 +146,31 @@ def is_union_type(type_hint: type | UnionType) -> bool: def is_type_in_union(union_type: type | UnionType, target_type: type) -> bool: - if get_origin(union_type) is UnionType: - return any(issubclass(arg, target_type) for arg in get_args(union_type)) + # Check for Union from typing module (Union[str, None]) or UnionType (str | None) + if get_origin(union_type) in (Union, UnionType): + args = get_args(union_type) + for arg in args: + # Handle None type specially since it's not a class + if arg is None or arg is type(None): + if target_type is type(None): + return True + # For generic types like dict[str, str], check their origin + elif get_origin(arg) is not None: + if get_origin(arg) == target_type: + return True + # Also check if target_type is a generic with same origin + elif get_origin(target_type) is not None and get_origin( + arg + ) == get_origin(target_type): + return True + # For actual classes, use issubclass + elif inspect.isclass(arg) and inspect.isclass(target_type): + if issubclass(arg, target_type): + return True + # For non-class types, use direct comparison + elif arg == target_type: + return True + return False return union_type == target_type diff --git a/pkgs/clan-cli/clan_lib/api/serde_deserialize_test.py b/pkgs/clan-cli/clan_lib/api/serde_deserialize_test.py index 6b8065154..817e5a86c 100644 --- a/pkgs/clan-cli/clan_lib/api/serde_deserialize_test.py +++ b/pkgs/clan-cli/clan_lib/api/serde_deserialize_test.py @@ -8,6 +8,7 @@ import pytest from clan_lib.api import dataclass_to_dict, from_dict from clan_lib.errors import ClanError from clan_lib.machines import machines +from clan_lib.api.serde import is_type_in_union def test_simple() -> None: @@ -216,6 +217,44 @@ def test_none_or_string() -> None: assert checked3 is None +def test_union_with_none_edge_cases() -> None: + """ + Test various union types with None to ensure issubclass() error is avoided. + This specifically tests the fix for the TypeError in is_type_in_union. + """ + # Test basic types with None + assert from_dict(str | None, None) is None + assert from_dict(str | None, "hello") == "hello" + + # Test dict with None - this was the specific case that failed + assert from_dict(dict[str, str] | None, None) is None + assert from_dict(dict[str, str] | None, {"key": "value"}) == {"key": "value"} + + # Test list with None + assert from_dict(list[str] | None, None) is None + assert from_dict(list[str] | None, ["a", "b"]) == ["a", "b"] + + # Test dataclass with None + @dataclass + class TestClass: + value: str + + assert from_dict(TestClass | None, None) is None + assert from_dict(TestClass | None, {"value": "test"}) == TestClass(value="test") + + # Test Path with None (since it's used in the original failing test) + assert from_dict(Path | None, None) is None + assert from_dict(Path | None, "/home/test") == Path("/home/test") + + # Test that the is_type_in_union function works correctly + # This is the core of what was fixed - ensuring None doesn't cause issubclass error + # These should not raise TypeError anymore + assert is_type_in_union(str | None, type(None)) is True + assert is_type_in_union(dict[str, str] | None, type(None)) is True + assert is_type_in_union(list[str] | None, type(None)) is True + assert is_type_in_union(Path | None, type(None)) is True + + def test_roundtrip_escape() -> None: assert from_dict(str, "\n") == "\n" assert dataclass_to_dict("\n") == "\n" From d58505200789c4ce66c3b619ebe6f3555c9992b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 4 Jul 2025 17:18:13 +0200 Subject: [PATCH 022/258] migrate all projects to python 3.13 linting --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cac6c997d..7c3b24e4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.mypy] -python_version = "3.12" +python_version = "3.13" pretty = true warn_redundant_casts = true disallow_untyped_calls = true @@ -8,7 +8,7 @@ no_implicit_optional = true exclude = "clan_cli.nixpkgs" [tool.ruff] -target-version = "py311" +target-version = "py313" line-length = 88 lint.select = [ "A", From 76503b2a92e241630cfbe719309f12d3ca0be72b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 4 Jul 2025 17:46:48 +0200 Subject: [PATCH 023/258] terminate_process_group: also properly yield iterator when we return early --- pkgs/clan-cli/clan_lib/cmd/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/clan-cli/clan_lib/cmd/__init__.py b/pkgs/clan-cli/clan_lib/cmd/__init__.py index 49d630d09..d31ec65d1 100644 --- a/pkgs/clan-cli/clan_lib/cmd/__init__.py +++ b/pkgs/clan-cli/clan_lib/cmd/__init__.py @@ -179,6 +179,7 @@ def terminate_process_group(process: subprocess.Popen) -> Iterator[None]: try: process_group = os.getpgid(process.pid) except ProcessLookupError: + yield return if process_group == os.getpgid(os.getpid()): msg = "Bug! Refusing to terminate the current process group" From e4c8aba5bcb106800a4a52fcce75d385845aeef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 4 Jul 2025 18:19:47 +0200 Subject: [PATCH 024/258] zerotierone: disable tests on macos --- pkgs/zerotierone/default.nix | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkgs/zerotierone/default.nix b/pkgs/zerotierone/default.nix index bca52047f..d90278743 100644 --- a/pkgs/zerotierone/default.nix +++ b/pkgs/zerotierone/default.nix @@ -1,7 +1,14 @@ -{ zerotierone, lib }: +{ + zerotierone, + stdenv, + lib, +}: # halalify zerotierone -zerotierone.overrideAttrs (_old: { - meta = _old.meta // { +zerotierone.overrideAttrs (old: { + # Maybe a sandbox issue? + # zerotierone> [phy] Binding UDP listen socket to 127.0.0.1/60002... FAILED. + doCheck = old.doCheck && !stdenv.hostPlatform.isDarwin; + meta = old.meta // { license = lib.licenses.apsl20; }; }) From 324e93420424d9ec26da972436d5d2eff0ce0c87 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" Date: Fri, 4 Jul 2025 16:50:11 +0000 Subject: [PATCH 025/258] chore(deps): update disko digest to da6109c --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 997733c48..939903749 100644 --- a/flake.lock +++ b/flake.lock @@ -34,11 +34,11 @@ ] }, "locked": { - "lastModified": 1750903843, - "narHash": "sha256-Ng9+f0H5/dW+mq/XOKvB9uwvGbsuiiO6HrPdAcVglCs=", + "lastModified": 1751607816, + "narHash": "sha256-5PtrwjqCIJ4DKQhzYdm8RFePBuwb+yTzjV52wWoGSt4=", "owner": "nix-community", "repo": "disko", - "rev": "83c4da299c1d7d300f8c6fd3a72ac46cb0d59aae", + "rev": "da6109c917b48abc1f76dd5c9bf3901c8c80f662", "type": "github" }, "original": { From 448e60f866c12cca104cfdb6e1b9401472c2c8a8 Mon Sep 17 00:00:00 2001 From: DavHau Date: Sat, 5 Jul 2025 15:26:31 +0700 Subject: [PATCH 026/258] refactor: remove Machine.vars_generators() method Replace all calls to machine.vars_generators() with direct calls to Generator.generators_from_flake() to make the dependency more explicit and remove unnecessary indirection. This reduces coupling to the Machine class, making the codebase more modular and easier to refactor in the future. --- pkgs/clan-cli/clan_cli/flash/flash.py | 6 ++++- pkgs/clan-cli/clan_cli/tests/test_modules.py | 7 +++++- pkgs/clan-cli/clan_cli/vars/check.py | 4 +++- pkgs/clan-cli/clan_cli/vars/fix.py | 4 +++- pkgs/clan-cli/clan_cli/vars/generate.py | 20 ++++++++++++---- pkgs/clan-cli/clan_cli/vars/list.py | 19 ++++++++++++--- .../vars/secret_modules/password_store.py | 13 ++++++++-- .../clan_cli/vars/secret_modules/sops.py | 24 +++++++++++++++---- pkgs/clan-cli/clan_lib/machines/machines.py | 7 +----- 9 files changed, 81 insertions(+), 23 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/flash/flash.py b/pkgs/clan-cli/clan_cli/flash/flash.py index b4b9734bb..5b409bf3b 100644 --- a/pkgs/clan-cli/clan_cli/flash/flash.py +++ b/pkgs/clan-cli/clan_cli/flash/flash.py @@ -91,7 +91,11 @@ def flash_machine( "users": {"root": {"openssh": {"authorizedKeys": {"keys": root_keys}}}} } - for generator in machine.vars_generators(): + from clan_cli.vars.generate import Generator + + for generator in Generator.generators_from_flake( + machine.name, machine.flake, machine + ): for file in generator.files: if file.needed_for == "partitioning": msg = f"Partitioning time secrets are not supported with `clan flash write`: clan.core.vars.generators.{generator.name}.files.{file.name}" diff --git a/pkgs/clan-cli/clan_cli/tests/test_modules.py b/pkgs/clan-cli/clan_cli/tests/test_modules.py index 78e9ab835..045fb8f97 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_modules.py +++ b/pkgs/clan-cli/clan_cli/tests/test_modules.py @@ -114,9 +114,14 @@ def test_add_module_to_inventory( name="machine1", flake=Flake(str(test_flake_with_core.path)) ) + from clan_cli.vars.generate import Generator + generator = None - for gen in machine.vars_generators(): + generators = Generator.generators_from_flake( + machine.name, machine.flake, machine + ) + for gen in generators: if gen.name == "borgbackup": generator = gen break diff --git a/pkgs/clan-cli/clan_cli/vars/check.py b/pkgs/clan-cli/clan_cli/vars/check.py index 09f0dd0c9..417dad36d 100644 --- a/pkgs/clan-cli/clan_cli/vars/check.py +++ b/pkgs/clan-cli/clan_cli/vars/check.py @@ -32,7 +32,9 @@ def vars_status(machine: Machine, generator_name: None | str = None) -> VarStatu # signals if a var needs to be updated (eg. needs re-encryption due to new users added) unfixed_secret_vars = [] invalid_generators = [] - generators = machine.vars_generators() + from clan_cli.vars.generate import Generator + + generators = Generator.generators_from_flake(machine.name, machine.flake, machine) if generator_name: for generator in generators: if generator_name == generator.name: diff --git a/pkgs/clan-cli/clan_cli/vars/fix.py b/pkgs/clan-cli/clan_cli/vars/fix.py index c6e1ee53d..9d5889fbd 100644 --- a/pkgs/clan-cli/clan_cli/vars/fix.py +++ b/pkgs/clan-cli/clan_cli/vars/fix.py @@ -9,7 +9,9 @@ log = logging.getLogger(__name__) def fix_vars(machine: Machine, generator_name: None | str = None) -> None: - generators = machine.vars_generators() + from clan_cli.vars.generate import Generator + + generators = Generator.generators_from_flake(machine.name, machine.flake, machine) if generator_name: for generator in generators: if generator_name == generator.name: diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 0367887e1..18b6a37fa 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -187,7 +187,10 @@ def decrypt_dependencies( decrypted_dependencies: dict[str, Any] = {} for generator_name in set(generator.dependencies): decrypted_dependencies[generator_name] = {} - for dep_generator in machine.vars_generators(): + generators = Generator.generators_from_flake( + machine.name, machine.flake, machine + ) + for dep_generator in generators: if generator_name == dep_generator.name: break else: @@ -395,7 +398,9 @@ def get_closure( ) -> list[Generator]: from . import graph - vars_generators = machine.vars_generators() + vars_generators = Generator.generators_from_flake( + machine.name, machine.flake, machine + ) generators: dict[str, Generator] = { generator.name: generator for generator in vars_generators } @@ -470,7 +475,11 @@ def generate_vars_for_machine( machine = Machine(name=machine_name, flake=Flake(str(base_dir))) generators_set = set(generators) - generators_ = [g for g in machine.vars_generators() if g.name in generators_set] + generators_ = [ + g + for g in Generator.generators_from_flake(machine_name, machine.flake, machine) + if g.name in generators_set + ] return _generate_vars_for_machine( machine=machine, @@ -489,7 +498,10 @@ def generate_vars_for_machine_interactive( ) -> bool: _generator = None if generator_name: - for generator in machine.vars_generators(): + generators = Generator.generators_from_flake( + machine.name, machine.flake, machine + ) + for generator in generators: if generator.name == generator_name: _generator = generator break diff --git a/pkgs/clan-cli/clan_cli/vars/list.py b/pkgs/clan-cli/clan_cli/vars/list.py index b48e8ffc4..75a655189 100644 --- a/pkgs/clan-cli/clan_cli/vars/list.py +++ b/pkgs/clan-cli/clan_cli/vars/list.py @@ -18,8 +18,12 @@ def get_vars(base_dir: str, machine_name: str) -> list[Var]: machine = Machine(name=machine_name, flake=Flake(base_dir)) pub_store = machine.public_vars_store sec_store = machine.secret_vars_store + from clan_cli.vars.generate import Generator + all_vars = [] - for generator in machine.vars_generators(): + for generator in Generator.generators_from_flake( + machine_name, machine.flake, machine + ): for var in generator.files: if var.secret: var.store(sec_store) @@ -49,8 +53,12 @@ def _get_previous_value( @API.register def get_generators(base_dir: str, machine_name: str) -> list[Generator]: + from clan_cli.vars.generate import Generator + machine = Machine(name=machine_name, flake=Flake(base_dir)) - generators: list[Generator] = machine.vars_generators() + generators: list[Generator] = Generator.generators_from_flake( + machine_name, machine.flake, machine + ) for generator in generators: for prompt in generator.prompts: prompt.previous_value = _get_previous_value(machine, generator, prompt) @@ -64,9 +72,14 @@ def get_generators(base_dir: str, machine_name: str) -> list[Generator]: def set_prompts( base_dir: str, machine_name: str, updates: list[GeneratorUpdate] ) -> None: + from clan_cli.vars.generate import Generator + machine = Machine(name=machine_name, flake=Flake(base_dir)) for update in updates: - for generator in machine.vars_generators(): + generators = Generator.generators_from_flake( + machine_name, machine.flake, machine + ) + for generator in generators: if generator.name == update.generator: break else: diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py index c04d5ce9c..f4c99465d 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py @@ -140,8 +140,13 @@ class SecretStore(StoreBase): # we sort the hashes to make sure that the order is always the same hashes.sort() + from clan_cli.vars.generate import Generator + manifest = [] - for generator in self.machine.vars_generators(): + generators = Generator.generators_from_flake( + self.machine.name, self.machine.flake, self.machine + ) + for generator in generators: for file in generator.files: manifest.append(f"{generator.name}/{file.name}".encode()) manifest += hashes @@ -165,7 +170,11 @@ class SecretStore(StoreBase): return local_hash.decode() != remote_hash def populate_dir(self, output_dir: Path, phases: list[str]) -> None: - vars_generators = self.machine.vars_generators() + from clan_cli.vars.generate import Generator + + vars_generators = Generator.generators_from_flake( + self.machine.name, self.machine.flake, self.machine + ) if "users" in phases: with tarfile.open( output_dir / "secrets_for_users.tar.gz", "w:gz" diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py index b2ba2e6c7..64f1f374e 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py @@ -52,7 +52,11 @@ class SecretStore(StoreBase): self.machine = machine # no need to generate keys if we don't manage secrets - vars_generators = self.machine.vars_generators() + from clan_cli.vars.generate import Generator + + vars_generators = Generator.generators_from_flake( + self.machine.name, self.machine.flake, self.machine + ) if not vars_generators: return has_secrets = False @@ -111,7 +115,11 @@ class SecretStore(StoreBase): """ if generator is None: - generators = self.machine.vars_generators() + from clan_cli.vars.generate import Generator + + generators = Generator.generators_from_flake( + self.machine.name, self.machine.flake, self.machine + ) else: generators = [generator] file_found = False @@ -184,7 +192,11 @@ class SecretStore(StoreBase): return [store_folder] def populate_dir(self, output_dir: Path, phases: list[str]) -> None: - vars_generators = self.machine.vars_generators() + from clan_cli.vars.generate import Generator + + vars_generators = Generator.generators_from_flake( + self.machine.name, self.machine.flake, self.machine + ) if "users" in phases or "services" in phases: key_name = f"{self.machine.name}-age.key" if not has_secret(sops_secrets_folder(self.machine.flake_dir) / key_name): @@ -295,7 +307,11 @@ class SecretStore(StoreBase): from clan_cli.secrets.secrets import update_keys if generator is None: - generators = self.machine.vars_generators() + from clan_cli.vars.generate import Generator + + generators = Generator.generators_from_flake( + self.machine.name, self.machine.flake, self.machine + ) else: generators = [generator] file_found = False diff --git a/pkgs/clan-cli/clan_lib/machines/machines.py b/pkgs/clan-cli/clan_lib/machines/machines.py index 194f6081b..3715308f6 100644 --- a/pkgs/clan-cli/clan_lib/machines/machines.py +++ b/pkgs/clan-cli/clan_lib/machines/machines.py @@ -21,7 +21,7 @@ from clan_lib.ssh.remote import Remote log = logging.getLogger(__name__) if TYPE_CHECKING: - from clan_cli.vars.generate import Generator + pass @dataclass(frozen=True) @@ -118,11 +118,6 @@ class Machine: return services return {} - def vars_generators(self) -> list["Generator"]: - from clan_cli.vars.generate import Generator - - return Generator.generators_from_flake(self.name, self.flake, self) - @property def secrets_upload_directory(self) -> str: return self.select("config.clan.core.facts.secretUploadDirectory") From d143359a2d0e277a8baf7503405b75180d87eee8 Mon Sep 17 00:00:00 2001 From: DavHau Date: Sat, 5 Jul 2025 15:54:37 +0700 Subject: [PATCH 027/258] refactor: reduce coupling to Machine class in vars module - Change Generator class to store machine name as string instead of Machine reference - Update Generator.generators_from_flake() to only require machine name and flake - Refactor check_vars() to accept machine name and flake instead of Machine object - Create Machine instances only when needed for specific operations This continues the effort to reduce dependencies on the Machine class, making the codebase more modular and easier to refactor. --- pkgs/clan-cli/clan_cli/flash/flash.py | 4 +- pkgs/clan-cli/clan_cli/tests/test_modules.py | 4 +- pkgs/clan-cli/clan_cli/tests/test_vars.py | 14 +++--- pkgs/clan-cli/clan_cli/vars/check.py | 21 ++++----- pkgs/clan-cli/clan_cli/vars/fix.py | 2 +- pkgs/clan-cli/clan_cli/vars/generate.py | 44 +++++++++---------- pkgs/clan-cli/clan_cli/vars/list.py | 10 ++--- .../vars/secret_modules/password_store.py | 4 +- .../clan_cli/vars/secret_modules/sops.py | 8 ++-- 9 files changed, 50 insertions(+), 61 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/flash/flash.py b/pkgs/clan-cli/clan_cli/flash/flash.py index 5b409bf3b..aa77db5f5 100644 --- a/pkgs/clan-cli/clan_cli/flash/flash.py +++ b/pkgs/clan-cli/clan_cli/flash/flash.py @@ -93,9 +93,7 @@ def flash_machine( from clan_cli.vars.generate import Generator - for generator in Generator.generators_from_flake( - machine.name, machine.flake, machine - ): + for generator in Generator.generators_from_flake(machine.name, machine.flake): for file in generator.files: if file.needed_for == "partitioning": msg = f"Partitioning time secrets are not supported with `clan flash write`: clan.core.vars.generators.{generator.name}.files.{file.name}" diff --git a/pkgs/clan-cli/clan_cli/tests/test_modules.py b/pkgs/clan-cli/clan_cli/tests/test_modules.py index 045fb8f97..94de8a815 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_modules.py +++ b/pkgs/clan-cli/clan_cli/tests/test_modules.py @@ -118,9 +118,7 @@ def test_add_module_to_inventory( generator = None - generators = Generator.generators_from_flake( - machine.name, machine.flake, machine - ) + generators = Generator.generators_from_flake(machine.name, machine.flake) for gen in generators: if gen.name == "borgbackup": generator = gen diff --git a/pkgs/clan-cli/clan_cli/tests/test_vars.py b/pkgs/clan-cli/clan_cli/tests/test_vars.py index 306f61615..181b2a543 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_vars.py +++ b/pkgs/clan-cli/clan_cli/tests/test_vars.py @@ -140,7 +140,7 @@ def test_generate_public_and_secret_vars( monkeypatch.chdir(flake.path) machine = Machine(name="my_machine", flake=Flake(str(flake.path))) - assert not check_vars(machine) + assert not check_vars(machine.name, machine.flake) vars_text = stringify_all_vars(machine) assert "my_generator/my_value: " in vars_text assert "my_generator/my_secret: " in vars_text @@ -158,7 +158,7 @@ def test_generate_public_and_secret_vars( assert json.loads(value_non_default) == "default_value" cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) - assert check_vars(machine) + assert check_vars(machine.name, machine.flake) # get last commit message commit_message = run( ["git", "log", "-3", "--pretty=%B"], @@ -351,10 +351,10 @@ def test_generated_shared_secret_sops( machine1 = Machine(name="machine1", flake=Flake(str(flake.path))) machine2 = Machine(name="machine2", flake=Flake(str(flake.path))) cli.run(["vars", "generate", "--flake", str(flake.path), "machine1"]) - assert check_vars(machine1) + assert check_vars(machine1.name, machine1.flake) cli.run(["vars", "generate", "--flake", str(flake.path), "machine2"]) - assert check_vars(machine2) - assert check_vars(machine2) + assert check_vars(machine2.name, machine2.flake) + assert check_vars(machine2.name, machine2.flake) m1_sops_store = sops.SecretStore(machine1) m2_sops_store = sops.SecretStore(machine2) assert m1_sops_store.exists( @@ -404,9 +404,9 @@ def test_generate_secret_var_password_store( monkeypatch.setenv("PASSWORD_STORE_DIR", str(password_store_dir)) machine = Machine(name="my_machine", flake=Flake(str(flake.path))) - assert not check_vars(machine) + assert not check_vars(machine.name, machine.flake) cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) - assert check_vars(machine) + assert check_vars(machine.name, machine.flake) store = password_store.SecretStore( Machine(name="my_machine", flake=Flake(str(flake.path))) ) diff --git a/pkgs/clan-cli/clan_cli/vars/check.py b/pkgs/clan-cli/clan_cli/vars/check.py index 417dad36d..0c3fe484b 100644 --- a/pkgs/clan-cli/clan_cli/vars/check.py +++ b/pkgs/clan-cli/clan_cli/vars/check.py @@ -4,6 +4,7 @@ import logging from clan_cli.completions import add_dynamic_completer, complete_machines from clan_lib.errors import ClanError from clan_lib.machines.machines import Machine +from clan_lib.flake import Flake from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -26,7 +27,10 @@ class VarStatus: self.invalid_generators = invalid_generators -def vars_status(machine: Machine, generator_name: None | str = None) -> VarStatus: +def vars_status( + machine_name: str, flake: Flake, generator_name: None | str = None +) -> VarStatus: + machine = Machine(name=machine_name, flake=flake) missing_secret_vars = [] missing_public_vars = [] # signals if a var needs to be updated (eg. needs re-encryption due to new users added) @@ -34,7 +38,7 @@ def vars_status(machine: Machine, generator_name: None | str = None) -> VarStatu invalid_generators = [] from clan_cli.vars.generate import Generator - generators = Generator.generators_from_flake(machine.name, machine.flake, machine) + generators = Generator.generators_from_flake(machine.name, machine.flake) if generator_name: for generator in generators: if generator_name == generator.name: @@ -47,7 +51,6 @@ def vars_status(machine: Machine, generator_name: None | str = None) -> VarStatu raise ClanError(err_msg) for generator in generators: - generator.machine(machine) for file in generator.files: file.store( machine.secret_vars_store if file.secret else machine.public_vars_store @@ -93,8 +96,10 @@ def vars_status(machine: Machine, generator_name: None | str = None) -> VarStatu ) -def check_vars(machine: Machine, generator_name: None | str = None) -> bool: - status = vars_status(machine, generator_name=generator_name) +def check_vars( + machine_name: str, flake: Flake, generator_name: None | str = None +) -> bool: + status = vars_status(machine_name, flake, generator_name=generator_name) return not ( status.missing_secret_vars or status.missing_public_vars @@ -104,11 +109,7 @@ def check_vars(machine: Machine, generator_name: None | str = None) -> bool: def check_command(args: argparse.Namespace) -> None: - machine = Machine( - name=args.machine, - flake=args.flake, - ) - ok = check_vars(machine, generator_name=args.generator) + ok = check_vars(args.machine, args.flake, generator_name=args.generator) if not ok: raise SystemExit(1) diff --git a/pkgs/clan-cli/clan_cli/vars/fix.py b/pkgs/clan-cli/clan_cli/vars/fix.py index 9d5889fbd..6ad0e6ab2 100644 --- a/pkgs/clan-cli/clan_cli/vars/fix.py +++ b/pkgs/clan-cli/clan_cli/vars/fix.py @@ -11,7 +11,7 @@ log = logging.getLogger(__name__) def fix_vars(machine: Machine, generator_name: None | str = None) -> None: from clan_cli.vars.generate import Generator - generators = Generator.generators_from_flake(machine.name, machine.flake, machine) + generators = Generator.generators_from_flake(machine.name, machine.flake) if generator_name: for generator in generators: if generator_name == generator.name: diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 18b6a37fa..925c97b1a 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -49,20 +49,17 @@ class Generator: migrate_fact: str | None = None - # TODO: remove this - _machine: "Machine | None" = None - - def machine(self, machine: "Machine") -> None: - self._machine = machine + machine: str | None = None + _flake: "Flake | None" = None @cached_property def exists(self) -> bool: - assert self._machine is not None - return check_vars(self._machine, generator_name=self.name) + assert self.machine is not None and self._flake is not None + return check_vars(self.machine, self._flake, generator_name=self.name) @classmethod def generators_from_flake( - cls: type["Generator"], machine_name: str, flake: "Flake", machine: "Machine" + cls: type["Generator"], machine_name: str, flake: "Flake" ) -> list["Generator"]: config = nix_config() system = config["system"] @@ -112,19 +109,21 @@ class Generator: dependencies=gen_data["dependencies"], migrate_fact=gen_data.get("migrateFact"), prompts=prompts, + machine=machine_name, + _flake=flake, ) - # Set the machine immediately - generator.machine(machine) generators.append(generator) return generators def final_script(self) -> Path: - assert self._machine is not None + assert self.machine is not None and self._flake is not None + from clan_lib.machines.machines import Machine from clan_lib.nix import nix_test_store + machine = Machine(name=self.machine, flake=self._flake) output = Path( - self._machine.select( + machine.select( f'config.clan.core.vars.generators."{self.name}".finalScript' ) ) @@ -133,8 +132,11 @@ class Generator: return output def validation(self) -> str | None: - assert self._machine is not None - return self._machine.select( + assert self.machine is not None and self._flake is not None + from clan_lib.machines.machines import Machine + + machine = Machine(name=self.machine, flake=self._flake) + return machine.select( f'config.clan.core.vars.generators."{self.name}".validationHash' ) @@ -187,9 +189,7 @@ def decrypt_dependencies( decrypted_dependencies: dict[str, Any] = {} for generator_name in set(generator.dependencies): decrypted_dependencies[generator_name] = {} - generators = Generator.generators_from_flake( - machine.name, machine.flake, machine - ) + generators = Generator.generators_from_flake(machine.name, machine.flake) for dep_generator in generators: if generator_name == dep_generator.name: break @@ -398,9 +398,7 @@ def get_closure( ) -> list[Generator]: from . import graph - vars_generators = Generator.generators_from_flake( - machine.name, machine.flake, machine - ) + vars_generators = Generator.generators_from_flake(machine.name, machine.flake) generators: dict[str, Generator] = { generator.name: generator for generator in vars_generators } @@ -477,7 +475,7 @@ def generate_vars_for_machine( generators_set = set(generators) generators_ = [ g - for g in Generator.generators_from_flake(machine_name, machine.flake, machine) + for g in Generator.generators_from_flake(machine_name, machine.flake) if g.name in generators_set ] @@ -498,9 +496,7 @@ def generate_vars_for_machine_interactive( ) -> bool: _generator = None if generator_name: - generators = Generator.generators_from_flake( - machine.name, machine.flake, machine - ) + generators = Generator.generators_from_flake(machine.name, machine.flake) for generator in generators: if generator.name == generator_name: _generator = generator diff --git a/pkgs/clan-cli/clan_cli/vars/list.py b/pkgs/clan-cli/clan_cli/vars/list.py index 75a655189..c880b6696 100644 --- a/pkgs/clan-cli/clan_cli/vars/list.py +++ b/pkgs/clan-cli/clan_cli/vars/list.py @@ -21,9 +21,7 @@ def get_vars(base_dir: str, machine_name: str) -> list[Var]: from clan_cli.vars.generate import Generator all_vars = [] - for generator in Generator.generators_from_flake( - machine_name, machine.flake, machine - ): + for generator in Generator.generators_from_flake(machine_name, machine.flake): for var in generator.files: if var.secret: var.store(sec_store) @@ -57,7 +55,7 @@ def get_generators(base_dir: str, machine_name: str) -> list[Generator]: machine = Machine(name=machine_name, flake=Flake(base_dir)) generators: list[Generator] = Generator.generators_from_flake( - machine_name, machine.flake, machine + machine_name, machine.flake ) for generator in generators: for prompt in generator.prompts: @@ -76,9 +74,7 @@ def set_prompts( machine = Machine(name=machine_name, flake=Flake(base_dir)) for update in updates: - generators = Generator.generators_from_flake( - machine_name, machine.flake, machine - ) + generators = Generator.generators_from_flake(machine_name, machine.flake) for generator in generators: if generator.name == update.generator: break diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py index f4c99465d..98060a997 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py @@ -144,7 +144,7 @@ class SecretStore(StoreBase): manifest = [] generators = Generator.generators_from_flake( - self.machine.name, self.machine.flake, self.machine + self.machine.name, self.machine.flake ) for generator in generators: for file in generator.files: @@ -173,7 +173,7 @@ class SecretStore(StoreBase): from clan_cli.vars.generate import Generator vars_generators = Generator.generators_from_flake( - self.machine.name, self.machine.flake, self.machine + self.machine.name, self.machine.flake ) if "users" in phases: with tarfile.open( diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py index 64f1f374e..5008d1b56 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py @@ -55,7 +55,7 @@ class SecretStore(StoreBase): from clan_cli.vars.generate import Generator vars_generators = Generator.generators_from_flake( - self.machine.name, self.machine.flake, self.machine + self.machine.name, self.machine.flake ) if not vars_generators: return @@ -118,7 +118,7 @@ class SecretStore(StoreBase): from clan_cli.vars.generate import Generator generators = Generator.generators_from_flake( - self.machine.name, self.machine.flake, self.machine + self.machine.name, self.machine.flake ) else: generators = [generator] @@ -195,7 +195,7 @@ class SecretStore(StoreBase): from clan_cli.vars.generate import Generator vars_generators = Generator.generators_from_flake( - self.machine.name, self.machine.flake, self.machine + self.machine.name, self.machine.flake ) if "users" in phases or "services" in phases: key_name = f"{self.machine.name}-age.key" @@ -310,7 +310,7 @@ class SecretStore(StoreBase): from clan_cli.vars.generate import Generator generators = Generator.generators_from_flake( - self.machine.name, self.machine.flake, self.machine + self.machine.name, self.machine.flake ) else: generators = [generator] From f7d6c23aaaf1d2d801a74573cf3facaee4a29841 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 5 Jul 2025 18:39:49 +0200 Subject: [PATCH 028/258] clan_cli machines update: remove caching of sometimes missing pass config This config value is not set if people don't use pass, it's also at the wrong location We could cache it with a maybe, but we plan to move it anyway --- pkgs/clan-cli/clan_cli/machines/update.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 76f5afb38..98bc0798b 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -110,7 +110,6 @@ def update_command(args: argparse.Namespace) -> None: f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.{{share,dependencies,migrateFact,prompts}}", f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.files.*.{{secret,deploy,owner,group,mode,neededFor}}", f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.secretUploadDirectory", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.vars.password-store.secretLocation", f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.passBackend", ] ) From daf843eeab2fc1a9633c674918982198daae1092 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 5 Jul 2025 19:47:35 +0200 Subject: [PATCH 029/258] clan_cli run: add trace runOption to disable verbose traces in debug mode --- pkgs/clan-cli/clan_lib/cmd/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_lib/cmd/__init__.py b/pkgs/clan-cli/clan_lib/cmd/__init__.py index d31ec65d1..17dc5faff 100644 --- a/pkgs/clan-cli/clan_lib/cmd/__init__.py +++ b/pkgs/clan-cli/clan_lib/cmd/__init__.py @@ -290,6 +290,7 @@ class RunOpts: # Ask for sudo password in a graphical way. # This is needed for GUI applications graphical_perm: bool = False + trace: bool = True def cmd_with_root(cmd: list[str], graphical: bool = False) -> list[str]: @@ -344,7 +345,7 @@ def run( # Use our sudo ask proxy here as well options.needs_user_terminal = True - if cmdlog.isEnabledFor(logging.DEBUG): + if cmdlog.isEnabledFor(logging.DEBUG) and options.trace: if options.input and isinstance(options.input, bytes): if any( not ch.isprintable() for ch in options.input.decode("ascii", "replace") From 0670f0ad32e669b049df1aa630b0c85777691063 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sun, 6 Jul 2025 00:48:26 +0200 Subject: [PATCH 030/258] clan_cli flake: remove apply from select, as it will break stuff in horrible ways Since apply changes the structure of the retuned value, the cache will be confused about the structure and in subsequent request will use this wrong structure. For example: we would use builtins.attrNames on inputs, the flake will forever think that inputs is a list of strings and will report errors whenever we try to fetch subkeys from it --- pkgs/clan-cli/clan_lib/flake/flake.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/flake/flake.py b/pkgs/clan-cli/clan_lib/flake/flake.py index 3a3b90310..529207860 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake.py +++ b/pkgs/clan-cli/clan_lib/flake/flake.py @@ -621,7 +621,9 @@ class Flake: return self._is_local def get_input_names(self) -> list[str]: - return self.select("inputs", apply="builtins.attrNames") + log.debug("flake.get_input_names is deprecated and will be removed") + flakes = self.select("inputs.*._type") + return list(flakes.keys()) @property def path(self) -> Path: @@ -710,7 +712,6 @@ class Flake: def get_from_nix( self, selectors: list[str], - apply: str = "v: v", ) -> None: """ Retrieves specific attributes from a Nix flake using the provided selectors. @@ -772,7 +773,7 @@ class Flake: result = builtins.toJSON [ {" ".join( [ - f"(({apply}) (selectLib.applySelectors (builtins.fromJSON ''{attr}'') flake))" + f"(selectLib.applySelectors (builtins.fromJSON ''{attr}'') flake)" for attr in str_selectors ] )} @@ -840,7 +841,6 @@ class Flake: def select( self, selector: str, - apply: str = "v: v", ) -> Any: """ Selects a value from the cache based on the provided selector string. @@ -856,6 +856,6 @@ class Flake: if not self._cache.is_cached(selector): log.debug(f"Cache miss for {selector}") - self.get_from_nix([selector], apply=apply) + self.get_from_nix([selector]) value = self._cache.select(selector) return value From 195134dd5e2d03a9fa5d2e130abb20220617de06 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 5 Jul 2025 19:48:38 +0200 Subject: [PATCH 031/258] clan_cli: better select debug output --- pkgs/clan-cli/clan_lib/flake/flake.py | 31 +++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/flake/flake.py b/pkgs/clan-cli/clan_lib/flake/flake.py index 529207860..2d82a8acb 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake.py +++ b/pkgs/clan-cli/clan_lib/flake/flake.py @@ -730,7 +730,7 @@ class Flake: ClanError: If the number of outputs does not match the number of selectors. AssertionError: If the cache or flake cache path is not properly initialized. """ - from clan_lib.cmd import Log, RunOpts, run + from clan_lib.cmd import run, RunOpts, Log from clan_lib.dirs import select_source from clan_lib.nix import ( nix_build, @@ -796,11 +796,38 @@ class Flake: ]; }} """ + if len(selectors) > 1: + log.debug(f""" +selecting: {selectors} +to debug run: +nix repl --expr 'rec {{ + flake = builtins.getFlake "self.identifier"; + selectLib = (builtins.getFlake "path:{select_source()}?narHash={select_hash}").lib; + query = [ + {" ".join( + [ + f"(selectLib.select ''{selector}'' flake)" + for selector in selectors + ] + )} + ]; +}}' + """) # fmt: on + elif len(selectors) == 1: + log.debug(f""" +selecting: {selectors[0]} +to debug run: +nix repl --expr 'rec {{ + flake = builtins.getFlake "{self.identifier}"; + selectLib = (builtins.getFlake "path:{select_source()}?narHash={select_hash}").lib; + query = selectLib.select '"''{selectors[0]}''"' flake; +}}' + """) build_output = Path( run( - nix_build(["--expr", nix_code, *nix_options]), RunOpts(log=Log.NONE) + nix_build(["--expr", nix_code, *nix_options]), RunOpts(log=Log.NONE, trace=False), ).stdout.strip() ) From 52b711667ee2236c05656935a81b3b8e47431bea Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Fri, 4 Jul 2025 10:02:06 +0200 Subject: [PATCH 032/258] lib/get_host: improve abstraction, turn missconfiguration into a warning Motivation: A warning should encourage consistent usage of inventory.machines setting targetHost inside the machine should be considered a custom override Changing the warning strings to avoid the term 'nix'/'json' both inventory and nixos machines are nix features --- pkgs/clan-cli/clan_lib/machines/machines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_lib/machines/machines.py b/pkgs/clan-cli/clan_lib/machines/machines.py index 3715308f6..400a336d8 100644 --- a/pkgs/clan-cli/clan_lib/machines/machines.py +++ b/pkgs/clan-cli/clan_lib/machines/machines.py @@ -188,7 +188,7 @@ def get_host( if host_str is None: machine.warn( - f"'{field}' is not set in `inventory.machines.${name}.deploy.targetHost` - falling back to _slower_ nixos option: `clan.core.networking.targetHost`" + f"'{field}' is not set in `inventory.machines.${name}.deploy.targetHost` - falling back to _slower_ nixos option: `clan.core.networking.{field}`" ) host_str = machine.select(f'config.clan.core.networking."{field}"') source = "machine" From d0613b403089ac1687ae2619672a3fb7c8f921fa Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 12:22:00 +0200 Subject: [PATCH 033/258] cli: return validated list from validate_machine_names --- pkgs/clan-cli/clan_lib/machines/suggestions.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/machines/suggestions.py b/pkgs/clan-cli/clan_lib/machines/suggestions.py index 5923fc881..81be38156 100644 --- a/pkgs/clan-cli/clan_lib/machines/suggestions.py +++ b/pkgs/clan-cli/clan_lib/machines/suggestions.py @@ -48,9 +48,13 @@ def get_available_machines(flake: Flake) -> list[str]: return list(machines.keys()) -def validate_machine_names(machine_names: list[str], flake: Flake) -> None: +def validate_machine_names(machine_names: list[str], flake: Flake) -> list[str]: + """ + Returns a list of valid machine names + that are guaranteed to exist in the referenced clan + """ if not machine_names: - return + return [] available_machines = get_available_machines(flake) invalid_machines = [ @@ -70,3 +74,5 @@ def validate_machine_names(machine_names: list[str], flake: Flake) -> None: error_lines.append(f"Machine '{machine_name}' not found. {suggestion_text}") raise ClanError("\n".join(error_lines)) + + return machine_names From bff3908bb12fa64cbf7e64176cd196ec62b12cd5 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 12:22:25 +0200 Subject: [PATCH 034/258] CLI: update requireExplicitUpdate in help --- pkgs/clan-cli/clan_cli/machines/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_cli/machines/cli.py b/pkgs/clan-cli/clan_cli/machines/cli.py index cf8935d0c..708291dfe 100644 --- a/pkgs/clan-cli/clan_cli/machines/cli.py +++ b/pkgs/clan-cli/clan_cli/machines/cli.py @@ -34,7 +34,7 @@ Examples: $ clan machines update [MACHINES] Will update the specified machines [MACHINES], if [MACHINES] is omitted, the command will attempt to update every configured machine. - To exclude machines being updated `clan.deployment.requireExplicitUpdate = true;` + To exclude machines being updated `clan.core.deployment.requireExplicitUpdate = true;` can be set in the machine config. $ clan machines update --tags [TAGS..] From f7faf2cd63577186e688105f965ebf62f1d3d50e Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 12:23:47 +0200 Subject: [PATCH 035/258] machines/list: remove duplicate query_machines_by_tags --- pkgs/clan-cli/clan_lib/machines/list.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/machines/list.py b/pkgs/clan-cli/clan_lib/machines/list.py index c595b8f69..07cb83e2f 100644 --- a/pkgs/clan-cli/clan_lib/machines/list.py +++ b/pkgs/clan-cli/clan_lib/machines/list.py @@ -31,26 +31,6 @@ def list_full_machines(flake: Flake) -> dict[str, Machine]: """ machines = list_machines(flake) - return convert_inventory_to_machines(flake, machines) - - -def query_machines_by_tags( - flake: Flake, tags: list[str] -) -> dict[str, InventoryMachine]: - """ - Query machines by their respective tags, if multiple tags are specified - then only machines that have those respective tags specified will be listed. - It is an intersection of the tags and machines. - """ - machines = list_machines(flake) - - filtered_machines = {} - for machine_name, machine in machines.items(): - machine_tags = machine.get("tags", []) - if all(tag in machine_tags for tag in tags): - filtered_machines[machine_name] = machine - - return filtered_machines @dataclass From a6b8ca06ab37f0113af5728eb4c458beeb76ca6d Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 12:24:12 +0200 Subject: [PATCH 036/258] machines/list: rename helper to instantiate_inventory_to_machines --- pkgs/clan-cli/clan_lib/machines/list.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/machines/list.py b/pkgs/clan-cli/clan_lib/machines/list.py index 07cb83e2f..0fc152f6e 100644 --- a/pkgs/clan-cli/clan_lib/machines/list.py +++ b/pkgs/clan-cli/clan_lib/machines/list.py @@ -16,12 +16,12 @@ from clan_lib.nix_models.clan import InventoryMachine log = logging.getLogger(__name__) -def convert_inventory_to_machines( +def instantiate_inventory_to_machines( flake: Flake, machines: dict[str, InventoryMachine] ) -> dict[str, Machine]: return { - name: Machine.from_inventory(name, flake, inventory_machine) - for name, inventory_machine in machines.items() + name: Machine.from_inventory(name, flake, _inventory_machine) + for name, _inventory_machine in machines.items() } @@ -31,6 +31,7 @@ def list_full_machines(flake: Flake) -> dict[str, Machine]: """ machines = list_machines(flake) + return instantiate_inventory_to_machines(flake, machines) @dataclass From 3baa43fd875d8fe76eeed6a39bad50958c791360 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 12:27:28 +0200 Subject: [PATCH 037/258] cli/update: refactor machine selection logic into 'get_machines_for_update' --- pkgs/clan-cli/clan_cli/machines/update.py | 236 ++++++++++-------- .../clan-cli/clan_cli/machines/update_test.py | 162 ++++++++++++ 2 files changed, 290 insertions(+), 108 deletions(-) create mode 100644 pkgs/clan-cli/clan_cli/machines/update_test.py diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 98bc0798b..e0f539446 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -4,7 +4,9 @@ import sys from clan_lib.async_run import AsyncContext, AsyncOpts, AsyncRuntime from clan_lib.errors import ClanError -from clan_lib.machines.list import list_full_machines, query_machines_by_tags +from clan_lib.flake.flake import Flake +from clan_lib.machines.actions import list_machines +from clan_lib.machines.list import instantiate_inventory_to_machines from clan_lib.machines.machines import Machine from clan_lib.machines.suggestions import validate_machine_names from clan_lib.machines.update import deploy_machine @@ -16,128 +18,141 @@ from clan_cli.completions import ( complete_machines, complete_tags, ) -from clan_cli.host_key_check import add_host_key_check_arg log = logging.getLogger(__name__) +def requires_explicit_update(m: Machine) -> bool: + try: + if m.select("config.clan.deployment.requireExplicitUpdate"): + return False + except Exception: + pass + + try: + # check if the machine has a target host set + m.target_host # noqa: B018 + except ClanError: + return False + + return True + + +def get_machines_for_update( + flake: Flake, + explicit_names: list[str], + filter_tags: list[str], +) -> list[Machine]: + all_machines = list_machines(flake) + machines_with_tags = list_machines(flake, {"filter": {"tags": filter_tags}}) + + if filter_tags and not machines_with_tags: + msg = f"No machines found with tags: {' AND '.join(filter_tags)}" + raise ClanError(msg) + + # Implicit update all machines / with tags + # Using tags is not an explizit update + if not explicit_names: + machines_to_update = list( + filter( + requires_explicit_update, + instantiate_inventory_to_machines(flake, machines_with_tags).values(), + ) + ) + # all machines that are in the clan but not included in the update list + machine_names_to_update = [m.name for m in machines_to_update] + ignored_machines = { + m_name for m_name in all_machines if m_name not in machine_names_to_update + } + + if not machines_to_update and ignored_machines: + print( + "WARNING: No machines to update.\n" + "The following defined machines were ignored because they\n" + "- Require explicit update (see 'requireExplicitUpdate')\n", + file=sys.stderr, + ) + for m in ignored_machines: + print(m, file=sys.stderr) + + return machines_to_update + + # Else: Explicit update + machines_to_update = [] + valid_names = validate_machine_names(explicit_names, flake) + for name in valid_names: + inventory_machine = machines_with_tags.get(name) + if not inventory_machine: + msg = "This is an internal bug" + raise ClanError(msg) + + machines_to_update.append( + Machine.from_inventory(name, flake, inventory_machine) + ) + + return machines_to_update + + def update_command(args: argparse.Namespace) -> None: try: if args.flake is None: msg = "Could not find clan flake toplevel directory" raise ClanError(msg) - all_machines: list[Machine] = [] - if args.tags: - tag_filtered_machines = query_machines_by_tags(args.flake, args.tags) - if args.machines: - selected_machines = [ - name for name in args.machines if name in tag_filtered_machines - ] - else: - selected_machines = list(tag_filtered_machines.keys()) - else: - selected_machines = ( - args.machines - if args.machines - else list(list_full_machines(args.flake).keys()) - ) + machines_to_update = get_machines_for_update( + args.flake, args.machines, args.tags + ) - if args.tags and not selected_machines: - msg = f"No machines found with tags: {', '.join(args.tags)}" - raise ClanError(msg) - - if args.machines: - validate_machine_names(args.machines, args.flake) - - for machine_name in selected_machines: - machine = Machine(name=machine_name, flake=args.flake) - all_machines.append(machine) - - if args.target_host is not None and len(all_machines) > 1: + if args.target_host is not None and len(machines_to_update) > 1: msg = "Target Host can only be set for one machines" raise ClanError(msg) - def filter_machine(m: Machine) -> bool: - try: - if m.select("config.clan.deployment.requireExplicitUpdate"): - return False - except Exception: - pass + # Prepopulate the cache + config = nix_config() + system = config["system"] + machine_names = [machine.name for machine in machines_to_update] + args.flake.precache( + [ + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.validationHash", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.deployment.requireExplicitUpdate", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.system.clan.deployment.nixosMobileWorkaround", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.secretModule", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.publicModule", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.secretModule", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.publicModule", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.services", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.{{share,dependencies,migrateFact,prompts}}", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.files.*.{{secret,deploy,owner,group,mode,neededFor}}", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.secretUploadDirectory", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.vars.password-store.secretLocation", + f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.passBackend", + ] + ) - try: - # check if the machine has a target host set - m.target_host # noqa: B018 - except ClanError: - return False - - return True - - machines_to_update = all_machines - implicit_all: bool = len(args.machines) == 0 and not args.tags - if implicit_all: - machines_to_update = list(filter(filter_machine, all_machines)) - - # machines that are in the list but not included in the update list - ignored_machines = {m.name for m in all_machines if m not in machines_to_update} - - if not machines_to_update and ignored_machines: - print( - "WARNING: No machines to update.\n" - "The following defined machines were ignored because they\n" - "- Require explicit update (see 'requireExplicitUpdate')\n", - "- Might not have the `clan.core.networking.targetHost` nixos option set:\n", - file=sys.stderr, - ) - for m in ignored_machines: - print(m, file=sys.stderr) - - if machines_to_update: - # Prepopulate the cache - config = nix_config() - system = config["system"] - machine_names = [machine.name for machine in machines_to_update] - args.flake.precache( - [ - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.validationHash", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.deployment.requireExplicitUpdate", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.system.clan.deployment.nixosMobileWorkaround", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.secretModule", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.publicModule", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.secretModule", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.publicModule", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.services", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.{{share,dependencies,migrateFact,prompts}}", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.files.*.{{secret,deploy,owner,group,mode,neededFor}}", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.secretUploadDirectory", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.passBackend", - ] - ) - - host_key_check = args.host_key_check - with AsyncRuntime() as runtime: - for machine in machines_to_update: - if args.target_host: - target_host = Remote.from_ssh_uri( - machine_name=machine.name, - address=args.target_host, - ).override(host_key_check=host_key_check) - else: - target_host = machine.target_host().override( - host_key_check=host_key_check - ) - runtime.async_run( - AsyncOpts( - tid=machine.name, - async_ctx=AsyncContext(prefix=machine.name), - ), - deploy_machine, - machine=machine, - target_host=target_host, - build_host=machine.build_host(), + host_key_check = args.host_key_check + with AsyncRuntime() as runtime: + for machine in machines_to_update: + if args.target_host: + target_host = Remote.from_ssh_uri( + machine_name=machine.name, + address=args.target_host, + ).override(host_key_check=host_key_check) + else: + target_host = machine.target_host().override( + host_key_check=host_key_check ) - runtime.join_all() - runtime.check_all() + runtime.async_run( + AsyncOpts( + tid=machine.name, + async_ctx=AsyncContext(prefix=machine.name), + ), + deploy_machine, + machine=machine, + target_host=target_host, + build_host=machine.build_host(), + ) + runtime.join_all() + runtime.check_all() except KeyboardInterrupt: log.warning("Interrupted by user") @@ -163,7 +178,12 @@ def register_update_parser(parser: argparse.ArgumentParser) -> None: ) add_dynamic_completer(tag_parser, complete_tags) - add_host_key_check_arg(parser) + parser.add_argument( + "--host-key-check", + choices=["strict", "ask", "tofu", "none"], + default="ask", + help="Host key (.ssh/known_hosts) check mode.", + ) parser.add_argument( "--target-host", type=str, diff --git a/pkgs/clan-cli/clan_cli/machines/update_test.py b/pkgs/clan-cli/clan_cli/machines/update_test.py new file mode 100644 index 000000000..ffb8b2922 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/machines/update_test.py @@ -0,0 +1,162 @@ +import pytest +from clan_lib.flake import Flake + +from clan_cli.machines.update import get_machines_for_update + +# Functions to test +from clan_cli.tests.fixtures_flakes import FlakeForTest + + +@pytest.mark.parametrize( + ("test_flake_with_core", "explicit_names", "filter_tags", "expected_names"), + [ + ( + { + "inventory_expr": r"""{ + machines.jon = { tags = [ "foo" "bar" ]; }; + machines.sara = { tags = [ "foo" "baz" ]; }; + }""" + }, + ["jon"], # explizit names + [], # filter tags + ["jon"], # expected + ) + ], + # Important! + # tells pytest to pass these values to the fixture + # So we can write it to the flake fixtures + indirect=["test_flake_with_core"], +) +@pytest.mark.with_core +def test_get_machines_for_update_single_name( + test_flake_with_core: FlakeForTest, + explicit_names: list[str], + filter_tags: list[str], + expected_names: list[str], +) -> None: + selected_for_update = get_machines_for_update( + Flake(str(test_flake_with_core.path)), + explicit_names=explicit_names, + filter_tags=filter_tags, + ) + names = [m.name for m in selected_for_update] + + print(explicit_names, filter_tags) + assert names == expected_names + + +@pytest.mark.parametrize( + ("test_flake_with_core", "explicit_names", "filter_tags", "expected_names"), + [ + ( + { + "inventory_expr": r"""{ + machines.jon = { tags = [ "foo" "bar" ]; }; + machines.sara = { tags = [ "foo" "baz" ]; }; + }""" + }, + [], # explizit names + ["foo"], # filter tags + ["jon", "sara"], # expected + ) + ], + # Important! + # tells pytest to pass these values to the fixture + # So we can write it to the flake fixtures + indirect=["test_flake_with_core"], +) +@pytest.mark.with_core +def test_get_machines_for_update_tags( + test_flake_with_core: FlakeForTest, + explicit_names: list[str], + filter_tags: list[str], + expected_names: list[str], +) -> None: + selected_for_update = get_machines_for_update( + Flake(str(test_flake_with_core.path)), + explicit_names=explicit_names, + filter_tags=filter_tags, + ) + names = [m.name for m in selected_for_update] + + print(explicit_names, filter_tags) + assert names == expected_names + + +@pytest.mark.parametrize( + ("test_flake_with_core", "explicit_names", "filter_tags", "expected_names"), + [ + ( + { + "inventory_expr": r"""{ + machines.jon = { tags = [ "foo" "bar" ]; }; + machines.sara = { tags = [ "foo" "baz" ]; }; + }""" + }, + ["sara"], # explizit names + ["foo"], # filter tags + ["sara"], # expected + ) + ], + # Important! + # tells pytest to pass these values to the fixture + # So we can write it to the flake fixtures + indirect=["test_flake_with_core"], +) +@pytest.mark.with_core +def test_get_machines_for_update_tags_and_name( + test_flake_with_core: FlakeForTest, + explicit_names: list[str], + filter_tags: list[str], + expected_names: list[str], +) -> None: + selected_for_update = get_machines_for_update( + Flake(str(test_flake_with_core.path)), + explicit_names=explicit_names, + filter_tags=filter_tags, + ) + names = [m.name for m in selected_for_update] + + print(explicit_names, filter_tags) + assert names == expected_names + + +@pytest.mark.parametrize( + ("test_flake_with_core", "explicit_names", "filter_tags", "expected_names"), + [ + ( + { + "inventory_expr": r"""{ + machines.jon = { tags = [ "foo" "bar" ]; }; + machines.sara = { tags = [ "foo" "baz" ]; }; + }""" + }, + [], # no explizit names + [], # no filter tags + ["jon", "sara", "vm1", "vm2"], # all machines + ), + ], + # Important! + # tells pytest to pass these values to the fixture + # So we can write it to the flake fixtures + indirect=["test_flake_with_core"], +) +@pytest.mark.with_core +def test_get_machines_for_update_implicit_all( + test_flake_with_core: FlakeForTest, + explicit_names: list[str], + filter_tags: list[str], + expected_names: list[str], +) -> None: + selected_for_update = get_machines_for_update( + Flake(str(test_flake_with_core.path)), + explicit_names=explicit_names, + filter_tags=filter_tags, + ) + names = [m.name for m in selected_for_update] + + print(explicit_names, filter_tags) + assert names == expected_names + + +# TODO: Add more tests for requireExplicitUpdate From 06613de825638932eb41510d01f961c7186a23da Mon Sep 17 00:00:00 2001 From: adeci Date: Sun, 6 Jul 2025 02:27:12 -0400 Subject: [PATCH 038/258] clan-cli: fix incorrect field name in deploy warning messages. The warning for missing buildHost/targetHost always showed targetHost in the path, even when buildHost was the missing field. --- pkgs/clan-cli/clan_lib/machines/machines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_lib/machines/machines.py b/pkgs/clan-cli/clan_lib/machines/machines.py index 400a336d8..489b0bccf 100644 --- a/pkgs/clan-cli/clan_lib/machines/machines.py +++ b/pkgs/clan-cli/clan_lib/machines/machines.py @@ -188,7 +188,7 @@ def get_host( if host_str is None: machine.warn( - f"'{field}' is not set in `inventory.machines.${name}.deploy.targetHost` - falling back to _slower_ nixos option: `clan.core.networking.{field}`" + f"'{field}' is not set in `inventory.machines.${machine.name}.deploy.{field}` - falling back to _slower_ nixos option: `clan.core.networking.{field}`" ) host_str = machine.select(f'config.clan.core.networking."{field}"') source = "machine" From bd82de60013b5f528f435809313083ba1c908ba5 Mon Sep 17 00:00:00 2001 From: lassulus Date: Fri, 4 Jul 2025 14:35:33 +0200 Subject: [PATCH 039/258] fix(flake): handle file paths with line numbers in cache existence check The is_cached method now correctly handles store paths that have line numbers appended (e.g., /nix/store/file.nix:123:456). Previously, these paths would fail the existence check because the exact path with line numbers doesn't exist as a file. The fix adds a helper method that: - First checks if the exact path exists - If not, and the path contains colons, validates that the suffix consists only of numbers (line:column format) - If valid, strips the line numbers and checks the base file path This ensures that cached references to specific file locations are properly validated while avoiding false positives with files that have colons in their names. --- pkgs/clan-cli/clan_lib/flake/flake.py | 21 ++++- .../clan_lib/flake/flake_cache_test.py | 78 +++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/flake/flake.py b/pkgs/clan-cli/clan_lib/flake/flake.py index 2d82a8acb..e4beaf2c2 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake.py +++ b/pkgs/clan-cli/clan_lib/flake/flake.py @@ -345,6 +345,23 @@ class FlakeCacheEntry: msg = f"Cannot insert {value} into cache, already have {self.value}" raise TypeError(msg) + def _check_path_exists(self, path_str: str) -> bool: + """Check if a path exists, handling potential line number suffixes.""" + path = Path(path_str) + if path.exists(): + return True + + # Try stripping line numbers if the path doesn't exist + # Handle format: /path/to/file:123 or /path/to/file:123:456 + if ":" in path_str: + parts = path_str.split(":") + if len(parts) >= 2: + # Check if all parts after the first colon are numbers + if all(part.isdigit() for part in parts[1:]): + base_path = parts[0] + return Path(base_path).exists() + return False + def is_cached(self, selectors: list[Selector]) -> bool: selector: Selector @@ -353,12 +370,12 @@ class FlakeCacheEntry: # Check if it's a regular nix store path nix_store_dir = os.environ.get("NIX_STORE_DIR", "/nix/store") if self.value.startswith(nix_store_dir): - return Path(self.value).exists() + return self._check_path_exists(self.value) # Check if it's a test store path test_store = os.environ.get("CLAN_TEST_STORE") if test_store and self.value.startswith(test_store): - return Path(self.value).exists() + return self._check_path_exists(self.value) # if self.value is not dict but we request more selectors, we assume we are cached and an error will be thrown in the select function if isinstance(self.value, str | float | int | None): diff --git a/pkgs/clan-cli/clan_lib/flake/flake_cache_test.py b/pkgs/clan-cli/clan_lib/flake/flake_cache_test.py index 625abf913..9849752ed 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake_cache_test.py +++ b/pkgs/clan-cli/clan_lib/flake/flake_cache_test.py @@ -296,3 +296,81 @@ def test_cache_gc(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: assert my_flake._cache.is_cached("testfile") # noqa: SLF001 subprocess.run(["nix-collect-garbage"], check=True) assert not my_flake._cache.is_cached("testfile") # noqa: SLF001 + + +def test_cache_path_with_line_numbers( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Test that is_cached correctly handles store paths with line numbers appended. + + This is a regression test for the bug where cached store paths with line numbers + (e.g., /nix/store/path:123) are not properly checked for existence. + """ + # Create a temporary store + test_store = tmp_path / "test-store" + test_store.mkdir() + + # Set CLAN_TEST_STORE environment variable + monkeypatch.setenv("CLAN_TEST_STORE", str(test_store)) + + # Create a fake store path + fake_store_path = test_store / "abc123-source-file.nix" + fake_store_path.write_text("# nix source file\n{ foo = 123; }") + + # Create cache entries for paths with line numbers + cache = FlakeCacheEntry() + + # Test single line number format + path_with_line = f"{fake_store_path}:42" + selectors = parse_selector("testPath1") + cache.insert(path_with_line, selectors) + + # Test line:column format + path_with_line_col = f"{fake_store_path}:42:10" + selectors2 = parse_selector("testPath2") + cache.insert(path_with_line_col, selectors2) + + # Test path with colon but non-numeric suffix (should not be treated as line number) + path_with_colon = test_store / "file:with:colons" + path_with_colon.write_text("test") + selectors3 = parse_selector("testPath3") + cache.insert(str(path_with_colon), selectors3) + + # Before the fix: These would return True even though the exact paths don't exist + # After the fix: They check the base file path exists + assert cache.is_cached(parse_selector("testPath1")), ( + "Path with line number should be cached when base file exists" + ) + assert cache.is_cached(parse_selector("testPath2")), ( + "Path with line:column should be cached when base file exists" + ) + assert cache.is_cached(parse_selector("testPath3")), ( + "Path with colons in filename should be cached when file exists" + ) + + # Now delete the base file + fake_store_path.unlink() + + # After deletion, paths with line numbers should not be cached + assert not cache.is_cached(parse_selector("testPath1")), ( + "Path with line number should not be cached when base file doesn't exist" + ) + assert not cache.is_cached(parse_selector("testPath2")), ( + "Path with line:column should not be cached when base file doesn't exist" + ) + + # Path with colons in name still exists + assert cache.is_cached(parse_selector("testPath3")), ( + "Path with colons in filename should still be cached" + ) + + # Test with regular /nix/store paths + monkeypatch.delenv("CLAN_TEST_STORE", raising=False) + cache2 = FlakeCacheEntry() + nix_path_with_line = "/nix/store/fake-source.nix:123" + cache2.insert(nix_path_with_line, parse_selector("nixPath")) + + # Should return False because neither the exact path nor base path exists + assert not cache2.is_cached(parse_selector("nixPath")), ( + "Nix store path with line number should not be cached when file doesn't exist" + ) From 8302f3ffde70a611f37267a06f4f5c58b6b793bc Mon Sep 17 00:00:00 2001 From: lassulus Date: Sun, 29 Jun 2025 10:51:18 +0200 Subject: [PATCH 040/258] vars/password-store: replace passBackend option with passPackage The `clan.core.vars.settings.passBackend` option has been replaced with `clan.vars.password-store.passPackage` to provide better type safety and clearer configuration. Changes: - Remove problematic mkRemovedOptionModule that caused circular dependency - Add proper option definition with assertion-based migration - Users setting the old option get clear migration instructions - Normal evaluation continues to work for users not using the old option Migration: Replace `clan.core.vars.settings.passBackend = "passage"` with `clan.vars.password-store.passPackage = pkgs.passage` --- nixosModules/clanCore/vars/default.nix | 12 ++ .../clanCore/vars/secret/password-store.nix | 9 +- nixosModules/clanCore/vars/settings-opts.nix | 22 +-- pkgs/clan-cli/clan_cli/machines/update.py | 2 - pkgs/clan-cli/clan_cli/tests/test_vars.py | 13 ++ pkgs/clan-cli/clan_cli/vars/generate.py | 9 +- .../vars/secret_modules/password_store.py | 157 +++++++++--------- pkgs/clan-cli/default.nix | 3 + 8 files changed, 129 insertions(+), 98 deletions(-) diff --git a/nixosModules/clanCore/vars/default.nix b/nixosModules/clanCore/vars/default.nix index 0fa5c6165..afe8962ad 100644 --- a/nixosModules/clanCore/vars/default.nix +++ b/nixosModules/clanCore/vars/default.nix @@ -40,6 +40,18 @@ in }; config = { + # Check for removed passBackend option usage + assertions = [ + { + assertion = config.clan.core.vars.settings.passBackend == null; + message = '' + The option `clan.core.vars.settings.passBackend' has been removed. + Use clan.vars.password-store.passPackage instead. + Set it to pkgs.pass for GPG or pkgs.passage for age encryption. + ''; + } + ]; + # check all that all non-secret files have no owner/group/mode set warnings = lib.foldl' ( warnings: generator: diff --git a/nixosModules/clanCore/vars/secret/password-store.nix b/nixosModules/clanCore/vars/secret/password-store.nix index 8b02be38d..d79c46cf4 100644 --- a/nixosModules/clanCore/vars/secret/password-store.nix +++ b/nixosModules/clanCore/vars/secret/password-store.nix @@ -62,6 +62,13 @@ in location where the tarball with the password-store secrets will be uploaded to and the manifest ''; }; + passPackage = lib.mkOption { + type = lib.types.package; + default = pkgs.pass; + description = '' + Password store package to use. Can be pkgs.pass for GPG-based storage or pkgs.passage for age-based storage. + ''; + }; }; config = { clan.core.vars.settings = @@ -76,7 +83,7 @@ in else if file.config.neededFor == "services" then "/run/secrets/${file.config.generatorName}/${file.config.name}" else if file.config.neededFor == "activation" then - "${config.clan.password-store.secretLocation}/activation/${file.config.generatorName}/${file.config.name}" + "${config.clan.vars.password-store.secretLocation}/activation/${file.config.generatorName}/${file.config.name}" else if file.config.neededFor == "partitioning" then "/run/partitioning-secrets/${file.config.generatorName}/${file.config.name}" else diff --git a/nixosModules/clanCore/vars/settings-opts.nix b/nixosModules/clanCore/vars/settings-opts.nix index 276da4e9e..226888111 100644 --- a/nixosModules/clanCore/vars/settings-opts.nix +++ b/nixosModules/clanCore/vars/settings-opts.nix @@ -15,17 +15,6 @@ ''; }; - passBackend = lib.mkOption { - type = lib.types.enum [ - "passage" - "pass" - ]; - default = "pass"; - description = '' - password-store backend to use. Valid options are `pass` and `passage` - ''; - }; - secretModule = lib.mkOption { type = lib.types.str; internal = true; @@ -65,4 +54,15 @@ the python import path to the public module ''; }; + + # Legacy option that guides migration + passBackend = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + visible = false; + description = '' + DEPRECATED: This option has been removed. Use clan.vars.password-store.passPackage instead. + Set it to pkgs.pass for GPG or pkgs.passage for age encryption. + ''; + }; } diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index e0f539446..75c543c8c 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -124,8 +124,6 @@ def update_command(args: argparse.Namespace) -> None: f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.{{share,dependencies,migrateFact,prompts}}", f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.generators.*.files.*.{{secret,deploy,owner,group,mode,neededFor}}", f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.facts.secretUploadDirectory", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.vars.password-store.secretLocation", - f"clanInternals.machines.{system}.{{{','.join(machine_names)}}}.config.clan.core.vars.settings.passBackend", ] ) diff --git a/pkgs/clan-cli/clan_cli/tests/test_vars.py b/pkgs/clan-cli/clan_cli/tests/test_vars.py index 181b2a543..274dd38fc 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_vars.py +++ b/pkgs/clan-cli/clan_cli/tests/test_vars.py @@ -403,6 +403,19 @@ def test_generate_secret_var_password_store( shutil.copytree(test_root / "data" / "password-store", password_store_dir) monkeypatch.setenv("PASSWORD_STORE_DIR", str(password_store_dir)) + # Initialize password store as a git repository + import subprocess + + subprocess.run(["git", "init"], cwd=password_store_dir, check=True) + subprocess.run( + ["git", "config", "user.email", "test@example.com"], + cwd=password_store_dir, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "Test User"], cwd=password_store_dir, check=True + ) + machine = Machine(name="my_machine", flake=Flake(str(flake.path))) assert not check_vars(machine.name, machine.flake) cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 925c97b1a..2d1ef9c08 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -82,11 +82,6 @@ class Generator: files = [] gen_files = files_data.get(gen_name, {}) for file_name, file_data in gen_files.items(): - # Handle mode conversion properly - mode = file_data["mode"] - if isinstance(mode, str): - mode = int(mode, 8) - var = Var( id=f"{gen_name}/{file_name}", name=file_name, @@ -94,7 +89,9 @@ class Generator: deploy=file_data["deploy"], owner=file_data["owner"], group=file_data["group"], - mode=mode, + mode=file_data["mode"] + if isinstance(file_data["mode"], int) + else int(file_data["mode"], 8), needed_for=file_data["neededFor"], ) files.append(var) diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py index 98060a997..d23cf18a4 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py @@ -1,9 +1,7 @@ import io import logging -import os import tarfile from collections.abc import Iterable -from itertools import chain from pathlib import Path from tempfile import TemporaryDirectory @@ -12,7 +10,6 @@ from clan_cli.vars._types import StoreBase from clan_cli.vars.generate import Generator, Var from clan_lib.cmd import CmdOut, Log, RunOpts, run from clan_lib.machines.machines import Machine -from clan_lib.nix import nix_shell from clan_lib.ssh.remote import Remote log = logging.getLogger(__name__) @@ -26,31 +23,64 @@ class SecretStore(StoreBase): def __init__(self, machine: Machine) -> None: self.machine = machine self.entry_prefix = "clan-vars" + self._store_dir: Path | None = None @property def store_name(self) -> str: return "password_store" @property - def _store_backend(self) -> str: - backend = self.machine.select("config.clan.core.vars.settings.passBackend") - return backend + def store_dir(self) -> Path: + """Get the password store directory, cached after first access.""" + if self._store_dir is None: + result = self._run_pass( + "git", "rev-parse", "--show-toplevel", options=RunOpts(check=False) + ) + if result.returncode != 0: + msg = "Password store must be a git repository" + raise ValueError(msg) + self._store_dir = Path(result.stdout.strip()) + return self._store_dir @property - def _password_store_dir(self) -> Path: - if self._store_backend == "passage": - lookup = os.environ.get("PASSAGE_DIR") - default = Path.home() / ".passage/store" - else: - lookup = os.environ.get("PASSWORD_STORE_DIR") - default = Path.home() / ".password-store" - return Path(lookup) if lookup else default + def _pass_command(self) -> str: + out_path = self.machine.select( + "config.clan.vars.password-store.passPackage.outPath" + ) + main_program = ( + self.machine.select( + "config.clan.vars.password-store.passPackage.?meta.?mainProgram" + ) + .get("meta", {}) + .get("mainProgram") + ) + + if main_program: + binary_path = Path(out_path) / "bin" / main_program + if binary_path.exists(): + return str(binary_path) + + # Look for common password store binaries + bin_dir = Path(out_path) / "bin" + if bin_dir.exists(): + for binary in ["pass", "passage"]: + binary_path = bin_dir / binary + if binary_path.exists(): + return str(binary_path) + + # If only one binary exists, use it + binaries = [f for f in bin_dir.iterdir() if f.is_file()] + if len(binaries) == 1: + return str(binaries[0]) + + msg = "Could not find password store binary in package" + raise ValueError(msg) def entry_dir(self, generator: Generator, name: str) -> Path: return Path(self.entry_prefix) / self.rel_dir(generator, name) def _run_pass(self, *args: str, options: RunOpts | None = None) -> CmdOut: - cmd = nix_shell(packages=["pass"], cmd=[self._store_backend, *args]) + cmd = [self._pass_command, *args] return run(cmd, options) def _set( @@ -68,9 +98,11 @@ class SecretStore(StoreBase): return self._run_pass("show", pass_name).stdout.encode() def exists(self, generator: Generator, name: str) -> bool: - extension = "age" if self._store_backend == "passage" else "gpg" - filename = f"{self.entry_dir(generator, name)}.{extension}" - return (self._password_store_dir / filename).exists() + pass_name = str(self.entry_dir(generator, name)) + # Check if the file exists with either .age or .gpg extension + age_file = self.store_dir / f"{pass_name}.age" + gpg_file = self.store_dir / f"{pass_name}.gpg" + return age_file.exists() or gpg_file.exists() def delete(self, generator: Generator, name: str) -> Iterable[Path]: pass_name = str(self.entry_dir(generator, name)) @@ -79,66 +111,31 @@ class SecretStore(StoreBase): def delete_store(self) -> Iterable[Path]: machine_dir = Path(self.entry_prefix) / "per-machine" / self.machine.name - if not (self._password_store_dir / machine_dir).exists(): - # The directory may not exist if the machine - # has no vars, or they have been deleted already. - return [] - pass_call = ["rm", "--force", "--recursive", str(machine_dir)] - self._run_pass(*pass_call, options=RunOpts(check=True)) + # Check if the directory exists in the password store before trying to delete + result = self._run_pass("ls", str(machine_dir), options=RunOpts(check=False)) + if result.returncode == 0: + self._run_pass( + "rm", + "--force", + "--recursive", + str(machine_dir), + options=RunOpts(check=True), + ) return [] def generate_hash(self) -> bytes: - hashes = [] - hashes.append( - run( - nix_shell( - ["git"], - [ - "git", - "-C", - str(self._password_store_dir), - "log", - "-1", - "--format=%H", - self.entry_prefix, - ], - ), - RunOpts(check=False), - ) - .stdout.strip() - .encode() + result = self._run_pass( + "git", + "log", + "-1", + "--format=%H", + self.entry_prefix, + options=RunOpts(check=False), ) - shared_dir = self._password_store_dir / self.entry_prefix / "shared" - machine_dir = ( - self._password_store_dir - / self.entry_prefix - / "per-machine" - / self.machine.name - ) - for symlink in chain(shared_dir.glob("**/*"), machine_dir.glob("**/*")): - if symlink.is_symlink(): - hashes.append( - run( - nix_shell( - ["git"], - [ - "git", - "-C", - str(self._password_store_dir), - "log", - "-1", - "--format=%H", - str(symlink), - ], - ), - RunOpts(check=False), - ) - .stdout.strip() - .encode() - ) + git_hash = result.stdout.strip().encode() - # we sort the hashes to make sure that the order is always the same - hashes.sort() + if not git_hash: + return b"" from clan_cli.vars.generate import Generator @@ -149,22 +146,24 @@ class SecretStore(StoreBase): for generator in generators: for file in generator.files: manifest.append(f"{generator.name}/{file.name}".encode()) - manifest += hashes + + manifest.append(git_hash) return b"\n".join(manifest) def needs_upload(self, host: Remote) -> bool: local_hash = self.generate_hash() + if not local_hash: + return True + remote_hash = host.run( - # TODO get the path to the secrets from the machine [ "cat", - f"{self.machine.select('config.clan.vars.password-store.secretLocation')}/.{self._store_backend}_info", + f"{self.machine.select('config.clan.vars.password-store.secretLocation')}/.pass_info", ], RunOpts(log=Log.STDERR, check=False), ).stdout.strip() if not remote_hash: - print("remote hash is empty") return True return local_hash.decode() != remote_hash @@ -233,7 +232,9 @@ class SecretStore(StoreBase): out_file.parent.mkdir(parents=True, exist_ok=True) out_file.write_bytes(self.get(generator, file.name)) - (output_dir / f".{self._store_backend}_info").write_bytes(self.generate_hash()) + hash_data = self.generate_hash() + if hash_data: + (output_dir / ".pass_info").write_bytes(hash_data) def upload(self, host: Remote, phases: list[str]) -> None: if "partitioning" in phases: diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 1e04822c7..be9dd935a 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -2,6 +2,7 @@ # callPackage args gnupg, installShellFiles, + pass, jq, lib, nix, @@ -58,6 +59,7 @@ let testDependencies = testRuntimeDependencies ++ [ gnupg + pass stdenv.cc # Compiler used for certain native extensions (pythonRuntime.withPackages pyTestDeps) ]; @@ -213,6 +215,7 @@ pythonRuntime.pkgs.buildPythonApplication { pkgs.shellcheck-minimal pkgs.mkpasswd pkgs.xkcdpass + pkgs.pass nix-select ]; }; From db6220b57b40cf494fe9680a755735dac1eb3075 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 14:37:03 +0200 Subject: [PATCH 041/258] Templates/list: display templates via exposed nix value --- lib/modules/clan/module.nix | 4 +- .../service-list-from-inputs.nix | 21 +++++++--- pkgs/clan-cli/clan_cli/clan/list.py | 39 ++++++++++++++----- .../clan_cli/tests/test_clan_nix_attrset.py | 5 --- pkgs/clan-cli/clan_lib/api/modules.py | 1 - pkgs/clan-cli/clan_lib/templates/__init__.py | 31 +++++---------- 6 files changed, 56 insertions(+), 45 deletions(-) diff --git a/lib/modules/clan/module.nix b/lib/modules/clan/module.nix index 917468e0f..7b0986649 100644 --- a/lib/modules/clan/module.nix +++ b/lib/modules/clan/module.nix @@ -229,8 +229,6 @@ in clanInternals = { inventoryClass = let - localModuleSet = - lib.filterAttrs (n: _: !inventory._legacyModules ? ${n}) inventory.modules // config.modules; flakeInputs = config.self.inputs; in { @@ -240,7 +238,7 @@ in imports = [ ../inventoryClass/builder/default.nix (lib.modules.importApply ../inventoryClass/service-list-from-inputs.nix { - inherit flakeInputs clanLib localModuleSet; + inherit flakeInputs clanLib; }) { inherit inventory directory; diff --git a/lib/modules/inventoryClass/service-list-from-inputs.nix b/lib/modules/inventoryClass/service-list-from-inputs.nix index a73d9c87a..7e913bed1 100644 --- a/lib/modules/inventoryClass/service-list-from-inputs.nix +++ b/lib/modules/inventoryClass/service-list-from-inputs.nix @@ -1,12 +1,9 @@ { flakeInputs, clanLib, - localModuleSet, }: { lib, config, ... }: - let - inspectModule = inputName: moduleName: module: let @@ -28,16 +25,30 @@ in { options.modulesPerSource = lib.mkOption { # { sourceName :: { moduleName :: {} }} + readOnly = true; + type = lib.types.raw; default = let inputsWithModules = lib.filterAttrs (_inputName: v: v ? clan.modules) flakeInputs; - in lib.mapAttrs ( inputName: v: lib.mapAttrs (inspectModule inputName) v.clan.modules ) inputsWithModules; }; options.localModules = lib.mkOption { - default = lib.mapAttrs (inspectModule "self") localModuleSet; + readOnly = true; + type = lib.types.raw; + default = config.modulesPerSource.self; + }; + options.templatesPerSource = lib.mkOption { + # { sourceName :: { moduleName :: {} }} + readOnly = true; + type = lib.types.raw; + default = + let + inputsWithTemplates = lib.filterAttrs (_inputName: v: v ? clan.templates) flakeInputs; + in + lib.mapAttrs (_inputName: v: lib.mapAttrs (_n: t: t) v.clan.templates) inputsWithTemplates; + }; } diff --git a/pkgs/clan-cli/clan_cli/clan/list.py b/pkgs/clan-cli/clan_cli/clan/list.py index ff80c20f1..58ae350d0 100644 --- a/pkgs/clan-cli/clan_cli/clan/list.py +++ b/pkgs/clan-cli/clan_cli/clan/list.py @@ -7,17 +7,38 @@ log = logging.getLogger(__name__) def list_command(args: argparse.Namespace) -> None: - template_list = list_templates("clan", args.flake) + templates = list_templates(args.flake) - print("Available local templates:") - for name, template in template_list.self.items(): - print(f" {name}: {template['description']}") + builtin_clan_templates = templates.builtins.get("clan", {}) - print("Available templates from inputs:") - for input_name, input_templates in template_list.inputs.items(): - print(f" {input_name}:") - for name, template in input_templates.items(): - print(f" {name}: {template['description']}") + print("Available templates") + print("β”œβ”€β”€ ") + for i, (name, template) in enumerate(builtin_clan_templates.items()): + is_last_template = i == len(builtin_clan_templates.items()) - 1 + if not is_last_template: + print(f"β”‚ β”œβ”€β”€ {name}: {template.get('description', 'no description')}") + else: + print(f"β”‚ └── {name}: {template.get('description', 'no description')}") + + for i, (input_name, input_templates) in enumerate(templates.custom.items()): + custom_clan_templates = input_templates.get("clan", {}) + is_last_input = i == len(templates.custom.items()) - 1 + prefix = "β”‚" if not is_last_input else " " + if not is_last_input: + print(f"β”œβ”€β”€ inputs.{input_name}:") + else: + print(f"└── inputs.{input_name}:") + + for i, (name, template) in enumerate(custom_clan_templates.items()): + is_last_template = i == len(builtin_clan_templates.items()) - 1 + if not is_last_template: + print( + f"{prefix} β”œβ”€β”€ {name}: {template.get('description', 'no description')}" + ) + else: + print( + f"{prefix} └── {name}: {template.get('description', 'no description')}" + ) def register_list_parser(parser: argparse.ArgumentParser) -> None: diff --git a/pkgs/clan-cli/clan_cli/tests/test_clan_nix_attrset.py b/pkgs/clan-cli/clan_cli/tests/test_clan_nix_attrset.py index 8cda0d002..0e7de5f9f 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_clan_nix_attrset.py +++ b/pkgs/clan-cli/clan_cli/tests/test_clan_nix_attrset.py @@ -17,7 +17,6 @@ from clan_lib.templates import ( TemplateName, get_clan_nix_attrset, get_template, - list_templates, ) from clan_lib.templates.filesystem import copy_from_nixstore @@ -96,10 +95,6 @@ def test_clan_core_templates( expected_templates = ["default", "flake-parts", "minimal", "minimal-flake-parts"] assert clan_core_template_keys == expected_templates - vlist_temps = list_templates("clan", clan_dir) - list_template_keys = list(vlist_temps.inputs[InputName("clan-core")].keys()) - assert list_template_keys == expected_templates - default_template = get_template( TemplateName("default"), "clan", diff --git a/pkgs/clan-cli/clan_lib/api/modules.py b/pkgs/clan-cli/clan_lib/api/modules.py index d6d624610..7e306a2c0 100644 --- a/pkgs/clan-cli/clan_lib/api/modules.py +++ b/pkgs/clan-cli/clan_lib/api/modules.py @@ -168,7 +168,6 @@ def list_modules(base_path: str) -> ModuleLists: modules = flake.select( "clanInternals.inventoryClass.{?modulesPerSource,?localModules}" ) - print("Modules found:", modules) return modules diff --git a/pkgs/clan-cli/clan_lib/templates/__init__.py b/pkgs/clan-cli/clan_lib/templates/__init__.py index 4b30131e5..ae2c556ae 100644 --- a/pkgs/clan-cli/clan_lib/templates/__init__.py +++ b/pkgs/clan-cli/clan_lib/templates/__init__.py @@ -1,10 +1,11 @@ import logging -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any, Literal, NewType, TypedDict, cast from clan_lib.dirs import clan_templates from clan_lib.errors import ClanCmdError, ClanError from clan_lib.flake import Flake +from clan_lib.nix_models.clan import ClanTemplatesType from clan_lib.templates.filesystem import realize_nix_path log = logging.getLogger(__name__) @@ -168,32 +169,18 @@ class InputPrio: @dataclass class TemplateList: - inputs: dict[InputName, dict[TemplateName, Template]] = field(default_factory=dict) - self: dict[TemplateName, Template] = field(default_factory=dict) + builtins: ClanTemplatesType + custom: dict[str, ClanTemplatesType] -def list_templates( - template_type: TemplateType, clan_dir: Flake | None = None -) -> TemplateList: +def list_templates(flake: Flake) -> TemplateList: """ - List all templates of a specific type from a flake, without a path attribute. - As these paths are not yet downloaded into the nix store, and thus cannot be used directly. + Show information about a module """ - clan_exports = get_clan_nix_attrset(clan_dir) - result = TemplateList() + custom_templates = flake.select("clanInternals.inventoryClass.templatesPerSource") + builtin_templates = flake.select("clanInternals.templates") - for template_name, template in clan_exports["self"]["templates"][ - template_type - ].items(): - result.self[template_name] = template - - for input_name, attrset in clan_exports["inputs"].items(): - for template_name, template in attrset["templates"][template_type].items(): - if input_name not in result.inputs: - result.inputs[input_name] = {} - result.inputs[input_name][template_name] = template - - return result + return TemplateList(builtin_templates, custom_templates) def get_template( From e72795904d247aa18f9f53a205750ff499bcbfee Mon Sep 17 00:00:00 2001 From: lassulus Date: Sun, 6 Jul 2025 14:14:29 +0200 Subject: [PATCH 042/258] Revert "make host key check an enum instead of an literal type" This reverts commit 543c518ed0573a23883aa1f0f0be0163bb098add. --- pkgs/clan-cli/clan_cli/host_key_check.py | 28 ------------------- pkgs/clan-cli/clan_cli/machines/hardware.py | 8 ++++-- pkgs/clan-cli/clan_cli/machines/install.py | 8 ++++-- pkgs/clan-cli/clan_cli/ssh/deploy_info.py | 11 +++++--- .../clan-cli/clan_cli/ssh/test_deploy_info.py | 9 ++---- pkgs/clan-cli/clan_cli/tests/hosts.py | 3 +- pkgs/clan-cli/clan_lib/ssh/host_key.py | 22 +++++++-------- pkgs/clan-cli/clan_lib/ssh/remote.py | 2 +- pkgs/clan-cli/clan_lib/ssh/remote_test.py | 5 ++-- pkgs/clan-cli/clan_lib/tests/test_create.py | 3 +- 10 files changed, 38 insertions(+), 61 deletions(-) delete mode 100644 pkgs/clan-cli/clan_cli/host_key_check.py diff --git a/pkgs/clan-cli/clan_cli/host_key_check.py b/pkgs/clan-cli/clan_cli/host_key_check.py deleted file mode 100644 index df1331574..000000000 --- a/pkgs/clan-cli/clan_cli/host_key_check.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Common argument types and utilities for host key checking in clan CLI commands.""" - -import argparse - -from clan_lib.ssh.host_key import HostKeyCheck - - -def host_key_check_type(value: str) -> HostKeyCheck: - """ - Argparse type converter for HostKeyCheck enum. - """ - try: - return HostKeyCheck(value) - except ValueError: - valid_values = [e.value for e in HostKeyCheck] - msg = f"Invalid host key check mode: {value}. Valid options: {', '.join(valid_values)}" - raise argparse.ArgumentTypeError(msg) from None - - -def add_host_key_check_arg( - parser: argparse.ArgumentParser, default: HostKeyCheck = HostKeyCheck.ASK -) -> None: - parser.add_argument( - "--host-key-check", - type=host_key_check_type, - default=default, - help=f"Host key (.ssh/known_hosts) check mode. Options: {', '.join([e.value for e in HostKeyCheck])}", - ) diff --git a/pkgs/clan-cli/clan_cli/machines/hardware.py b/pkgs/clan-cli/clan_cli/machines/hardware.py index cc36cdf0d..4db7c0e92 100644 --- a/pkgs/clan-cli/clan_cli/machines/hardware.py +++ b/pkgs/clan-cli/clan_cli/machines/hardware.py @@ -12,7 +12,6 @@ from clan_lib.machines.suggestions import validate_machine_names from clan_lib.ssh.remote import Remote from clan_cli.completions import add_dynamic_completer, complete_machines -from clan_cli.host_key_check import add_host_key_check_arg from .types import machine_name_type @@ -56,7 +55,12 @@ def register_update_hardware_config(parser: argparse.ArgumentParser) -> None: nargs="?", help="ssh address to install to in the form of user@host:2222", ) - add_host_key_check_arg(parser) + parser.add_argument( + "--host-key-check", + choices=["strict", "ask", "tofu", "none"], + default="ask", + help="Host key (.ssh/known_hosts) check mode.", + ) parser.add_argument( "--password", help="Pre-provided password the cli will prompt otherwise if needed.", diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index a6222e88f..b93a7331d 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -13,7 +13,6 @@ from clan_cli.completions import ( complete_machines, complete_target_host, ) -from clan_cli.host_key_check import add_host_key_check_arg from clan_cli.machines.hardware import HardwareConfig from clan_cli.ssh.deploy_info import DeployInfo, find_reachable_host, ssh_command_parse @@ -98,7 +97,12 @@ def register_install_parser(parser: argparse.ArgumentParser) -> None: help="do not reboot after installation (deprecated)", default=False, ) - add_host_key_check_arg(parser) + parser.add_argument( + "--host-key-check", + choices=["strict", "ask", "tofu", "none"], + default="ask", + help="Host key (.ssh/known_hosts) check mode.", + ) parser.add_argument( "--build-on", choices=[x.value for x in BuildOn], diff --git a/pkgs/clan-cli/clan_cli/ssh/deploy_info.py b/pkgs/clan-cli/clan_cli/ssh/deploy_info.py index 3734b47a7..9ba609c95 100644 --- a/pkgs/clan-cli/clan_cli/ssh/deploy_info.py +++ b/pkgs/clan-cli/clan_cli/ssh/deploy_info.py @@ -8,14 +8,12 @@ from typing import Any from clan_lib.cmd import run from clan_lib.errors import ClanError from clan_lib.nix import nix_shell -from clan_lib.ssh.host_key import HostKeyCheck -from clan_lib.ssh.remote import Remote +from clan_lib.ssh.remote import HostKeyCheck, Remote from clan_cli.completions import ( add_dynamic_completer, complete_machines, ) -from clan_cli.host_key_check import add_host_key_check_arg from clan_cli.ssh.tor import TorTarget, spawn_tor, ssh_tor_reachable log = logging.getLogger(__name__) @@ -183,5 +181,10 @@ def register_parser(parser: argparse.ArgumentParser) -> None: "--png", help="specify the json file for ssh data as the qrcode image (generated by starting the clan installer)", ) - add_host_key_check_arg(parser, default=HostKeyCheck.TOFU) + parser.add_argument( + "--host-key-check", + choices=["strict", "ask", "tofu", "none"], + default="tofu", + help="Host key (.ssh/known_hosts) check mode.", + ) parser.set_defaults(func=ssh_command) diff --git a/pkgs/clan-cli/clan_cli/ssh/test_deploy_info.py b/pkgs/clan-cli/clan_cli/ssh/test_deploy_info.py index 5efa7cbfb..3a4996f7d 100644 --- a/pkgs/clan-cli/clan_cli/ssh/test_deploy_info.py +++ b/pkgs/clan-cli/clan_cli/ssh/test_deploy_info.py @@ -4,7 +4,6 @@ from pathlib import Path import pytest from clan_lib.cmd import RunOpts, run from clan_lib.nix import nix_shell -from clan_lib.ssh.host_key import HostKeyCheck from clan_lib.ssh.remote import Remote from clan_cli.ssh.deploy_info import DeployInfo, find_reachable_host @@ -24,7 +23,7 @@ def test_qrcode_scan(temp_dir: Path) -> None: run(cmd, RunOpts(input=data.encode())) # Call the qrcode_scan function - deploy_info = DeployInfo.from_qr_code(img_path, HostKeyCheck.NONE) + deploy_info = DeployInfo.from_qr_code(img_path, "none") host = deploy_info.addrs[0] assert host.address == "192.168.122.86" @@ -47,7 +46,7 @@ def test_qrcode_scan(temp_dir: Path) -> None: def test_from_json() -> None: data = '{"pass":"scabbed-defender-headlock","tor":"qjeerm4r6t55hcfum4pinnvscn5njlw2g3k7ilqfuu7cdt3ahaxhsbid.onion","addrs":["192.168.122.86"]}' - deploy_info = DeployInfo.from_json(json.loads(data), HostKeyCheck.NONE) + deploy_info = DeployInfo.from_json(json.loads(data), "none") host = deploy_info.addrs[0] assert host.password == "scabbed-defender-headlock" @@ -70,9 +69,7 @@ def test_from_json() -> None: @pytest.mark.with_core def test_find_reachable_host(hosts: list[Remote]) -> None: host = hosts[0] - deploy_info = DeployInfo.from_hostnames( - ["172.19.1.2", host.ssh_url()], HostKeyCheck.NONE - ) + deploy_info = DeployInfo.from_hostnames(["172.19.1.2", host.ssh_url()], "none") assert deploy_info.addrs[0].address == "172.19.1.2" diff --git a/pkgs/clan-cli/clan_cli/tests/hosts.py b/pkgs/clan-cli/clan_cli/tests/hosts.py index f40c79e63..84c32e720 100644 --- a/pkgs/clan-cli/clan_cli/tests/hosts.py +++ b/pkgs/clan-cli/clan_cli/tests/hosts.py @@ -4,7 +4,6 @@ from pathlib import Path import pytest from clan_cli.tests.sshd import Sshd -from clan_lib.ssh.host_key import HostKeyCheck from clan_lib.ssh.remote import Remote @@ -17,7 +16,7 @@ def hosts(sshd: Sshd) -> list[Remote]: port=sshd.port, user=login, private_key=Path(sshd.key), - host_key_check=HostKeyCheck.NONE, + host_key_check="none", command_prefix="local_test", ) ] diff --git a/pkgs/clan-cli/clan_lib/ssh/host_key.py b/pkgs/clan-cli/clan_lib/ssh/host_key.py index e97932ef3..3f2e6968c 100644 --- a/pkgs/clan-cli/clan_lib/ssh/host_key.py +++ b/pkgs/clan-cli/clan_lib/ssh/host_key.py @@ -1,15 +1,15 @@ # Adapted from https://github.com/numtide/deploykit -from enum import Enum +from typing import Literal from clan_lib.errors import ClanError - -class HostKeyCheck(Enum): - STRICT = "strict" # Strictly check ssh host keys, prompt for unknown ones - ASK = "ask" # Ask for confirmation on first use - TOFU = "tofu" # Trust on ssh keys on first use - NONE = "none" # Do not check ssh host keys +HostKeyCheck = Literal[ + "strict", # Strictly check ssh host keys, prompt for unknown ones + "ask", # Ask for confirmation on first use + "tofu", # Trust on ssh keys on first use + "none", # Do not check ssh host keys +] def hostkey_to_ssh_opts(host_key_check: HostKeyCheck) -> list[str]: @@ -17,13 +17,13 @@ def hostkey_to_ssh_opts(host_key_check: HostKeyCheck) -> list[str]: Convert a HostKeyCheck value to SSH options. """ match host_key_check: - case HostKeyCheck.STRICT: + case "strict": return ["-o", "StrictHostKeyChecking=yes"] - case HostKeyCheck.ASK: + case "ask": return [] - case HostKeyCheck.TOFU: + case "tofu": return ["-o", "StrictHostKeyChecking=accept-new"] - case HostKeyCheck.NONE: + case "none": return [ "-o", "StrictHostKeyChecking=no", diff --git a/pkgs/clan-cli/clan_lib/ssh/remote.py b/pkgs/clan-cli/clan_lib/ssh/remote.py index 86f4aceff..8f4d4f8af 100644 --- a/pkgs/clan-cli/clan_lib/ssh/remote.py +++ b/pkgs/clan-cli/clan_lib/ssh/remote.py @@ -38,7 +38,7 @@ class Remote: private_key: Path | None = None password: str | None = None forward_agent: bool = True - host_key_check: HostKeyCheck = HostKeyCheck.ASK + host_key_check: HostKeyCheck = "ask" verbose_ssh: bool = False ssh_options: dict[str, str] = field(default_factory=dict) tor_socks: bool = False diff --git a/pkgs/clan-cli/clan_lib/ssh/remote_test.py b/pkgs/clan-cli/clan_lib/ssh/remote_test.py index 7eeb5feac..6d8a094b7 100644 --- a/pkgs/clan-cli/clan_lib/ssh/remote_test.py +++ b/pkgs/clan-cli/clan_lib/ssh/remote_test.py @@ -8,7 +8,6 @@ import pytest from clan_lib.async_run import AsyncRuntime from clan_lib.cmd import ClanCmdTimeoutError, Log, RunOpts from clan_lib.errors import ClanError, CmdOut -from clan_lib.ssh.host_key import HostKeyCheck from clan_lib.ssh.remote import Remote from clan_lib.ssh.sudo_askpass_proxy import SudoAskpassProxy @@ -114,7 +113,7 @@ def test_parse_deployment_address( result = Remote.from_ssh_uri( machine_name=machine_name, address=test_addr, - ).override(host_key_check=HostKeyCheck.STRICT) + ).override(host_key_check="strict") if expected_exception: return @@ -132,7 +131,7 @@ def test_parse_deployment_address( def test_parse_ssh_options() -> None: addr = "root@example.com:2222?IdentityFile=/path/to/private/key&StrictRemoteKeyChecking=yes" host = Remote.from_ssh_uri(machine_name="foo", address=addr).override( - host_key_check=HostKeyCheck.STRICT + host_key_check="strict" ) assert host.address == "example.com" assert host.port == 2222 diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index 16467d69f..29b460a19 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -33,7 +33,6 @@ from clan_lib.nix_models.clan import ( ) from clan_lib.nix_models.clan import InventoryMachineDeploy as MachineDeploy from clan_lib.persist.util import set_value_by_path -from clan_lib.ssh.host_key import HostKeyCheck from clan_lib.ssh.remote import Remote, can_ssh_login log = logging.getLogger(__name__) @@ -189,7 +188,7 @@ def test_clan_create_api( clan_dir_flake.invalidate_cache() target_host = machine.target_host().override( - private_key=private_key, host_key_check=HostKeyCheck.NONE + private_key=private_key, host_key_check="none" ) result = can_ssh_login(target_host) assert result == "Online", f"Machine {machine.name} is not online" From 74d2ae0619cb765a479ed0944417d8557fb197d0 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 12:39:17 +0200 Subject: [PATCH 043/258] templates_url: add clan template url test --- .../clan_lib/templates/template_url_test.py | 61 ++++++++++++++----- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/templates/template_url_test.py b/pkgs/clan-cli/clan_lib/templates/template_url_test.py index 3193f0447..de8a7e7b9 100644 --- a/pkgs/clan-cli/clan_lib/templates/template_url_test.py +++ b/pkgs/clan-cli/clan_lib/templates/template_url_test.py @@ -5,7 +5,7 @@ import pytest from clan_lib.errors import ClanError from clan_lib.templates.template_url import transform_url -template_type = "machine" +machine_template_type = "machine" class DummyFlake: @@ -23,7 +23,18 @@ def test_transform_url_self_explizit_dot() -> None: user_input = ".#new-machine" expected_selector = 'clan.templates.machine."new-machine"' - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) + assert flake_ref == str(local_path.path) + assert selector == expected_selector + + +def test_default_clan_template() -> None: + user_input = ".#default" + expected_selector = 'clan.templates.clan."default"' + + flake_ref, selector = transform_url("clan", user_input, flake=local_path) assert flake_ref == str(local_path.path) assert selector == expected_selector @@ -32,7 +43,9 @@ def test_transform_url_self_no_dot() -> None: user_input = "#new-machine" expected_selector = 'clan.templates.machine."new-machine"' - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == str(local_path.path) assert selector == expected_selector @@ -41,7 +54,9 @@ def test_transform_url_builtin_template() -> None: user_input = "new-machine" expected_selector = 'clanInternals.templates.machine."new-machine"' - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == str(local_path.path) assert selector == expected_selector @@ -50,7 +65,9 @@ def test_transform_url_remote_template() -> None: user_input = "github:/org/repo#new-machine" expected_selector = 'clan.templates.machine."new-machine"' - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == "github:/org/repo" assert selector == expected_selector @@ -60,7 +77,9 @@ def test_transform_url_explicit_path() -> None: user_input = ".#clan.templates.machine.new-machine" expected_selector = "clan.templates.machine.new-machine" - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == str(local_path.path) assert selector == expected_selector @@ -69,7 +88,9 @@ def test_transform_url_explicit_path() -> None: def test_transform_url_quoted_selector() -> None: user_input = '.#"new.machine"' expected_selector = '"new.machine"' - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == str(local_path.path) assert selector == expected_selector @@ -77,7 +98,9 @@ def test_transform_url_quoted_selector() -> None: def test_single_quote_selector() -> None: user_input = ".#'new.machine'" expected_selector = "'new.machine'" - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == str(local_path.path) assert selector == expected_selector @@ -86,7 +109,9 @@ def test_custom_template_path() -> None: user_input = "github:/org/repo#my.templates.custom.machine" expected_selector = "my.templates.custom.machine" - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == "github:/org/repo" assert selector == expected_selector @@ -96,7 +121,9 @@ def test_full_url_query_and_fragment() -> None: expected_flake_ref = "github:/org/repo?query=param" expected_selector = "my.templates.custom.machine" - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == expected_flake_ref assert selector == expected_selector @@ -114,7 +141,7 @@ def test_malformed_identifier() -> None: user_input = "github:/org/repo#my.templates.custom.machine#extra" with pytest.raises(ClanError) as exc_info: _flake_ref, _selector = transform_url( - template_type, user_input, flake=local_path + machine_template_type, user_input, flake=local_path ) assert isinstance(exc_info.value, ClanError) @@ -128,7 +155,9 @@ def test_locked_input_template() -> None: user_input = "locked-input#new-machine" expected_selector = 'inputs.locked-input.clan.templates.machine."new-machine"' - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert flake_ref == str(local_path.path) assert selector == expected_selector @@ -137,7 +166,9 @@ def test_locked_input_template_no_quotes() -> None: user_input = 'locked-input#"new.machine"' expected_selector = 'inputs.locked-input."new.machine"' - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert selector == expected_selector assert flake_ref == str(local_path.path) @@ -146,6 +177,8 @@ def test_locked_input_template_no_dot() -> None: user_input = "locked-input#new.machine" expected_selector = "inputs.locked-input.new.machine" - flake_ref, selector = transform_url(template_type, user_input, flake=local_path) + flake_ref, selector = transform_url( + machine_template_type, user_input, flake=local_path + ) assert selector == expected_selector assert flake_ref == str(local_path.path) From 38f98645acee4ef8b193fcd7778e45458902f866 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 13:03:17 +0200 Subject: [PATCH 044/258] Templates: replace leftover MachineID, by Machine --- pkgs/clan-cli/clan_lib/machines/actions.py | 12 ++---------- pkgs/clan-cli/clan_lib/machines/machines.py | 4 +++- pkgs/clan-cli/clan_lib/templates/handler.py | 5 +++-- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/machines/actions.py b/pkgs/clan-cli/clan_lib/machines/actions.py index 0fd7748ce..4c19e7a23 100644 --- a/pkgs/clan-cli/clan_lib/machines/actions.py +++ b/pkgs/clan-cli/clan_lib/machines/actions.py @@ -1,9 +1,9 @@ -from dataclasses import dataclass from typing import TypedDict from clan_lib.api import API from clan_lib.errors import ClanError from clan_lib.flake.flake import Flake +from clan_lib.machines.machines import Machine from clan_lib.nix_models.clan import ( InventoryMachine, ) @@ -65,16 +65,8 @@ def get_machine(flake: Flake, name: str) -> InventoryMachine: return InventoryMachine(**machine_inv) -# TODO: remove this machine, once the Machine class is refactored -# We added this now, to allow for dispatching actions. To require only 'name' and 'flake' of a machine. -@dataclass(frozen=True) -class MachineID: - name: str - flake: Flake - - @API.register -def set_machine(machine: MachineID, update: InventoryMachine) -> None: +def set_machine(machine: Machine, update: InventoryMachine) -> None: """ Update the machine information in the inventory. """ diff --git a/pkgs/clan-cli/clan_lib/machines/machines.py b/pkgs/clan-cli/clan_lib/machines/machines.py index 489b0bccf..ce4a51b6d 100644 --- a/pkgs/clan-cli/clan_lib/machines/machines.py +++ b/pkgs/clan-cli/clan_lib/machines/machines.py @@ -13,7 +13,6 @@ from clan_cli.vars._types import StoreBase from clan_lib.api import API from clan_lib.errors import ClanCmdError, ClanError from clan_lib.flake import Flake -from clan_lib.machines.actions import get_machine from clan_lib.nix import nix_config from clan_lib.nix_models.clan import InventoryMachine from clan_lib.ssh.remote import Remote @@ -39,6 +38,9 @@ class Machine: return cls(name=name, flake=flake) def get_inv_machine(self) -> "InventoryMachine": + # Import on demand to avoid circular imports + from clan_lib.machines.actions import get_machine + return get_machine(self.flake, self.name) def get_id(self) -> str: diff --git a/pkgs/clan-cli/clan_lib/templates/handler.py b/pkgs/clan-cli/clan_lib/templates/handler.py index 72d28a627..74ce167d6 100644 --- a/pkgs/clan-cli/clan_lib/templates/handler.py +++ b/pkgs/clan-cli/clan_lib/templates/handler.py @@ -7,7 +7,8 @@ from pathlib import Path from clan_lib.dirs import specific_machine_dir from clan_lib.errors import ClanError from clan_lib.flake import Flake -from clan_lib.machines.actions import MachineID, list_machines +from clan_lib.machines.actions import list_machines +from clan_lib.machines.machines import Machine from clan_lib.templates.filesystem import copy_from_nixstore, realize_nix_path from clan_lib.templates.template_url import transform_url @@ -84,7 +85,7 @@ def machine_template( description="Template machine must contain a configuration.nix", ) - tmp_machine = MachineID(flake=flake, name=dst_machine_name) + tmp_machine = Machine(flake=flake, name=dst_machine_name) dst_machine_dir = specific_machine_dir(tmp_machine) From cce02072259e2cb524d7d4c0bcfaffd5b6e36794 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 13:03:46 +0200 Subject: [PATCH 045/258] Templates: remove outdated check for 'configuration.nix' in machine templates --- pkgs/clan-cli/clan_lib/templates/handler.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/templates/handler.py b/pkgs/clan-cli/clan_lib/templates/handler.py index 74ce167d6..185552243 100644 --- a/pkgs/clan-cli/clan_lib/templates/handler.py +++ b/pkgs/clan-cli/clan_lib/templates/handler.py @@ -77,14 +77,6 @@ def machine_template( msg = f"Template {printable_template_ref} is not a directory at {src_path}" raise ClanError(msg) - # TODO: Do we really need to check for a specific file in the template? - if not (src_path / "configuration.nix").exists(): - msg = f"Template {printable_template_ref} does not contain a configuration.nix" - raise ClanError( - msg, - description="Template machine must contain a configuration.nix", - ) - tmp_machine = Machine(flake=flake, name=dst_machine_name) dst_machine_dir = specific_machine_dir(tmp_machine) From 1502cfa4a7756a7bd9b083a7b89875c94d478459 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 13:31:46 +0200 Subject: [PATCH 046/258] Templates: migrate clan templates to flake identifiers --- pkgs/clan-cli/clan_cli/clan/create.py | 36 ++--------- pkgs/clan-cli/clan_lib/clan/create.py | 67 ++++++++------------ pkgs/clan-cli/clan_lib/templates/handler.py | 70 +++++++++++++++++++++ pkgs/clan-cli/clan_lib/tests/test_create.py | 2 +- 4 files changed, 100 insertions(+), 75 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/clan/create.py b/pkgs/clan-cli/clan_cli/clan/create.py index 2604456e0..8d8cf2def 100644 --- a/pkgs/clan-cli/clan_cli/clan/create.py +++ b/pkgs/clan-cli/clan_cli/clan/create.py @@ -4,36 +4,17 @@ import logging from pathlib import Path from clan_lib.clan.create import CreateOptions, create_clan -from clan_lib.templates import ( - InputPrio, -) log = logging.getLogger(__name__) def register_create_parser(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "--input", - type=str, - help="""Flake input name to use as template source - can be specified multiple times, inputs are tried in order of definition - Example: --input clan --input clan-core - """, - action="append", - default=[], - ) - - parser.add_argument( - "--no-self", - help="Do not look into own flake for templates", - action="store_true", - default=False, - ) - parser.add_argument( "--template", type=str, - help="Clan template name", + help="""Reference to the template to use for the clan. default="default". In the format '#template_name' Where is a flake reference (e.g. github:org/repo) or a local path (e.g. '.' ). + Omitting '#' will use the builtin templates (e.g. just 'default' from clan-core ). + """, default="default", ) @@ -59,19 +40,10 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None: ) def create_flake_command(args: argparse.Namespace) -> None: - if len(args.input) == 0: - args.input = ["clan", "clan-core"] - - if args.no_self: - input_prio = InputPrio.try_inputs(tuple(args.input)) - else: - input_prio = InputPrio.try_self_then_inputs(tuple(args.input)) - create_clan( CreateOptions( - input_prio=input_prio, dest=args.path, - template_name=args.template, + template=args.template, setup_git=not args.no_git, src_flake=args.flake, update_clan=not args.no_update, diff --git a/pkgs/clan-cli/clan_lib/clan/create.py b/pkgs/clan-cli/clan_lib/clan/create.py index ddea36969..81272977b 100644 --- a/pkgs/clan-cli/clan_lib/clan/create.py +++ b/pkgs/clan-cli/clan_lib/clan/create.py @@ -4,16 +4,12 @@ from pathlib import Path from clan_lib.api import API from clan_lib.cmd import RunOpts, run +from clan_lib.dirs import clan_templates from clan_lib.errors import ClanError from clan_lib.flake import Flake from clan_lib.nix import nix_command, nix_metadata, nix_shell from clan_lib.persist.inventory_store import InventorySnapshot, InventoryStore -from clan_lib.templates import ( - InputPrio, - TemplateName, - get_template, -) -from clan_lib.templates.filesystem import copy_from_nixstore +from clan_lib.templates.handler import clan_template log = logging.getLogger(__name__) @@ -21,9 +17,9 @@ log = logging.getLogger(__name__) @dataclass class CreateOptions: dest: Path - template_name: str + template: str + src_flake: Flake | None = None - input_prio: InputPrio | None = None setup_git: bool = True initial: InventorySnapshot | None = None update_clan: bool = True @@ -47,44 +43,31 @@ def create_clan(opts: CreateOptions) -> None: log.warning("Setting src_flake to None") opts.src_flake = None - template = get_template( - TemplateName(opts.template_name), - "clan", - input_prio=opts.input_prio, - clan_dir=opts.src_flake, - ) - log.info(f"Found template '{template.name}' in '{template.input_variant}'") + if opts.src_flake is None: + opts.src_flake = Flake(str(clan_templates())) - if dest.exists(): - dest /= template.name + with clan_template( + opts.src_flake, template_ident=opts.template, dst_dir=opts.dest + ) as _clan_dir: + if opts.setup_git: + run(git_command(dest, "init")) + run(git_command(dest, "add", ".")) - if dest.exists(): - msg = f"Destination directory {dest} already exists" - raise ClanError(msg) + # check if username is set + has_username = run( + git_command(dest, "config", "user.name"), RunOpts(check=False) + ) + if has_username.returncode != 0: + run(git_command(dest, "config", "user.name", "clan-tool")) - src = Path(template.src["path"]) + has_username = run( + git_command(dest, "config", "user.email"), RunOpts(check=False) + ) + if has_username.returncode != 0: + run(git_command(dest, "config", "user.email", "clan@example.com")) - copy_from_nixstore(src, dest) - - if opts.setup_git: - run(git_command(dest, "init")) - run(git_command(dest, "add", ".")) - - # check if username is set - has_username = run( - git_command(dest, "config", "user.name"), RunOpts(check=False) - ) - if has_username.returncode != 0: - run(git_command(dest, "config", "user.name", "clan-tool")) - - has_username = run( - git_command(dest, "config", "user.email"), RunOpts(check=False) - ) - if has_username.returncode != 0: - run(git_command(dest, "config", "user.email", "clan@example.com")) - - if opts.update_clan: - run(nix_command(["flake", "update"]), RunOpts(cwd=dest)) + if opts.update_clan: + run(nix_command(["flake", "update"]), RunOpts(cwd=dest)) if opts.initial: inventory_store = InventoryStore(flake=Flake(str(opts.dest))) diff --git a/pkgs/clan-cli/clan_lib/templates/handler.py b/pkgs/clan-cli/clan_lib/templates/handler.py index 185552243..dd44f0438 100644 --- a/pkgs/clan-cli/clan_lib/templates/handler.py +++ b/pkgs/clan-cli/clan_lib/templates/handler.py @@ -99,3 +99,73 @@ def machine_template( finally: # If no error occurred, the machine directory is kept pass + + +@contextmanager +def clan_template(flake: Flake, template_ident: str, dst_dir: Path) -> Iterator[Path]: + """ + Create a clan from a template. + This function will copy the template files to a new clan directory + + :param flake: The flake to create the machine in. + :param template_ident: The identifier of the template to use. Example ".#template_name" + :param dst: The name of the directory to create. + + + Example usage: + + >>> with clan_template( + ... Flake("/home/johannes/git/clan-core"), ".#new-machine", "my-machine" + ... ) as clan_dir: + ... # Use `clan_dir` here if you want to access the created directory + + ... The directory is removed if the context raised any errors. + ... Only if the context is exited without errors, it is kept. + """ + + # Get the clan template from the specifier + [flake_ref, template_selector] = transform_url("clan", template_ident, flake=flake) + # For pretty error messages + printable_template_ref = f"{flake_ref}#{template_selector}" + + template_flake = Flake(flake_ref) + + try: + template = template_flake.select(template_selector) + except ClanError as e: + msg = f"Failed to select template '{template_ident}' from flake '{flake_ref}' (via attribute path: {printable_template_ref})" + raise ClanError(msg) from e + + src = template.get("path") + if not src: + msg = f"Malformed template: {printable_template_ref} does not have a 'path' attribute" + raise ClanError(msg) + + src_path = Path(src).resolve() + + realize_nix_path(template_flake, str(src_path)) + + if not src_path.exists(): + msg = f"Template {printable_template_ref} does not exist at {src_path}" + raise ClanError(msg) + + if not src_path.is_dir(): + msg = f"Template {printable_template_ref} is not a directory at {src_path}" + raise ClanError(msg) + + if dst_dir.exists(): + msg = f"Destination directory {dst_dir} already exists" + raise ClanError(msg) + + copy_from_nixstore(src_path, dst_dir) + + try: + yield dst_dir + except Exception as e: + log.error(f"An error occurred inside the 'clan_template' context: {e}") + log.info(f"Removing left-over directory: {dst_dir}") + shutil.rmtree(dst_dir, ignore_errors=True) + raise + finally: + # If no error occurred, the directory is kept + pass diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index 29b460a19..ed43e442f 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -137,7 +137,7 @@ def test_clan_create_api( # TODO: We need to generate a lock file for the templates clan_cli.clan.create.create_clan( clan_cli.clan.create.CreateOptions( - template_name="minimal", dest=dest_clan_dir, update_clan=False + template="minimal", dest=dest_clan_dir, update_clan=False ) ) assert dest_clan_dir.is_dir() From 94919dc9b823106a2e34266fb35b3fe4d32fc03f Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 15:48:35 +0200 Subject: [PATCH 047/258] Fix/ui: update create argument --- pkgs/clan-app/ui/src/routes/clans/create.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-app/ui/src/routes/clans/create.tsx b/pkgs/clan-app/ui/src/routes/clans/create.tsx index 002109c27..6a1b8fa39 100644 --- a/pkgs/clan-app/ui/src/routes/clans/create.tsx +++ b/pkgs/clan-app/ui/src/routes/clans/create.tsx @@ -49,7 +49,7 @@ export const CreateClan = () => { const r = await callApi("create_clan", { opts: { dest: target_dir[0], - template_name: template, + template: template, initial: { meta, services: {}, From 7ad8ed1af0f05d88fef603fc33dd8aa2fd6b22f8 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 16:43:38 +0200 Subject: [PATCH 048/258] Templates: fix invalid mock flake --- pkgs/clan-cli/clan_lib/flake/flake.py | 9 ++++++--- templates/flake.nix | 10 ++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/flake/flake.py b/pkgs/clan-cli/clan_lib/flake/flake.py index e4beaf2c2..4aed6e820 100644 --- a/pkgs/clan-cli/clan_lib/flake/flake.py +++ b/pkgs/clan-cli/clan_lib/flake/flake.py @@ -832,7 +832,8 @@ nix repl --expr 'rec {{ """) # fmt: on elif len(selectors) == 1: - log.debug(f""" + log.debug( + f""" selecting: {selectors[0]} to debug run: nix repl --expr 'rec {{ @@ -840,11 +841,13 @@ nix repl --expr 'rec {{ selectLib = (builtins.getFlake "path:{select_source()}?narHash={select_hash}").lib; query = selectLib.select '"''{selectors[0]}''"' flake; }}' - """) + """ + ) build_output = Path( run( - nix_build(["--expr", nix_code, *nix_options]), RunOpts(log=Log.NONE, trace=False), + nix_build(["--expr", nix_code, *nix_options]), + RunOpts(log=Log.NONE, trace=False), ).stdout.strip() ) diff --git a/templates/flake.nix b/templates/flake.nix index e349af18b..c7e76fb92 100644 --- a/templates/flake.nix +++ b/templates/flake.nix @@ -1,8 +1,8 @@ { outputs = { ... }: - { - clan.templates = { + let + templates = { disko = { single-disk = { description = "A simple ext4 disk with a single partition"; @@ -41,5 +41,11 @@ }; }; }; + in + rec { + inherit (clan) clanInternals; + + clan.clanInternals.templates = templates; + clan.templates = templates; }; } From a082fd2ed9a648c0f240b903a22a239213b6df44 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" Date: Sun, 6 Jul 2025 15:00:31 +0000 Subject: [PATCH 049/258] Lock file maintenance --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 939903749..3d53e729f 100644 --- a/flake.lock +++ b/flake.lock @@ -164,10 +164,10 @@ "nixpkgs": { "locked": { "lastModified": 315532800, - "narHash": "sha256-0HRxGUoOMtOYnwlMWY0AkuU88WHaI3Q5GEILmsWpI8U=", - "rev": "a48741b083d4f36dd79abd9f760c84da6b4dc0e5", + "narHash": "sha256-mUlYenGbsUFP0A3EhfKJXmUl5+MQGJLhoEop2t3g5p4=", + "rev": "ceb24d94c6feaa4e8737a8e2bd3cf71c3a7eaaa0", "type": "tarball", - "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre823094.a48741b083d4/nixexprs.tar.xz" + "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre826033.ceb24d94c6fe/nixexprs.tar.xz" }, "original": { "type": "tarball", From 4df4f5220bf1c6985f68729584f140860719d401 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 19:07:08 +0200 Subject: [PATCH 050/258] Templates: remove InputPrio and related classes --- .../clan_cli/tests/test_clan_nix_attrset.py | 228 ++--------------- pkgs/clan-cli/clan_lib/templates/__init__.py | 233 ------------------ 2 files changed, 26 insertions(+), 435 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/tests/test_clan_nix_attrset.py b/pkgs/clan-cli/clan_cli/tests/test_clan_nix_attrset.py index 0e7de5f9f..2b8a99269 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_clan_nix_attrset.py +++ b/pkgs/clan-cli/clan_cli/tests/test_clan_nix_attrset.py @@ -1,62 +1,15 @@ -# mypy: disable-error-code="var-annotated" - -import json -from pathlib import Path -from typing import Any - import pytest +from pathlib import Path + from clan_cli.tests.fixtures_flakes import FlakeForTest from clan_lib.cmd import run from clan_lib.flake import Flake -from clan_lib.git import commit_file -from clan_lib.locked_open import locked_open from clan_lib.nix import nix_command -from clan_lib.templates import ( - ClanExports, - InputName, - TemplateName, - get_clan_nix_attrset, - get_template, -) + +from clan_lib.templates import list_templates from clan_lib.templates.filesystem import copy_from_nixstore -# Function to write clan attributes to a file -def write_clan_attr(clan_attrset: dict[str, Any], flake: FlakeForTest) -> None: - file = flake.path / "clan_attrs.json" - with locked_open(file, "w") as cfile: - json.dump(clan_attrset, cfile, indent=2) - - commit_file(file, flake.path, "Add clan attributes") - - -# Common function to test clan nix attrset -def nix_attr_tester( - test_flake_with_core: FlakeForTest, - injected: dict[str, Any], - expected_self: dict[str, Any], - test_number: int, -) -> ClanExports: - write_clan_attr(injected, test_flake_with_core) - clan_dir = Flake(str(test_flake_with_core.path)) - nix_attrset = get_clan_nix_attrset(clan_dir) - - def recursive_sort(item: Any) -> Any: - if isinstance(item, dict): - return {k: recursive_sort(item[k]) for k in sorted(item)} - if isinstance(item, list): - return sorted(recursive_sort(elem) for elem in item) - return item - - returned_sorted = recursive_sort(nix_attrset["self"]) - expected_sorted = recursive_sort(expected_self["self"]) - - assert json.dumps(returned_sorted, indent=2) == json.dumps( - expected_sorted, indent=2 - ) - return nix_attrset - - @pytest.mark.impure def test_copy_from_nixstore_symlink( monkeypatch: pytest.MonkeyPatch, temporary_home: Path @@ -85,166 +38,37 @@ def test_clan_core_templates( temporary_home: Path, ) -> None: clan_dir = Flake(str(test_flake_with_core.path)) - nix_attrset = get_clan_nix_attrset(clan_dir) - clan_core_templates = nix_attrset["inputs"][InputName("clan-core")]["templates"][ - "clan" + templates = list_templates(clan_dir) + + assert list(templates.builtins.get("clan", {}).keys()) == [ + "default", + "flake-parts", + "minimal", + "minimal-flake-parts", ] - clan_core_template_keys = list(clan_core_templates.keys()) - expected_templates = ["default", "flake-parts", "minimal", "minimal-flake-parts"] - assert clan_core_template_keys == expected_templates + # clan.default + default_template = templates.builtins.get("clan", {}).get("default") + assert default_template is not None - default_template = get_template( - TemplateName("default"), - "clan", - input_prio=None, - clan_dir=clan_dir, - ) + template_path = default_template.get("path", None) + assert template_path is not None new_clan = temporary_home / "new_clan" + copy_from_nixstore( - Path(default_template.src["path"]), + Path(template_path), new_clan, ) - assert (new_clan / "flake.nix").exists() - assert (new_clan / "machines").is_dir() - assert (new_clan / "machines" / "jon").is_dir() - config_nix_p = new_clan / "machines" / "jon" / "configuration.nix" - assert (config_nix_p).is_file() - # Test if we can write to the configuration.nix file - with config_nix_p.open("r+") as f: + flake_nix = new_clan / "flake.nix" + assert (flake_nix).exists() + assert (flake_nix).is_file() + + assert (new_clan / "machines").is_dir() + + # Test if we can write to the flake.nix file + with flake_nix.open("r+") as f: data = f.read() f.write(data) - - -# Test Case 1: Minimal input with empty templates -@pytest.mark.with_core -def test_clan_get_nix_attrset_case_1( - monkeypatch: pytest.MonkeyPatch, - temporary_home: Path, - test_flake_with_core: FlakeForTest, -) -> None: - test_number = 1 - injected = {"templates": {"disko": {}, "machine": {}}} - expected = { - "inputs": {}, - "self": {"templates": {"disko": {}, "machine": {}, "clan": {}}}, - } - nix_attr_tester(test_flake_with_core, injected, expected, test_number) - - -# Test Case 2: Input with one template under 'clan' -@pytest.mark.with_core -def test_clan_get_nix_attrset_case_2( - monkeypatch: pytest.MonkeyPatch, - temporary_home: Path, - test_flake_with_core: FlakeForTest, -) -> None: - test_number = 2 - injected = { - "templates": { - "clan": { - "example_template": { - "description": "An example clan template.", - "path": "/example/path", - } - } - } - } - expected = { - "inputs": {}, - "self": { - "templates": { - "clan": { - "example_template": { - "description": "An example clan template.", - "path": "/example/path", - } - }, - "disko": {}, - "machine": {}, - }, - }, - } - - nix_attrset = nix_attr_tester(test_flake_with_core, injected, expected, test_number) - - assert "default" in list( - nix_attrset["inputs"][InputName("clan-core")]["templates"]["clan"].keys() - ) - - -# Test Case 3: Input with templates under multiple types -@pytest.mark.with_core -def test_clan_get_nix_attrset_case_3( - monkeypatch: pytest.MonkeyPatch, - temporary_home: Path, - test_flake_with_core: FlakeForTest, -) -> None: - test_number = 3 - injected = { - "templates": { - "clan": { - "clan_template": { - "description": "A clan template.", - "path": "/clan/path", - } - }, - "disko": { - "disko_template": { - "description": "A disko template.", - "path": "/disko/path", - } - }, - "machine": { - "machine_template": { - "description": "A machine template.", - "path": "/machine/path", - } - }, - } - } - expected = { - "inputs": {}, - "self": { - "templates": { - "clan": { - "clan_template": { - "description": "A clan template.", - "path": "/clan/path", - } - }, - "disko": { - "disko_template": { - "description": "A disko template.", - "path": "/disko/path", - } - }, - "machine": { - "machine_template": { - "description": "A machine template.", - "path": "/machine/path", - } - }, - }, - }, - } - nix_attr_tester(test_flake_with_core, injected, expected, test_number) - - -# Test Case 6: Input with missing 'templates' and 'modules' (empty clan attrset) -@pytest.mark.with_core -def test_clan_get_nix_attrset_case_6( - monkeypatch: pytest.MonkeyPatch, - temporary_home: Path, - test_flake_with_core: FlakeForTest, -) -> None: - test_number = 6 - injected = {} - expected = { - "inputs": {}, - "self": {"templates": {"disko": {}, "machine": {}, "clan": {}}}, - } - nix_attr_tester(test_flake_with_core, injected, expected, test_number) diff --git a/pkgs/clan-cli/clan_lib/templates/__init__.py b/pkgs/clan-cli/clan_lib/templates/__init__.py index ae2c556ae..2e61406da 100644 --- a/pkgs/clan-cli/clan_lib/templates/__init__.py +++ b/pkgs/clan-cli/clan_lib/templates/__init__.py @@ -1,172 +1,12 @@ import logging from dataclasses import dataclass -from typing import Any, Literal, NewType, TypedDict, cast -from clan_lib.dirs import clan_templates -from clan_lib.errors import ClanCmdError, ClanError from clan_lib.flake import Flake from clan_lib.nix_models.clan import ClanTemplatesType -from clan_lib.templates.filesystem import realize_nix_path log = logging.getLogger(__name__) -InputName = NewType("InputName", str) - - -@dataclass -class InputVariant: - input_name: InputName | None - - def is_self(self) -> bool: - return self.input_name is None - - def __str__(self) -> str: - return self.input_name or "self" - - -TemplateName = NewType("TemplateName", str) -TemplateType = Literal["clan", "disko", "machine"] - - -class Template(TypedDict): - description: str - - -class TemplatePath(Template): - path: str - - -@dataclass -class FoundTemplate: - input_variant: InputVariant - name: TemplateName - src: TemplatePath - - -class TemplateTypeDict(TypedDict): - disko: dict[TemplateName, TemplatePath] - clan: dict[TemplateName, TemplatePath] - machine: dict[TemplateName, TemplatePath] - - -class ClanAttrset(TypedDict): - templates: TemplateTypeDict - - -class ClanExports(TypedDict): - inputs: dict[InputName, ClanAttrset] - self: ClanAttrset - - -def apply_fallback_structure(attrset: dict[str, Any]) -> ClanAttrset: - """Ensure the attrset has all required fields with defaults when missing.""" - # Deep copy not needed since we're constructing the dict from scratch - result: dict[str, Any] = {} - - # Ensure templates field exists - if "templates" not in attrset: - result["templates"] = {"disko": {}, "clan": {}, "machine": {}} - else: - templates = attrset["templates"] - result["templates"] = { - "disko": templates.get("disko", {}), - "clan": templates.get("clan", {}), - "machine": templates.get("machine", {}), - } - - return cast(ClanAttrset, result) - - -def get_clan_nix_attrset(clan_dir: Flake | None = None) -> ClanExports: - """ - Get the clan nix attrset from a flake, with fallback structure applied. - Path inside the attrsets have NOT YET been realized in the nix store. - """ - if not clan_dir: - clan_dir = Flake(str(clan_templates())) - - log.debug(f"Evaluating flake {clan_dir} for Clan attrsets") - - raw_clan_exports: dict[str, Any] = {"self": {"clan": {}}, "inputs": {"clan": {}}} - - maybe_templates = clan_dir.select("?clan.?templates") - if "clan" in maybe_templates: - raw_clan_exports["self"] = maybe_templates["clan"] - else: - log.info("Current flake does not export the 'clan' attribute") - - # FIXME: flake.select destroys lazy evaluation - # this is why if one input has a template with a non existant path - # the whole evaluation will fail - try: - # FIXME: We expect here that if the input exports the clan attribute it also has clan.templates - # this is not always the case if we just want to export clan.modules for example - # However, there is no way to fix this, as clan.select does not support two optional selectors - # and we cannot eval the clan attribute as clan.modules can be non JSON serializable because - # of import statements. - # This needs to be fixed in clan.select - # For now always define clan.templates or no clan attribute at all - temp = clan_dir.select("inputs.*.?clan.templates") - - # FIXME: We need this because clan.select removes the templates attribute - # but not the clan and other attributes leading up to templates - for input_name, attrset in temp.items(): - if "clan" in attrset: - raw_clan_exports["inputs"][input_name] = { - "clan": {"templates": {**attrset["clan"]}} - } - - except ClanCmdError as e: - msg = "Failed to evaluate flake inputs" - raise ClanError(msg) from e - - inputs_with_fallback = {} - for input_name, attrset in raw_clan_exports["inputs"].items(): - # FIXME: flake.select("inputs.*.{clan}") returns the wrong attrset - # depth when used with conditional fields - # this is why we have to do a attrset.get here - inputs_with_fallback[input_name] = apply_fallback_structure( - attrset.get("clan", {}) - ) - - # Apply fallback structure to self - self_with_fallback = apply_fallback_structure(raw_clan_exports["self"]) - - # Construct the final result - clan_exports: ClanExports = { - "inputs": inputs_with_fallback, - "self": self_with_fallback, - } - - return clan_exports - - -@dataclass -class InputPrio: - """ - Strategy for prioritizing inputs when searching for a template - """ - - input_names: tuple[str, ...] # Tuple of input names (ordered priority list) - prioritize_self: bool = True # Whether to prioritize "self" first - - @staticmethod - def self_only() -> "InputPrio": - # Only consider "self" (no external inputs) - return InputPrio(prioritize_self=True, input_names=()) - - @staticmethod - def try_inputs(input_names: tuple[str, ...]) -> "InputPrio": - # Only consider the specified external inputs - return InputPrio(prioritize_self=False, input_names=input_names) - - @staticmethod - def try_self_then_inputs(input_names: tuple[str, ...]) -> "InputPrio": - # Consider "self" first, then the specified external inputs - return InputPrio(prioritize_self=True, input_names=input_names) - - @dataclass class TemplateList: builtins: ClanTemplatesType @@ -181,76 +21,3 @@ def list_templates(flake: Flake) -> TemplateList: builtin_templates = flake.select("clanInternals.templates") return TemplateList(builtin_templates, custom_templates) - - -def get_template( - template_name: TemplateName, - template_type: TemplateType, - *, - input_prio: InputPrio | None = None, - clan_dir: Flake | None = None, -) -> FoundTemplate: - """ - Find a specific template by name and type within a flake and then ensures it is realized in the nix store. - """ - - if not clan_dir: - clan_dir = Flake(str(clan_templates())) - - log.info(f"Get template in {clan_dir}") - - log.info(f"Searching for template '{template_name}' of type '{template_type}'") - - # Set default priority strategy if none is provided - if input_prio is None: - input_prio = InputPrio.try_self_then_inputs(("clan-core",)) - - # Helper function to search for a specific template within a dictionary of templates - def find_template( - template_name: TemplateName, templates: dict[TemplateName, TemplatePath] - ) -> TemplatePath | None: - if template_name in templates: - return templates[template_name] - return None - - # Initialize variables for the search results - template: TemplatePath | None = None - input_name: InputName | None = None - clan_exports = get_clan_nix_attrset(clan_dir) - - # Step 1: Check "self" first, if prioritize_self is enabled - if input_prio.prioritize_self: - log.info(f"Checking 'self' for template '{template_name}'") - template = find_template( - template_name, clan_exports["self"]["templates"][template_type] - ) - - # Step 2: Otherwise, check the external inputs if no match is found - if not template and input_prio.input_names: - log.info(f"Checking external inputs for template '{template_name}'") - for input_name_str in input_prio.input_names: - input_name = InputName(input_name_str) - log.debug(f"Searching in '{input_name}' for template '{template_name}'") - - if input_name not in clan_exports["inputs"]: - log.debug(f"Skipping input '{input_name}', not found in '{clan_dir}'") - continue - - template = find_template( - template_name, - clan_exports["inputs"][input_name]["templates"][template_type], - ) - if template: - log.debug(f"Found template '{template_name}' in input '{input_name}'") - break - - # Step 3: Raise an error if the template wasn't found - if not template: - msg = f"Template '{template_name}' could not be found in '{clan_dir}'" - raise ClanError(msg) - - realize_nix_path(clan_dir, template["path"]) - - return FoundTemplate( - input_variant=InputVariant(input_name), src=template, name=template_name - ) From d0ec4fd8e6671ef1fd46ba3eb795fbe7954ffdb1 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 19:36:57 +0200 Subject: [PATCH 051/258] Templates/cli: move display command into it own category --- pkgs/clan-cli/clan_cli/__init__.py | 8 +++ pkgs/clan-cli/clan_cli/clan/__init__.py | 3 - pkgs/clan-cli/clan_cli/clan/list.py | 45 --------------- pkgs/clan-cli/clan_cli/templates/__init__.py | 15 +++++ pkgs/clan-cli/clan_cli/templates/list.py | 60 ++++++++++++++++++++ 5 files changed, 83 insertions(+), 48 deletions(-) delete mode 100644 pkgs/clan-cli/clan_cli/clan/list.py create mode 100644 pkgs/clan-cli/clan_cli/templates/__init__.py create mode 100644 pkgs/clan-cli/clan_cli/templates/list.py diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index 762502d30..b45d2c37e 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -15,6 +15,7 @@ from . import ( clan, secrets, select, + templates, state, vms, ) @@ -195,6 +196,13 @@ For more detailed information, visit: {help_hyperlink("getting-started", "https: clan.register_parser(parser_flake) + parser_templates = subparsers.add_parser( + "templates", + help="Subcommands to interact with templates", + formatter_class=argparse.RawTextHelpFormatter, + ) + templates.register_parser(parser_templates) + parser_flash = subparsers.add_parser( "flash", help="Flashes your machine to an USB drive", diff --git a/pkgs/clan-cli/clan_cli/clan/__init__.py b/pkgs/clan-cli/clan_cli/clan/__init__.py index d55091a94..41c8ae718 100644 --- a/pkgs/clan-cli/clan_cli/clan/__init__.py +++ b/pkgs/clan-cli/clan_cli/clan/__init__.py @@ -4,7 +4,6 @@ import argparse from clan_cli.clan.inspect import register_inspect_parser from .create import register_create_parser -from .list import register_list_parser # takes a (sub)parser and configures it @@ -19,5 +18,3 @@ def register_parser(parser: argparse.ArgumentParser) -> None: register_create_parser(create_parser) inspect_parser = subparser.add_parser("inspect", help="Inspect a clan ") register_inspect_parser(inspect_parser) - list_parser = subparser.add_parser("list", help="List clan templates") - register_list_parser(list_parser) diff --git a/pkgs/clan-cli/clan_cli/clan/list.py b/pkgs/clan-cli/clan_cli/clan/list.py deleted file mode 100644 index 58ae350d0..000000000 --- a/pkgs/clan-cli/clan_cli/clan/list.py +++ /dev/null @@ -1,45 +0,0 @@ -import argparse -import logging - -from clan_lib.templates import list_templates - -log = logging.getLogger(__name__) - - -def list_command(args: argparse.Namespace) -> None: - templates = list_templates(args.flake) - - builtin_clan_templates = templates.builtins.get("clan", {}) - - print("Available templates") - print("β”œβ”€β”€ ") - for i, (name, template) in enumerate(builtin_clan_templates.items()): - is_last_template = i == len(builtin_clan_templates.items()) - 1 - if not is_last_template: - print(f"β”‚ β”œβ”€β”€ {name}: {template.get('description', 'no description')}") - else: - print(f"β”‚ └── {name}: {template.get('description', 'no description')}") - - for i, (input_name, input_templates) in enumerate(templates.custom.items()): - custom_clan_templates = input_templates.get("clan", {}) - is_last_input = i == len(templates.custom.items()) - 1 - prefix = "β”‚" if not is_last_input else " " - if not is_last_input: - print(f"β”œβ”€β”€ inputs.{input_name}:") - else: - print(f"└── inputs.{input_name}:") - - for i, (name, template) in enumerate(custom_clan_templates.items()): - is_last_template = i == len(builtin_clan_templates.items()) - 1 - if not is_last_template: - print( - f"{prefix} β”œβ”€β”€ {name}: {template.get('description', 'no description')}" - ) - else: - print( - f"{prefix} └── {name}: {template.get('description', 'no description')}" - ) - - -def register_list_parser(parser: argparse.ArgumentParser) -> None: - parser.set_defaults(func=list_command) diff --git a/pkgs/clan-cli/clan_cli/templates/__init__.py b/pkgs/clan-cli/clan_cli/templates/__init__.py new file mode 100644 index 000000000..5e052f111 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/templates/__init__.py @@ -0,0 +1,15 @@ +# !/usr/bin/env python3 +import argparse +from .list import register_list_parser + + +# takes a (sub)parser and configures it +def register_parser(parser: argparse.ArgumentParser) -> None: + subparser = parser.add_subparsers( + title="command", + description="the command to run", + help="the command to run", + required=True, + ) + list_parser = subparser.add_parser("list", help="List avilable templates") + register_list_parser(list_parser) diff --git a/pkgs/clan-cli/clan_cli/templates/list.py b/pkgs/clan-cli/clan_cli/templates/list.py new file mode 100644 index 000000000..b9e26dc10 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/templates/list.py @@ -0,0 +1,60 @@ +import argparse +import logging + +from clan_lib.nix_models.clan import TemplateClanType +from clan_lib.templates import list_templates + +log = logging.getLogger(__name__) + + +def list_command(args: argparse.Namespace) -> None: + templates = list_templates(args.flake) + + # Display all templates + for i, (template_type, _builtin_template_set) in enumerate( + templates.builtins.items() + ): + builtin_template_set: TemplateClanType | None = templates.builtins.get( + template_type, None + ) # type: ignore + if not builtin_template_set: + continue + + print(f"Avilable '{template_type}' templates") + print("β”œβ”€β”€ ") + for i, (name, template) in enumerate(builtin_template_set.items()): + description = template.get("description", "no description") + is_last_template = i == len(builtin_template_set.items()) - 1 + if not is_last_template: + print(f"β”‚ β”œβ”€β”€ {name}: {description}") + else: + print(f"β”‚ └── {name}: {description}") + + for i, (input_name, input_templates) in enumerate(templates.custom.items()): + custom_templates: TemplateClanType | None = input_templates.get( + template_type, None + ) # type: ignore + if not custom_templates: + continue + + is_last_input = i == len(templates.custom.items()) - 1 + prefix = "β”‚" if not is_last_input else " " + if not is_last_input: + print(f"β”œβ”€β”€ inputs.{input_name}:") + else: + print(f"└── inputs.{input_name}:") + + for i, (name, template) in enumerate(custom_templates.items()): + is_last_template = i == len(custom_templates.items()) - 1 + if not is_last_template: + print( + f"{prefix} β”œβ”€β”€ {name}: {template.get('description', 'no description')}" + ) + else: + print( + f"{prefix} └── {name}: {template.get('description', 'no description')}" + ) + + +def register_list_parser(parser: argparse.ArgumentParser) -> None: + parser.set_defaults(func=list_command) From 2b3e847c28e8e87ca1755b8bdfad776a6837f553 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 19:47:58 +0200 Subject: [PATCH 052/258] machine: rename standalone 'get_host' to 'get_machine_host' --- pkgs/clan-cli/clan_lib/machines/machines.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/machines/machines.py b/pkgs/clan-cli/clan_lib/machines/machines.py index ce4a51b6d..be4c81c32 100644 --- a/pkgs/clan-cli/clan_lib/machines/machines.py +++ b/pkgs/clan-cli/clan_lib/machines/machines.py @@ -129,7 +129,7 @@ class Machine: return self.flake.path def target_host(self) -> Remote: - remote = get_host(self.name, self.flake, field="targetHost") + remote = get_machine_host(self.name, self.flake, field="targetHost") if remote is None: msg = f"'targetHost' is not set for machine '{self.name}'" raise ClanError( @@ -144,7 +144,7 @@ class Machine: The host where the machine is built and deployed from. Can be the same as the target host. """ - remote = get_host(self.name, self.flake, field="buildHost") + remote = get_machine_host(self.name, self.flake, field="buildHost") if remote: data = remote.data @@ -176,7 +176,7 @@ class RemoteSource: @API.register -def get_host( +def get_machine_host( name: str, flake: Flake, field: Literal["targetHost", "buildHost"] ) -> RemoteSource | None: """ From cd04686663d6ba99fe7087cb67be2740498d5297 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 20:06:17 +0200 Subject: [PATCH 053/258] Docs: update index --- docs/mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index ce47a30c7..206fe9002 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -154,6 +154,7 @@ nav: - reference/cli/show.md - reference/cli/ssh.md - reference/cli/state.md + - reference/cli/templates.md - reference/cli/vars.md - reference/cli/vms.md - NixOS Modules: From a2c2d73e49749795b447a170f50ebc3ae496aeee Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 19:58:04 +0200 Subject: [PATCH 054/258] Vars: rename 'keygen' to 'create_secrets_user' --- pkgs/clan-app/ui-2d/src/routes/clans/create.tsx | 2 +- pkgs/clan-app/ui/src/routes/clans/create.tsx | 2 +- pkgs/clan-cli/clan_cli/tests/test_vars.py | 4 +++- pkgs/clan-cli/clan_cli/vars/keygen.py | 11 ++++++++--- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pkgs/clan-app/ui-2d/src/routes/clans/create.tsx b/pkgs/clan-app/ui-2d/src/routes/clans/create.tsx index 67899a1f1..ea82a0e74 100644 --- a/pkgs/clan-app/ui-2d/src/routes/clans/create.tsx +++ b/pkgs/clan-app/ui-2d/src/routes/clans/create.tsx @@ -66,7 +66,7 @@ export const CreateClan = () => { } // Will generate a key if it doesn't exist, and add a user to the clan - const k = await callApi("keygen", { + const k = await callApi("create_secrets_user", { flake_dir: target_dir[0], }).promise; diff --git a/pkgs/clan-app/ui/src/routes/clans/create.tsx b/pkgs/clan-app/ui/src/routes/clans/create.tsx index 6a1b8fa39..92cac1f65 100644 --- a/pkgs/clan-app/ui/src/routes/clans/create.tsx +++ b/pkgs/clan-app/ui/src/routes/clans/create.tsx @@ -65,7 +65,7 @@ export const CreateClan = () => { } // Will generate a key if it doesn't exist, and add a user to the clan - const k = await callApi("keygen", { + const k = await callApi("create_secrets_user", { flake_dir: target_dir[0], }).promise; diff --git a/pkgs/clan-cli/clan_cli/tests/test_vars.py b/pkgs/clan-cli/clan_cli/tests/test_vars.py index 274dd38fc..fa084f6f1 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_vars.py +++ b/pkgs/clan-cli/clan_cli/tests/test_vars.py @@ -900,7 +900,9 @@ def test_fails_when_files_are_left_from_other_backend( @pytest.mark.with_core -def test_keygen(monkeypatch: pytest.MonkeyPatch, flake: ClanFlake) -> None: +def test_create_sops_age_secrets( + monkeypatch: pytest.MonkeyPatch, flake: ClanFlake +) -> None: monkeypatch.chdir(flake.path) cli.run(["vars", "keygen", "--flake", str(flake.path), "--user", "user"]) # check public key exists diff --git a/pkgs/clan-cli/clan_cli/vars/keygen.py b/pkgs/clan-cli/clan_cli/vars/keygen.py index 4f1bdd931..9475a771f 100644 --- a/pkgs/clan-cli/clan_cli/vars/keygen.py +++ b/pkgs/clan-cli/clan_cli/vars/keygen.py @@ -13,11 +13,16 @@ log = logging.getLogger(__name__) @API.register -def keygen(flake_dir: Path, user: str | None = None, force: bool = False) -> None: +def create_secrets_user( + flake_dir: Path, user: str | None = None, force: bool = False +) -> None: + """ + initialize sops keys for vars + """ if user is None: user = os.getenv("USER", None) if not user: - msg = "No user provided and $USER is not set. Please provide a user via --user." + msg = "No user provided and environment variable: '$USER' is not set. Please provide an explizit username via argument" raise ClanError(msg) pub_keys = maybe_get_admin_public_keys() if not pub_keys: @@ -34,7 +39,7 @@ def keygen(flake_dir: Path, user: str | None = None, force: bool = False) -> Non def _command( args: argparse.Namespace, ) -> None: - keygen( + create_secrets_user( flake_dir=args.flake.path, user=args.user, force=args.force, From 0589c71601e6515ba78b0fb9b22576b109da1b99 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 20:04:02 +0200 Subject: [PATCH 055/258] Vars: rename public functions into 'create_machine_vars' --- pkgs/clan-cli/clan_cli/tests/test_vars.py | 26 ++++++++++----------- pkgs/clan-cli/clan_cli/vars/generate.py | 14 ++++++----- pkgs/clan-cli/clan_lib/tests/test_create.py | 4 ++-- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/tests/test_vars.py b/pkgs/clan-cli/clan_cli/tests/test_vars.py index fa084f6f1..08d4f095d 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_vars.py +++ b/pkgs/clan-cli/clan_cli/tests/test_vars.py @@ -10,8 +10,8 @@ from clan_cli.tests.helpers import cli from clan_cli.vars.check import check_vars from clan_cli.vars.generate import ( Generator, - generate_vars_for_machine, - generate_vars_for_machine_interactive, + create_machine_vars, + create_machine_vars_interactive, get_generators_closure, ) from clan_cli.vars.get import get_var @@ -668,7 +668,7 @@ def test_api_set_prompts( monkeypatch.chdir(flake.path) - generate_vars_for_machine( + create_machine_vars( machine_name="my_machine", base_dir=flake.path, generators=["my_generator"], @@ -682,7 +682,7 @@ def test_api_set_prompts( store = in_repo.FactStore(machine) assert store.exists(Generator("my_generator"), "prompt1") assert store.get(Generator("my_generator"), "prompt1").decode() == "input1" - generate_vars_for_machine( + create_machine_vars( machine_name="my_machine", base_dir=flake.path, generators=["my_generator"], @@ -727,11 +727,11 @@ def test_stdout_of_generate( flake_.refresh() monkeypatch.chdir(flake_.path) flake = Flake(str(flake_.path)) - from clan_cli.vars.generate import generate_vars_for_machine_interactive + from clan_cli.vars.generate import create_machine_vars_interactive # with capture_output as output: with caplog.at_level(logging.INFO): - generate_vars_for_machine_interactive( + create_machine_vars_interactive( Machine(name="my_machine", flake=flake), "my_generator", regenerate=False, @@ -744,7 +744,7 @@ def test_stdout_of_generate( set_var("my_machine", "my_generator/my_value", b"world", flake) with caplog.at_level(logging.INFO): - generate_vars_for_machine_interactive( + create_machine_vars_interactive( Machine(name="my_machine", flake=flake), "my_generator", regenerate=True, @@ -755,7 +755,7 @@ def test_stdout_of_generate( caplog.clear() # check the output when nothing gets regenerated with caplog.at_level(logging.INFO): - generate_vars_for_machine_interactive( + create_machine_vars_interactive( Machine(name="my_machine", flake=flake), "my_generator", regenerate=True, @@ -764,7 +764,7 @@ def test_stdout_of_generate( assert "hello" in caplog.text caplog.clear() with caplog.at_level(logging.INFO): - generate_vars_for_machine_interactive( + create_machine_vars_interactive( Machine(name="my_machine", flake=flake), "my_secret_generator", regenerate=False, @@ -779,7 +779,7 @@ def test_stdout_of_generate( Flake(str(flake.path)), ) with caplog.at_level(logging.INFO): - generate_vars_for_machine_interactive( + create_machine_vars_interactive( Machine(name="my_machine", flake=flake), "my_secret_generator", regenerate=True, @@ -869,7 +869,7 @@ def test_fails_when_files_are_left_from_other_backend( flake.refresh() monkeypatch.chdir(flake.path) for generator in ["my_secret_generator", "my_value_generator"]: - generate_vars_for_machine_interactive( + create_machine_vars_interactive( Machine(name="my_machine", flake=Flake(str(flake.path))), generator, regenerate=False, @@ -886,13 +886,13 @@ def test_fails_when_files_are_left_from_other_backend( # This should raise an error if generator == "my_secret_generator": with pytest.raises(ClanError): - generate_vars_for_machine_interactive( + create_machine_vars_interactive( Machine(name="my_machine", flake=Flake(str(flake.path))), generator, regenerate=False, ) else: - generate_vars_for_machine_interactive( + create_machine_vars_interactive( Machine(name="my_machine", flake=Flake(str(flake.path))), generator, regenerate=False, diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index 2d1ef9c08..b70f18223 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -89,9 +89,11 @@ class Generator: deploy=file_data["deploy"], owner=file_data["owner"], group=file_data["group"], - mode=file_data["mode"] - if isinstance(file_data["mode"], int) - else int(file_data["mode"], 8), + mode=( + file_data["mode"] + if isinstance(file_data["mode"], int) + else int(file_data["mode"], 8) + ), needed_for=file_data["neededFor"], ) files.append(var) @@ -459,7 +461,7 @@ def _generate_vars_for_machine( @API.register -def generate_vars_for_machine( +def create_machine_vars( machine_name: str, generators: list[str], all_prompt_values: dict[str, dict[str, str]], @@ -484,7 +486,7 @@ def generate_vars_for_machine( ) -def generate_vars_for_machine_interactive( +def create_machine_vars_interactive( machine: "Machine", generator_name: str | None, regenerate: bool, @@ -538,7 +540,7 @@ def generate_vars( for machine in machines: errors = [] try: - was_regenerated |= generate_vars_for_machine_interactive( + was_regenerated |= create_machine_vars_interactive( machine, generator_name, regenerate, diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index ed43e442f..4f56a3192 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -14,7 +14,7 @@ from clan_cli.machines.create import create_machine from clan_cli.secrets.key import generate_key from clan_cli.secrets.sops import maybe_get_admin_public_keys from clan_cli.secrets.users import add_user -from clan_cli.vars.generate import generate_vars_for_machine, get_generators_closure +from clan_cli.vars.generate import create_machine_vars, get_generators_closure from clan_lib.api.disk import hw_main_disk_options, set_machine_disk_schema from clan_lib.api.modules import list_modules @@ -234,7 +234,7 @@ def test_clan_create_api( raise ClanError(msg) all_prompt_values[generator.name] = prompt_values - generate_vars_for_machine( + create_machine_vars( machine_name=machine.name, base_dir=machine.flake.path, generators=[gen.name for gen in generators], From f48c596617ff2a6d9bc29973e50145bd8f1ccaaa Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 20:10:58 +0200 Subject: [PATCH 056/258] vars/api: rename, unregister some unused vars functions --- pkgs/clan-cli/clan_cli/tests/test_vars.py | 32 ++++++------ pkgs/clan-cli/clan_cli/vars/get.py | 9 ++-- pkgs/clan-cli/clan_cli/vars/list.py | 63 +---------------------- pkgs/clan-cli/clan_cli/vars/set.py | 6 +-- 4 files changed, 25 insertions(+), 85 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/tests/test_vars.py b/pkgs/clan-cli/clan_cli/tests/test_vars.py index 08d4f095d..5b049f87e 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_vars.py +++ b/pkgs/clan-cli/clan_cli/tests/test_vars.py @@ -14,7 +14,7 @@ from clan_cli.vars.generate import ( create_machine_vars_interactive, get_generators_closure, ) -from clan_cli.vars.get import get_var +from clan_cli.vars.get import get_machine_var from clan_cli.vars.graph import all_missing_closure, requested_closure from clan_cli.vars.list import stringify_all_vars from clan_cli.vars.public_modules import in_repo @@ -172,13 +172,13 @@ def test_generate_public_and_secret_vars( in commit_message ) assert ( - get_var( + get_machine_var( str(machine.flake.path), machine.name, "my_generator/my_value" ).printable_value == "public" ) assert ( - get_var( + get_machine_var( str(machine.flake.path), machine.name, "my_shared_generator/my_shared_value" ).printable_value == "shared" @@ -343,9 +343,9 @@ def test_generated_shared_secret_sops( shared_generator["script"] = 'echo hello > "$out"/my_shared_secret' m2_config = flake.machines["machine2"] m2_config["nixpkgs"]["hostPlatform"] = "x86_64-linux" - m2_config["clan"]["core"]["vars"]["generators"]["my_shared_generator"] = ( - shared_generator.copy() - ) + m2_config["clan"]["core"]["vars"]["generators"][ + "my_shared_generator" + ] = shared_generator.copy() flake.refresh() monkeypatch.chdir(flake.path) machine1 = Machine(name="machine1", flake=Flake(str(flake.path))) @@ -803,9 +803,9 @@ def test_migration( my_service = config["clan"]["core"]["facts"]["services"]["my_service"] my_service["public"]["my_value"] = {} my_service["secret"]["my_secret"] = {} - my_service["generator"]["script"] = ( - 'echo -n hello > "$facts"/my_value && echo -n hello > "$secrets"/my_secret' - ) + my_service["generator"][ + "script" + ] = 'echo -n hello > "$facts"/my_value && echo -n hello > "$secrets"/my_secret' my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"] my_generator["files"]["my_value"]["secret"] = False my_generator["files"]["my_secret"]["secret"] = True @@ -875,9 +875,9 @@ def test_fails_when_files_are_left_from_other_backend( regenerate=False, ) # Will raise. It was secret before, but now it's not. - my_secret_generator["files"]["my_secret"]["secret"] = ( - False # secret -> public (NOT OK) - ) + my_secret_generator["files"]["my_secret"][ + "secret" + ] = False # secret -> public (NOT OK) # WIll not raise. It was not secret before, and it's secret now. my_value_generator["files"]["my_value"]["secret"] = True # public -> secret (OK) flake.refresh() @@ -932,12 +932,12 @@ def test_invalidation( monkeypatch.chdir(flake.path) cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) machine = Machine(name="my_machine", flake=Flake(str(flake.path))) - value1 = get_var( + value1 = get_machine_var( str(machine.flake.path), machine.name, "my_generator/my_value" ).printable_value # generate again and make sure nothing changes without the invalidation data being set cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) - value1_new = get_var( + value1_new = get_machine_var( str(machine.flake.path), machine.name, "my_generator/my_value" ).printable_value assert value1 == value1_new @@ -946,13 +946,13 @@ def test_invalidation( flake.refresh() # generate again and make sure the value changes cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) - value2 = get_var( + value2 = get_machine_var( str(machine.flake.path), machine.name, "my_generator/my_value" ).printable_value assert value1 != value2 # generate again without changing invalidation data -> value should not change cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"]) - value2_new = get_var( + value2_new = get_machine_var( str(machine.flake.path), machine.name, "my_generator/my_value" ).printable_value assert value2 == value2_new diff --git a/pkgs/clan-cli/clan_cli/vars/get.py b/pkgs/clan-cli/clan_cli/vars/get.py index c95de5fb3..446be36af 100644 --- a/pkgs/clan-cli/clan_cli/vars/get.py +++ b/pkgs/clan-cli/clan_cli/vars/get.py @@ -8,14 +8,13 @@ from clan_lib.errors import ClanError from clan_lib.flake import Flake from .generate import Var -from .list import get_vars +from .list import get_machine_vars log = logging.getLogger(__name__) -@API.register -def get_var(base_dir: str, machine_name: str, var_id: str) -> Var: - vars_ = get_vars(base_dir=base_dir, machine_name=machine_name) +def get_machine_var(base_dir: str, machine_name: str, var_id: str) -> Var: + vars_ = get_machine_vars(base_dir=base_dir, machine_name=machine_name) results = [] for var in vars_: if var.id == var_id: @@ -41,7 +40,7 @@ def get_var(base_dir: str, machine_name: str, var_id: str) -> Var: def get_command(machine_name: str, var_id: str, flake: Flake) -> None: - var = get_var(str(flake.path), machine_name, var_id) + var = get_machine_var(str(flake.path), machine_name, var_id) if not var.exists: msg = f"Var {var.id} has not been generated yet" raise ClanError(msg) diff --git a/pkgs/clan-cli/clan_cli/vars/list.py b/pkgs/clan-cli/clan_cli/vars/list.py index c880b6696..d3db6c396 100644 --- a/pkgs/clan-cli/clan_cli/vars/list.py +++ b/pkgs/clan-cli/clan_cli/vars/list.py @@ -13,8 +13,7 @@ from .generate import Generator, Prompt, Var, execute_generator log = logging.getLogger(__name__) -@API.register -def get_vars(base_dir: str, machine_name: str) -> list[Var]: +def get_machine_vars(base_dir: str, machine_name: str) -> list[Var]: machine = Machine(name=machine_name, flake=Flake(base_dir)) pub_store = machine.public_vars_store sec_store = machine.secret_vars_store @@ -32,70 +31,12 @@ def get_vars(base_dir: str, machine_name: str) -> list[Var]: return all_vars -def _get_previous_value( - machine: Machine, - generator: Generator, - prompt: Prompt, -) -> str | None: - if not prompt.persist: - return None - - pub_store = machine.public_vars_store - if pub_store.exists(generator, prompt.name): - return pub_store.get(generator, prompt.name).decode() - sec_store = machine.secret_vars_store - if sec_store.exists(generator, prompt.name): - return sec_store.get(generator, prompt.name).decode() - return None - - -@API.register -def get_generators(base_dir: str, machine_name: str) -> list[Generator]: - from clan_cli.vars.generate import Generator - - machine = Machine(name=machine_name, flake=Flake(base_dir)) - generators: list[Generator] = Generator.generators_from_flake( - machine_name, machine.flake - ) - for generator in generators: - for prompt in generator.prompts: - prompt.previous_value = _get_previous_value(machine, generator, prompt) - return generators - - -# TODO: Ensure generator dependencies are met (executed in correct order etc.) -# TODO: for missing prompts, default to existing values -# TODO: raise error if mandatory prompt not provided -@API.register -def set_prompts( - base_dir: str, machine_name: str, updates: list[GeneratorUpdate] -) -> None: - from clan_cli.vars.generate import Generator - - machine = Machine(name=machine_name, flake=Flake(base_dir)) - for update in updates: - generators = Generator.generators_from_flake(machine_name, machine.flake) - for generator in generators: - if generator.name == update.generator: - break - else: - msg = f"Generator '{update.generator}' not found in machine {machine.name}" - raise ClanError(msg) - execute_generator( - machine, - generator, - secret_vars_store=machine.secret_vars_store, - public_vars_store=machine.public_vars_store, - prompt_values=update.prompt_values, - ) - - def stringify_vars(_vars: list[Var]) -> str: return "\n".join([str(var) for var in _vars]) def stringify_all_vars(machine: Machine) -> str: - return stringify_vars(get_vars(str(machine.flake), machine.name)) + return stringify_vars(get_machine_vars(str(machine.flake), machine.name)) def list_command(args: argparse.Namespace) -> None: diff --git a/pkgs/clan-cli/clan_cli/vars/set.py b/pkgs/clan-cli/clan_cli/vars/set.py index ceeb4f46e..cd67b7522 100644 --- a/pkgs/clan-cli/clan_cli/vars/set.py +++ b/pkgs/clan-cli/clan_cli/vars/set.py @@ -3,7 +3,7 @@ import logging import sys from clan_cli.completions import add_dynamic_completer, complete_machines -from clan_cli.vars.get import get_var +from clan_cli.vars.get import get_machine_var from clan_cli.vars.prompt import PromptType from clan_lib.flake import Flake from clan_lib.git import commit_files @@ -21,7 +21,7 @@ def set_var(machine: str | Machine, var: str | Var, value: bytes, flake: Flake) else: _machine = machine if isinstance(var, str): - _var = get_var(str(flake.path), _machine.name, var) + _var = get_machine_var(str(flake.path), _machine.name, var) else: _var = var path = _var.set(value) @@ -35,7 +35,7 @@ def set_var(machine: str | Machine, var: str | Var, value: bytes, flake: Flake) def set_via_stdin(machine_name: str, var_id: str, flake: Flake) -> None: machine = Machine(name=machine_name, flake=flake) - var = get_var(str(flake.path), machine_name, var_id) + var = get_machine_var(str(flake.path), machine_name, var_id) if sys.stdin.isatty(): new_value = ask( var.id, From 9635fb03b70e73b505be8bc88fc73e4e87158bc9 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 20:17:13 +0200 Subject: [PATCH 057/258] api/flash: refactor into 'list_flash_options' --- pkgs/clan-app/ui-2d/src/routes/flash/view.tsx | 4 ++-- pkgs/clan-cli/clan_cli/flash/flash.py | 6 +++--- pkgs/clan-cli/clan_cli/flash/list.py | 19 ++++++++++++++----- pkgs/clan-cli/clan_cli/tests/test_vars.py | 18 +++++++++--------- pkgs/clan-cli/clan_cli/vars/get.py | 1 - pkgs/clan-cli/clan_cli/vars/list.py | 5 +---- 6 files changed, 29 insertions(+), 24 deletions(-) diff --git a/pkgs/clan-app/ui-2d/src/routes/flash/view.tsx b/pkgs/clan-app/ui-2d/src/routes/flash/view.tsx index c5d0a645d..7bea3a960 100644 --- a/pkgs/clan-app/ui-2d/src/routes/flash/view.tsx +++ b/pkgs/clan-app/ui-2d/src/routes/flash/view.tsx @@ -110,7 +110,7 @@ export const Flash = () => { const keymapQuery = createQuery(() => ({ queryKey: ["list_keymaps"], queryFn: async () => { - const result = await callApi("list_possible_keymaps", {}).promise; + const result = await callApi("list_keymaps", {}).promise; if (result.status === "error") throw new Error("Failed to fetch data"); return result.data; }, @@ -120,7 +120,7 @@ export const Flash = () => { const langQuery = createQuery(() => ({ queryKey: ["list_languages"], queryFn: async () => { - const result = await callApi("list_possible_languages", {}).promise; + const result = await callApi("list_languages", {}).promise; if (result.status === "error") throw new Error("Failed to fetch data"); return result.data; }, diff --git a/pkgs/clan-cli/clan_cli/flash/flash.py b/pkgs/clan-cli/clan_cli/flash/flash.py index aa77db5f5..be2eb683e 100644 --- a/pkgs/clan-cli/clan_cli/flash/flash.py +++ b/pkgs/clan-cli/clan_cli/flash/flash.py @@ -17,7 +17,7 @@ from clan_cli.vars.generate import generate_vars from clan_cli.vars.upload import populate_secret_vars from .automount import pause_automounting -from .list import list_possible_keymaps, list_possible_languages +from .list import list_keymaps, list_languages log = logging.getLogger(__name__) @@ -59,7 +59,7 @@ def flash_machine( generate_vars([machine]) if system_config.language: - if system_config.language not in list_possible_languages(): + if system_config.language not in list_languages(): msg = ( f"Language '{system_config.language}' is not a valid language. " f"Run 'clan flash list languages' to see a list of possible languages." @@ -68,7 +68,7 @@ def flash_machine( system_config_nix["i18n"] = {"defaultLocale": system_config.language} if system_config.keymap: - if system_config.keymap not in list_possible_keymaps(): + if system_config.keymap not in list_keymaps(): msg = ( f"Keymap '{system_config.keymap}' is not a valid keymap. " f"Run 'clan flash list keymaps' to see a list of possible keymaps." diff --git a/pkgs/clan-cli/clan_cli/flash/list.py b/pkgs/clan-cli/clan_cli/flash/list.py index ac356f8d8..c60a6ee7d 100644 --- a/pkgs/clan-cli/clan_cli/flash/list.py +++ b/pkgs/clan-cli/clan_cli/flash/list.py @@ -2,6 +2,7 @@ import argparse import logging import os from pathlib import Path +from typing import TypedDict from clan_lib.api import API from clan_lib.cmd import Log, RunOpts, run @@ -11,8 +12,17 @@ from clan_lib.nix import nix_build log = logging.getLogger(__name__) +class FlashOptions(TypedDict): + languages: list[str] + keymaps: list[str] + + @API.register -def list_possible_languages() -> list[str]: +def show_flash_options() -> FlashOptions: + return {"languages": list_languages(), "keymaps": list_keymaps()} + + +def list_languages() -> list[str]: cmd = nix_build(["nixpkgs#glibcLocales"]) result = run(cmd, RunOpts(log=Log.STDERR, error_msg="Failed to find glibc locales")) locale_file = Path(result.stdout.strip()) / "share" / "i18n" / "SUPPORTED" @@ -37,8 +47,7 @@ def list_possible_languages() -> list[str]: return languages -@API.register -def list_possible_keymaps() -> list[str]: +def list_keymaps() -> list[str]: cmd = nix_build(["nixpkgs#kbd"]) result = run(cmd, RunOpts(log=Log.STDERR, error_msg="Failed to find kbdinfo")) keymaps_dir = Path(result.stdout.strip()) / "share" / "keymaps" @@ -61,11 +70,11 @@ def list_possible_keymaps() -> list[str]: def list_command(args: argparse.Namespace) -> None: if args.cmd == "languages": - languages = list_possible_languages() + languages = list_languages() for language in languages: print(language) elif args.cmd == "keymaps": - keymaps = list_possible_keymaps() + keymaps = list_keymaps() for keymap in keymaps: print(keymap) diff --git a/pkgs/clan-cli/clan_cli/tests/test_vars.py b/pkgs/clan-cli/clan_cli/tests/test_vars.py index 5b049f87e..08e1eea19 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_vars.py +++ b/pkgs/clan-cli/clan_cli/tests/test_vars.py @@ -343,9 +343,9 @@ def test_generated_shared_secret_sops( shared_generator["script"] = 'echo hello > "$out"/my_shared_secret' m2_config = flake.machines["machine2"] m2_config["nixpkgs"]["hostPlatform"] = "x86_64-linux" - m2_config["clan"]["core"]["vars"]["generators"][ - "my_shared_generator" - ] = shared_generator.copy() + m2_config["clan"]["core"]["vars"]["generators"]["my_shared_generator"] = ( + shared_generator.copy() + ) flake.refresh() monkeypatch.chdir(flake.path) machine1 = Machine(name="machine1", flake=Flake(str(flake.path))) @@ -803,9 +803,9 @@ def test_migration( my_service = config["clan"]["core"]["facts"]["services"]["my_service"] my_service["public"]["my_value"] = {} my_service["secret"]["my_secret"] = {} - my_service["generator"][ - "script" - ] = 'echo -n hello > "$facts"/my_value && echo -n hello > "$secrets"/my_secret' + my_service["generator"]["script"] = ( + 'echo -n hello > "$facts"/my_value && echo -n hello > "$secrets"/my_secret' + ) my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"] my_generator["files"]["my_value"]["secret"] = False my_generator["files"]["my_secret"]["secret"] = True @@ -875,9 +875,9 @@ def test_fails_when_files_are_left_from_other_backend( regenerate=False, ) # Will raise. It was secret before, but now it's not. - my_secret_generator["files"]["my_secret"][ - "secret" - ] = False # secret -> public (NOT OK) + my_secret_generator["files"]["my_secret"]["secret"] = ( + False # secret -> public (NOT OK) + ) # WIll not raise. It was not secret before, and it's secret now. my_value_generator["files"]["my_value"]["secret"] = True # public -> secret (OK) flake.refresh() diff --git a/pkgs/clan-cli/clan_cli/vars/get.py b/pkgs/clan-cli/clan_cli/vars/get.py index 446be36af..0966e187f 100644 --- a/pkgs/clan-cli/clan_cli/vars/get.py +++ b/pkgs/clan-cli/clan_cli/vars/get.py @@ -3,7 +3,6 @@ import logging import sys from clan_cli.completions import add_dynamic_completer, complete_machines -from clan_lib.api import API from clan_lib.errors import ClanError from clan_lib.flake import Flake diff --git a/pkgs/clan-cli/clan_cli/vars/list.py b/pkgs/clan-cli/clan_cli/vars/list.py index d3db6c396..5bd8cfcda 100644 --- a/pkgs/clan-cli/clan_cli/vars/list.py +++ b/pkgs/clan-cli/clan_cli/vars/list.py @@ -2,13 +2,10 @@ import argparse import logging from clan_cli.completions import add_dynamic_completer, complete_machines -from clan_lib.api import API -from clan_lib.errors import ClanError from clan_lib.flake import Flake from clan_lib.machines.machines import Machine -from ._types import GeneratorUpdate -from .generate import Generator, Prompt, Var, execute_generator +from .generate import Var log = logging.getLogger(__name__) From d1abebf068ac013903d516cafe9f2cec7d7c597e Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 20:21:58 +0200 Subject: [PATCH 058/258] api/inventory: remove 'inventory' from api entirely --- pkgs/clan-app/ui-2d/src/queries/index.ts | 23 +------------------ .../clan-app/ui-2d/src/routes/modules/add.tsx | 21 ++--------------- .../ui-2d/src/routes/modules/details.tsx | 1 - pkgs/clan-cli/clan_lib/inventory/__init__.py | 11 --------- pkgs/clan-cli/clan_lib/machines/delete.py | 4 ++-- pkgs/clan-cli/clan_lib/tests/test_create.py | 2 +- 6 files changed, 6 insertions(+), 56 deletions(-) diff --git a/pkgs/clan-app/ui-2d/src/queries/index.ts b/pkgs/clan-app/ui-2d/src/queries/index.ts index 220d1bd07..29ea86116 100644 --- a/pkgs/clan-app/ui-2d/src/queries/index.ts +++ b/pkgs/clan-app/ui-2d/src/queries/index.ts @@ -33,27 +33,6 @@ export const createModulesQuery = ( }, })); -export const tagsQuery = (uri: string | undefined) => - useQuery(() => ({ - queryKey: [uri, "tags"], - placeholderData: [], - queryFn: async () => { - if (!uri) return []; - - const response = await callApi("get_inventory", { - flake: { identifier: uri }, - }).promise; - if (response.status === "error") { - console.error("Failed to fetch data"); - } else { - const machines = response.data.machines || {}; - const tags = Object.values(machines).flatMap((m) => m.tags || []); - return tags; - } - return []; - }, - })); - export const machinesQuery = (uri: string | undefined) => useQuery(() => ({ queryKey: [uri, "machines"], @@ -61,7 +40,7 @@ export const machinesQuery = (uri: string | undefined) => queryFn: async () => { if (!uri) return []; - const response = await callApi("get_inventory", { + const response = await callApi("list_machines", { flake: { identifier: uri }, }).promise; if (response.status === "error") { diff --git a/pkgs/clan-app/ui-2d/src/routes/modules/add.tsx b/pkgs/clan-app/ui-2d/src/routes/modules/add.tsx index d732312ee..e87133bb3 100644 --- a/pkgs/clan-app/ui-2d/src/routes/modules/add.tsx +++ b/pkgs/clan-app/ui-2d/src/routes/modules/add.tsx @@ -1,5 +1,5 @@ import { BackButton } from "@/src/components/BackButton"; -import { createModulesQuery, machinesQuery, tagsQuery } from "@/src/queries"; +import { createModulesQuery, machinesQuery } from "@/src/queries"; import { useParams } from "@solidjs/router"; import { For, Match, Switch } from "solid-js"; import { ModuleInfo } from "./list"; @@ -34,28 +34,11 @@ interface AddModuleProps { const AddModule = (props: AddModuleProps) => { const { activeClanURI } = useClanContext(); - const tags = tagsQuery(activeClanURI()); const machines = machinesQuery(activeClanURI()); return (
Add to your clan
- - - {(tags) => ( - - {(role) => ( - <> -
{role}s
- - - )} - - )} - - + Removed
); }; diff --git a/pkgs/clan-app/ui-2d/src/routes/modules/details.tsx b/pkgs/clan-app/ui-2d/src/routes/modules/details.tsx index de8b35f9f..3e40a9f8d 100644 --- a/pkgs/clan-app/ui-2d/src/routes/modules/details.tsx +++ b/pkgs/clan-app/ui-2d/src/routes/modules/details.tsx @@ -62,7 +62,6 @@ const Details = (props: DetailsProps) => { navigate(`/modules/add/${props.id}`); // const uri = activeURI(); // if (!uri) return; - // const res = await callApi("get_inventory", { base_path: uri }); // if (res.status === "error") { // toast.error("Failed to fetch inventory"); // return; diff --git a/pkgs/clan-cli/clan_lib/inventory/__init__.py b/pkgs/clan-cli/clan_lib/inventory/__init__.py index 7409ce8fe..e2fd4eba4 100644 --- a/pkgs/clan-cli/clan_lib/inventory/__init__.py +++ b/pkgs/clan-cli/clan_lib/inventory/__init__.py @@ -10,14 +10,3 @@ Which is an abstraction over the inventory Interacting with 'clan_lib.inventory' is NOT recommended and will be removed """ - -from clan_lib.api import API -from clan_lib.flake import Flake -from clan_lib.persist.inventory_store import InventorySnapshot, InventoryStore - - -@API.register -def get_inventory(flake: Flake) -> InventorySnapshot: - inventory_store = InventoryStore(flake) - inventory = inventory_store.read() - return inventory diff --git a/pkgs/clan-cli/clan_lib/machines/delete.py b/pkgs/clan-cli/clan_lib/machines/delete.py index 68f210cb8..7d642a1d7 100644 --- a/pkgs/clan-cli/clan_lib/machines/delete.py +++ b/pkgs/clan-cli/clan_lib/machines/delete.py @@ -9,7 +9,7 @@ from clan_cli.secrets.secrets import ( list_secrets, ) -from clan_lib import inventory +from clan_lib.persist.inventory_store import InventoryStore from clan_lib.api import API from clan_lib.dirs import specific_machine_dir from clan_lib.machines.machines import Machine @@ -19,7 +19,7 @@ log = logging.getLogger(__name__) @API.register def delete_machine(machine: Machine) -> None: - inventory_store = inventory.InventoryStore(machine.flake) + inventory_store = InventoryStore(machine.flake) try: inventory_store.delete( {f"machines.{machine.name}"}, diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index 4f56a3192..9b5ab7b81 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -22,7 +22,7 @@ from clan_lib.cmd import RunOpts, run from clan_lib.dirs import specific_machine_dir from clan_lib.errors import ClanError from clan_lib.flake import Flake -from clan_lib.inventory import InventoryStore +from clan_lib.persist.inventory_store import InventoryStore from clan_lib.machines.machines import Machine from clan_lib.nix import nix_command from clan_lib.nix_models.clan import ( From a2c016718a6e4344be0891736d48e5e388034f11 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 20:31:53 +0200 Subject: [PATCH 059/258] api/hardware: consolidate into 'describe_machine_hardware' --- pkgs/clan-cli/clan_lib/api/disk.py | 4 +- pkgs/clan-cli/clan_lib/machines/hardware.py | 43 ++++++++++++++------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/api/disk.py b/pkgs/clan-cli/clan_lib/api/disk.py index 28a14befa..0fb083402 100644 --- a/pkgs/clan-cli/clan_lib/api/disk.py +++ b/pkgs/clan-cli/clan_lib/api/disk.py @@ -10,7 +10,7 @@ from clan_lib.api.modules import Frontmatter, extract_frontmatter from clan_lib.dirs import TemplateType, clan_templates from clan_lib.errors import ClanError from clan_lib.git import commit_file -from clan_lib.machines.hardware import HardwareConfig, show_machine_hardware_config +from clan_lib.machines.hardware import HardwareConfig, get_machine_hardware_config from clan_lib.machines.machines import Machine log = logging.getLogger(__name__) @@ -137,7 +137,7 @@ def set_machine_disk_schema( Set the disk placeholders of the template """ # Assert the hw-config must exist before setting the disk - hw_config = show_machine_hardware_config(machine) + hw_config = get_machine_hardware_config(machine) hw_config_path = hw_config.config_path(machine) if not hw_config_path.exists(): diff --git a/pkgs/clan-cli/clan_lib/machines/hardware.py b/pkgs/clan-cli/clan_lib/machines/hardware.py index 8324a7953..402213faa 100644 --- a/pkgs/clan-cli/clan_lib/machines/hardware.py +++ b/pkgs/clan-cli/clan_lib/machines/hardware.py @@ -3,6 +3,7 @@ import logging from dataclasses import dataclass from enum import Enum from pathlib import Path +from typing import TypedDict from clan_lib.api import API from clan_lib.cmd import RunOpts, run @@ -40,19 +41,7 @@ class HardwareConfig(Enum): return HardwareConfig.NONE -@API.register -def show_machine_hardware_config(machine: Machine) -> HardwareConfig: - """ - Show hardware information for a machine returns None if none exist. - """ - return HardwareConfig.detect_type(machine) - - -@API.register -def show_machine_hardware_platform(machine: Machine) -> str | None: - """ - Show hardware information for a machine returns None if none exist. - """ +def get_machine_target_platform(machine: Machine) -> str | None: config = nix_config() system = config["system"] cmd = nix_eval( @@ -132,7 +121,7 @@ def generate_machine_hardware_info( f"machines/{opts.machine}/{hw_file.name}: update hardware configuration", ) try: - show_machine_hardware_platform(opts.machine) + get_machine_target_platform(opts.machine) if backup_file: backup_file.unlink(missing_ok=True) except ClanCmdError as e: @@ -150,3 +139,29 @@ def generate_machine_hardware_info( ) from e return opts.backend + + +def get_machine_hardware_config(machine: Machine) -> HardwareConfig: + """ + Detect and return the full hardware configuration for the given machine. + + Returns: + HardwareConfig: Structured hardware information, or None if unavailable. + """ + return HardwareConfig.detect_type(machine) + + +class MachineHardwareBrief(TypedDict): + hardware_config: HardwareConfig + platform: str | None + + +@API.register +def describe_machine_hardware(machine: Machine) -> MachineHardwareBrief: + """ + Return a high-level summary of hardware config and platform type. + """ + return { + "hardware_config": get_machine_hardware_config(machine), + "platform": get_machine_target_platform(machine), + } From 00df0326359166dbed7159fafe5a814a56c3fadd Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sun, 6 Jul 2025 20:45:07 +0200 Subject: [PATCH 060/258] vars/api: rename 'get_generators_closure' into 'get_machine_generators' --- pkgs/clan-cli/clan_cli/tests/test_vars.py | 4 ++-- pkgs/clan-cli/clan_cli/vars/generate.py | 2 +- pkgs/clan-cli/clan_lib/tests/test_create.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/tests/test_vars.py b/pkgs/clan-cli/clan_cli/tests/test_vars.py index 08e1eea19..899050f7d 100644 --- a/pkgs/clan-cli/clan_cli/tests/test_vars.py +++ b/pkgs/clan-cli/clan_cli/tests/test_vars.py @@ -12,7 +12,7 @@ from clan_cli.vars.generate import ( Generator, create_machine_vars, create_machine_vars_interactive, - get_generators_closure, + get_machine_generators, ) from clan_cli.vars.get import get_machine_var from clan_cli.vars.graph import all_missing_closure, requested_closure @@ -694,7 +694,7 @@ def test_api_set_prompts( ) assert store.get(Generator("my_generator"), "prompt1").decode() == "input2" - generators = get_generators_closure( + generators = get_machine_generators( machine_name="my_machine", base_dir=flake.path, full_closure=True, diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py index b70f18223..67c0fa071 100644 --- a/pkgs/clan-cli/clan_cli/vars/generate.py +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -423,7 +423,7 @@ def get_closure( @API.register -def get_generators_closure( +def get_machine_generators( machine_name: str, base_dir: Path, full_closure: bool = False, diff --git a/pkgs/clan-cli/clan_lib/tests/test_create.py b/pkgs/clan-cli/clan_lib/tests/test_create.py index 9b5ab7b81..ac2758374 100644 --- a/pkgs/clan-cli/clan_lib/tests/test_create.py +++ b/pkgs/clan-cli/clan_lib/tests/test_create.py @@ -14,7 +14,7 @@ from clan_cli.machines.create import create_machine from clan_cli.secrets.key import generate_key from clan_cli.secrets.sops import maybe_get_admin_public_keys from clan_cli.secrets.users import add_user -from clan_cli.vars.generate import create_machine_vars, get_generators_closure +from clan_cli.vars.generate import create_machine_vars, get_machine_generators from clan_lib.api.disk import hw_main_disk_options, set_machine_disk_schema from clan_lib.api.modules import list_modules @@ -221,7 +221,7 @@ def test_clan_create_api( # Invalidate cache because of new inventory clan_dir_flake.invalidate_cache() - generators = get_generators_closure(machine.name, machine.flake.path) + generators = get_machine_generators(machine.name, machine.flake.path) all_prompt_values = {} for generator in generators: prompt_values = {} From ca69864a20286587a7961fb686e3f35b0d21ee4e Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 7 Jul 2025 00:46:06 +0200 Subject: [PATCH 061/258] rename lingering clan.vars -> clan.core.vars --- nixosModules/clanCore/vars/default.nix | 2 +- nixosModules/clanCore/vars/secret/password-store.nix | 12 ++++++------ .../clan_cli/vars/secret_modules/password_store.py | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/nixosModules/clanCore/vars/default.nix b/nixosModules/clanCore/vars/default.nix index afe8962ad..42532b284 100644 --- a/nixosModules/clanCore/vars/default.nix +++ b/nixosModules/clanCore/vars/default.nix @@ -46,7 +46,7 @@ in assertion = config.clan.core.vars.settings.passBackend == null; message = '' The option `clan.core.vars.settings.passBackend' has been removed. - Use clan.vars.password-store.passPackage instead. + Use clan.core.vars.password-store.passPackage instead. Set it to pkgs.pass for GPG or pkgs.passage for age encryption. ''; } diff --git a/nixosModules/clanCore/vars/secret/password-store.nix b/nixosModules/clanCore/vars/secret/password-store.nix index d79c46cf4..9654b43c8 100644 --- a/nixosModules/clanCore/vars/secret/password-store.nix +++ b/nixosModules/clanCore/vars/secret/password-store.nix @@ -54,7 +54,7 @@ in { _class = "nixos"; - options.clan.vars.password-store = { + options.clan.core.vars.password-store = { secretLocation = lib.mkOption { type = lib.types.path; default = "/etc/secret-vars"; @@ -83,7 +83,7 @@ in else if file.config.neededFor == "services" then "/run/secrets/${file.config.generatorName}/${file.config.name}" else if file.config.neededFor == "activation" then - "${config.clan.vars.password-store.secretLocation}/activation/${file.config.generatorName}/${file.config.name}" + "${config.clan.core.vars.password-store.secretLocation}/activation/${file.config.generatorName}/${file.config.name}" else if file.config.neededFor == "partitioning" then "/run/partitioning-secrets/${file.config.generatorName}/${file.config.name}" else @@ -102,7 +102,7 @@ in ] '' [ -e /run/current-system ] || echo setting up secrets... - ${installSecretTarball}/bin/install-secret-tarball ${config.clan.vars.password-store.secretLocation}/secrets_for_users.tar.gz /run/user-secrets + ${installSecretTarball}/bin/install-secret-tarball ${config.clan.core.vars.password-store.secretLocation}/secrets_for_users.tar.gz /run/user-secrets '' // lib.optionalAttrs (config.system ? dryActivationScript) { supportsDryActivation = true; @@ -118,7 +118,7 @@ in ] '' [ -e /run/current-system ] || echo setting up secrets... - ${installSecretTarball}/bin/install-secret-tarball ${config.clan.vars.password-store.secretLocation}/secrets.tar.gz /run/secrets + ${installSecretTarball}/bin/install-secret-tarball ${config.clan.core.vars.password-store.secretLocation}/secrets.tar.gz /run/secrets '' // lib.optionalAttrs (config.system ? dryActivationScript) { supportsDryActivation = true; @@ -136,7 +136,7 @@ in serviceConfig = { Type = "oneshot"; ExecStart = [ - "${installSecretTarball}/bin/install-secret-tarball ${config.clan.vars.password-store.secretLocation}/secrets_for_users.tar.gz /run/user-secrets" + "${installSecretTarball}/bin/install-secret-tarball ${config.clan.core.vars.password-store.secretLocation}/secrets_for_users.tar.gz /run/user-secrets" ]; RemainAfterExit = true; }; @@ -149,7 +149,7 @@ in serviceConfig = { Type = "oneshot"; ExecStart = [ - "${installSecretTarball}/bin/install-secret-tarball ${config.clan.vars.password-store.secretLocation}/secrets.tar.gz /run/secrets" + "${installSecretTarball}/bin/install-secret-tarball ${config.clan.core.vars.password-store.secretLocation}/secrets.tar.gz /run/secrets" ]; RemainAfterExit = true; }; diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py index d23cf18a4..071729681 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py @@ -45,11 +45,11 @@ class SecretStore(StoreBase): @property def _pass_command(self) -> str: out_path = self.machine.select( - "config.clan.vars.password-store.passPackage.outPath" + "config.clan.core.vars.password-store.passPackage.outPath" ) main_program = ( self.machine.select( - "config.clan.vars.password-store.passPackage.?meta.?mainProgram" + "config.clan.core.vars.password-store.passPackage.?meta.?mainProgram" ) .get("meta", {}) .get("mainProgram") @@ -158,7 +158,7 @@ class SecretStore(StoreBase): remote_hash = host.run( [ "cat", - f"{self.machine.select('config.clan.vars.password-store.secretLocation')}/.pass_info", + f"{self.machine.select('config.clan.core.vars.password-store.secretLocation')}/.pass_info", ], RunOpts(log=Log.STDERR, check=False), ).stdout.strip() @@ -247,6 +247,8 @@ class SecretStore(StoreBase): pass_dir = Path(_tempdir).resolve() self.populate_dir(pass_dir, phases) upload_dir = Path( - self.machine.select("config.clan.vars.password-store.secretLocation") + self.machine.select( + "config.clan.core.vars.password-store.secretLocation" + ) ) upload(host, pass_dir, upload_dir) From b690515dd7a6b79c901e4815bf0ff075fd959454 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" Date: Mon, 7 Jul 2025 00:10:13 +0000 Subject: [PATCH 062/258] Update data-mesher digest to a2166c1 --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 3d53e729f..896483170 100644 --- a/flake.lock +++ b/flake.lock @@ -16,11 +16,11 @@ ] }, "locked": { - "lastModified": 1751413887, - "narHash": "sha256-+ut7DrSwamExIvaCFdiTYD88NTSYJFG2CEOvCha59vI=", - "rev": "246f0d66547d073af6249e4f7852466197e871ed", + "lastModified": 1751846468, + "narHash": "sha256-h0mpWZIOIAKj4fmLNyI2HDG+c0YOkbYmyJXSj/bQ9s0=", + "rev": "a2166c13b0cb3febdaf36391cd2019aa2ccf4366", "type": "tarball", - "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/246f0d66547d073af6249e4f7852466197e871ed.tar.gz" + "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/a2166c13b0cb3febdaf36391cd2019aa2ccf4366.tar.gz" }, "original": { "type": "tarball", From ddc105979981f024e609ed0a7b79bd3050a0cfb6 Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 7 Jul 2025 02:34:39 +0200 Subject: [PATCH 063/258] vars password-store: fix secret mangling due to string encoding --- .../vars/secret_modules/password_store.py | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py index 071729681..2358d838e 100644 --- a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py @@ -1,6 +1,7 @@ import io import logging import tarfile +import subprocess from collections.abc import Iterable from pathlib import Path from tempfile import TemporaryDirectory @@ -8,7 +9,6 @@ from tempfile import TemporaryDirectory from clan_cli.ssh.upload import upload from clan_cli.vars._types import StoreBase from clan_cli.vars.generate import Generator, Var -from clan_lib.cmd import CmdOut, Log, RunOpts, run from clan_lib.machines.machines import Machine from clan_lib.ssh.remote import Remote @@ -33,13 +33,11 @@ class SecretStore(StoreBase): def store_dir(self) -> Path: """Get the password store directory, cached after first access.""" if self._store_dir is None: - result = self._run_pass( - "git", "rev-parse", "--show-toplevel", options=RunOpts(check=False) - ) + result = self._run_pass("git", "rev-parse", "--show-toplevel", check=False) if result.returncode != 0: msg = "Password store must be a git repository" raise ValueError(msg) - self._store_dir = Path(result.stdout.strip()) + self._store_dir = Path(result.stdout.strip().decode()) return self._store_dir @property @@ -79,9 +77,20 @@ class SecretStore(StoreBase): def entry_dir(self, generator: Generator, name: str) -> Path: return Path(self.entry_prefix) / self.rel_dir(generator, name) - def _run_pass(self, *args: str, options: RunOpts | None = None) -> CmdOut: + def _run_pass( + self, *args: str, input: bytes | None = None, check: bool = True + ) -> subprocess.CompletedProcess[bytes]: cmd = [self._pass_command, *args] - return run(cmd, options) + # We need bytes support here, so we can not use clan cmd. + # If you change this to run( add bytes support to it first! + # otherwise we mangle binary secrets (which is annoying to debug) + return subprocess.run( + cmd, + input=input, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=check, + ) def _set( self, @@ -90,12 +99,12 @@ class SecretStore(StoreBase): value: bytes, ) -> Path | None: pass_call = ["insert", "-m", str(self.entry_dir(generator, var.name))] - self._run_pass(*pass_call, options=RunOpts(input=value, check=True)) + self._run_pass(*pass_call, input=value, check=True) return None # we manage the files outside of the git repo def get(self, generator: Generator, name: str) -> bytes: pass_name = str(self.entry_dir(generator, name)) - return self._run_pass("show", pass_name).stdout.encode() + return self._run_pass("show", pass_name).stdout def exists(self, generator: Generator, name: str) -> bool: pass_name = str(self.entry_dir(generator, name)) @@ -106,33 +115,22 @@ class SecretStore(StoreBase): def delete(self, generator: Generator, name: str) -> Iterable[Path]: pass_name = str(self.entry_dir(generator, name)) - self._run_pass("rm", "--force", pass_name, options=RunOpts(check=True)) + self._run_pass("rm", "--force", pass_name, check=True) return [] def delete_store(self) -> Iterable[Path]: machine_dir = Path(self.entry_prefix) / "per-machine" / self.machine.name # Check if the directory exists in the password store before trying to delete - result = self._run_pass("ls", str(machine_dir), options=RunOpts(check=False)) + result = self._run_pass("ls", str(machine_dir), check=False) if result.returncode == 0: - self._run_pass( - "rm", - "--force", - "--recursive", - str(machine_dir), - options=RunOpts(check=True), - ) + self._run_pass("rm", "--force", "--recursive", str(machine_dir), check=True) return [] def generate_hash(self) -> bytes: result = self._run_pass( - "git", - "log", - "-1", - "--format=%H", - self.entry_prefix, - options=RunOpts(check=False), + "git", "log", "-1", "--format=%H", self.entry_prefix, check=False ) - git_hash = result.stdout.strip().encode() + git_hash = result.stdout.strip() if not git_hash: return b"" @@ -155,6 +153,8 @@ class SecretStore(StoreBase): if not local_hash: return True + from clan_lib.cmd import RunOpts, Log + remote_hash = host.run( [ "cat", @@ -166,7 +166,7 @@ class SecretStore(StoreBase): if not remote_hash: return True - return local_hash.decode() != remote_hash + return local_hash != remote_hash.encode() def populate_dir(self, output_dir: Path, phases: list[str]) -> None: from clan_cli.vars.generate import Generator From 08c15b3d9b1522df9485b3aa4faefe29ba00d0c0 Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Mon, 7 Jul 2025 13:30:00 +1000 Subject: [PATCH 064/258] docs: remove colon from headings --- clanModules/importer/README.md | 2 +- clanServices/importer/README.md | 2 +- docs/site/decisions/04-fetching-nix-from-python.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clanModules/importer/README.md b/clanModules/importer/README.md index c75039925..80c55699a 100644 --- a/clanModules/importer/README.md +++ b/clanModules/importer/README.md @@ -7,7 +7,7 @@ The importer module allows users to configure importing modules in a flexible an It exposes the `extraModules` functionality of the inventory, without any added configuration. -## Usage: +## Usage ```nix inventory.services = { diff --git a/clanServices/importer/README.md b/clanServices/importer/README.md index 6ef1618d6..32538f532 100644 --- a/clanServices/importer/README.md +++ b/clanServices/importer/README.md @@ -1,7 +1,7 @@ The importer module allows users to configure importing modules in a flexible and structured way. It exposes the `extraModules` functionality of the inventory, without any added configuration. -## Usage: +## Usage ```nix inventory.instances = { diff --git a/docs/site/decisions/04-fetching-nix-from-python.md b/docs/site/decisions/04-fetching-nix-from-python.md index 5a5a39ef3..124bcaca1 100644 --- a/docs/site/decisions/04-fetching-nix-from-python.md +++ b/docs/site/decisions/04-fetching-nix-from-python.md @@ -28,7 +28,7 @@ Benefits: * Caching mechanism is very simple. -### Method 2: Direct access: +### Method 2: Direct access Directly calling the evaluator / build sandbox via `nix build` and `nix eval`within the Python code From 700f5715984279d258ea8b2eddefa5685a327089 Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Mon, 7 Jul 2025 13:30:00 +1000 Subject: [PATCH 065/258] docs: fix highlighting in code block --- clanServices/users/README.md | 49 ++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/clanServices/users/README.md b/clanServices/users/README.md index 2262fb18c..ad7a586f6 100644 --- a/clanServices/users/README.md +++ b/clanServices/users/README.md @@ -1,30 +1,31 @@ ## Usage -``` -inventory.instances = { - - # Deploy user alice on all machines. Don't prompt for password (will be - # auto-generated). - - user-alice = { - module = { - name = "users"; - input = "clan"; +```nix +{ + inventory.instances = { + # Deploy user alice on all machines. Don't prompt for password (will be + # auto-generated). + user-alice = { + module = { + name = "users"; + input = "clan"; + }; + roles.default.tags.all = { }; + roles.default.settings = { + user = "alice"; + prompt = false; + }; }; - roles.default.tags.all = { }; - roles.default.settings = { - user = "alice"; - prompt = false; + + # Deploy user bob only on his laptop. Prompt for a password. + user-bob = { + module = { + name = "users"; + input = "clan"; + }; + roles.default.machines.bobs-laptop = { }; + roles.default.settings.user = "bob"; }; }; - - # Deploy user bob only on his laptop. Prompt for a password. - user-bob = { - module = { - name = "users"; - input = "clan"; - }; - roles.default.machines.bobs-laptop = { }; - roles.default.settings.user = "bob"; - }; +} ``` From a8a08e21e46665738e6a23bc974c12681ead5841 Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Mon, 7 Jul 2025 13:30:00 +1000 Subject: [PATCH 066/258] clanServices/sshd: add README --- clanServices/sshd/README.md | 36 +++++++++++++++++++++++++++++++++++ clanServices/sshd/default.nix | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 clanServices/sshd/README.md diff --git a/clanServices/sshd/README.md b/clanServices/sshd/README.md new file mode 100644 index 000000000..0535da100 --- /dev/null +++ b/clanServices/sshd/README.md @@ -0,0 +1,36 @@ +The `sshd` Clan service manages SSH to make it easy to securely access your machines over the internet. The service uses `vars` to store the SSH host keys for each machine to ensure they remain stable across deployments. + +`sshd` also generates SSH certificates for both servers and clients allowing for certificate-based authentication for SSH. + +The service also disables password-based authentication over SSH, to access your machines you'll need to use public key authentication or certificate-based authentication. + +## Usage + +```nix +{ + inventory.instances = { + # By default this service only generates ed25519 host keys + sshd-basic = { + module = { + name = "sshd"; + input = "clan-core"; + }; + roles.server.tags.all = { }; + roles.client.tags.all = { }; + }; + + # Also generate RSA host keys for all servers + sshd-with-rsa = { + module = { + name = "sshd"; + input = "clan-core"; + }; + roles.server.tags.all = { }; + roles.server.settings = { + hostKeys.rsa.enable = true; + }; + roles.client.tags.all = { }; + }; + }; +} +``` diff --git a/clanServices/sshd/default.nix b/clanServices/sshd/default.nix index 9cb878db2..b4799b917 100644 --- a/clanServices/sshd/default.nix +++ b/clanServices/sshd/default.nix @@ -2,7 +2,7 @@ { _class = "clan.service"; manifest.name = "clan-core/sshd"; - manifest.description = "Enables secure remote access to the machine over ssh."; + manifest.description = "Enables secure remote access to the machine over SSH"; manifest.categories = [ "System" "Network" From e6785fa1d010228398a45003dfe0b6c10fc3b03f Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Mon, 7 Jul 2025 14:07:40 +1000 Subject: [PATCH 067/258] treewide: don't generate SSH keys with builder hostname --- clanModules/borgbackup/roles/client.nix | 2 +- clanModules/sshd/roles/server.nix | 4 ++-- clanModules/sshd/shared.nix | 2 +- clanServices/borgbackup/default.nix | 2 +- clanServices/sshd/default.nix | 8 ++++---- docs/site/guides/disk-encryption.md | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/clanModules/borgbackup/roles/client.nix b/clanModules/borgbackup/roles/client.nix index e2cb8e591..1887fb902 100644 --- a/clanModules/borgbackup/roles/client.nix +++ b/clanModules/borgbackup/roles/client.nix @@ -196,7 +196,7 @@ in pkgs.xkcdpass ]; script = '' - ssh-keygen -t ed25519 -N "" -f "$out"/borgbackup.ssh + ssh-keygen -t ed25519 -N "" -C "" -f "$out"/borgbackup.ssh xkcdpass -n 4 -d - > "$out"/borgbackup.repokey ''; }; diff --git a/clanModules/sshd/roles/server.nix b/clanModules/sshd/roles/server.nix index 8aa4d45f8..bfc6b1bb9 100644 --- a/clanModules/sshd/roles/server.nix +++ b/clanModules/sshd/roles/server.nix @@ -54,7 +54,7 @@ in pkgs.openssh ]; script = '' - ssh-keygen -t ed25519 -N "" -f "$out"/ssh.id_ed25519 + ssh-keygen -t ed25519 -N "" -C "" -f "$out"/ssh.id_ed25519 ''; }; @@ -74,7 +74,7 @@ in pkgs.openssh ]; script = '' - ssh-keygen -t rsa -b 4096 -N "" -f "$out"/ssh.id_rsa + ssh-keygen -t rsa -b 4096 -N "" -C "" -f "$out"/ssh.id_rsa ''; }; diff --git a/clanModules/sshd/shared.nix b/clanModules/sshd/shared.nix index 298b6f9ab..439495f8f 100644 --- a/clanModules/sshd/shared.nix +++ b/clanModules/sshd/shared.nix @@ -36,7 +36,7 @@ pkgs.openssh ]; script = '' - ssh-keygen -t ed25519 -N "" -f "$out"/id_ed25519 + ssh-keygen -t ed25519 -N "" -C "" -f "$out"/id_ed25519 ''; }; diff --git a/clanServices/borgbackup/default.nix b/clanServices/borgbackup/default.nix index b292a0bdb..41bac47e6 100644 --- a/clanServices/borgbackup/default.nix +++ b/clanServices/borgbackup/default.nix @@ -256,7 +256,7 @@ pkgs.xkcdpass ]; script = '' - ssh-keygen -t ed25519 -N "" -f "$out"/borgbackup.ssh + ssh-keygen -t ed25519 -N "" -C "" -f "$out"/borgbackup.ssh xkcdpass -n 4 -d - > "$out"/borgbackup.repokey ''; }; diff --git a/clanServices/sshd/default.nix b/clanServices/sshd/default.nix index b4799b917..799358969 100644 --- a/clanServices/sshd/default.nix +++ b/clanServices/sshd/default.nix @@ -49,7 +49,7 @@ pkgs.openssh ]; script = '' - ssh-keygen -t ed25519 -N "" -f "$out"/id_ed25519 + ssh-keygen -t ed25519 -N "" -C "" -f "$out"/id_ed25519 ''; }; @@ -109,7 +109,7 @@ pkgs.openssh ]; script = '' - ssh-keygen -t ed25519 -N "" -f "$out"/id_ed25519 + ssh-keygen -t ed25519 -N "" -C "" -f "$out"/id_ed25519 ''; }; @@ -151,7 +151,7 @@ pkgs.openssh ]; script = '' - ssh-keygen -t rsa -b 4096 -N "" -f "$out"/ssh.id_rsa + ssh-keygen -t rsa -b 4096 -N "" -C "" -f "$out"/ssh.id_rsa ''; }; @@ -164,7 +164,7 @@ pkgs.openssh ]; script = '' - ssh-keygen -t ed25519 -N "" -f "$out"/ssh.id_ed25519 + ssh-keygen -t ed25519 -N "" -C "" -f "$out"/ssh.id_ed25519 ''; }; }; diff --git a/docs/site/guides/disk-encryption.md b/docs/site/guides/disk-encryption.md index e38c4ae13..190101fa2 100644 --- a/docs/site/guides/disk-encryption.md +++ b/docs/site/guides/disk-encryption.md @@ -122,8 +122,8 @@ CTRL+D 4. Locally generate ssh host keys. You only need to generate ones for the algorithms you're using in `authorizedKeys`. ```bash -ssh-keygen -q -N "" -t ed25519 -f ./initrd_host_ed25519_key -ssh-keygen -q -N "" -t rsa -b 4096 -f ./initrd_host_rsa_key +ssh-keygen -q -N "" -C "" -t ed25519 -f ./initrd_host_ed25519_key +ssh-keygen -q -N "" -C "" -t rsa -b 4096 -f ./initrd_host_rsa_key ``` 5. Securely copy your local initrd ssh host keys to the installer's `/mnt` directory: From 4aa536a1bfe68f73460414be27edd04f8878f7b9 Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Mon, 7 Jul 2025 15:22:47 +1000 Subject: [PATCH 068/258] cli: don't log every public key we find --- pkgs/clan-cli/clan_cli/secrets/sops.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/sops.py b/pkgs/clan-cli/clan_cli/secrets/sops.py index 34f91c870..c87775d61 100644 --- a/pkgs/clan-cli/clan_cli/secrets/sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/sops.py @@ -62,7 +62,7 @@ class KeyType(enum.Enum): try: for public_key in get_public_age_keys(content): - log.info( + log.debug( f"Found age public key from a private key " f"in {key_path}: {public_key}" ) @@ -85,7 +85,7 @@ class KeyType(enum.Enum): try: for public_key in get_public_age_keys(content): - log.info( + log.debug( f"Found age public key from a private key " f"in the environment (SOPS_AGE_KEY): {public_key}" ) @@ -107,7 +107,7 @@ class KeyType(enum.Enum): if pgp_fingerprints := os.environ.get("SOPS_PGP_FP"): for fp in pgp_fingerprints.strip().split(","): msg = f"Found PGP public key in the environment (SOPS_PGP_FP): {fp}" - log.info(msg) + log.debug(msg) keyring.append(fp) return keyring From 885103bfa4f3e1e74251da5bcd4ab3046f8f228b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" Date: Mon, 7 Jul 2025 05:40:16 +0000 Subject: [PATCH 069/258] chore(deps): lock file maintenance --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 896483170..e60bb3566 100644 --- a/flake.lock +++ b/flake.lock @@ -34,11 +34,11 @@ ] }, "locked": { - "lastModified": 1751607816, - "narHash": "sha256-5PtrwjqCIJ4DKQhzYdm8RFePBuwb+yTzjV52wWoGSt4=", + "lastModified": 1751854533, + "narHash": "sha256-U/OQFplExOR1jazZY4KkaQkJqOl59xlh21HP9mI79Vc=", "owner": "nix-community", "repo": "disko", - "rev": "da6109c917b48abc1f76dd5c9bf3901c8c80f662", + "rev": "16b74a1e304197248a1bc663280f2548dbfcae3c", "type": "github" }, "original": { From 3577c689bd563bfec3b14f6347dbd7a0be626f48 Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Mon, 7 Jul 2025 00:48:32 -0700 Subject: [PATCH 070/258] Add missing f to f-string --- pkgs/zerotier-members/zerotier-members.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/zerotier-members/zerotier-members.py b/pkgs/zerotier-members/zerotier-members.py index d69642862..322a6fc96 100755 --- a/pkgs/zerotier-members/zerotier-members.py +++ b/pkgs/zerotier-members/zerotier-members.py @@ -15,7 +15,7 @@ class ClanError(Exception): def compute_zerotier_ip(network_id: str, identity: str) -> ipaddress.IPv6Address: assert len(network_id) == 16, ( - "network_id must be 16 characters long, got {network_id}" + f"network_id must be 16 characters long, got {network_id}" ) nwid = int(network_id, 16) node_id = int(identity, 16) From e0da5752010d54cd20ff68f3c61487544a76c3eb Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Mon, 7 Jul 2025 00:49:45 -0700 Subject: [PATCH 071/258] Fix bug? `member_id` -> `member_ip` (I stumbled across this while reading code, I haven't tested this at all.) --- pkgs/zerotier-members/zerotier-members.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/zerotier-members/zerotier-members.py b/pkgs/zerotier-members/zerotier-members.py index d69642862..a5c22bb9b 100755 --- a/pkgs/zerotier-members/zerotier-members.py +++ b/pkgs/zerotier-members/zerotier-members.py @@ -66,7 +66,7 @@ def get_network_id() -> str: def allow_member(args: argparse.Namespace) -> None: if args.member_ip: - member_id = compute_member_id(args.member_id) + member_id = compute_member_id(args.member_ip) else: member_id = args.member_id network_id = get_network_id() From 0e10122d541d612041f66a6f92597400bde0c12b Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 7 Jul 2025 09:47:27 +0200 Subject: [PATCH 072/258] api/clan: rename 'show_clan_meta' -> 'get_clan_details' --- pkgs/clan-app/ui-2d/src/queries/clan-meta.ts | 2 +- .../ui-2d/src/routes/clans/create.tsx | 2 +- .../ui-2d/src/routes/clans/details.tsx | 2 +- pkgs/clan-app/ui-2d/src/routes/flash/view.tsx | 2 +- pkgs/clan-app/ui/src/queries/clan-meta.ts | 2 +- pkgs/clan-app/ui/src/routes/clans/create.tsx | 2 +- pkgs/clan-app/ui/src/routes/clans/details.tsx | 2 +- pkgs/clan-cli/clan_cli/clan/show.py | 4 +- pkgs/clan-cli/clan_lib/api/directory.py | 2 +- pkgs/clan-cli/clan_lib/clan/get.py | 4 +- templates/clan/flake-parts/clan.nix | 81 ++++++++++++++++++ templates/clan/flake-parts/flake.nix | 85 ++----------------- 12 files changed, 98 insertions(+), 92 deletions(-) create mode 100644 templates/clan/flake-parts/clan.nix diff --git a/pkgs/clan-app/ui-2d/src/queries/clan-meta.ts b/pkgs/clan-app/ui-2d/src/queries/clan-meta.ts index e71790182..1562bd997 100644 --- a/pkgs/clan-app/ui-2d/src/queries/clan-meta.ts +++ b/pkgs/clan-app/ui-2d/src/queries/clan-meta.ts @@ -13,7 +13,7 @@ export const clanMetaQuery = (uri: string | undefined = undefined) => queryFn: async () => { console.log("fetching clan meta", clanURI); - const result = await callApi("show_clan_meta", { + const result = await callApi("get_clan_details", { flake: { identifier: clanURI! }, }).promise; diff --git a/pkgs/clan-app/ui-2d/src/routes/clans/create.tsx b/pkgs/clan-app/ui-2d/src/routes/clans/create.tsx index ea82a0e74..f732499bc 100644 --- a/pkgs/clan-app/ui-2d/src/routes/clans/create.tsx +++ b/pkgs/clan-app/ui-2d/src/routes/clans/create.tsx @@ -203,6 +203,6 @@ export const CreateClan = () => { }; type Meta = Extract< - OperationResponse<"show_clan_meta">, + OperationResponse<"get_clan_details">, { status: "success" } >["data"]; diff --git a/pkgs/clan-app/ui-2d/src/routes/clans/details.tsx b/pkgs/clan-app/ui-2d/src/routes/clans/details.tsx index f087b70a3..7ef28306c 100644 --- a/pkgs/clan-app/ui-2d/src/routes/clans/details.tsx +++ b/pkgs/clan-app/ui-2d/src/routes/clans/details.tsx @@ -128,7 +128,7 @@ const EditClanForm = (props: EditClanFormProps) => { ); }; -type GeneralData = SuccessQuery<"show_clan_meta">["data"]; +type GeneralData = SuccessQuery<"get_clan_details">["data"]; export const ClanDetails = () => { const params = useParams(); diff --git a/pkgs/clan-app/ui-2d/src/routes/flash/view.tsx b/pkgs/clan-app/ui-2d/src/routes/flash/view.tsx index 7bea3a960..e99b5ce93 100644 --- a/pkgs/clan-app/ui-2d/src/routes/flash/view.tsx +++ b/pkgs/clan-app/ui-2d/src/routes/flash/view.tsx @@ -100,7 +100,7 @@ export const Flash = () => { const deviceQuery = createQuery(() => ({ queryKey: ["block_devices"], queryFn: async () => { - const result = await callApi("show_block_devices", {}).promise; + const result = await callApi("list_block_devices", {}).promise; if (result.status === "error") throw new Error("Failed to fetch data"); return result.data; }, diff --git a/pkgs/clan-app/ui/src/queries/clan-meta.ts b/pkgs/clan-app/ui/src/queries/clan-meta.ts index e71790182..1562bd997 100644 --- a/pkgs/clan-app/ui/src/queries/clan-meta.ts +++ b/pkgs/clan-app/ui/src/queries/clan-meta.ts @@ -13,7 +13,7 @@ export const clanMetaQuery = (uri: string | undefined = undefined) => queryFn: async () => { console.log("fetching clan meta", clanURI); - const result = await callApi("show_clan_meta", { + const result = await callApi("get_clan_details", { flake: { identifier: clanURI! }, }).promise; diff --git a/pkgs/clan-app/ui/src/routes/clans/create.tsx b/pkgs/clan-app/ui/src/routes/clans/create.tsx index 92cac1f65..1071f7084 100644 --- a/pkgs/clan-app/ui/src/routes/clans/create.tsx +++ b/pkgs/clan-app/ui/src/routes/clans/create.tsx @@ -202,6 +202,6 @@ export const CreateClan = () => { }; type Meta = Extract< - OperationResponse<"show_clan_meta">, + OperationResponse<"get_clan_details">, { status: "success" } >["data"]; diff --git a/pkgs/clan-app/ui/src/routes/clans/details.tsx b/pkgs/clan-app/ui/src/routes/clans/details.tsx index f087b70a3..7ef28306c 100644 --- a/pkgs/clan-app/ui/src/routes/clans/details.tsx +++ b/pkgs/clan-app/ui/src/routes/clans/details.tsx @@ -128,7 +128,7 @@ const EditClanForm = (props: EditClanFormProps) => { ); }; -type GeneralData = SuccessQuery<"show_clan_meta">["data"]; +type GeneralData = SuccessQuery<"get_clan_details">["data"]; export const ClanDetails = () => { const params = useParams(); diff --git a/pkgs/clan-cli/clan_cli/clan/show.py b/pkgs/clan-cli/clan_cli/clan/show.py index 0af6b599e..1b7e0beac 100644 --- a/pkgs/clan-cli/clan_cli/clan/show.py +++ b/pkgs/clan-cli/clan_cli/clan/show.py @@ -1,14 +1,14 @@ import argparse import logging -from clan_lib.clan.get import show_clan_meta +from clan_lib.clan.get import get_clan_details log = logging.getLogger(__name__) def show_command(args: argparse.Namespace) -> None: flake_path = args.flake.path - meta = show_clan_meta(flake_path) + meta = get_clan_details(flake_path) print(f"Name: {meta.get('name')}") print(f"Description: {meta.get('description', '-')}") diff --git a/pkgs/clan-cli/clan_lib/api/directory.py b/pkgs/clan-cli/clan_lib/api/directory.py index f87c46b42..af533eaeb 100644 --- a/pkgs/clan-cli/clan_lib/api/directory.py +++ b/pkgs/clan-cli/clan_lib/api/directory.py @@ -122,7 +122,7 @@ def blk_from_dict(data: dict) -> BlkInfo: @API.register -def show_block_devices() -> Blockdevices: +def list_block_devices() -> Blockdevices: """ Api method to show local block devices. diff --git a/pkgs/clan-cli/clan_lib/clan/get.py b/pkgs/clan-cli/clan_lib/clan/get.py index 3b8c58ef8..5fb43f31d 100644 --- a/pkgs/clan-cli/clan_lib/clan/get.py +++ b/pkgs/clan-cli/clan_lib/clan/get.py @@ -1,12 +1,12 @@ from clan_lib.api import API from clan_lib.errors import ClanError from clan_lib.flake import Flake -from clan_lib.nix_models.clan import InventoryMeta as Meta +from clan_lib.nix_models.clan import InventoryMeta from clan_lib.persist.inventory_store import InventoryStore @API.register -def show_clan_meta(flake: Flake) -> Meta: +def get_clan_details(flake: Flake) -> InventoryMeta: if flake.is_local and not flake.path.exists(): msg = f"Path {flake} does not exist" raise ClanError(msg, description="clan directory does not exist") diff --git a/templates/clan/flake-parts/clan.nix b/templates/clan/flake-parts/clan.nix new file mode 100644 index 000000000..a29e89894 --- /dev/null +++ b/templates/clan/flake-parts/clan.nix @@ -0,0 +1,81 @@ +{ self }: +{ + meta.name = "__CHANGE_ME__"; # Ensure this is unique among all clans you want to use. + + inherit self; + machines = { + # "jon" will be the hostname of the machine + jon = + { pkgs, ... }: + { + imports = [ + ./modules/shared.nix + ./modules/disko.nix + ./machines/jon/configuration.nix + ]; + + nixpkgs.hostPlatform = "x86_64-linux"; + + # Set this for clan commands use ssh i.e. `clan machines update` + # If you change the hostname, you need to update this line to root@ + # This only works however if you have avahi running on your admin machine else use IP + clan.core.networking.targetHost = pkgs.lib.mkDefault "root@jon"; + + # You can get your disk id by running the following command on the installer: + # Replace with the IP of the installer printed on the screen or by running the `ip addr` command. + # ssh root@ lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT + disko.devices.disk.main = { + device = "/dev/disk/by-id/__CHANGE_ME__"; + }; + + # IMPORTANT! Add your SSH key here + # e.g. > cat ~/.ssh/id_ed25519.pub + users.users.root.openssh.authorizedKeys.keys = throw '' + Don't forget to add your SSH key here! + users.users.root.openssh.authorizedKeys.keys = [ "" ] + ''; + + # Zerotier needs one controller to accept new nodes. Once accepted + # the controller can be offline and routing still works. + clan.core.networking.zerotier.controller.enable = true; + }; + # "sara" will be the hostname of the machine + sara = + { pkgs, ... }: + { + imports = [ + ./modules/shared.nix + ./modules/disko.nix + ./machines/sara/configuration.nix + ]; + + nixpkgs.hostPlatform = "x86_64-linux"; + + # Set this for clan commands use ssh i.e. `clan machines update` + # If you change the hostname, you need to update this line to root@ + # This only works however if you have avahi running on your admin machine else use IP + clan.core.networking.targetHost = pkgs.lib.mkDefault "root@sara"; + + # You can get your disk id by running the following command on the installer: + # Replace with the IP of the installer printed on the screen or by running the `ip addr` command. + # ssh root@ lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT + disko.devices.disk.main = { + device = "/dev/disk/by-id/__CHANGE_ME__"; + }; + + # IMPORTANT! Add your SSH key here + # e.g. > cat ~/.ssh/id_ed25519.pub + users.users.root.openssh.authorizedKeys.keys = throw '' + Don't forget to add your SSH key here! + users.users.root.openssh.authorizedKeys.keys = [ "" ] + ''; + + /* + After jon is deployed, uncomment the following line + This will allow sara to share the VPN overlay network with jon + The networkId is generated by the first deployment of jon + */ + # clan.core.networking.zerotier.networkId = builtins.readFile ../../vars/per-machine/jon/zerotier/zerotier-network-id/value; + }; + }; +} diff --git a/templates/clan/flake-parts/flake.nix b/templates/clan/flake-parts/flake.nix index 2d61576bc..340c9b2a2 100644 --- a/templates/clan/flake-parts/flake.nix +++ b/templates/clan/flake-parts/flake.nix @@ -17,88 +17,13 @@ "x86_64-darwin" "aarch64-darwin" ]; - imports = [ inputs.clan-core.flakeModules.default ]; + imports = [ + inputs.clan-core.flakeModules.default + ]; + # https://docs.clan.lol/guides/getting-started/flake-parts/ - clan = { - meta.name = "__CHANGE_ME__"; # Ensure this is unique among all clans you want to use. + clan = import ./clan.nix { inherit self; }; - inherit self; - machines = { - # "jon" will be the hostname of the machine - jon = - { pkgs, ... }: - { - imports = [ - ./modules/shared.nix - ./modules/disko.nix - ./machines/jon/configuration.nix - ]; - - nixpkgs.hostPlatform = "x86_64-linux"; - - # Set this for clan commands use ssh i.e. `clan machines update` - # If you change the hostname, you need to update this line to root@ - # This only works however if you have avahi running on your admin machine else use IP - clan.core.networking.targetHost = pkgs.lib.mkDefault "root@jon"; - - # You can get your disk id by running the following command on the installer: - # Replace with the IP of the installer printed on the screen or by running the `ip addr` command. - # ssh root@ lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT - disko.devices.disk.main = { - device = "/dev/disk/by-id/__CHANGE_ME__"; - }; - - # IMPORTANT! Add your SSH key here - # e.g. > cat ~/.ssh/id_ed25519.pub - users.users.root.openssh.authorizedKeys.keys = throw '' - Don't forget to add your SSH key here! - users.users.root.openssh.authorizedKeys.keys = [ "" ] - ''; - - # Zerotier needs one controller to accept new nodes. Once accepted - # the controller can be offline and routing still works. - clan.core.networking.zerotier.controller.enable = true; - }; - # "sara" will be the hostname of the machine - sara = - { pkgs, ... }: - { - imports = [ - ./modules/shared.nix - ./modules/disko.nix - ./machines/sara/configuration.nix - ]; - - nixpkgs.hostPlatform = "x86_64-linux"; - - # Set this for clan commands use ssh i.e. `clan machines update` - # If you change the hostname, you need to update this line to root@ - # This only works however if you have avahi running on your admin machine else use IP - clan.core.networking.targetHost = pkgs.lib.mkDefault "root@sara"; - - # You can get your disk id by running the following command on the installer: - # Replace with the IP of the installer printed on the screen or by running the `ip addr` command. - # ssh root@ lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT - disko.devices.disk.main = { - device = "/dev/disk/by-id/__CHANGE_ME__"; - }; - - # IMPORTANT! Add your SSH key here - # e.g. > cat ~/.ssh/id_ed25519.pub - users.users.root.openssh.authorizedKeys.keys = throw '' - Don't forget to add your SSH key here! - users.users.root.openssh.authorizedKeys.keys = [ "" ] - ''; - - /* - After jon is deployed, uncomment the following line - This will allow sara to share the VPN overlay network with jon - The networkId is generated by the first deployment of jon - */ - # clan.core.networking.zerotier.networkId = builtins.readFile ../../vars/per-machine/jon/zerotier/zerotier-network-id/value; - }; - }; - }; perSystem = { pkgs, inputs', ... }: { From 84703fa29375f015eaa40c7792547f76024f12a0 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 7 Jul 2025 10:46:26 +0200 Subject: [PATCH 073/258] docs: improve docstring for 'list_block_devices' --- pkgs/clan-cli/clan_lib/api/directory.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_lib/api/directory.py b/pkgs/clan-cli/clan_lib/api/directory.py index af533eaeb..a0cbcb0e9 100644 --- a/pkgs/clan-cli/clan_lib/api/directory.py +++ b/pkgs/clan-cli/clan_lib/api/directory.py @@ -124,9 +124,10 @@ def blk_from_dict(data: dict) -> BlkInfo: @API.register def list_block_devices() -> Blockdevices: """ - Api method to show local block devices. + List local block devices by running `lsblk`. - It must return a list of block devices. + Returns: + A list of detected block devices with metadata like size, path, type, etc. """ cmd = nix_shell( From 1cb2156d8740b31774cdf37f6998d068765826e8 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 7 Jul 2025 10:48:14 +0200 Subject: [PATCH 074/258] api: rename to get_flash_options --- pkgs/clan-cli/clan_cli/flash/list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_cli/flash/list.py b/pkgs/clan-cli/clan_cli/flash/list.py index c60a6ee7d..146f0bcb4 100644 --- a/pkgs/clan-cli/clan_cli/flash/list.py +++ b/pkgs/clan-cli/clan_cli/flash/list.py @@ -18,7 +18,7 @@ class FlashOptions(TypedDict): @API.register -def show_flash_options() -> FlashOptions: +def get_flash_options() -> FlashOptions: return {"languages": list_languages(), "keymaps": list_keymaps()} From e1b4f296e385a7d6c92ceccffdc687285168dfde Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 7 Jul 2025 10:49:46 +0200 Subject: [PATCH 075/258] api: rename 'show_mdns' -> 'list_mdns_services' --- pkgs/clan-app/ui-2d/src/routes/hosts/view.tsx | 4 ++-- pkgs/clan-app/ui/src/routes/hosts/view.tsx | 4 ++-- pkgs/clan-cli/clan_lib/api/mdns_discovery.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkgs/clan-app/ui-2d/src/routes/hosts/view.tsx b/pkgs/clan-app/ui-2d/src/routes/hosts/view.tsx index d649ab71d..737e7e6fb 100644 --- a/pkgs/clan-app/ui-2d/src/routes/hosts/view.tsx +++ b/pkgs/clan-app/ui-2d/src/routes/hosts/view.tsx @@ -4,7 +4,7 @@ import { Button } from "../../components/Button/Button"; import Icon from "@/src/components/icon"; type ServiceModel = Extract< - OperationResponse<"show_mdns">, + OperationResponse<"list_mdns_services">, { status: "success" } >["data"]["services"]; @@ -16,7 +16,7 @@ export const HostList: Component = () => {
diff --git a/pkgs/clan-app/ui/src/routes/hosts/view.tsx b/pkgs/clan-app/ui/src/routes/hosts/view.tsx index d649ab71d..737e7e6fb 100644 --- a/pkgs/clan-app/ui/src/routes/hosts/view.tsx +++ b/pkgs/clan-app/ui/src/routes/hosts/view.tsx @@ -4,7 +4,7 @@ import { Button } from "../../components/Button/Button"; import Icon from "@/src/components/icon"; type ServiceModel = Extract< - OperationResponse<"show_mdns">, + OperationResponse<"list_mdns_services">, { status: "success" } >["data"]["services"]; @@ -16,7 +16,7 @@ export const HostList: Component = () => {
diff --git a/pkgs/clan-cli/clan_lib/api/mdns_discovery.py b/pkgs/clan-cli/clan_lib/api/mdns_discovery.py index 8bcdd7e11..93c1bc2c4 100644 --- a/pkgs/clan-cli/clan_lib/api/mdns_discovery.py +++ b/pkgs/clan-cli/clan_lib/api/mdns_discovery.py @@ -88,7 +88,7 @@ def parse_avahi_output(output: str) -> DNSInfo: @API.register -def show_mdns() -> DNSInfo: +def list_mdns_services() -> DNSInfo: cmd = nix_shell( ["avahi"], [ @@ -107,7 +107,7 @@ def show_mdns() -> DNSInfo: def mdns_command(args: argparse.Namespace) -> None: - dns_info = show_mdns() + dns_info = list_mdns_services() for name, info in dns_info.services.items(): print(f"Hostname: {name} - ip: {info.ip}") From 972adc7a7c9fc4e2d1ee9c7e20ce594ff9d5e9d9 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 7 Jul 2025 10:53:32 +0200 Subject: [PATCH 076/258] api: chore rename outdated reference --- .../ui-2d/src/routes/machines/install/hardware-step.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-app/ui-2d/src/routes/machines/install/hardware-step.tsx b/pkgs/clan-app/ui-2d/src/routes/machines/install/hardware-step.tsx index 291937e21..2696a12bc 100644 --- a/pkgs/clan-app/ui-2d/src/routes/machines/install/hardware-step.tsx +++ b/pkgs/clan-app/ui-2d/src/routes/machines/install/hardware-step.tsx @@ -71,7 +71,7 @@ export const HWStep = (props: StepProps) => { const hwReportQuery = useQuery(() => ({ queryKey: [props.dir, props.machine_id, "hw_report"], queryFn: async () => { - const result = await callApi("show_machine_hardware_config", { + const result = await callApi("describe_machine_hardware", { machine: { flake: { identifier: props.dir, From e1796e19e42c6c437148c6e1d9a6e4350b7b2b32 Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Mon, 7 Jul 2025 10:51:43 +0100 Subject: [PATCH 077/258] feat(ui): refine Fieldset API --- .../components/v2/Form/Fieldset.stories.tsx | 4 +- .../ui/src/components/v2/Form/Fieldset.tsx | 62 ++++++++++++------- 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/pkgs/clan-app/ui/src/components/v2/Form/Fieldset.stories.tsx b/pkgs/clan-app/ui/src/components/v2/Form/Fieldset.stories.tsx index 7320afe05..c157c041f 100644 --- a/pkgs/clan-app/ui/src/components/v2/Form/Fieldset.stories.tsx +++ b/pkgs/clan-app/ui/src/components/v2/Form/Fieldset.stories.tsx @@ -40,7 +40,7 @@ export type Story = StoryObj; export const Default: Story = { args: { legend: "Signup", - fields: (props: FieldProps) => ( + children: (props: FieldProps) => ( <> ( + children: (props: FieldProps) => ( <> & { error?: string; - fields: (props: FieldProps) => JSX.Element; + disabled?: boolean; +}; + +export interface FieldsetProps + extends Pick { + legend?: string; + disabled?: boolean; + error?: string; + children: (props: FieldsetFieldProps) => JSX.Element; } export const Fieldset = (props: FieldsetProps) => { const orientation = () => props.orientation || "vertical"; + const [fieldProps] = splitProps(props, [ + "orientation", + "inverted", + "disabled", + "error", + ]); + return (
- - - {props.legend} - - -
- {props.fields({ ...props, orientation: orientation() })} -
+ {props.legend && ( + + + {props.legend} + + + )} +
{props.children(fieldProps)}
{props.error && (