clan-cli: Increase test coverage for clan flash list

checks/flash/flake-module.nix

@@ -29,9 +29,20 @@
         imports = [ self.nixosModules.test-install-machine-without-system ];
 
         clan.core.vars.generators.test = lib.mkForce { };
-
         disko.devices.disk.main.preCreateHook = lib.mkForce "";
+
+        # Every option here should match the options set through `clan flash write`
+        # if you get a mass rebuild on the disko derivation, this means you need to
+        # adjust something here. Also make sure that the injected json in clan flash write
+        # is up to date.
+        i18n.defaultLocale = "de_DE.UTF-8";
+        console.keyMap = "de";
+        services.xserver.xkb.layout = "de";
+        users.users.root.openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIRWUusawhlIorx7VFeQJHmMkhl9X3QpnvOdhnV/bQNG root@target\n"
+        ];
       };
+
   };
 
   perSystem =
@@ -44,6 +55,8 @@
       dependencies = [
         pkgs.disko
         pkgs.buildPackages.xorg.lndir
+        pkgs.glibcLocales
+        pkgs.kbd.out
         self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".pkgs.perlPackages.ConfigIniFiles
         self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".pkgs.perlPackages.FileSlurp
 
@@ -83,10 +96,10 @@
           };
           testScript = ''
             start_all()
-
+            machine.succeed("echo 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIRWUusawhlIorx7VFeQJHmMkhl9X3QpnvOdhnV/bQNG root@target' > ./test_id_ed25519.pub")
             # Some distros like to automount disks with spaces
             machine.succeed('mkdir -p "/mnt/with spaces" && mkfs.ext4 /dev/vdc && mount /dev/vdc "/mnt/with spaces"')
-            machine.succeed("clan flash write --debug --flake ${self.checks.x86_64-linux.clan-core-for-checks} --yes --disk main /dev/vdc test-flash-machine-${pkgs.hostPlatform.system}")
+            machine.succeed("clan flash write --ssh-pubkey ./test_id_ed25519.pub --keymap de --language de_DE.UTF-8 --debug --flake ${self.checks.x86_64-linux.clan-core-for-checks} --yes --disk main /dev/vdc test-flash-machine-${pkgs.hostPlatform.system}")
           '';
         } { inherit pkgs self; };
       };

pkgs/clan-cli/clan_lib/flash/flash.py

@@ -43,6 +43,48 @@ class Disk:
 installer_machine = Machine(name="flash-installer", flake=Flake(str(clan_core_flake())))
 
 
+def build_system_config_nix(system_config: SystemConfig) -> dict[str, Any]:
+    """Translate ``SystemConfig`` to the structure expected by disko-install."""
+    system_config_nix: dict[str, Any] = {}
+
+    if system_config.language:
+        languages = list_languages()
+        if system_config.language not in languages:
+            msg = (
+                f"Language '{system_config.language}' is not a valid language. "
+                "Run 'clan flash list languages' to see a list of possible languages."
+            )
+            raise ClanError(msg)
+        system_config_nix["i18n"] = {"defaultLocale": system_config.language}
+
+    if system_config.keymap:
+        keymaps = list_keymaps()
+        if system_config.keymap not in keymaps:
+            msg = (
+                f"Keymap '{system_config.keymap}' is not a valid keymap. "
+                "Run 'clan flash list keymaps' to see a list of possible keymaps."
+            )
+            raise ClanError(msg)
+        system_config_nix["console"] = {"keyMap": system_config.keymap}
+        system_config_nix["services"] = {
+            "xserver": {"xkb": {"layout": system_config.keymap}},
+        }
+
+    if system_config.ssh_keys_path:
+        root_keys = []
+        for key_path in (Path(x) for x in system_config.ssh_keys_path):
+            try:
+                root_keys.append(key_path.read_text())
+            except OSError as e:
+                msg = f"Cannot read SSH public key file: {key_path}: {e}"
+                raise ClanError(msg) from e
+        system_config_nix["users"] = {
+            "users": {"root": {"openssh": {"authorizedKeys": {"keys": root_keys}}}},
+        }
+
+    return system_config_nix
+
+
 # TODO: unify this with machine install
 @API.register
 def run_machine_flash(
@@ -79,43 +121,11 @@ def run_machine_flash(
     with pause_automounting(devices, machine, request_graphical=graphical):
         if extra_args is None:
             extra_args = []
-        system_config_nix: dict[str, Any] = {}
 
         generate_facts([machine])
         run_generators([machine], generators=None, full_closure=False)
 
-        if system_config.language:
+        system_config_nix = build_system_config_nix(system_config)
-            if system_config.language not in list_languages():
-                msg = (
-                    f"Language '{system_config.language}' is not a valid language. "
-                    f"Run 'clan flash list languages' to see a list of possible languages."
-                )
-                raise ClanError(msg)
-            system_config_nix["i18n"] = {"defaultLocale": system_config.language}
-
-        if system_config.keymap:
-            if system_config.keymap not in list_keymaps():
-                msg = (
-                    f"Keymap '{system_config.keymap}' is not a valid keymap. "
-                    f"Run 'clan flash list keymaps' to see a list of possible keymaps."
-                )
-                raise ClanError(msg)
-            system_config_nix["console"] = {"keyMap": system_config.keymap}
-            system_config_nix["services"] = {
-                "xserver": {"xkb": {"layout": system_config.keymap}},
-            }
-
-        if system_config.ssh_keys_path:
-            root_keys = []
-            for key_path in (Path(x) for x in system_config.ssh_keys_path):
-                try:
-                    root_keys.append(key_path.read_text())
-                except OSError as e:
-                    msg = f"Cannot read SSH public key file: {key_path}: {e}"
-                    raise ClanError(msg) from e
-            system_config_nix["users"] = {
-                "users": {"root": {"openssh": {"authorizedKeys": {"keys": root_keys}}}},
-            }
 
         for generator in Generator.get_machine_generators(
             [machine.name], machine.flake

pkgs/clan-cli/clan_lib/flash/list.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import re
 from pathlib import Path
 from typing import TypedDict
 
@@ -43,17 +44,26 @@ def list_languages() -> list[str]:
     with locale_file.open() as f:
         lines = f.readlines()
 
-    languages = []
+    langs: set[str] = set()
+    base = r"[A-Za-z0-9]*_[A-Za-z0-9]*.UTF-8"
+    pattern = re.compile(base)
     for line in lines:
-        if line.startswith("#"):
+        s = line.strip()
-            continue
-        if "SUPPORTED-LOCALES" in line:
-            continue
-        # Split by '/' and take the first part
-        language = line.split("/")[0].strip()
-        languages.append(language)
 
-    return languages
+        if not s:
+            continue
+        if s.startswith("#"):
+            continue
+        if "SUPPORTED-LOCALES" in s:
+            continue
+
+        tok = s.removesuffix("\\").strip()
+        tok = tok.split("/")[0]
+
+        if pattern.match(tok):
+            langs.add(tok)
+
+    return sorted(langs)
 
 
 def list_keymaps() -> list[str]:

pkgs/clan-cli/clan_lib/flash/test_list.py

@@ -0,0 +1,92 @@
+import logging
+import os
+from pathlib import Path
+
+import pytest
+from clan_cli.tests.fixtures_flakes import ClanFlake
+
+from clan_lib.errors import ClanCmdError, ClanError
+from clan_lib.flake import ClanSelectError, Flake
+from clan_lib.flash.flash import SystemConfig, build_system_config_nix
+from clan_lib.flash.list import list_keymaps, list_languages
+from clan_lib.machines.machines import Machine
+from clan_lib.nix import nix_config
+
+log = logging.getLogger(__name__)
+
+
+@pytest.mark.with_core
+def test_language_list() -> None:
+    languages = list_languages()
+    assert isinstance(languages, list)
+    assert "en_US.UTF-8" in languages  # Common locale
+    assert "fr_FR.UTF-8" in languages  # Common locale
+    assert "de_DE.UTF-8" in languages  # Common locale
+
+
+@pytest.mark.with_core
+def test_flash_config(flake: ClanFlake, test_root: Path) -> None:
+    languages = list_languages()
+    keymaps = list_keymaps()
+
+    host_key = test_root / "data" / "ssh_host_ed25519_key"
+
+    test_langs = list(
+        filter(
+            lambda x: "zh_CN" in x,
+            languages,
+        )
+    )
+
+    for test_lang in test_langs:
+        log.info(f"Testing language: {test_lang}")
+        sys_config = SystemConfig(
+            language=test_lang,
+            keymap=keymaps[3],
+            ssh_keys_path=[str(host_key)],
+        )
+
+        result = build_system_config_nix(sys_config)
+
+        config = flake.machines["my_machine"]
+        config["nixpkgs"]["hostPlatform"] = nix_config()["system"]
+        config["boot"]["loader"]["grub"]["devices"] = ["/dev/vda"]
+        config["fileSystems"]["/"]["device"] = "/dev/vda"
+        config.update(result)
+        flake.refresh()
+
+        # In the sandbox, building fails due to network restrictions (can't download dependencies)
+        # Outside the sandbox, the build should succeed
+        in_sandbox = os.environ.get("IN_NIX_SANDBOX") == "1"
+
+        machine = Machine(name="my_machine", flake=Flake(str(flake.path)))
+        if in_sandbox:
+            # In sandbox: expect build to fail due to network restrictions
+            with pytest.raises(ClanSelectError) as select_error:
+                Path(machine.select("config.system.build.toplevel"))
+            # The error should be a select_error without a failed_attr
+            cmd_error = select_error.value.__cause__
+            assert cmd_error is not None
+            assert isinstance(cmd_error, ClanCmdError)
+            assert "nixos-system-my_machine" in str(cmd_error.cmd.stderr)
+        else:
+            try:
+                # Outside sandbox: build should succeed
+                toplevel_path = Path(machine.select("config.system.build.toplevel"))
+                assert toplevel_path.exists()
+            except ClanSelectError as e:
+                if "Error: unsupported locales detected" in str(e.__cause__):
+                    msg = f"Locale '{sys_config.language}' is not supported on this system."
+                    raise ClanError(msg) from e
+                raise
+            # Verify it's a NixOS system by checking for expected content
+            assert "nixos-system-my_machine" in str(toplevel_path)
+
+
+@pytest.mark.with_core
+def test_list_keymaps() -> None:
+    keymaps = list_keymaps()
+    assert isinstance(keymaps, list)
+    assert "us" in keymaps  # Common keymap
+    assert "uk" in keymaps  # Common keymap
+    assert "de" in keymaps  # Common keymap

pkgs/clan-cli/clan_lib/nix/allowed-packages.json

@@ -24,6 +24,8 @@
   "qemu",
   "qrencode",
   "rsync",
+  "kbd",
+  "glibcLocales",
   "shellcheck-minimal",
   "sops",
   "sshpass",

pkgs/clan-cli/default.nix

@@ -193,7 +193,7 @@ pythonRuntime.pkgs.buildPythonApplication {
           # limit build cores to 16
           jobs="$((NIX_BUILD_CORES>16 ? 16 : NIX_BUILD_CORES))"
 
-          python -m pytest -m "not impure and not with_core" -n $jobs ./clan_cli ./clan_lib
+          python -m pytest -m "not impure and not with_core" -n "$jobs"  ./clan_cli ./clan_lib
           touch $out
         '';
   }
@@ -227,7 +227,7 @@ pythonRuntime.pkgs.buildPythonApplication {
               ../../nixosModules/clanCore/zerotier/generate.py
 
               # needed by flash list tests
-              pkgs.kbd
+              pkgs.kbd.out
               pkgs.glibcLocales
 
               # Pre-built VMs for impure tests
@@ -272,7 +272,7 @@ pythonRuntime.pkgs.buildPythonApplication {
           jobs="$((NIX_BUILD_CORES>16 ? 16 : NIX_BUILD_CORES))"
 
           # Run all tests with core marker
-          python -m pytest -m "not impure and with_core" -n $jobs ./clan_cli ./clan_lib
+          python -m pytest -m "not impure and with_core" -n "$jobs" ./clan_cli ./clan_lib
           touch $out
         '';
   };