container test: with writeable nix store

checks/backups/flake-module.nix

@@ -27,7 +27,6 @@
         ];
         clan.core.networking.targetHost = "machine";
         networking.hostName = "machine";
-        services.openssh.settings.UseDns = false;
         nixpkgs.hostPlatform = "x86_64-linux";
 
         programs.ssh.knownHosts = {
@@ -37,6 +36,8 @@
 
         services.openssh = {
           enable = true;
+          settings.UsePAM = false;
+          settings.UseDns = false;
           hostKeys = [
             {
               path = "/root/.ssh/id_ed25519";
@@ -47,6 +48,10 @@
 
         users.users.root.openssh.authorizedKeys.keyFiles = [ ../lib/ssh/pubkey ];
 
+        # This is needed to unlock the user for sshd
+        # Because we use sshd without setuid binaries
+        users.users.borg.initialPassword = "hello";
+
         systemd.tmpfiles.settings."vmsecrets" = {
           "/root/.ssh/id_ed25519" = {
             C.argument = "${../lib/ssh/privkey}";
@@ -161,15 +166,19 @@
       # vm-test-run-test-backups> qemu-kvm: No machine specified, and there is no default
       # vm-test-run-test-backups> Use -machine help to list supported machines
       checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux && pkgs.stdenv.hostPlatform.system != "aarch64-linux") {
-        test-backups = (import ../lib/test-base.nix) {
+        test-backups = (import ../lib/container-test.nix) {
           name = "test-backups";
           nodes.machine = {
             imports = [
               self.nixosModules.clanCore
               self.nixosModules.test-backup
             ];
-            virtualisation.emptyDiskImages = [ 256 ];
             clan.core.settings.directory = ./.;
+            environment.systemPackages = [
+              (pkgs.writeShellScriptBin "foo" ''
+                echo ${self}
+              '')
+            ];
           };
 
           testScript = ''

checks/lib/container-driver/module.nix

@@ -7,9 +7,19 @@
 let
   testDriver = hostPkgs.python3.pkgs.callPackage ./package.nix {
     inherit (config) extraPythonPackages;
-    inherit (hostPkgs.pkgs) util-linux systemd;
+    inherit (hostPkgs.pkgs) util-linux systemd nix;
   };
-  containers = map (m: m.system.build.toplevel) (lib.attrValues config.nodes);
+  containers =
+    testScript:
+    map (m: [
+      m.system.build.toplevel
+      (hostPkgs.closureInfo {
+        rootPaths = [
+          m.system.build.toplevel
+          (hostPkgs.writeText "testScript" testScript)
+        ];
+      })
+    ]) (lib.attrValues config.nodes);
   pythonizeName =
     name:
     let
@@ -44,8 +54,6 @@ in
       ''
         mkdir -p $out/bin
 
-        containers=(${toString containers})
-
         ${lib.optionalString (!config.skipTypeCheck) ''
           # prepend type hints so the test script can be type checked with mypy
           cat "${./test-script-prepend.py}" >> testScriptWithTypes
@@ -66,7 +74,13 @@ in
         ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver
 
         wrapProgram $out/bin/nixos-test-driver \
-          ${lib.concatStringsSep " " (map (name: "--add-flags '--container ${name}'") containers)} \
+          ${
+            lib.concatStringsSep " " (
+              map (container: "--add-flags '--container ${builtins.toString container}'") (
+                containers config.testScriptString
+              )
+            )
+          } \
           --add-flags "--test-script '$out/test-script'"
       ''
   );

checks/lib/container-driver/package.nix

@@ -5,6 +5,7 @@
   setuptools,
   util-linux,
   systemd,
+  nix,
   colorama,
   junit-xml,
 }:
@@ -16,6 +17,7 @@ buildPythonApplication {
     systemd
     colorama
     junit-xml
+    nix
   ] ++ extraPythonPackages python3Packages;
   nativeBuildInputs = [ setuptools ];
   format = "pyproject";

checks/lib/container-driver/test_driver/__init__.py

@@ -1,4 +1,5 @@
 import argparse
+import ctypes
 import os
 import re
 import subprocess
@@ -12,6 +13,55 @@ from typing import Any
 
 from .logger import AbstractLogger, CompositeLogger, TerminalLogger
 
+# Load the C library
+libc = ctypes.CDLL("libc.so.6", use_errno=True)
+
+# Define the mount function
+libc.mount.argtypes = [
+    ctypes.c_char_p,  # source
+    ctypes.c_char_p,  # target
+    ctypes.c_char_p,  # filesystemtype
+    ctypes.c_ulong,  # mountflags
+    ctypes.c_void_p,  # data
+]
+libc.mount.restype = ctypes.c_int
+
+MS_BIND = 0x1000
+MS_REC = 0x4000
+
+
+def mount(
+    source: Path,
+    target: Path,
+    filesystemtype: str,
+    mountflags: int = 0,
+    data: str | None = None,
+) -> None:
+    """
+    A Python wrapper for the mount system call.
+
+    :param source: The source of the file system (e.g., device name, remote filesystem).
+    :param target: The mount point (an existing directory).
+    :param filesystemtype: The filesystem type (e.g., "ext4", "nfs").
+    :param mountflags: Mount options flags.
+    :param data: File system-specific data (e.g., options like "rw").
+    :raises OSError: If the mount system call fails.
+    """
+    # Convert Python strings to C-compatible strings
+    source_c = ctypes.c_char_p(str(source).encode("utf-8"))
+    target_c = ctypes.c_char_p(str(target).encode("utf-8"))
+    fstype_c = ctypes.c_char_p(filesystemtype.encode("utf-8"))
+    data_c = ctypes.c_char_p(data.encode("utf-8")) if data else None
+
+    # Call the mount system call
+    result = libc.mount(
+        source_c, target_c, fstype_c, ctypes.c_ulong(mountflags), data_c
+    )
+
+    if result != 0:
+        errno = ctypes.get_errno()
+        raise OSError(errno, os.strerror(errno))
+
 
 class Error(Exception):
     pass
@@ -71,7 +121,7 @@ class Machine:
             self.rootdir,
             "--register=no",
             "--resolv-conf=off",
-            "--bind-ro=/nix/store",
+            "--bind=/nix",
             "--bind",
             self.out_dir,
             "--bind=/proc:/run/host/proc",
@@ -273,7 +323,9 @@ class Machine:
     def succeed(self, command: str, timeout: int | None = None) -> str:
         res = self.execute(command, timeout=timeout)
         if res.returncode != 0:
-            msg = f"Failed to run command {command}"
+            msg = f"Failed to run command {command}\n"
+            msg += f"Exit code: {res.returncode}\n"
+            msg += f"Stdout: {res.stdout}"
             raise RuntimeError(msg)
         return res.stdout
 
@@ -290,6 +342,12 @@ class Machine:
         self.shutdown()
 
 
+NIX_DIR = Path("/nix")
+NIX_STORE = Path("/nix/store/")
+NEW_NIX_DIR = Path("/.nix-rw")
+NEW_NIX_STORE_DIR = NEW_NIX_DIR / "store"
+
+
 def setup_filesystems() -> None:
     # We don't care about cleaning up the mount points, since we're running in a nix sandbox.
     Path("/run").mkdir(parents=True, exist_ok=True)
@@ -298,6 +356,32 @@ def setup_filesystems() -> None:
     Path("/etc").chmod(0o755)
     Path("/etc/os-release").touch()
     Path("/etc/machine-id").write_text("a5ea3f98dedc0278b6f3cc8c37eeaeac")
+    NEW_NIX_STORE_DIR.mkdir(parents=True)
+    # Read /proc/mounts and replicate every bind mount
+    with Path("/proc/self/mounts").open() as f:
+        for line in f:
+            columns = line.split(" ")
+            source = Path(columns[1])
+            if source.parent != NIX_STORE:
+                continue
+            target = NEW_NIX_STORE_DIR / source.name
+            if source.is_dir():
+                target.mkdir()
+            else:
+                target.touch()
+            try:
+                mount(source, target, "none", MS_BIND)
+            except OSError as e:
+                msg = f"mount({source}, {target}) failed"
+                raise Error(msg) from e
+    out = Path(os.environ["out"])
+    (NEW_NIX_STORE_DIR / out.name).mkdir()
+    mount(NEW_NIX_DIR, NIX_DIR, "none", MS_BIND | MS_REC)
+
+
+def load_nix_db(closure_info: Path) -> None:
+    with (closure_info / "registration").open() as f:
+        subprocess.run(["nix-store", "--load-db"], stdin=f, check=True, text=True)
 
 
 class Driver:
@@ -305,7 +389,7 @@ class Driver:
 
     def __init__(
         self,
-        containers: list[Path],
+        containers: list[tuple[Path, Path]],
         logger: AbstractLogger,
         testscript: str,
         out_dir: str,
@@ -315,21 +399,24 @@ class Driver:
         self.out_dir = out_dir
         self.logger = logger
         setup_filesystems()
+        # TODO: this won't work for multiple containers
+        assert len(containers) == 1, "Only one container is supported at the moment"
+        load_nix_db(containers[0][1])
 
         self.tempdir = TemporaryDirectory()
         tempdir_path = Path(self.tempdir.name)
 
         self.machines = []
         for container in containers:
-            name_match = re.match(r".*-nixos-system-(.+)-(.+)", container.name)
+            name_match = re.match(r".*-nixos-system-(.+)-(.+)", container[0].name)
             if not name_match:
-                msg = f"Unable to extract hostname from {container.name}"
+                msg = f"Unable to extract hostname from {container[0].name}"
                 raise Error(msg)
             name = name_match.group(1)
             self.machines.append(
                 Machine(
                     name=name,
-                    toplevel=container,
+                    toplevel=container[0],
                     rootdir=tempdir_path / name,
                     out_dir=self.out_dir,
                     logger=self.logger,
@@ -401,9 +488,11 @@ def main() -> None:
     arg_parser = argparse.ArgumentParser(prog="nixos-test-driver")
     arg_parser.add_argument(
         "--containers",
-        nargs="+",
+        nargs=2,
+        action="append",
         type=Path,
-        help="container system toplevel paths",
+        metavar=("TOPLEVEL_STORE_DIR", "CLOSURE_INFO"),
+        help="container system toplevel store dir and closure info",
     )
     arg_parser.add_argument(
         "--test-script",

checks/lib/container-test.nix

@@ -25,6 +25,9 @@ in
       networking.interfaces = lib.mkForce { };
       #networking.primaryIPAddress = lib.mkForce null;
       systemd.services.backdoor.enable = false;
+
+      # we don't have permission to set cpu scheduler in our container
+      systemd.services.nix-daemon.serviceConfig.CPUSchedulingPolicy = lib.mkForce "";
     };
     # to accept external dependencies such as disko
     node.specialArgs.self = self;