use recursive-nix for integration tests

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/3097

.envrc

@@ -1,11 +1,13 @@
+# shellcheck shell=bash
 if ! has nix_direnv_version || ! nix_direnv_version 3.0.4; then
   source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.4/direnvrc" "sha256-DzlYZ33mWF/Gs8DDeyjr8mnVmQGx7ASYqA5WlxwvBG4="
 fi
 
 watch_file .direnv/selected-shell 
+watch_file formatter.nix
 
 if [ -e .direnv/selected-shell ]; then
-  use flake .#$(cat .direnv/selected-shell)
+  use flake ".#$(cat .direnv/selected-shell)"
 else
   use flake
 fi

.gitea/workflows/checks.yaml

@@ -8,5 +8,5 @@ jobs:
   checks-impure:
     runs-on: nix
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - run: nix run .#impure-checks

.gitea/workflows/deploy.yaml

@@ -1,13 +1,13 @@
 name: deploy
 on:
- push:
+  push:
     branches:
       - main
 jobs:
   deploy-docs:
     runs-on: nix
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - run: nix run .#deploy-docs
     env:
-      SSH_HOMEPAGE_KEY: ${{ secrets.SSH_HOMEPAGE_KEY }}
+      SSH_HOMEPAGE_KEY: ${{ secrets.SSH_HOMEPAGE_KEY }}

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/repo-sync.yml

@@ -0,0 +1,28 @@
+name: Github<->Gitea sync
+on:
+  schedule:
+    - cron: "39 * * * *"
+  workflow_dispatch:
+permissions:
+  contents: write
+jobs:
+  repo-sync:
+    if: github.repository_owner == 'clan-lol'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.CI_APP_ID }}
+          private-key: ${{ secrets.CI_PRIVATE_KEY }}
+      - name: repo-sync
+        uses: repo-sync/github-sync@v2
+        env:
+          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
+        with:
+          source_repo: "https://git.clan.lol/clan/clan-core.git"
+          source_branch: "main"
+          destination_branch: "main"

.gitignore

@@ -1,17 +1,20 @@
 .direnv
+**/.nixos-test-history
 ***/.hypothesis
 out.log
 .coverage.*
 **/qubeclan
+pkgs/repro-hook
 **/testdir
 democlan
 example_clan
-result*
+**/result
 /pkgs/clan-cli/clan_cli/nixpkgs
 /pkgs/clan-cli/clan_cli/webui/assets
 nixos.qcow2
 **/*.glade~
 /docs/out
+**/.local.env
 
 # dream2nix
 .dream2nix
@@ -35,4 +38,4 @@ repo
 # node
 node_modules
 dist
-.webui
+.webui

CONTRIBUTING.md

@@ -1,21 +1,4 @@
 # Contributing to Clan
 
-## Live-reloading documentation
-
-Enter the `docs` directory:
-
-```shell-session
-cd docs
-```
-
-Enter the development shell or enable `direnv`:
-
-```shell-session
-direnv allow
-```
-
-Run a local server:
-
-```shell-session
-mkdocs serve
-```
+<!-- Local file: docs/CONTRIBUTING.md -->
+Go to the Contributing guide at https://docs.clan.lol/manual/contribute/

LICENSE.md

@@ -0,0 +1,19 @@
+Copyright 2023-2024 Clan contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

README.md

@@ -1,6 +1,6 @@
-# Clan Core Repository
+# Clan core repository
 
-Welcome to the Clan Core Repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo is the foundation of Clan, a revolutionary open-source project aimed at restoring fun, freedom, and functionality to computing. Here, you'll find all the essential packages, NixOS modules, CLI tools, and tests needed to contribute to and work with the Clan project. Clan leverages the Nix system to ensure reliability, security, and seamless management of digital environments, putting the power back into the hands of users.
+Welcome to the Clan core repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo is the foundation of Clan, a revolutionary open-source project aimed at restoring fun, freedom, and functionality to computing. Here, you'll find all the essential packages, NixOS modules, CLI tools, and tests needed to contribute to and work with the Clan project. Clan leverages the Nix system to ensure reliability, security, and seamless management of digital environments, putting the power back into the hands of users.
 
 ## Why Clan?
 
@@ -14,13 +14,13 @@ Our mission is simple: to democratize computing by providing tools that empower
 - **Robust Backup Management:** Long-term, self-hosted data preservation.
 - **Intuitive Secret Management:** Simplified encryption and password management processes.
 
-## Getting Started with Clan
+## Getting started with Clan
 
 If you're new to Clan and eager to dive in, start with our quickstart guide and explore the core functionalities that Clan offers:
 
 - **Quickstart Guide**: Check out [getting started](https://docs.clan.lol/#starting-with-a-new-clan-project)<!-- [docs/site/index.md](docs/site/index.md) --> to get up and running with Clan in no time.
 
-### Managing Secrets
+### Managing secrets
 
 In the Clan ecosystem, security is paramount. Learn how to handle secrets effectively:
 
@@ -32,14 +32,14 @@ The Clan project thrives on community contributions. We welcome everyone to cont
 
 - **Contribution Guidelines**: Make a meaningful impact by following the steps in [contributing](https://docs.clan.lol/contributing/contributing/)<!-- [contributing.md](docs/CONTRIBUTING.md) -->.
 
-## Join the Revolution
+## Join the revolution
 
 Clan is more than a tool; it's a movement towards a better digital future. By contributing to the Clan project, you're part of changing technology for the better, together.
 
-### Community and Support
+### Community and support
 
 Connect with us and the Clan community for support and discussion:
 
-- [Matrix channel](https://matrix.to/#/#clan:lassul.us) for live discussions.
-- IRC bridges (coming soon) for real-time chat support.
+- [Matrix channel](https://matrix.to/#/#clan:clan.lol) for live discussions.
+- IRC bridge on [hackint#clan](https://chat.hackint.org/#/connect?join=clan) for real-time chat support.
 

checks/backups/flake-module.nix

@@ -5,12 +5,17 @@
     fileSystems."/".device = "/dev/null";
     boot.loader.grub.device = "/dev/null";
   };
+  clan.inventory.services = {
+    borgbackup.test-backup = {
+      roles.client.machines = [ "test-backup" ];
+      roles.server.machines = [ "test-backup" ];
+    };
+  };
   flake.nixosModules = {
     test-backup =
       {
         pkgs,
         lib,
-        config,
         ...
       }:
       let
@@ -23,21 +28,44 @@
       in
       {
         imports = [
-          self.clanModules.borgbackup
+          # Do not import inventory modules. They should be configured via 'clan.inventory'
+          #
+          # TODO: Configure localbackup via inventory
           self.clanModules.localbackup
-          self.clanModules.sshd
         ];
-        clan.networking.targetHost = "machine";
+        # Borgbackup overrides
+        services.borgbackup.repos.test-backups = {
+          path = "/var/lib/borgbackup/test-backups";
+          authorizedKeys = [ (builtins.readFile ../lib/ssh/pubkey) ];
+        };
+        clan.borgbackup.destinations.test-backup.repo = lib.mkForce "borg@machine:.";
+
+        clan.core.networking.targetHost = "machine";
         networking.hostName = "machine";
-        services.openssh.settings.UseDns = false;
 
         programs.ssh.knownHosts = {
           machine.hostNames = [ "machine" ];
           machine.publicKey = builtins.readFile ../lib/ssh/pubkey;
         };
 
+        services.openssh = {
+          enable = true;
+          settings.UsePAM = false;
+          settings.UseDns = false;
+          hostKeys = [
+            {
+              path = "/root/.ssh/id_ed25519";
+              type = "ed25519";
+            }
+          ];
+        };
+
         users.users.root.openssh.authorizedKeys.keyFiles = [ ../lib/ssh/pubkey ];
 
+        # This is needed to unlock the user for sshd
+        # Because we use sshd without setuid binaries
+        users.users.borg.initialPassword = "hello";
+
         systemd.tmpfiles.settings."vmsecrets" = {
           "/root/.ssh/id_ed25519" = {
             C.argument = "${../lib/ssh/privkey}";
@@ -53,14 +81,14 @@
               user = "root";
             };
           };
-          "/etc/secrets/borgbackup.ssh" = {
+          "/etc/secrets/borgbackup/borgbackup.ssh" = {
             C.argument = "${../lib/ssh/privkey}";
             z = {
               mode = "0400";
               user = "root";
             };
           };
-          "/etc/secrets/borgbackup.repokey" = {
+          "/etc/secrets/borgbackup/borgbackup.repokey" = {
             C.argument = builtins.toString (pkgs.writeText "repokey" "repokey12345");
             z = {
               mode = "0400";
@@ -68,17 +96,10 @@
             };
           };
         };
-        clanCore.facts.secretStore = "vm";
+        clan.core.facts.secretStore = "vm";
+        clan.core.vars.settings.secretStore = "vm";
 
-        environment.systemPackages = [
-          self.packages.${pkgs.system}.clan-cli
-          (pkgs.writeShellScriptBin "pre-restore-command" ''
-            touch /var/test-service/pre-restore-command
-          '')
-          (pkgs.writeShellScriptBin "post-restore-command" ''
-            touch /var/test-service/post-restore-command
-          '')
-        ];
+        environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
         environment.etc.install-closure.source = "${closureInfo}/store-paths";
         nix.settings = {
           substituters = lib.mkForce [ ];
@@ -87,14 +108,20 @@
           flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
         };
         system.extraDependencies = dependencies;
-        clanCore.state.test-backups.folders = [ "/var/test-backups" ];
+        clan.core.state.test-backups.folders = [ "/var/test-backups" ];
 
-        clanCore.state.test-service = {
-          preRestoreCommand = "pre-restore-command";
-          postRestoreCommand = "post-restore-command";
+        clan.core.state.test-service = {
+          preBackupScript = ''
+            touch /var/test-service/pre-backup-command
+          '';
+          preRestoreScript = ''
+            touch /var/test-service/pre-restore-command
+          '';
+          postRestoreScript = ''
+            touch /var/test-service/post-restore-command
+          '';
           folders = [ "/var/test-service" ];
         };
-        clan.borgbackup.destinations.test-backup.repo = "borg@machine:.";
 
         fileSystems."/mnt/external-disk" = {
           device = "/dev/vdb"; # created in tests with virtualisation.emptyDisks
@@ -115,25 +142,55 @@
             touch /run/unmount-external-disk
           '';
         };
-
-        services.borgbackup.repos.test-backups = {
-          path = "/var/lib/borgbackup/test-backups";
-          authorizedKeys = [ (builtins.readFile ../lib/ssh/pubkey) ];
-        };
       };
   };
   perSystem =
-    { nodes, pkgs, ... }:
+    { pkgs, ... }:
+    let
+      clanCore = self.filter {
+        include = [
+          "checks/backups"
+          "checks/flake-module.nix"
+          "clanModules/borgbackup"
+          "clanModules/flake-module.nix"
+          "clanModules/localbackup"
+          "clanModules/packages"
+          "clanModules/single-disk"
+          "clanModules/zerotier"
+          "flake.lock"
+          "flakeModules"
+          "inventory.json"
+          "lib/build-clan"
+          "lib/default.nix"
+          "lib/select.nix"
+          "lib/flake-module.nix"
+          "lib/frontmatter"
+          "lib/inventory"
+          "lib/constraints"
+          "nixosModules"
+        ];
+      };
+    in
     {
-      checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) {
-        test-backups = (import ../lib/test-base.nix) {
+      checks = pkgs.lib.mkIf pkgs.stdenv.isLinux {
+        test-backups = (import ../lib/container-test.nix) {
           name = "test-backups";
           nodes.machine = {
-            imports = [
-              self.nixosModules.clanCore
-              self.nixosModules.test-backup
+            imports =
+              [
+                self.nixosModules.clanCore
+                # Some custom overrides for the backup tests
+                self.nixosModules.test-backup
+              ]
+              ++
+              # import the inventory generated nixosModules
+              self.clanInternals.inventoryClass.machines.test-backup.machineImports;
+            clan.core.settings.directory = ./.;
+            environment.systemPackages = [
+              (pkgs.writeShellScriptBin "foo" ''
+                echo ${clanCore}
+              '')
             ];
-            virtualisation.emptyDiskImages = [ 256 ];
           };
 
           testScript = ''
@@ -145,14 +202,14 @@
             machine.succeed("echo testing > /var/test-backups/somefile")
 
             # create
-            machine.succeed("clan --debug --flake ${self} backups create test-backup")
+            machine.succeed("clan backups create --debug --flake ${clanCore} test-backup")
             machine.wait_until_succeeds("! systemctl is-active borgbackup-job-test-backup >&2")
             machine.succeed("test -f /run/mount-external-disk")
             machine.succeed("test -f /run/unmount-external-disk")
 
             # list
             backup_id = json.loads(machine.succeed("borg-job-test-backup list --json"))["archives"][0]["archive"]
-            out = machine.succeed("clan --debug --flake ${self} backups list test-backup").strip()
+            out = machine.succeed("clan backups list --debug --flake ${clanCore} test-backup").strip()
             print(out)
             assert backup_id in out, f"backup {backup_id} not found in {out}"
             localbackup_id = "hdd::/mnt/external-disk/snapshot.0"
@@ -160,17 +217,19 @@
 
             ## borgbackup restore
             machine.succeed("rm -f /var/test-backups/somefile")
-            machine.succeed(f"clan --debug --flake ${self} backups restore test-backup borgbackup 'test-backup::borg@machine:.::{backup_id}' >&2")
+            machine.succeed(f"clan backups restore --debug --flake ${clanCore} test-backup borgbackup 'test-backup::borg@machine:.::{backup_id}' >&2")
             assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
             machine.succeed("test -f /var/test-service/pre-restore-command")
             machine.succeed("test -f /var/test-service/post-restore-command")
+            machine.succeed("test -f /var/test-service/pre-backup-command")
 
             ## localbackup restore
-            machine.succeed("rm -f /var/test-backups/somefile /var/test-service/{pre,post}-restore-command")
-            machine.succeed(f"clan --debug --flake ${self} backups restore test-backup localbackup '{localbackup_id}' >&2")
+            machine.succeed("rm -rf /var/test-backups/somefile /var/test-service/ && mkdir -p /var/test-service")
+            machine.succeed(f"clan backups restore --debug --flake ${clanCore} test-backup localbackup '{localbackup_id}' >&2")
             assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
             machine.succeed("test -f /var/test-service/pre-restore-command")
             machine.succeed("test -f /var/test-service/post-restore-command")
+            machine.succeed("test -f /var/test-service/pre-backup-command")
           '';
         } { inherit pkgs self; };
       };

checks/borgbackup/default.nix

@@ -16,19 +16,19 @@
             };
           }
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
-            clanCore.state.testState.folders = [ "/etc/state" ];
+            clan.core.settings.machine.name = "machine";
+            clan.core.settings.directory = ./.;
+            clan.core.state.testState.folders = [ "/etc/state" ];
             environment.etc.state.text = "hello world";
             systemd.tmpfiles.settings."vmsecrets" = {
-              "/etc/secrets/borgbackup.ssh" = {
+              "/etc/secrets/borgbackup/borgbackup.ssh" = {
                 C.argument = "${../lib/ssh/privkey}";
                 z = {
                   mode = "0400";
                   user = "root";
                 };
               };
-              "/etc/secrets/borgbackup.repokey" = {
+              "/etc/secrets/borgbackup/borgbackup.repokey" = {
                 C.argument = builtins.toString (pkgs.writeText "repokey" "repokey12345");
                 z = {
                   mode = "0400";
@@ -36,7 +36,8 @@
                 };
               };
             };
-            clanCore.facts.secretStore = "vm";
+            # clan.core.facts.secretStore = "vm";
+            clan.core.vars.settings.secretStore = "vm";
 
             clan.borgbackup.destinations.test.repo = "borg@localhost:.";
           }

checks/container/default.nix

@@ -1,7 +1,7 @@
 (import ../lib/container-test.nix) (
   { ... }:
   {
-    name = "secrets";
+    name = "container";
 
     nodes.machine =
       { ... }:

checks/deltachat/default.nix

@@ -1,7 +1,7 @@
 (import ../lib/container-test.nix) (
   { pkgs, ... }:
   {
-    name = "secrets";
+    name = "deltachat";
 
     nodes.machine =
       { self, ... }:
@@ -10,8 +10,8 @@
           self.clanModules.deltachat
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
+            clan.core.settings.machine.name = "machine";
+            clan.core.settings.directory = ./.;
           }
         ];
       };

checks/devshell/flake-module.nix

@@ -0,0 +1,22 @@
+{ ... }:
+{
+  perSystem =
+    { self', pkgs, ... }:
+    {
+      checks.devshell =
+        pkgs.runCommand "check-devshell-not-depends-on-clan-cli"
+          {
+            exportReferencesGraph = [
+              "graph"
+              self'.devShells.default
+            ];
+          }
+          ''
+            if grep -q "${self'.packages.clan-cli}" ./graph; then
+              echo "devshell depends on clan-cli, which is not allowed";
+              exit 1;
+            fi
+            mkdir $out
+          '';
+    };
+}

checks/flake-module.nix

@@ -1,10 +1,20 @@
-{ self, ... }:
+{ self, lib, ... }:
+let
+  inherit (lib)
+    filter
+    pathExists
+    ;
+in
 {
-  imports = [
-    ./impure/flake-module.nix
+  imports = filter pathExists [
     ./backups/flake-module.nix
-    ./installation/flake-module.nix
+    ./devshell/flake-module.nix
     ./flash/flake-module.nix
+    ./impure/flake-module.nix
+    ./installation/flake-module.nix
+    ./installation-without-system/flake-module.nix
+    ./morph/flake-module.nix
+    ./nixos-documentation/flake-module.nix
   ];
   perSystem =
     {
@@ -16,19 +26,6 @@
     {
       checks =
         let
-          # ensure all options can be rendered after importing clan into nixos
-          renderClanOptions =
-            let
-              docs = pkgs.nixosOptionsDoc {
-                options =
-                  (pkgs.nixos {
-                    imports = [ self.nixosModules.clanCore ];
-                    clanCore.clanDir = ./.;
-                  }).options;
-                warningsAreErrors = false;
-              };
-            in
-            docs.optionsJSON;
           nixosTestArgs = {
             # reference to nixpkgs for the current system
             inherit pkgs;
@@ -39,25 +36,28 @@
             # import our test
             secrets = import ./secrets nixosTestArgs;
             container = import ./container nixosTestArgs;
-            deltachat = import ./deltachat nixosTestArgs;
-            matrix-synapse = import ./matrix-synapse nixosTestArgs;
-            zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs;
+            # Deltachat is currently marked as broken
+            # deltachat = import ./deltachat nixosTestArgs;
             borgbackup = import ./borgbackup nixosTestArgs;
+            matrix-synapse = import ./matrix-synapse nixosTestArgs;
+            mumble = import ./mumble nixosTestArgs;
             syncthing = import ./syncthing nixosTestArgs;
+            zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs;
+            postgresql = import ./postgresql nixosTestArgs;
             wayland-proxy-virtwl = import ./wayland-proxy-virtwl nixosTestArgs;
           };
 
           flakeOutputs =
             lib.mapAttrs' (
               name: config: lib.nameValuePair "nixos-${name}" config.config.system.build.toplevel
-            ) self.nixosConfigurations
+            ) (lib.filterAttrs (n: _: !lib.hasPrefix "test-" n) self.nixosConfigurations)
             // lib.mapAttrs' (n: lib.nameValuePair "package-${n}") self'.packages
             // lib.mapAttrs' (n: lib.nameValuePair "devShell-${n}") self'.devShells
             // lib.mapAttrs' (name: config: lib.nameValuePair "home-manager-${name}" config.activation-script) (
               self'.legacyPackages.homeConfigurations or { }
             );
         in
-        { inherit renderClanOptions; } // nixosTests // flakeOutputs;
+        nixosTests // flakeOutputs;
       legacyPackages = {
         nixosTests =
           let

checks/flash/flake-module.nix

@@ -1,33 +1,87 @@
-{ ... }:
 {
-  perSystem =
-    { ... }:
-    {
-      # checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) {
-      #   flash = (import ../lib/test-base.nix) {
-      #     name = "flash";
-      #     nodes.target = {
-      #       virtualisation.emptyDiskImages = [ 4096 ];
-      #       virtualisation.memorySize = 3000;
-      #       environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
-      #       environment.etc."install-closure".source = "${closureInfo}/store-paths"; 
+  config,
+  self,
+  lib,
+  ...
+}:
+{
+  clan.machines = lib.listToAttrs (
+    lib.map (
+      system:
+      lib.nameValuePair "test-flash-machine-${system}" {
+        clan.core.networking.targetHost = "test-flash-machine";
+        fileSystems."/".device = lib.mkDefault "/dev/vda";
+        boot.loader.grub.device = lib.mkDefault "/dev/vda";
 
-      #       nix.settings = {
-      #         substituters = lib.mkForce [ ];
-      #         hashed-mirrors = null;
-      #         connect-timeout = lib.mkForce 3;
-      #         flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
-      #         experimental-features = [
-      #           "nix-command"
-      #           "flakes"
-      #         ];
-      #       };
-      #     };
-      #     testScript = ''
-      #       start_all()
-      #       machine.succeed("clan --debug --flake ${../..} flash --yes --disk main /dev/vdb test_install_machine")
-      #     '';
-      #   } { inherit pkgs self; };
-      # };
+        # We need to use `mkForce` because we inherit from `test-install-machine`
+        # which currently hardcodes `nixpkgs.hostPlatform`
+        nixpkgs.hostPlatform = lib.mkForce system;
+
+        imports = [ self.nixosModules.test-flash-machine ];
+      }
+    ) (lib.filter (lib.hasSuffix "linux") config.systems)
+  );
+
+  flake.nixosModules = {
+    test-flash-machine =
+      { lib, ... }:
+      {
+        imports = [ self.nixosModules.test-install-machine ];
+
+        clan.core.vars.generators.test = lib.mkForce { };
+
+        disko.devices.disk.main.preCreateHook = lib.mkForce "";
+      };
+  };
+
+  perSystem =
+    {
+      nodes,
+      pkgs,
+      lib,
+      ...
+    }:
+    let
+      dependencies = [
+        pkgs.disko
+        self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".pkgs.perlPackages.ConfigIniFiles
+        self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".pkgs.perlPackages.FileSlurp
+
+        self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.build.toplevel
+        self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.build.diskoScript
+        self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.build.diskoScript.drvPath
+        self.nixosConfigurations."test-flash-machine-${pkgs.hostPlatform.system}".config.system.clan.deployment.file
+
+      ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
+      closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
+    in
+    {
+      checks = pkgs.lib.mkIf pkgs.stdenv.isLinux {
+        test-flash = (import ../lib/test-base.nix) {
+          name = "flash";
+          nodes.target = {
+            virtualisation.emptyDiskImages = [ 4096 ];
+            virtualisation.memorySize = 3000;
+            environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
+            environment.etc."install-closure".source = "${closureInfo}/store-paths";
+
+            nix.settings = {
+              substituters = lib.mkForce [ ];
+              hashed-mirrors = null;
+              connect-timeout = lib.mkForce 3;
+              flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
+              experimental-features = [
+                "nix-command"
+                "flakes"
+              ];
+            };
+          };
+          testScript = ''
+            start_all()
+
+            machine.succeed("clan flash write --debug --flake ${../..} --yes --disk main /dev/vdb test-flash-machine-${pkgs.hostPlatform.system}")
+          '';
+        } { inherit pkgs self; };
+      };
     };
 }

checks/impure/flake-module.nix

@@ -1,6 +1,11 @@
 {
   perSystem =
-    { pkgs, lib, ... }:
+    {
+      pkgs,
+      lib,
+      self',
+      ...
+    }:
     {
       # a script that executes all other checks
       packages.impure-checks = pkgs.writeShellScriptBin "impure-checks" ''
@@ -10,15 +15,22 @@
         unset CLAN_DIR
 
         export PATH="${
-          lib.makeBinPath [
-            pkgs.gitMinimal
-            pkgs.nix
-            pkgs.rsync # needed to have rsync installed on the dummy ssh server
-          ]
+          lib.makeBinPath (
+            [
+              pkgs.gitMinimal
+              pkgs.nix
+              pkgs.rsync # needed to have rsync installed on the dummy ssh server
+            ]
+            ++ self'.packages.clan-cli-full.runtimeDependencies
+          )
         }"
         ROOT=$(git rev-parse --show-toplevel)
         cd "$ROOT/pkgs/clan-cli"
-        nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -s -m impure ./tests $@"
+
+        # this disables dynamic dependency loading in clan-cli
+        export CLAN_NO_DYNAMIC_DEPS=1
+
+        nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -m impure ./tests $@"
       '';
     };
 }

checks/installation-without-system/flake-module.nix

@@ -0,0 +1,236 @@
+{
+  self,
+  lib,
+  ...
+}:
+{
+  # The purpose of this test is to ensure `clan machines install` works
+  # for machines that don't have a hardware config yet.
+
+  # If this test starts failing it could be due to the `facter.json` being out of date
+  # you can get a new one by adding
+  # client.fail("cat test-flake/machines/test-install-machine/facter.json >&2")
+  # to the installation test.
+  clan.machines.test-install-machine-without-system = {
+    fileSystems."/".device = lib.mkDefault "/dev/vda";
+    boot.loader.grub.device = lib.mkDefault "/dev/vda";
+
+    imports = [ self.nixosModules.test-install-machine-without-system ];
+  };
+  clan.machines.test-install-machine-with-system =
+    { pkgs, ... }:
+    {
+      # https://git.clan.lol/clan/test-fixtures
+      facter.reportPath = builtins.fetchurl {
+        url = "https://git.clan.lol/clan/test-fixtures/raw/commit/4a2bc56d886578124b05060d3fb7eddc38c019f8/nixos-vm-facter-json/${pkgs.hostPlatform.system}.json";
+        sha256 =
+          {
+            aarch64-linux = "sha256:1rlfymk03rmfkm2qgrc8l5kj5i20srx79n1y1h4nzlpwaz0j7hh2";
+            x86_64-linux = "sha256:16myh0ll2gdwsiwkjw5ba4dl23ppwbsanxx214863j7nvzx42pws";
+          }
+          .${pkgs.hostPlatform.system};
+      };
+
+      fileSystems."/".device = lib.mkDefault "/dev/vda";
+      boot.loader.grub.device = lib.mkDefault "/dev/vda";
+
+      imports = [ self.nixosModules.test-install-machine-without-system ];
+    };
+  flake.nixosModules = {
+    test-install-machine-without-system =
+      { lib, modulesPath, ... }:
+      {
+        imports = [
+          (modulesPath + "/testing/test-instrumentation.nix") # we need these 2 modules always to be able to run the tests
+          (modulesPath + "/profiles/qemu-guest.nix")
+          ../lib/minify.nix
+        ];
+
+        networking.hostName = "test-install-machine";
+
+        environment.etc."install-successful".text = "ok";
+
+        boot.consoleLogLevel = lib.mkForce 100;
+        boot.kernelParams = [ "boot.shell_on_fail" ];
+
+        # disko config
+        boot.loader.grub.efiSupport = lib.mkDefault true;
+        boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
+        clan.core.vars.settings.secretStore = "vm";
+        clan.core.vars.generators.test = {
+          files.test.neededFor = "partitioning";
+          script = ''
+            echo "notok" > $out/test
+          '';
+        };
+        disko.devices = {
+          disk = {
+            main = {
+              type = "disk";
+              device = "/dev/vda";
+
+              preCreateHook = ''
+                test -e /run/partitioning-secrets/test/test
+              '';
+
+              content = {
+                type = "gpt";
+                partitions = {
+                  boot = {
+                    size = "1M";
+                    type = "EF02"; # for grub MBR
+                    priority = 1;
+                  };
+                  ESP = {
+                    size = "512M";
+                    type = "EF00";
+                    content = {
+                      type = "filesystem";
+                      format = "vfat";
+                      mountpoint = "/boot";
+                      mountOptions = [ "umask=0077" ];
+                    };
+                  };
+                  root = {
+                    size = "100%";
+                    content = {
+                      type = "filesystem";
+                      format = "ext4";
+                      mountpoint = "/";
+                    };
+                  };
+                };
+              };
+            };
+          };
+        };
+      };
+  };
+  perSystem =
+    {
+      pkgs,
+      lib,
+      ...
+    }:
+    let
+      dependencies = [
+        self
+        self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.toplevel
+        self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.build.diskoScript
+        self.clanInternals.machines.${pkgs.hostPlatform.system}.test-install-machine-with-system.config.system.clan.deployment.file
+        pkgs.stdenv.drvPath
+        pkgs.bash.drvPath
+        pkgs.nixos-anywhere
+        pkgs.bubblewrap
+      ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
+      closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
+      # with Nix 2.24 we get:
+      # vm-test-run-test-installation> client # error: sized: unexpected end-of-file
+      # vm-test-run-test-installation> client # error: unexpected end-of-file
+      # This seems to be fixed with Nix 2.26
+      # Remove this line once `pkgs.nix` is 2.26+
+      nixPackage =
+        assert
+          lib.versionOlder pkgs.nix.version "2.26"
+          && lib.versionAtLeast pkgs.nixVersions.latest.version "2.26";
+        pkgs.nixVersions.latest;
+    in
+    {
+      # On aarch64-linux, hangs on reboot with after installation:
+      # vm-test-run-test-installation-without-system> installer # [  288.002871] reboot: Restarting system
+      # vm-test-run-test-installation-without-system> client # [test-install-machine] ### Done! ###
+      # vm-test-run-test-installation-without-system> client # [test-install-machine] + step 'Done!'
+      # vm-test-run-test-installation-without-system> client # [test-install-machine] + echo '### Done! ###'
+      # vm-test-run-test-installation-without-system> client # [test-install-machine] + rm -rf /tmp/tmp.qb16EAq7hJ
+      # vm-test-run-test-installation-without-system> (finished: must succeed: clan machines install --debug --flake test-flake --yes test-install-machine-without-system --target-host root@installer --update-hardware-config nixos-facter >&2, in 154.62 seconds)
+      # vm-test-run-test-installation-without-system> target: starting vm
+      # vm-test-run-test-installation-without-system> target: QEMU running (pid 144)
+      # vm-test-run-test-installation-without-system> target: waiting for unit multi-user.target
+      # vm-test-run-test-installation-without-system> target: waiting for the VM to finish booting
+      # vm-test-run-test-installation-without-system> target: Guest root shell did not produce any data yet...
+      # vm-test-run-test-installation-without-system> target:   To debug, enter the VM and run 'systemctl status backdoor.service'.
+      checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux && !pkgs.stdenv.isAarch64) {
+        test-installation-without-system = (import ../lib/test-base.nix) {
+          name = "test-installation-without-system";
+          nodes.target = {
+            services.openssh.enable = true;
+            virtualisation.diskImage = "./target.qcow2";
+            virtualisation.useBootLoader = true;
+            nix.package = nixPackage;
+          };
+          nodes.installer =
+            { modulesPath, ... }:
+            {
+              imports = [
+                (modulesPath + "/../tests/common/auto-format-root-device.nix")
+              ];
+              services.openssh.enable = true;
+              users.users.root.openssh.authorizedKeys.keyFiles = [ ../lib/ssh/pubkey ];
+              system.nixos.variant_id = "installer";
+              environment.systemPackages = [ pkgs.nixos-facter ];
+              virtualisation.emptyDiskImages = [ 512 ];
+              virtualisation.diskSize = 8 * 1024;
+              virtualisation.rootDevice = "/dev/vdb";
+              # both installer and target need to use the same diskImage
+              virtualisation.diskImage = "./target.qcow2";
+              nix.package = nixPackage;
+              nix.settings = {
+                substituters = lib.mkForce [ ];
+                hashed-mirrors = null;
+                connect-timeout = lib.mkForce 3;
+                flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
+                experimental-features = [
+                  "nix-command"
+                  "flakes"
+                ];
+              };
+              system.extraDependencies = dependencies;
+            };
+          nodes.client = {
+            environment.systemPackages = [
+              self.packages.${pkgs.system}.clan-cli
+            ] ++ self.packages.${pkgs.system}.clan-cli.runtimeDependencies;
+            environment.etc."install-closure".source = "${closureInfo}/store-paths";
+            virtualisation.memorySize = 3048;
+            nix.package = nixPackage;
+            nix.settings = {
+              substituters = lib.mkForce [ ];
+              hashed-mirrors = null;
+              connect-timeout = lib.mkForce 3;
+              flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
+              experimental-features = [
+                "nix-command"
+                "flakes"
+              ];
+            };
+            system.extraDependencies = dependencies;
+          };
+
+          testScript = ''
+            client.start()
+            installer.start()
+
+            client.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../lib/ssh/privkey} /root/.ssh/id_ed25519")
+            client.wait_until_succeeds("timeout 2 ssh -o StrictHostKeyChecking=accept-new -v root@installer hostname")
+            client.succeed("cp -r ${../..} test-flake && chmod -R +w test-flake")
+            client.fail("test -f test-flake/machines/test-install-machine-without-system/hardware-configuration.nix")
+            client.fail("test -f test-flake/machines/test-install-machine-without-system/facter.json")
+            client.succeed("clan machines update-hardware-config --flake test-flake test-install-machine-without-system root@installer >&2")
+            client.succeed("test -f test-flake/machines/test-install-machine-without-system/facter.json")
+            client.succeed("rm test-flake/machines/test-install-machine-without-system/facter.json")
+            client.succeed("clan machines install --debug --flake test-flake --yes test-install-machine-without-system --target-host root@installer --update-hardware-config nixos-facter >&2")
+            try:
+              installer.shutdown()
+            except BrokenPipeError:
+              # qemu has already exited
+              pass
+
+            target.state_dir = installer.state_dir
+            target.start()
+            target.wait_for_unit("multi-user.target")
+            assert(target.succeed("cat /etc/install-successful").strip() == "ok")
+          '';
+        } { inherit pkgs self; };
+      };
+    };
+}

checks/installation/flake-module.nix

@@ -1,32 +1,87 @@
-{ self, lib, ... }:
 {
-  clan.machines.test_install_machine = {
-    clan.networking.targetHost = "test_install_machine";
-    fileSystems."/".device = lib.mkDefault "/dev/null";
-    boot.loader.grub.device = lib.mkDefault "/dev/null";
+  self,
+  lib,
+  ...
+}:
+{
+  clan.machines.test-install-machine = {
+    clan.core.networking.targetHost = "test-install-machine";
+    fileSystems."/".device = lib.mkDefault "/dev/vda";
+    boot.loader.grub.device = lib.mkDefault "/dev/vda";
 
-    imports = [ self.nixosModules.test_install_machine ];
+    imports = [ self.nixosModules.test-install-machine ];
   };
   flake.nixosModules = {
-    test_install_machine =
+    test-install-machine =
       { lib, modulesPath, ... }:
       {
         imports = [
-          self.clanModules.disk-layouts
           (modulesPath + "/testing/test-instrumentation.nix") # we need these 2 modules always to be able to run the tests
           (modulesPath + "/profiles/qemu-guest.nix")
+          ../lib/minify.nix
         ];
-        clan.disk-layouts.singleDiskExt4.device = "/dev/vdb";
 
         environment.etc."install-successful".text = "ok";
 
+        nixpkgs.hostPlatform = "x86_64-linux";
         boot.consoleLogLevel = lib.mkForce 100;
         boot.kernelParams = [ "boot.shell_on_fail" ];
+
+        # disko config
+        boot.loader.grub.efiSupport = lib.mkDefault true;
+        boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
+        clan.core.vars.settings.secretStore = "vm";
+        clan.core.vars.generators.test = {
+          files.test.neededFor = "partitioning";
+          script = ''
+            echo "notok" > $out/test
+          '';
+        };
+        disko.devices = {
+          disk = {
+            main = {
+              type = "disk";
+              device = "/dev/vda";
+
+              preCreateHook = ''
+                test -e /run/partitioning-secrets/test/test
+              '';
+
+              content = {
+                type = "gpt";
+                partitions = {
+                  boot = {
+                    size = "1M";
+                    type = "EF02"; # for grub MBR
+                    priority = 1;
+                  };
+                  ESP = {
+                    size = "512M";
+                    type = "EF00";
+                    content = {
+                      type = "filesystem";
+                      format = "vfat";
+                      mountpoint = "/boot";
+                      mountOptions = [ "umask=0077" ];
+                    };
+                  };
+                  root = {
+                    size = "100%";
+                    content = {
+                      type = "filesystem";
+                      format = "ext4";
+                      mountpoint = "/";
+                    };
+                  };
+                };
+              };
+            };
+          };
+        };
       };
   };
   perSystem =
     {
-      nodes,
       pkgs,
       lib,
       ...
@@ -34,40 +89,86 @@
     let
       dependencies = [
         self
-        self.nixosConfigurations.test_install_machine.config.system.build.toplevel
-        self.nixosConfigurations.test_install_machine.config.system.build.diskoScript
-        self.nixosConfigurations.test_install_machine.config.system.clan.deployment.file
+        self.nixosConfigurations.test-install-machine.config.system.build.toplevel
+        self.nixosConfigurations.test-install-machine.config.system.build.diskoScript
+        self.nixosConfigurations.test-install-machine.config.system.clan.deployment.file
+        pkgs.bash.drvPath
         pkgs.stdenv.drvPath
         pkgs.nixos-anywhere
+        pkgs.bubblewrap
       ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
       closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
+
+      # with Nix 2.24 we get:
+      # vm-test-run-test-installation> client # error: sized: unexpected end-of-file
+      # vm-test-run-test-installation> client # error: unexpected end-of-file
+      # This seems to be fixed with Nix 2.26
+      # Remove this line once `pkgs.nix` is 2.26+
+      nixPackage =
+        assert
+          lib.versionOlder pkgs.nix.version "2.26"
+          && lib.versionAtLeast pkgs.nixVersions.latest.version "2.26";
+        pkgs.nixVersions.latest;
     in
     {
-      checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) {
+      # On aarch64-linux, hangs on reboot with after installation:
+      # vm-test-run-test-installation> (finished: waiting for the VM to power off, in 1.97 seconds)
+      # vm-test-run-test-installation>
+      # vm-test-run-test-installation> new_machine: must succeed: cat /etc/install-successful
+      # vm-test-run-test-installation> new_machine: waiting for the VM to finish booting
+      # vm-test-run-test-installation> new_machine: starting vm
+      # vm-test-run-test-installation> new_machine: QEMU running (pid 80)
+      # vm-test-run-test-installation> new_machine: Guest root shell did not produce any data yet...
+      # vm-test-run-test-installation> new_machine:   To debug, enter the VM and run 'systemctl status backdoor.service'.
+      checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux && !pkgs.stdenv.isAarch64) {
         test-installation = (import ../lib/test-base.nix) {
           name = "test-installation";
           nodes.target = {
             services.openssh.enable = true;
-            users.users.root.openssh.authorizedKeys.keyFiles = [ ../lib/ssh/pubkey ];
-            system.nixos.variant_id = "installer";
-            virtualisation.emptyDiskImages = [ 4096 ];
-            nix.settings = {
-              substituters = lib.mkForce [ ];
-              hashed-mirrors = null;
-              connect-timeout = lib.mkForce 3;
-              flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
-              experimental-features = [
-                "nix-command"
-                "flakes"
-              ];
-            };
+            virtualisation.diskImage = "./target.qcow2";
+            virtualisation.useBootLoader = true;
+            nix.package = nixPackage;
+
+            # virtualisation.fileSystems."/" = {
+            #   device = "/dev/disk/by-label/this-is-not-real-and-will-never-be-used";
+            #   fsType = "ext4";
+            # };
           };
+          nodes.installer =
+            { modulesPath, ... }:
+            {
+              imports = [
+                (modulesPath + "/../tests/common/auto-format-root-device.nix")
+              ];
+              services.openssh.enable = true;
+              users.users.root.openssh.authorizedKeys.keyFiles = [ ../lib/ssh/pubkey ];
+              system.nixos.variant_id = "installer";
+              environment.systemPackages = [ pkgs.nixos-facter ];
+              virtualisation.emptyDiskImages = [ 512 ];
+              virtualisation.diskSize = 8 * 1024;
+              virtualisation.rootDevice = "/dev/vdb";
+              # both installer and target need to use the same diskImage
+              virtualisation.diskImage = "./target.qcow2";
+              nix.package = nixPackage;
+              nix.settings = {
+                substituters = lib.mkForce [ ];
+                hashed-mirrors = null;
+                connect-timeout = lib.mkForce 3;
+                flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
+                experimental-features = [
+                  "nix-command"
+                  "flakes"
+                ];
+              };
+              system.extraDependencies = dependencies;
+            };
           nodes.client = {
             environment.systemPackages = [
               self.packages.${pkgs.system}.clan-cli
             ] ++ self.packages.${pkgs.system}.clan-cli.runtimeDependencies;
             environment.etc."install-closure".source = "${closureInfo}/store-paths";
-            virtualisation.memorySize = 2048;
+            virtualisation.memorySize = 3048;
+            nix.package = nixPackage;
             nix.settings = {
               substituters = lib.mkForce [ ];
               hashed-mirrors = null;
@@ -82,31 +183,35 @@
           };
 
           testScript = ''
-            def create_test_machine(oldmachine=None, args={}): # taken from <nixpkgs/nixos/tests/installer.nix>
-                startCommand = "${pkgs.qemu_test}/bin/qemu-kvm"
-                startCommand += " -cpu max -m 1024 -virtfs local,path=/nix/store,security_model=none,mount_tag=nix-store"
-                startCommand += f' -drive file={oldmachine.state_dir}/empty0.qcow2,id=drive1,if=none,index=1,werror=report'
-                startCommand += ' -device virtio-blk-pci,drive=drive1'
-                machine = create_machine({
-                  "startCommand": startCommand,
-                } | args)
-                driver.machines.append(machine)
-                return machine
-
-            start_all()
+            client.start()
+            installer.start()
 
             client.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../lib/ssh/privkey} /root/.ssh/id_ed25519")
-            client.wait_until_succeeds("ssh -o StrictHostKeyChecking=accept-new -v root@target hostname")
+            client.wait_until_succeeds("timeout 2 ssh -o StrictHostKeyChecking=accept-new -v root@installer hostname")
+            client.succeed("cp -r ${../..} test-flake && chmod -R +w test-flake")
 
-            client.succeed("clan --debug --flake ${../..} machines install --yes test_install_machine root@target >&2")
+            # test that we can generate hardware configurations
+            client.fail("test -f test-flake/machines/test-install-machine/facter.json")
+            client.fail("test -f test-flake/machines/test-install-machine/hardware-configuration.nix")
+            client.succeed("clan machines update-hardware-config --flake test-flake test-install-machine root@installer >&2")
+            client.succeed("test -f test-flake/machines/test-install-machine/facter.json")
+            client.succeed("clan machines update-hardware-config --backend nixos-generate-config --flake test-flake test-install-machine root@installer>&2")
+            client.succeed("test -f test-flake/machines/test-install-machine/hardware-configuration.nix")
+
+            # but we don't use them because they're not cached
+            client.succeed("rm test-flake/machines/test-install-machine/hardware-configuration.nix test-flake/machines/test-install-machine/facter.json")
+
+            client.succeed("clan machines install --debug --flake test-flake --yes test-install-machine --target-host root@installer >&2")
             try:
-              target.shutdown()
+              installer.shutdown()
             except BrokenPipeError:
               # qemu has already exited
               pass
 
-            new_machine = create_test_machine(oldmachine=target, args={ "name": "new_machine" })
-            assert(new_machine.succeed("cat /etc/install-successful").strip() == "ok")
+            target.state_dir = installer.state_dir
+            target.start()
+            target.wait_for_unit("multi-user.target")
+            assert(target.succeed("cat /etc/install-successful").strip() == "ok")
           '';
         } { inherit pkgs self; };
       };

checks/lib/container-driver/module.nix

@@ -7,9 +7,19 @@
 let
   testDriver = hostPkgs.python3.pkgs.callPackage ./package.nix {
     inherit (config) extraPythonPackages;
-    inherit (hostPkgs.pkgs) util-linux systemd;
+    inherit (hostPkgs.pkgs) util-linux systemd nix;
   };
-  containers = map (m: m.system.build.toplevel) (lib.attrValues config.nodes);
+  containers =
+    testScript:
+    map (m: [
+      m.system.build.toplevel
+      (hostPkgs.closureInfo {
+        rootPaths = [
+          m.system.build.toplevel
+          (hostPkgs.writeText "testScript" testScript)
+        ];
+      })
+    ]) (lib.attrValues config.nodes);
   pythonizeName =
     name:
     let
@@ -44,8 +54,6 @@ in
       ''
         mkdir -p $out/bin
 
-        containers=(${toString containers})
-
         ${lib.optionalString (!config.skipTypeCheck) ''
           # prepend type hints so the test script can be type checked with mypy
           cat "${./test-script-prepend.py}" >> testScriptWithTypes
@@ -66,7 +74,13 @@ in
         ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver
 
         wrapProgram $out/bin/nixos-test-driver \
-          ${lib.concatStringsSep " " (map (name: "--add-flags '--container ${name}'") containers)} \
+          ${
+            lib.concatStringsSep " " (
+              map (container: "--add-flags '--container ${builtins.toString container}'") (
+                containers config.testScriptString
+              )
+            )
+          } \
           --add-flags "--test-script '$out/test-script'"
       ''
   );

checks/lib/container-driver/package.nix

@@ -5,6 +5,9 @@
   setuptools,
   util-linux,
   systemd,
+  nix,
+  colorama,
+  junit-xml,
 }:
 buildPythonApplication {
   pname = "test-driver";
@@ -12,6 +15,9 @@ buildPythonApplication {
   propagatedBuildInputs = [
     util-linux
     systemd
+    colorama
+    junit-xml
+    nix
   ] ++ extraPythonPackages python3Packages;
   nativeBuildInputs = [ setuptools ];
   format = "pyproject";

checks/lib/container-driver/pyproject.toml

@@ -14,16 +14,8 @@ find = {}
 
 [tool.setuptools.package-data]
 test_driver = ["py.typed"]
-
-[tool.ruff]
-target-version = "py311"
-line-length = 88
-
-lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
-lint.ignore = ["E501", "ANN101", "ANN401", "A003"]
-
 [tool.mypy]
-python_version = "3.11"
+python_version = "3.12"
 warn_redundant_casts = true
 disallow_untyped_calls = true
 disallow_untyped_defs = true

checks/lib/container-driver/test_driver/__init__.py

@@ -1,13 +1,71 @@
 import argparse
+import ctypes
 import os
 import re
 import subprocess
 import time
+import types
 from collections.abc import Callable
+from contextlib import _GeneratorContextManager
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any
 
+from .logger import AbstractLogger, CompositeLogger, TerminalLogger
+
+# Load the C library
+libc = ctypes.CDLL("libc.so.6", use_errno=True)
+
+# Define the mount function
+libc.mount.argtypes = [
+    ctypes.c_char_p,  # source
+    ctypes.c_char_p,  # target
+    ctypes.c_char_p,  # filesystemtype
+    ctypes.c_ulong,  # mountflags
+    ctypes.c_void_p,  # data
+]
+libc.mount.restype = ctypes.c_int
+
+MS_BIND = 0x1000
+MS_REC = 0x4000
+
+
+def mount(
+    source: Path,
+    target: Path,
+    filesystemtype: str,
+    mountflags: int = 0,
+    data: str | None = None,
+) -> None:
+    """
+    A Python wrapper for the mount system call.
+
+    :param source: The source of the file system (e.g., device name, remote filesystem).
+    :param target: The mount point (an existing directory).
+    :param filesystemtype: The filesystem type (e.g., "ext4", "nfs").
+    :param mountflags: Mount options flags.
+    :param data: File system-specific data (e.g., options like "rw").
+    :raises OSError: If the mount system call fails.
+    """
+    # Convert Python strings to C-compatible strings
+    source_c = ctypes.c_char_p(str(source).encode("utf-8"))
+    target_c = ctypes.c_char_p(str(target).encode("utf-8"))
+    fstype_c = ctypes.c_char_p(filesystemtype.encode("utf-8"))
+    data_c = ctypes.c_char_p(data.encode("utf-8")) if data else None
+
+    # Call the mount system call
+    result = libc.mount(
+        source_c, target_c, fstype_c, ctypes.c_ulong(mountflags), data_c
+    )
+
+    if result != 0:
+        errno = ctypes.get_errno()
+        raise OSError(errno, os.strerror(errno))
+
+
+class Error(Exception):
+    pass
+
 
 def prepare_machine_root(machinename: str, root: Path) -> None:
     root.mkdir(parents=True, exist_ok=True)
@@ -32,16 +90,25 @@ def retry(fn: Callable, timeout: int = 900) -> None:
         time.sleep(1)
 
     if not fn(True):
-        raise Exception(f"action timed out after {timeout} seconds")
+        msg = f"action timed out after {timeout} seconds"
+        raise Error(msg)
 
 
 class Machine:
-    def __init__(self, name: str, toplevel: Path, rootdir: Path, out_dir: str) -> None:
+    def __init__(
+        self,
+        name: str,
+        toplevel: Path,
+        logger: AbstractLogger,
+        rootdir: Path,
+        out_dir: str,
+    ) -> None:
         self.name = name
         self.toplevel = toplevel
         self.out_dir = out_dir
         self.process: subprocess.Popen | None = None
         self.rootdir: Path = rootdir
+        self.logger = logger
 
     def start(self) -> None:
         prepare_machine_root(self.name, self.rootdir)
@@ -54,7 +121,7 @@ class Machine:
             self.rootdir,
             "--register=no",
             "--resolv-conf=off",
-            "--bind-ro=/nix/store",
+            "--bind=/nix",
             "--bind",
             self.out_dir,
             "--bind=/proc:/run/host/proc",
@@ -72,30 +139,36 @@ class Machine:
         assert self.process.stdout is not None, "Machine has no stdout"
         for line in self.process.stdout:
             print(line, end="")
-            if line.startswith("systemd[1]: Startup finished in"):
+            if (
+                line.startswith("systemd[1]: Startup finished in")
+                or "Welcome to NixOS" in line
+            ):
                 break
         else:
-            raise RuntimeError(f"Failed to start container {self.name}")
+            msg = f"Failed to start container {self.name}"
+            raise RuntimeError(msg)
         childs = (
             Path(f"/proc/{self.process.pid}/task/{self.process.pid}/children")
             .read_text()
             .split()
         )
-        assert (
-            len(childs) == 1
-        ), f"Expected exactly one child process for systemd-nspawn, got {childs}"
+        assert len(childs) == 1, (
+            f"Expected exactly one child process for systemd-nspawn, got {childs}"
+        )
         try:
             return int(childs[0])
-        except ValueError:
-            raise RuntimeError(f"Failed to parse child process id {childs[0]}")
+        except ValueError as e:
+            msg = f"Failed to parse child process id {childs[0]}"
+            raise RuntimeError(msg) from e
 
     def get_unit_info(self, unit: str) -> dict[str, str]:
         proc = self.systemctl(f'--no-pager show "{unit}"')
         if proc.returncode != 0:
-            raise Exception(
+            msg = (
                 f'retrieving systemctl info for unit "{unit}"'
-                + f" failed with exit code {proc.returncode}"
+                f" failed with exit code {proc.returncode}"
             )
+            raise Error(msg)
 
         line_pattern = re.compile(r"^([^=]+)=(.*)$")
 
@@ -151,7 +224,7 @@ class Machine:
         """
 
         # Always run command with shell opts
-        command = f"set -euo pipefail; {command}"
+        command = f"set -eo pipefail; source /etc/profile; set -u; {command}"
 
         proc = subprocess.run(
             [
@@ -175,6 +248,15 @@ class Machine:
         )
         return proc
 
+    def nested(
+        self, msg: str, attrs: dict[str, str] | None = None
+    ) -> _GeneratorContextManager:
+        if attrs is None:
+            attrs = {}
+        my_attrs = {"machine": self.name}
+        my_attrs.update(attrs)
+        return self.logger.nested(msg, my_attrs)
+
     def systemctl(self, q: str) -> subprocess.CompletedProcess:
         """
         Runs `systemctl` commands with optional support for
@@ -191,6 +273,25 @@ class Machine:
         """
         return self.execute(f"systemctl {q}")
 
+    def wait_until_succeeds(self, command: str, timeout: int = 900) -> str:
+        """
+        Repeat a shell command with 1-second intervals until it succeeds.
+        Has a default timeout of 900 seconds which can be modified, e.g.
+        `wait_until_succeeds(cmd, timeout=10)`. See `execute` for details on
+        command execution.
+        Throws an exception on timeout.
+        """
+        output = ""
+
+        def check_success(_: Any) -> bool:
+            nonlocal output
+            result = self.execute(command, timeout=timeout)
+            return result.returncode == 0
+
+        with self.nested(f"waiting for success: {command}"):
+            retry(check_success, timeout)
+            return output
+
     def wait_for_unit(self, unit: str, timeout: int = 900) -> None:
         """
         Wait for a systemd unit to get into "active" state.
@@ -202,16 +303,18 @@ class Machine:
             info = self.get_unit_info(unit)
             state = info["ActiveState"]
             if state == "failed":
-                raise Exception(f'unit "{unit}" reached state "{state}"')
+                proc = self.systemctl(f"--lines 0 status {unit}")
+                journal = self.execute(f"journalctl -u {unit} --no-pager")
+                msg = f'unit "{unit}" reached state "{state}":\n{proc.stdout}\n{journal.stdout}'
+                raise Error(msg)
 
             if state == "inactive":
                 proc = self.systemctl("list-jobs --full 2>&1")
                 if "No jobs" in proc.stdout:
                     info = self.get_unit_info(unit)
                     if info["ActiveState"] == state:
-                        raise Exception(
-                            f'unit "{unit}" is inactive and there are no pending jobs'
-                        )
+                        msg = f'unit "{unit}" is inactive and there are no pending jobs'
+                        raise Error(msg)
 
             return state == "active"
 
@@ -220,7 +323,10 @@ class Machine:
     def succeed(self, command: str, timeout: int | None = None) -> str:
         res = self.execute(command, timeout=timeout)
         if res.returncode != 0:
-            raise RuntimeError(f"Failed to run command {command}")
+            msg = f"Failed to run command {command}\n"
+            msg += f"Exit code: {res.returncode}\n"
+            msg += f"Stdout: {res.stdout}"
+            raise RuntimeError(msg)
         return res.stdout
 
     def shutdown(self) -> None:
@@ -236,6 +342,12 @@ class Machine:
         self.shutdown()
 
 
+NIX_DIR = Path("/nix")
+NIX_STORE = Path("/nix/store/")
+NEW_NIX_DIR = Path("/.nix-rw")
+NEW_NIX_STORE_DIR = NEW_NIX_DIR / "store"
+
+
 def setup_filesystems() -> None:
     # We don't care about cleaning up the mount points, since we're running in a nix sandbox.
     Path("/run").mkdir(parents=True, exist_ok=True)
@@ -244,30 +356,70 @@ def setup_filesystems() -> None:
     Path("/etc").chmod(0o755)
     Path("/etc/os-release").touch()
     Path("/etc/machine-id").write_text("a5ea3f98dedc0278b6f3cc8c37eeaeac")
+    NEW_NIX_STORE_DIR.mkdir(parents=True)
+    # Read /proc/mounts and replicate every bind mount
+    with Path("/proc/self/mounts").open() as f:
+        for line in f:
+            columns = line.split(" ")
+            source = Path(columns[1])
+            if source.parent != NIX_STORE:
+                continue
+            target = NEW_NIX_STORE_DIR / source.name
+            if source.is_dir():
+                target.mkdir()
+            else:
+                target.touch()
+            try:
+                mount(source, target, "none", MS_BIND)
+            except OSError as e:
+                msg = f"mount({source}, {target}) failed"
+                raise Error(msg) from e
+    out = Path(os.environ["out"])
+    (NEW_NIX_STORE_DIR / out.name).mkdir()
+    mount(NEW_NIX_DIR, NIX_DIR, "none", MS_BIND | MS_REC)
+
+
+def load_nix_db(closure_info: Path) -> None:
+    with (closure_info / "registration").open() as f:
+        subprocess.run(["nix-store", "--load-db"], stdin=f, check=True, text=True)
 
 
 class Driver:
-    def __init__(self, containers: list[Path], testscript: str, out_dir: str) -> None:
+    logger: AbstractLogger
+
+    def __init__(
+        self,
+        containers: list[tuple[Path, Path]],
+        logger: AbstractLogger,
+        testscript: str,
+        out_dir: str,
+    ) -> None:
         self.containers = containers
         self.testscript = testscript
         self.out_dir = out_dir
+        self.logger = logger
         setup_filesystems()
+        # TODO: this won't work for multiple containers
+        assert len(containers) == 1, "Only one container is supported at the moment"
+        load_nix_db(containers[0][1])
 
         self.tempdir = TemporaryDirectory()
         tempdir_path = Path(self.tempdir.name)
 
         self.machines = []
         for container in containers:
-            name_match = re.match(r".*-nixos-system-(.+)-(.+)", container.name)
+            name_match = re.match(r".*-nixos-system-(.+)-(.+)", container[0].name)
             if not name_match:
-                raise ValueError(f"Unable to extract hostname from {container.name}")
+                msg = f"Unable to extract hostname from {container[0].name}"
+                raise Error(msg)
             name = name_match.group(1)
             self.machines.append(
                 Machine(
                     name=name,
-                    toplevel=container,
+                    toplevel=container[0],
                     rootdir=tempdir_path / name,
                     out_dir=self.out_dir,
+                    logger=self.logger,
                 )
             )
 
@@ -276,12 +428,12 @@ class Driver:
             machine.start()
 
     def test_symbols(self) -> dict[str, Any]:
-        general_symbols = dict(
-            start_all=self.start_all,
-            machines=self.machines,
-            driver=self,
-            Machine=Machine,  # for typing
-        )
+        general_symbols = {
+            "start_all": self.start_all,
+            "machines": self.machines,
+            "driver": self,
+            "Machine": Machine,  # for typing
+        }
         machine_symbols = {pythonize_name(m.name): m for m in self.machines}
         # If there's exactly one machine, make it available under the name
         # "machine", even if it's not called that.
@@ -289,7 +441,7 @@ class Driver:
             (machine_symbols["machine"],) = self.machines
         print(
             "additionally exposed symbols:\n    "
-            + ", ".join(map(lambda m: m.name, self.machines))
+            + ", ".join(m.name for m in self.machines)
             + ",\n    "
             + ", ".join(list(general_symbols.keys()))
         )
@@ -306,7 +458,12 @@ class Driver:
     def __enter__(self) -> "Driver":
         return self
 
-    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: types.TracebackType | None,
+    ) -> None:
         for machine in self.machines:
             machine.release()
 
@@ -319,9 +476,11 @@ def writeable_dir(arg: str) -> Path:
     """
     path = Path(arg)
     if not path.is_dir():
-        raise argparse.ArgumentTypeError(f"{path} is not a directory")
+        msg = f"{path} is not a directory"
+        raise argparse.ArgumentTypeError(msg)
     if not os.access(path, os.W_OK):
-        raise argparse.ArgumentTypeError(f"{path} is not a writeable directory")
+        msg = f"{path} is not a writeable directory"
+        raise argparse.ArgumentTypeError(msg)
     return path
 
 
@@ -329,9 +488,11 @@ def main() -> None:
     arg_parser = argparse.ArgumentParser(prog="nixos-test-driver")
     arg_parser.add_argument(
         "--containers",
-        nargs="+",
+        nargs=2,
+        action="append",
         type=Path,
-        help="container system toplevel paths",
+        metavar=("TOPLEVEL_STORE_DIR", "CLOSURE_INFO"),
+        help="container system toplevel store dir and closure info",
     )
     arg_parser.add_argument(
         "--test-script",
@@ -346,9 +507,11 @@ def main() -> None:
         type=writeable_dir,
     )
     args = arg_parser.parse_args()
+    logger = CompositeLogger([TerminalLogger()])
     with Driver(
-        args.containers,
-        args.test_script.read_text(),
-        args.output_directory.resolve(),
+        containers=args.containers,
+        testscript=args.test_script.read_text(),
+        out_dir=args.output_directory.resolve(),
+        logger=logger,
     ) as driver:
         driver.run_tests()

checks/lib/container-driver/test_driver/logger.py

@@ -0,0 +1,335 @@
+import atexit
+import codecs
+import os
+import sys
+import time
+import unicodedata
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from contextlib import ExitStack, contextmanager
+from pathlib import Path
+from queue import Empty, Queue
+from typing import Any
+from xml.sax.saxutils import XMLGenerator
+from xml.sax.xmlreader import AttributesImpl
+
+from colorama import Fore, Style
+from junit_xml import TestCase, TestSuite
+
+
+class AbstractLogger(ABC):
+    @abstractmethod
+    def log(self, message: str, attributes: dict[str, str] | None = None) -> None:
+        pass
+
+    @abstractmethod
+    @contextmanager
+    def subtest(
+        self, name: str, attributes: dict[str, str] | None = None
+    ) -> Iterator[None]:
+        pass
+
+    @abstractmethod
+    @contextmanager
+    def nested(
+        self, message: str, attributes: dict[str, str] | None = None
+    ) -> Iterator[None]:
+        pass
+
+    @abstractmethod
+    def info(self, *args: Any, **kwargs: Any) -> None:  # type: ignore
+        pass
+
+    @abstractmethod
+    def warning(self, *args: Any, **kwargs: Any) -> None:  # type: ignore
+        pass
+
+    @abstractmethod
+    def error(self, *args: Any, **kwargs: Any) -> None:  # type: ignore
+        pass
+
+    @abstractmethod
+    def log_serial(self, message: str, machine: str) -> None:
+        pass
+
+    @abstractmethod
+    def print_serial_logs(self, enable: bool) -> None:
+        pass
+
+
+class JunitXMLLogger(AbstractLogger):
+    class TestCaseState:
+        def __init__(self) -> None:
+            self.stdout = ""
+            self.stderr = ""
+            self.failure = False
+
+    def __init__(self, outfile: Path) -> None:
+        self.tests: dict[str, JunitXMLLogger.TestCaseState] = {
+            "main": self.TestCaseState()
+        }
+        self.currentSubtest = "main"
+        self.outfile: Path = outfile
+        self._print_serial_logs = True
+        atexit.register(self.close)
+
+    def log(self, message: str, attributes: dict[str, str] | None = None) -> None:
+        self.tests[self.currentSubtest].stdout += message + os.linesep
+
+    @contextmanager
+    def subtest(
+        self, name: str, attributes: dict[str, str] | None = None
+    ) -> Iterator[None]:
+        old_test = self.currentSubtest
+        self.tests.setdefault(name, self.TestCaseState())
+        self.currentSubtest = name
+
+        yield
+
+        self.currentSubtest = old_test
+
+    @contextmanager
+    def nested(
+        self, message: str, attributes: dict[str, str] | None = None
+    ) -> Iterator[None]:
+        self.log(message)
+        yield
+
+    def info(self, *args: Any, **kwargs: Any) -> None:
+        self.tests[self.currentSubtest].stdout += args[0] + os.linesep
+
+    def warning(self, *args: Any, **kwargs: Any) -> None:
+        self.tests[self.currentSubtest].stdout += args[0] + os.linesep
+
+    def error(self, *args: Any, **kwargs: Any) -> None:
+        self.tests[self.currentSubtest].stderr += args[0] + os.linesep
+        self.tests[self.currentSubtest].failure = True
+
+    def log_serial(self, message: str, machine: str) -> None:
+        if not self._print_serial_logs:
+            return
+
+        self.log(f"{machine} # {message}")
+
+    def print_serial_logs(self, enable: bool) -> None:
+        self._print_serial_logs = enable
+
+    def close(self) -> None:
+        with Path.open(self.outfile, "w") as f:
+            test_cases = []
+            for name, test_case_state in self.tests.items():
+                tc = TestCase(
+                    name,
+                    stdout=test_case_state.stdout,
+                    stderr=test_case_state.stderr,
+                )
+                if test_case_state.failure:
+                    tc.add_failure_info("test case failed")
+
+                test_cases.append(tc)
+            ts = TestSuite("NixOS integration test", test_cases)
+            f.write(TestSuite.to_xml_string([ts]))
+
+
+class CompositeLogger(AbstractLogger):
+    def __init__(self, logger_list: list[AbstractLogger]) -> None:
+        self.logger_list = logger_list
+
+    def add_logger(self, logger: AbstractLogger) -> None:
+        self.logger_list.append(logger)
+
+    def log(self, message: str, attributes: dict[str, str] | None = None) -> None:
+        for logger in self.logger_list:
+            logger.log(message, attributes)
+
+    @contextmanager
+    def subtest(
+        self, name: str, attributes: dict[str, str] | None = None
+    ) -> Iterator[None]:
+        with ExitStack() as stack:
+            for logger in self.logger_list:
+                stack.enter_context(logger.subtest(name, attributes))
+            yield
+
+    @contextmanager
+    def nested(
+        self, message: str, attributes: dict[str, str] | None = None
+    ) -> Iterator[None]:
+        with ExitStack() as stack:
+            for logger in self.logger_list:
+                stack.enter_context(logger.nested(message, attributes))
+            yield
+
+    def info(self, *args: Any, **kwargs: Any) -> None:  # type: ignore
+        for logger in self.logger_list:
+            logger.info(*args, **kwargs)
+
+    def warning(self, *args: Any, **kwargs: Any) -> None:  # type: ignore
+        for logger in self.logger_list:
+            logger.warning(*args, **kwargs)
+
+    def error(self, *args: Any, **kwargs: Any) -> None:  # type: ignore
+        for logger in self.logger_list:
+            logger.error(*args, **kwargs)
+        sys.exit(1)
+
+    def print_serial_logs(self, enable: bool) -> None:
+        for logger in self.logger_list:
+            logger.print_serial_logs(enable)
+
+    def log_serial(self, message: str, machine: str) -> None:
+        for logger in self.logger_list:
+            logger.log_serial(message, machine)
+
+
+class TerminalLogger(AbstractLogger):
+    def __init__(self) -> None:
+        self._print_serial_logs = True
+
+    def maybe_prefix(self, message: str, attributes: dict[str, str] | None) -> str:
+        if attributes and "machine" in attributes:
+            return f"{attributes['machine']}: {message}"
+        return message
+
+    @staticmethod
+    def _eprint(*args: object, **kwargs: Any) -> None:
+        print(*args, file=sys.stderr, **kwargs)
+
+    def log(self, message: str, attributes: dict[str, str] | None = None) -> None:
+        self._eprint(self.maybe_prefix(message, attributes))
+
+    @contextmanager
+    def subtest(
+        self, name: str, attributes: dict[str, str] | None = None
+    ) -> Iterator[None]:
+        with self.nested("subtest: " + name, attributes):
+            yield
+
+    @contextmanager
+    def nested(
+        self, message: str, attributes: dict[str, str] | None = None
+    ) -> Iterator[None]:
+        self._eprint(
+            self.maybe_prefix(
+                Style.BRIGHT + Fore.GREEN + message + Style.RESET_ALL, attributes
+            )
+        )
+
+        tic = time.time()
+        yield
+        toc = time.time()
+        self.log(f"(finished: {message}, in {toc - tic:.2f} seconds)")
+
+    def info(self, *args: Any, **kwargs: Any) -> None:  # type: ignore
+        self.log(*args, **kwargs)
+
+    def warning(self, *args: Any, **kwargs: Any) -> None:  # type: ignore
+        self.log(*args, **kwargs)
+
+    def error(self, *args: Any, **kwargs: Any) -> None:  # type: ignore
+        self.log(*args, **kwargs)
+
+    def print_serial_logs(self, enable: bool) -> None:
+        self._print_serial_logs = enable
+
+    def log_serial(self, message: str, machine: str) -> None:
+        if not self._print_serial_logs:
+            return
+
+        self._eprint(Style.DIM + f"{machine} # {message}" + Style.RESET_ALL)
+
+
+class XMLLogger(AbstractLogger):
+    def __init__(self, outfile: str) -> None:
+        self.logfile_handle = codecs.open(outfile, "wb")  # noqa: SIM115
+        self.xml = XMLGenerator(self.logfile_handle, encoding="utf-8")
+        self.queue: Queue[dict[str, str]] = Queue()
+
+        self._print_serial_logs = True
+
+        self.xml.startDocument()
+        self.xml.startElement("logfile", attrs=AttributesImpl({}))
+
+    def close(self) -> None:
+        self.xml.endElement("logfile")
+        self.xml.endDocument()
+        self.logfile_handle.close()
+
+    def sanitise(self, message: str) -> str:
+        return "".join(ch for ch in message if unicodedata.category(ch)[0] != "C")
+
+    def maybe_prefix(
+        self, message: str, attributes: dict[str, str] | None = None
+    ) -> str:
+        if attributes and "machine" in attributes:
+            return f"{attributes['machine']}: {message}"
+        return message
+
+    def log_line(self, message: str, attributes: dict[str, str]) -> None:
+        self.xml.startElement("line", attrs=AttributesImpl(attributes))
+        self.xml.characters(message)
+        self.xml.endElement("line")
+
+    def info(self, *args: Any, **kwargs: Any) -> None:  # type: ignore
+        self.log(*args, **kwargs)
+
+    def warning(self, *args: Any, **kwargs: Any) -> None:  # type: ignore
+        self.log(*args, **kwargs)
+
+    def error(self, *args: Any, **kwargs: Any) -> None:  # type: ignore
+        self.log(*args, **kwargs)
+
+    def log(self, message: str, attributes: dict[str, str] | None = None) -> None:
+        if attributes is None:
+            attributes = {}
+        self.drain_log_queue()
+        self.log_line(message, attributes)
+
+    def print_serial_logs(self, enable: bool) -> None:
+        self._print_serial_logs = enable
+
+    def log_serial(self, message: str, machine: str) -> None:
+        if not self._print_serial_logs:
+            return
+
+        self.enqueue({"msg": message, "machine": machine, "type": "serial"})
+
+    def enqueue(self, item: dict[str, str]) -> None:
+        self.queue.put(item)
+
+    def drain_log_queue(self) -> None:
+        try:
+            while True:
+                item = self.queue.get_nowait()
+                msg = self.sanitise(item["msg"])
+                del item["msg"]
+                self.log_line(msg, item)
+        except Empty:
+            pass
+
+    @contextmanager
+    def subtest(
+        self, name: str, attributes: dict[str, str] | None = None
+    ) -> Iterator[None]:
+        with self.nested("subtest: " + name, attributes):
+            yield
+
+    @contextmanager
+    def nested(
+        self, message: str, attributes: dict[str, str] | None = None
+    ) -> Iterator[None]:
+        if attributes is None:
+            attributes = {}
+        self.xml.startElement("nest", attrs=AttributesImpl({}))
+        self.xml.startElement("head", attrs=AttributesImpl(attributes))
+        self.xml.characters(message)
+        self.xml.endElement("head")
+
+        tic = time.time()
+        self.drain_log_queue()
+        yield
+        self.drain_log_queue()
+        toc = time.time()
+        self.log(f"(finished: {message}, in {toc - tic:.2f} seconds)")
+
+        self.xml.endElement("nest")

checks/lib/container-test.nix

@@ -10,9 +10,15 @@ in
     hostPkgs = pkgs;
     # speed-up evaluation
     defaults = {
+      imports = [
+        ./minify.nix
+      ];
       documentation.enable = lib.mkDefault false;
       boot.isContainer = true;
 
+      # needed since nixpkgs 7fb2f407c01b017737eafc26b065d7f56434a992 removed the getty unit by default
+      console.enable = true;
+
       # undo qemu stuff
       system.build.initialRamdisk = "";
       virtualisation.sharedDirectories = lib.mkForce { };
@@ -22,9 +28,13 @@ in
       networking.interfaces = lib.mkForce { };
       #networking.primaryIPAddress = lib.mkForce null;
       systemd.services.backdoor.enable = false;
+
+      # we don't have permission to set cpu scheduler in our container
+      systemd.services.nix-daemon.serviceConfig.CPUSchedulingPolicy = lib.mkForce "";
     };
     # to accept external dependencies such as disko
     node.specialArgs.self = self;
+    _module.args = { inherit self; };
     imports = [
       test
       ./container-driver/module.nix

checks/lib/minify.nix

@@ -0,0 +1,8 @@
+{ lib, ... }:
+{
+  nixpkgs.flake.setFlakeRegistry = false;
+  nixpkgs.flake.setNixPath = false;
+  nix.registry = lib.mkForce { };
+  documentation.doc.enable = false;
+  documentation.man.enable = false;
+}

checks/lib/test-base.nix

@@ -7,11 +7,19 @@ in
 (nixos-lib.runTest {
   hostPkgs = pkgs;
   # speed-up evaluation
-  defaults = {
-    documentation.enable = lib.mkDefault false;
-    nix.settings.min-free = 0;
-  };
+  defaults = (
+    { config, ... }:
+    {
+      imports = [
+        ./minify.nix
+      ];
+      documentation.enable = lib.mkDefault false;
+      nix.settings.min-free = 0;
+      system.stateVersion = config.system.nixos.release;
+    }
+  );
 
+  _module.args = { inherit self; };
   # to accept external dependencies such as disko
   node.specialArgs.self = self;
   imports = [ test ];

checks/matrix-synapse/default.nix

@@ -4,26 +4,67 @@
     name = "matrix-synapse";
 
     nodes.machine =
-      { self, lib, ... }:
+      {
+        config,
+        self,
+        lib,
+        ...
+      }:
       {
         imports = [
           self.clanModules.matrix-synapse
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
-            clan.matrix-synapse = {
-              enable = true;
-              domain = "clan.test";
-            };
-          }
-          {
-            # secret override
-            clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path = "${./synapse-registration_shared_secret}";
+            clan.core.settings.machine.name = "machine";
+            clan.core.settings.directory = ./.;
+
             services.nginx.virtualHosts."matrix.clan.test" = {
               enableACME = lib.mkForce false;
               forceSSL = lib.mkForce false;
             };
+            clan.nginx.acme.email = "admins@clan.lol";
+            clan.matrix-synapse = {
+              server_tld = "clan.test";
+              app_domain = "matrix.clan.test";
+            };
+            clan.matrix-synapse.users.admin.admin = true;
+            clan.matrix-synapse.users.someuser = { };
+
+            clan.core.facts.secretStore = "vm";
+            clan.core.vars.settings.secretStore = "vm";
+            clan.core.vars.settings.publicStore = "in_repo";
+
+            # because we use systemd-tmpfiles to copy the secrets, we need to a separate systemd-tmpfiles call to provision them.
+            boot.postBootCommands = "${config.systemd.package}/bin/systemd-tmpfiles --create /etc/tmpfiles.d/00-vmsecrets.conf";
+
+            systemd.tmpfiles.settings."00-vmsecrets" = {
+              # run before 00-nixos.conf
+              "/etc/secrets" = {
+                d.mode = "0700";
+                z.mode = "0700";
+              };
+              "/etc/secrets/matrix-synapse/synapse-registration_shared_secret" = {
+                f.argument = "supersecret";
+                z = {
+                  mode = "0400";
+                  user = "root";
+                };
+              };
+              "/etc/secrets/matrix-password-admin/matrix-password-admin" = {
+                f.argument = "matrix-password1";
+                z = {
+                  mode = "0400";
+                  user = "root";
+                };
+              };
+              "/etc/secrets/matrix-password-someuser/matrix-password-someuser" = {
+                f.argument = "matrix-password2";
+                z = {
+                  mode = "0400";
+                  user = "root";
+                };
+              };
+            };
           }
         ];
       };
@@ -31,6 +72,12 @@
       start_all()
       machine.wait_for_unit("matrix-synapse")
       machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 8008")
+      machine.wait_until_succeeds("${pkgs.curl}/bin/curl -Ssf -L http://localhost/_matrix/static/ -H 'Host: matrix.clan.test'")
+
+      machine.systemctl("restart matrix-synapse >&2") # check if user creation is idempotent
+      machine.execute("journalctl -u matrix-synapse --no-pager >&2")
+      machine.wait_for_unit("matrix-synapse")
+      machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 8008")
       machine.succeed("${pkgs.curl}/bin/curl -Ssf -L http://localhost/_matrix/static/ -H 'Host: matrix.clan.test'")
     '';
   }

checks/morph/flake-module.nix

@@ -0,0 +1,62 @@
+{
+  self,
+  ...
+}:
+{
+  clan.machines.test-morph-machine = {
+    imports = [
+      ./template/configuration.nix
+      self.nixosModules.clanCore
+    ];
+    nixpkgs.hostPlatform = "x86_64-linux";
+    environment.etc."testfile".text = "morphed";
+  };
+
+  clan.templates.machine.test-morph-template = {
+    description = "Morph a machine";
+    path = ./template;
+  };
+
+  perSystem =
+    {
+      pkgs,
+      ...
+    }:
+    {
+      checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux && !pkgs.stdenv.isAarch64) {
+        test-morph = (import ../lib/test-base.nix) {
+          name = "morph";
+
+          nodes = {
+            actual =
+              { pkgs, ... }:
+              let
+                dependencies = [
+                  self
+                  pkgs.nixos-anywhere
+                  pkgs.stdenv.drvPath
+                  pkgs.stdenvNoCC
+                  self.nixosConfigurations.test-morph-machine.config.system.build.toplevel
+                  self.nixosConfigurations.test-morph-machine.config.system.clan.deployment.file
+                ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
+                closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
+              in
+
+              {
+                environment.etc."install-closure".source = "${closureInfo}/store-paths";
+                system.extraDependencies = dependencies;
+                virtualisation.memorySize = 2048;
+                environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
+              };
+          };
+          testScript = ''
+            start_all()
+            actual.fail("cat /etc/testfile")
+            actual.succeed("env CLAN_DIR=${self} clan machines morph test-morph-template --i-will-be-fired-for-using-this --debug --name test-morph-machine")
+            assert actual.succeed("cat /etc/testfile") == "morphed"
+          '';
+        } { inherit pkgs self; };
+      };
+
+    };
+}

checks/morph/template/configuration.nix

@@ -0,0 +1,12 @@
+{ modulesPath, ... }:
+{
+  imports = [
+    # we need these 2 modules always to be able to run the tests
+    (modulesPath + "/testing/test-instrumentation.nix")
+    (modulesPath + "/virtualisation/qemu-vm.nix")
+
+    (modulesPath + "/profiles/minimal.nix")
+  ];
+
+  clan.core.enableRecommendedDefaults = false;
+}

checks/mumble/default.nix

@@ -0,0 +1,145 @@
+(import ../lib/test-base.nix) (
+  { ... }:
+  let
+    common =
+      { self, pkgs, ... }:
+      {
+        imports = [
+          self.clanModules.mumble
+          {
+            clan.services.mumble.user = "alice";
+          }
+          self.nixosModules.clanCore
+          (self.inputs.nixpkgs + "/nixos/tests/common/x11.nix")
+          {
+            clan.core.settings.directory = ./.;
+            environment.systemPackages = [ pkgs.killall ];
+            clan.core.facts.services.mumble.secret."mumble-key".path = "/etc/mumble-key";
+            clan.core.facts.services.mumble.public."mumble-cert".path = "/etc/mumble-cert";
+          }
+        ];
+
+      };
+  in
+  {
+    name = "mumble";
+
+    enableOCR = true;
+
+    nodes.peer1 =
+      { ... }:
+      {
+        imports = [
+          common
+          {
+            clan.core.settings.machine.name = "peer1";
+            environment.etc = {
+              "mumble-key".source = ./peer_1/peer_1_test_key;
+              "mumble-cert".source = ./peer_1/peer_1_test_cert;
+            };
+            systemd.tmpfiles.settings."vmsecrets" = {
+              "/var/lib/murmur/sslKey" = {
+                C.argument = "${./peer_1/peer_1_test_key}";
+                z = {
+                  mode = "0400";
+                  user = "murmur";
+                };
+              };
+              "/var/lib/murmur/sslCert" = {
+                C.argument = "${./peer_1/peer_1_test_cert}";
+                z = {
+                  mode = "0400";
+                  user = "murmur";
+                };
+              };
+            };
+            clan.core.facts.services.mumble.secret."mumble-key".path = "/etc/mumble-key";
+            clan.core.facts.services.mumble.public."mumble-cert".path = "/etc/mumble-cert";
+          }
+        ];
+      };
+    nodes.peer2 =
+      { ... }:
+      {
+        imports = [
+          common
+          {
+            clan.core.settings.machine.name = "peer2";
+            environment.etc = {
+              "mumble-key".source = ./peer_2/peer_2_test_key;
+              "mumble-cert".source = ./peer_2/peer_2_test_cert;
+            };
+            systemd.tmpfiles.settings."vmsecrets" = {
+              "/var/lib/murmur/sslKey" = {
+                C.argument = "${./peer_2/peer_2_test_key}";
+                z = {
+                  mode = "0400";
+                  user = "murmur";
+                };
+              };
+              "/var/lib/murmur/sslCert" = {
+                C.argument = "${./peer_2/peer_2_test_cert}";
+                z = {
+                  mode = "0400";
+                  user = "murmur";
+                };
+              };
+            };
+          }
+        ];
+      };
+    testScript = ''
+      start_all()
+
+      with subtest("Waiting for x"):
+        peer1.wait_for_x()
+        peer2.wait_for_x()
+
+      with subtest("Waiting for murmur"):
+        peer1.wait_for_unit("murmur.service")
+        peer2.wait_for_unit("murmur.service")
+
+      with subtest("Starting Mumble"):
+        # starting mumble is blocking
+        peer1.execute("mumble >&2 &")
+        peer2.execute("mumble >&2 &")
+
+      with subtest("Wait for Mumble"):
+        peer1.wait_for_window(r"^Mumble$")
+        peer2.wait_for_window(r"^Mumble$")
+
+      with subtest("Wait for certificate creation"):
+        peer1.wait_for_window(r"^Mumble$")
+        peer1.sleep(3) # mumble is slow to register handlers
+        peer1.send_chars("\n")
+        peer1.send_chars("\n")
+        peer2.wait_for_window(r"^Mumble$")
+        peer2.sleep(3) # mumble is slow to register handlers
+        peer2.send_chars("\n")
+        peer2.send_chars("\n")
+
+      with subtest("Wait for server connect"):
+        peer1.wait_for_window(r"^Mumble Server Connect$")
+        peer2.wait_for_window(r"^Mumble Server Connect$")
+
+      with subtest("Check validity of server certificates"):
+        peer1.execute("killall .mumble-wrapped")
+        peer1.sleep(1)
+        peer1.execute("mumble mumble://peer2 >&2 &")
+        peer1.wait_for_window(r"^Mumble$")
+        peer1.sleep(3) # mumble is slow to register handlers
+        peer1.send_chars("\n")
+        peer1.send_chars("\n")
+        peer1.wait_for_text("Connected.")
+
+        peer2.execute("killall .mumble-wrapped")
+        peer2.sleep(1)
+        peer2.execute("mumble mumble://peer1 >&2 &")
+        peer2.wait_for_window(r"^Mumble$")
+        peer2.sleep(3) # mumble is slow to register handlers
+        peer2.send_chars("\n")
+        peer2.send_chars("\n")
+        peer2.wait_for_text("Connected.")
+    '';
+  }
+)

checks/mumble/machines/peer1/facts/mumble-cert

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDazCCAlOgAwIBAgIUCUjfNkF0CDhTKbO3nNczcsCW4qEwDQYJKoZIhvcNAQEL
+BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDA2MjcwOTM2NDZaFw0yNDA3
+MjcwOTM2NDZaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQDCcdZEJvXJIeOKO5pF5XUFvUeJtCCiwfWvWS662bxc
+R/5MZucRLqfTNYo9aBv4NITw5kxZsTaaubmS4zSGQoTEAVzqzVdi3a/gNvsdVLb+
+7CivpmweLllX/OGsTL0kHPEI+74AYiTBjXfdWV1Y5T1tuwc3G8ATrguQ33Uo5vvF
+vcqsbTKcRZC0pB9O/nn4q03GsRdvlpaKakIhjMpRG/uZ3u7wtbyZ+WqjsjxZNfnY
+aMyPoaipFqX1v+L7GKlOj2NpyEZFVVwa2ZqhVSYXyDfpAWQFznwKGzD5mjtcyKym
+gnv/5LwrpH4Xj+JMt48hN+rPnu5vfXT8Y4KnID30OQW7AgMBAAGjUzBRMB0GA1Ud
+DgQWBBQBBO8Wp975pAGioMjkaxANAVInfzAfBgNVHSMEGDAWgBQBBO8Wp975pAGi
+oMjkaxANAVInfzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAg
+F40MszTZXpR/A1z9B1CcXH47tNK67f8bCMR2dhvXODbpatwSihyxhQjtLb5R6kYH
+5Yq/B4yrh303j0CXaobCQ4nQH7zI7fhViww+TzW7vDhgM7ueEyyXrqCXt6JY8avg
+TuvIRtJSeWSQJ5aLNaYqmiwMf/tj9W3BMDpctGyLqu1WTSrbpYa9mA5Vudud70Yz
+DgZ/aqHilB07cVNqzVYZzRZ56WJlTjGzVevRgnHZqPiZNVrU13H6gtWa3r8aV4Gj
+i4F663eRAttj166cRgfl1QqpSG2IprNyV9UfuS2LlUaVNT3y0idawiJ4HhaA8pGB
+ZqMUUkA4DSucb6xxEcTK
+-----END CERTIFICATE-----
+

checks/mumble/machines/peer1/key.age

@@ -0,0 +1 @@
+AGE-SECRET-KEY-1UCXEUJH6JXF8LFKWFHDM4N9AQE2CCGQZGXLUNV4TKR5KY0KC8FDQ2TY4NX

checks/mumble/machines/peer1/peer_1_test_cert

@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICHTCCAaKgAwIBAgIIT2gZuvqVFP0wCgYIKoZIzj0EAwIwSjESMBAGA1UEChMJ
+U3luY3RoaW5nMSAwHgYDVQQLExdBdXRvbWF0aWNhbGx5IEdlbmVyYXRlZDESMBAG
+A1UEAxMJc3luY3RoaW5nMB4XDTIzMTIwNjAwMDAwMFoXDTQzMTIwMTAwMDAwMFow
+SjESMBAGA1UEChMJU3luY3RoaW5nMSAwHgYDVQQLExdBdXRvbWF0aWNhbGx5IEdl
+bmVyYXRlZDESMBAGA1UEAxMJc3luY3RoaW5nMHYwEAYHKoZIzj0CAQYFK4EEACID
+YgAEBAr1CsciwCa0vi7eC6xxuSGijY3txbjtsyFanec/fge4oJBD3rVpaLKFETb3
+TvHHsuvblzElcP483MEVq6FMUoxwuL9CzTtpJrRhtwSmAs8AHLFu8irVn8sZjgkL
+sXMho1UwUzAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG
+AQUFBwMCMAwGA1UdEwEB/wQCMAAwFAYDVR0RBA0wC4IJc3luY3RoaW5nMAoGCCqG
+SM49BAMCA2kAMGYCMQDbrtLgfcyMMIkNQn+PJe9DHYAqj8C47LQcWuIY/nekhOu0
+aUfKctEAwyBtI60Y5zcCMQCEdgD/6CNBh7Qqq3z3CKPhlrpxHtCO5tNw17k0jfdH
+haCwJInHZvZgclHk4EtFpTw=
+-----END CERTIFICATE-----

checks/mumble/machines/peer1/peer_1_test_key

@@ -0,0 +1,6 @@
+-----BEGIN EC PRIVATE KEY-----
+MIGkAgEBBDA14Nqo17Xs/xRLGH2KLuyzjKp4eW9iWFobVNM93RZZbECT++W3XcQc
+cEc5WVtiPmWgBwYFK4EEACKhZANiAAQECvUKxyLAJrS+Lt4LrHG5IaKNje3FuO2z
+IVqd5z9+B7igkEPetWlosoURNvdO8cey69uXMSVw/jzcwRWroUxSjHC4v0LNO2km
+tGG3BKYCzwAcsW7yKtWfyxmOCQuxcyE=
+-----END EC PRIVATE KEY-----

checks/mumble/machines/peer2/facts/mumble-cert

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDazCCAlOgAwIBAgIUfENbTtH5nr7giuawwQpDYqUpWJswDQYJKoZIhvcNAQEL
+BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDA2MjcwOTQxNDNaFw0yNDA3
+MjcwOTQxNDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQCfP6cZhCs9jOnWqyQP12vrOOxlBrWofYZFf9amUA24
+AfE7oGcSfkylanmkxzvGqQkhgLAvkHZj/GEvHujKyy8PgcEGP+pwmsfWNQMvU0Dz
+j3syjWOTi3eIC/3DoUnHlWCT2qCil/bjqxgU1l7fO/OXUlq5kyvIjln7Za4sUHun
+ixe/m96Er6l8a4Mh2pxh2C5pkLCvulkQhjjGG+R6MccH8wwQwmLg5oVBkFEZrnRE
+pnRKBI0DvA+wk1aJFAPOI4d8Q5T7o/MyxH3f8TYGHqbeMQFCKwusnlWPRtrNdaIc
+gaLvSpR0LVlroXGu8tYmRpvHPByoKGDbgVvO0Bwx8fmRAgMBAAGjUzBRMB0GA1Ud
+DgQWBBR7r+mQWNUZ0TpQNwrwjgxgngvOjTAfBgNVHSMEGDAWgBR7r+mQWNUZ0TpQ
+NwrwjgxgngvOjTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCO
+7B4s6uQEGE8jg3CQgy76oU/D8sazGcP8+/E4JLHSc0Nj49w4ztSpkOVk2HyEtzbm
+uR3TreIw+SfqpbiOI/ivVNDbEBsb/vEeq7qPzDH1Bi72plHZNRVhNGGV5rd7ibga
+TkfXHKPM9yt8ffffHHiu1ROvb8gg2B6JbQwboU4hvvmmorW7onyTFSYEzZVdNSpv
+pUtKPldxYjTnLlbsJdXC4xyCC4PrJt2CC0n0jsWfICJ77LMxIxTODh8oZNjbPg6r
+RdI7U/DsD+R072DjbIcrivvigotJM+jihzz5inZwbO8o0WQOHAbJLIG3C3BnRW3A
+Ek4u3+HXZMl5a0LGJ76u
+-----END CERTIFICATE-----
+

checks/mumble/machines/peer2/peer_2_test_cert

@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICHjCCAaOgAwIBAgIJAKbMWefkf1rVMAoGCCqGSM49BAMCMEoxEjAQBgNVBAoT
+CVN5bmN0aGluZzEgMB4GA1UECxMXQXV0b21hdGljYWxseSBHZW5lcmF0ZWQxEjAQ
+BgNVBAMTCXN5bmN0aGluZzAeFw0yMzEyMDYwMDAwMDBaFw00MzEyMDEwMDAwMDBa
+MEoxEjAQBgNVBAoTCVN5bmN0aGluZzEgMB4GA1UECxMXQXV0b21hdGljYWxseSBH
+ZW5lcmF0ZWQxEjAQBgNVBAMTCXN5bmN0aGluZzB2MBAGByqGSM49AgEGBSuBBAAi
+A2IABFZTMt4RfsfBue0va7QuNdjfXMI4HfZzJCEcG+b9MtV7FlDmwMKX5fgGykD9
+FBbC7yiza3+xCobdMb5bakz1qYJ7nUFCv1mwSDo2eNM+/XE+rJmlre8NwkwGmvzl
+h1uhyqNVMFMwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggr
+BgEFBQcDAjAMBgNVHRMBAf8EAjAAMBQGA1UdEQQNMAuCCXN5bmN0aGluZzAKBggq
+hkjOPQQDAgNpADBmAjEAwzhsroN6R4/quWeXj6dO5gt5CfSTLkLee6vrcuIP5i1U
+rZvJ3OKQVmmGG6IWYe7iAjEAyuq3X2wznaqiw2YK3IDI4qVeYWpCUap0fwRNq7/x
+4dC4k+BOzHcuJOwNBIY/bEuK
+-----END CERTIFICATE-----

checks/mumble/machines/peer2/peer_2_test_key

@@ -0,0 +1,6 @@
+-----BEGIN EC PRIVATE KEY-----
+MIGkAgEBBDCXHGpvumKjjDRxB6SsjZOb7duw3w+rdlGQCJTIvRThLjD6zwjnyImi
+7c3PD5nWtLqgBwYFK4EEACKhZANiAARWUzLeEX7HwbntL2u0LjXY31zCOB32cyQh
+HBvm/TLVexZQ5sDCl+X4BspA/RQWwu8os2t/sQqG3TG+W2pM9amCe51BQr9ZsEg6
+NnjTPv1xPqyZpa3vDcJMBpr85Ydboco=
+-----END EC PRIVATE KEY-----

checks/mumble/peer_1/key.age

@@ -0,0 +1 @@
+AGE-SECRET-KEY-1UCXEUJH6JXF8LFKWFHDM4N9AQE2CCGQZGXLUNV4TKR5KY0KC8FDQ2TY4NX

checks/mumble/peer_1/peer_1_test_cert

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDazCCAlOgAwIBAgIUCUjfNkF0CDhTKbO3nNczcsCW4qEwDQYJKoZIhvcNAQEL
+BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDA2MjcwOTM2NDZaFw0yNDA3
+MjcwOTM2NDZaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQDCcdZEJvXJIeOKO5pF5XUFvUeJtCCiwfWvWS662bxc
+R/5MZucRLqfTNYo9aBv4NITw5kxZsTaaubmS4zSGQoTEAVzqzVdi3a/gNvsdVLb+
+7CivpmweLllX/OGsTL0kHPEI+74AYiTBjXfdWV1Y5T1tuwc3G8ATrguQ33Uo5vvF
+vcqsbTKcRZC0pB9O/nn4q03GsRdvlpaKakIhjMpRG/uZ3u7wtbyZ+WqjsjxZNfnY
+aMyPoaipFqX1v+L7GKlOj2NpyEZFVVwa2ZqhVSYXyDfpAWQFznwKGzD5mjtcyKym
+gnv/5LwrpH4Xj+JMt48hN+rPnu5vfXT8Y4KnID30OQW7AgMBAAGjUzBRMB0GA1Ud
+DgQWBBQBBO8Wp975pAGioMjkaxANAVInfzAfBgNVHSMEGDAWgBQBBO8Wp975pAGi
+oMjkaxANAVInfzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAg
+F40MszTZXpR/A1z9B1CcXH47tNK67f8bCMR2dhvXODbpatwSihyxhQjtLb5R6kYH
+5Yq/B4yrh303j0CXaobCQ4nQH7zI7fhViww+TzW7vDhgM7ueEyyXrqCXt6JY8avg
+TuvIRtJSeWSQJ5aLNaYqmiwMf/tj9W3BMDpctGyLqu1WTSrbpYa9mA5Vudud70Yz
+DgZ/aqHilB07cVNqzVYZzRZ56WJlTjGzVevRgnHZqPiZNVrU13H6gtWa3r8aV4Gj
+i4F663eRAttj166cRgfl1QqpSG2IprNyV9UfuS2LlUaVNT3y0idawiJ4HhaA8pGB
+ZqMUUkA4DSucb6xxEcTK
+-----END CERTIFICATE-----
+

checks/mumble/peer_1/peer_1_test_key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDCcdZEJvXJIeOK
+O5pF5XUFvUeJtCCiwfWvWS662bxcR/5MZucRLqfTNYo9aBv4NITw5kxZsTaaubmS
+4zSGQoTEAVzqzVdi3a/gNvsdVLb+7CivpmweLllX/OGsTL0kHPEI+74AYiTBjXfd
+WV1Y5T1tuwc3G8ATrguQ33Uo5vvFvcqsbTKcRZC0pB9O/nn4q03GsRdvlpaKakIh
+jMpRG/uZ3u7wtbyZ+WqjsjxZNfnYaMyPoaipFqX1v+L7GKlOj2NpyEZFVVwa2Zqh
+VSYXyDfpAWQFznwKGzD5mjtcyKymgnv/5LwrpH4Xj+JMt48hN+rPnu5vfXT8Y4Kn
+ID30OQW7AgMBAAECggEAGVKn+/Iy+kG+l2cRvV6XseqnoWhjA69M5swviMgIfuAl
+Xx/boeI4mwoS+dJQKi/0zEbB1MB+gwIDB/0s/vs0vS4MQswBQG/skr+2TmiU+Hgb
+CF0dIYUZv5rAbScFTumx/mCCqxwc+1QIMzyLKqOYL203EFc92ZJGEVT4th321haZ
+8Wd+dllcYAb7BbEeBhCrTqRe9T3zt5reZgtZTquTF5hGm8EAyBp6rLjZK7dyZ9dd
+gyIsDbWgPC9vkRc6x/eANn70hgDbYOuoXwAP/qIFnWLL1Zzy8LKUyOsSgQ91S3S3
+Il4Lt6lEyU3+61MsCYss7jDoP/7REEjz5h6gfxlFSQKBgQD9u8nhHuwte4/d9VNU
+rhSBW9h8IJzwPif/eS8vh9VaS2SjR2dDCcHg6rGYKnexeEzUcx56aQMA+p3nRJwy
+Uwnx5BfEWs9FO6yPR8VEI0a2sBp+hoWKJX/Lvat+QCs6IFuGmlQpczD7/RYAkhG4
+mwyt/ymqzjukb9mFaeYIltOfPwKBgQDELnkH1ChTUH5u3HgDoelFbzR18okz6dxH
+urMbfZMAl8W5h2zAvHsAX5qxyHHankOUsiH2y3BrAgqQtTuIA2a5W7j+yHBkYiEZ
+EUNeI9YNA0KU+wwZpVVvRGUsRB5SUBo5LlcSYmX/V32f0oU5Np44i0vjl3Ju8esx
+2MLfj1A2hQKBgQDCxtZZZ0h8Pb8Z7wpSFfQNvXi5CLwQvFYuClQLk6VXVErkAJsn
+XiUjyGYeXnNVm/i2mcyKwXQZ20k90HBrPU2ED8mi5Ob5ya5Uqw6mmMHe2d7sw81d
+WB37RBWSrCXC0DYSZQQ4cYHn3sd2Fqtd4EBijV7qDLjCKU582OdKLqYzNwKBgH31
+UKQkJZgIkIThbPT4GewI0GgCRvFb76DmUGUQJTg2Oi86siq1WUwOFiabie5RuxZX
+oNLyH8W008/BbO2RMX1FVOvRCciJ8LJFkTl6TM6iDzfUUBqPOuFryoG3Yrh60btw
+81rMbqyZIgFhi0QGu2OWnC0Oadyt2tJwV/5t55R5AoGBAPspZttDmOzVkAJDSn9Z
+iByYt1KmwBQ6l7LpFg33a7ds9zWqW4+i6r0PzXvSewf/z69L0cAywSk5CaJJjDso
+dTlNMqwux01wd6V+nQGR871xnsOg+qzgJ565TJZelWgRmNRUooi4DMp5POJA33xp
+rqAISUfW0w2S+q7/5Lm0QiJE
+-----END PRIVATE KEY-----

checks/mumble/peer_2/peer_2_test_cert

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDazCCAlOgAwIBAgIUfENbTtH5nr7giuawwQpDYqUpWJswDQYJKoZIhvcNAQEL
+BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDA2MjcwOTQxNDNaFw0yNDA3
+MjcwOTQxNDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQCfP6cZhCs9jOnWqyQP12vrOOxlBrWofYZFf9amUA24
+AfE7oGcSfkylanmkxzvGqQkhgLAvkHZj/GEvHujKyy8PgcEGP+pwmsfWNQMvU0Dz
+j3syjWOTi3eIC/3DoUnHlWCT2qCil/bjqxgU1l7fO/OXUlq5kyvIjln7Za4sUHun
+ixe/m96Er6l8a4Mh2pxh2C5pkLCvulkQhjjGG+R6MccH8wwQwmLg5oVBkFEZrnRE
+pnRKBI0DvA+wk1aJFAPOI4d8Q5T7o/MyxH3f8TYGHqbeMQFCKwusnlWPRtrNdaIc
+gaLvSpR0LVlroXGu8tYmRpvHPByoKGDbgVvO0Bwx8fmRAgMBAAGjUzBRMB0GA1Ud
+DgQWBBR7r+mQWNUZ0TpQNwrwjgxgngvOjTAfBgNVHSMEGDAWgBR7r+mQWNUZ0TpQ
+NwrwjgxgngvOjTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCO
+7B4s6uQEGE8jg3CQgy76oU/D8sazGcP8+/E4JLHSc0Nj49w4ztSpkOVk2HyEtzbm
+uR3TreIw+SfqpbiOI/ivVNDbEBsb/vEeq7qPzDH1Bi72plHZNRVhNGGV5rd7ibga
+TkfXHKPM9yt8ffffHHiu1ROvb8gg2B6JbQwboU4hvvmmorW7onyTFSYEzZVdNSpv
+pUtKPldxYjTnLlbsJdXC4xyCC4PrJt2CC0n0jsWfICJ77LMxIxTODh8oZNjbPg6r
+RdI7U/DsD+R072DjbIcrivvigotJM+jihzz5inZwbO8o0WQOHAbJLIG3C3BnRW3A
+Ek4u3+HXZMl5a0LGJ76u
+-----END CERTIFICATE-----
+

checks/mumble/peer_2/peer_2_test_key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCfP6cZhCs9jOnW
+qyQP12vrOOxlBrWofYZFf9amUA24AfE7oGcSfkylanmkxzvGqQkhgLAvkHZj/GEv
+HujKyy8PgcEGP+pwmsfWNQMvU0Dzj3syjWOTi3eIC/3DoUnHlWCT2qCil/bjqxgU
+1l7fO/OXUlq5kyvIjln7Za4sUHunixe/m96Er6l8a4Mh2pxh2C5pkLCvulkQhjjG
+G+R6MccH8wwQwmLg5oVBkFEZrnREpnRKBI0DvA+wk1aJFAPOI4d8Q5T7o/MyxH3f
+8TYGHqbeMQFCKwusnlWPRtrNdaIcgaLvSpR0LVlroXGu8tYmRpvHPByoKGDbgVvO
+0Bwx8fmRAgMBAAECggEACAkjOnNj5zA0IIP0RuRc6rqtmw9ynTTwUJN51lyVxKI8
+dQDMEq/S2En+J2VyS7z92/XtbgkBIFx83u7VWl5UWpj2j4UsJFB7IwD7zyiJT4D+
++3cM/kX8Wx4XyQZbfbm47N0MXAgFCkn45hxHH0acLReXwmN9wxoDyl7AIjZRdwvG
+Qq0rnOnIc8kkkew7L6AiFwQS8b77eyzua3d6moKXN9hU/kfiJ6YUFG/WLe0pmQA1
+HbF27YghfeLnYUt50oDuX6jF6CzQhflchWVq/wn8/cxEpg/RMicWE8ulrTk7o27l
+JwCrHrhYEBsPuZO4mxX/DHrAMmhTeFjLaV5bQlz0PQKBgQDgRPSOEixYnKz9iPs/
+EDTlji5LA3Rm6TytRCNsjYY6Trw60KcvYqwyDUCiEjruvOQ9mqgBiQm1VHSalrG3
+RcbVfpEMouyZbEwmTjS8KdOi5x4Z6AX+4yWDN31jX3b8sktgbxV/HRdg3sA3q7MJ
+vExTUuoXg57W+FepIZ+XlhSoQwKBgQC1x6UMAlAeW45/yUUm/LFRcCgb/bdCQx+e
+hSb8w3jdvVoNWgx1j7RsjjFKaZUnseK3qQvVfCm4Qjvlz6MpKDxslaUYuR162Ku0
+e153z/xc7XRoXyPyPLdGZFlWii30jirB7ZqPdyz6mwlWwqdImNerbUqdFt9R8bId
+pYsyHB5zmwKBgBjYCq9iW/9E+/TqI8sMpI95fK9app5v4AThs3rnAqOa7Ucmrh6V
+s7Wnui06D8U6r54Tb+EbqTOpM3Gcl/tRg4FLEA5yTfuA/76Ok1D04Tj+mVsNVPyz
+dQhgMUe835WGusroA12df2V/x5NjNeYyMdJZMQ2ByyrNQAjAbMmCGq+5AoGBAIj8
+ERFysMOfxUvg9b7CkDFJrsAhOzew86P2vYGfIHchGTqUkG0LRTDFGrnzxNXsBGjY
++DUB40Kajx7IkTETxC0jvA1ceq23l/VjPrZVQt0YiC+a+rCyNn7SYkyHxsfTVr9b
+ea0BZyDXMntyJrPbkjL6Ik8tDE9pLwuOU84ISJ5fAoGAZ2+Ams/VhdZj/wpRpMky
+K4jtS4nzbCmJzzTa6vdVV7Kjer5kFxSFFqMrS/FtJ/RxHeHvxdze9dfGu9jIdTKK
+vSzbyQdHFfZgRkmAKfcoN9u567z7Oc74AQ9UgFEGdEVFQUbfWOevmr8KIPt8nDQK
+J9HuVfILi1kH0jzDd/64TvA=
+-----END PRIVATE KEY-----

checks/nixos-documentation/flake-module.nix

@@ -0,0 +1,27 @@
+{ self, ... }:
+let
+  documentationModule = {
+    # This is how some downstream users currently generate documentation
+    # If this breaks notify them via matrix since we spent ~5 hrs for bughunting last time.
+    documentation.nixos.enable = true;
+    documentation.nixos.extraModules = [
+      self.nixosModules.clanCore
+      # This is the only option that is not part of the
+      # module because it is usually set by flake-parts
+      { clan.core.settings.directory = ./.; }
+    ];
+  };
+in
+{
+  clan = {
+    machines.test-documentation = {
+      # Dummy file system
+      fileSystems."/".device = "/dev/null";
+      boot.loader.grub.device = "/dev/null";
+      nixpkgs.hostPlatform = "x86_64-linux";
+      imports = [
+        documentationModule
+      ];
+    };
+  };
+}

checks/postgresql/default.nix

@@ -0,0 +1,73 @@
+(import ../lib/container-test.nix) ({
+  name = "postgresql";
+
+  nodes.machine =
+    { self, config, ... }:
+    {
+      imports = [
+        self.nixosModules.clanCore
+        self.clanModules.postgresql
+        self.clanModules.localbackup
+      ];
+      clan.postgresql.users.test = { };
+      clan.postgresql.databases.test.create.options.OWNER = "test";
+      clan.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ];
+      clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
+      clan.core.settings.directory = ./.;
+
+      systemd.services.sample-service = {
+        wantedBy = [ "multi-user.target" ];
+        script = ''
+          while true; do
+            echo "Hello, world!"
+            sleep 5
+          done
+        '';
+      };
+
+      environment.systemPackages = [ config.services.postgresql.package ];
+    };
+  testScript =
+    { nodes, ... }:
+    ''
+      start_all()
+      machine.wait_for_unit("postgresql")
+      machine.wait_for_unit("sample-service")
+      # Create a test table
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
+
+      machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
+      timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
+
+      machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
+      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+
+      machine.succeed("rm -rf /var/backup/postgres")
+
+      machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
+      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+
+      machine.succeed("""
+      set -x
+      ${nodes.machine.clan.core.state.test.postRestoreCommand}
+      """)
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
+
+      timestamp_after = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
+      assert timestamp_before < timestamp_after, f"{timestamp_before} >= {timestamp_after}: expected sample-service to be restarted after restore"
+
+      # Check that the table is still there
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
+      output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
+      owner = output.split("\n")[1]
+      assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
+
+      # check if restore works if the database does not exist
+      machine.succeed("runuser -u postgres -- dropdb test")
+      machine.succeed("${nodes.machine.clan.core.state.test.postRestoreCommand}")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
+    '';
+})

checks/secrets/default.nix

@@ -10,8 +10,8 @@
       environment.etc."group-secret".source = config.sops.secrets.group-secret.path;
       sops.age.keyFile = "/etc/privkey.age";
 
-      clanCore.clanDir = "${./.}";
-      clanCore.machineName = "machine";
+      clan.core.settings.directory = "${./.}";
+      clan.core.settings.machine.name = "machine";
 
       networking.hostName = "machine";
     };

checks/secrets/sops/machines/machine/key.json

@@ -1,4 +1,4 @@
 {
   "publickey": "age15x8u838dwqflr3t6csf4tlghxm4tx77y379ncqxav7y2n8qp7yzqgrwt00",
   "type": "age"
-}
+}

checks/secrets/sops/users/admin/key.json

@@ -1,4 +1,4 @@
 {
   "publickey": "age15x8u838dwqflr3t6csf4tlghxm4tx77y379ncqxav7y2n8qp7yzqgrwt00",
   "type": "age"
-}
+}

checks/syncthing/default.nix

@@ -12,14 +12,14 @@
           self.clanModules.syncthing
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "introducer";
-            clanCore.clanDir = ./.;
+            clan.core.settings.machine.name = "introducer";
+            clan.core.settings.directory = ./.;
             environment.etc = {
               "syncthing.pam".source = ./introducer/introducer_test_cert;
               "syncthing.key".source = ./introducer/introducer_test_key;
               "syncthing.api".source = ./introducer/introducer_test_api;
             };
-            clanCore.facts.services.syncthing.secret."syncthing.api".path = "/etc/syncthing.api";
+            clan.core.facts.services.syncthing.secret."syncthing.api".path = "/etc/syncthing.api";
             services.syncthing.cert = "/etc/syncthing.pam";
             services.syncthing.key = "/etc/syncthing.key";
             # Doesn't test zerotier!
@@ -53,8 +53,8 @@
           self.clanModules.syncthing
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "peer1";
-            clanCore.clanDir = ./.;
+            clan.core.settings.machine.name = "peer1";
+            clan.core.settings.directory = ./.;
             clan.syncthing.introducer = lib.strings.removeSuffix "\n" (
               builtins.readFile ./introducer/introducer_device_id
             );
@@ -75,8 +75,8 @@
           self.clanModules.syncthing
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "peer2";
-            clanCore.clanDir = ./.;
+            clan.core.settings.machine.name = "peer2";
+            clan.core.settings.directory = ./.;
             clan.syncthing.introducer = lib.strings.removeSuffix "\n" (
               builtins.readFile ./introducer/introducer_device_id
             );

checks/wayland-proxy-virtwl/default.nix

@@ -14,8 +14,8 @@ import ../lib/test-base.nix (
         imports = [
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
+            clan.core.settings.machine.name = "machine";
+            clan.core.settings.directory = ./.;
           }
         ];
         services.wayland-proxy-virtwl.enable = true;

checks/zt-tcp-relay/default.nix

@@ -10,8 +10,8 @@
           self.nixosModules.clanCore
           self.clanModules.zt-tcp-relay
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
+            clan.core.settings.machine.name = "machine";
+            clan.core.settings.directory = ./.;
           }
         ];
       };

clanModules/admin/README.md

@@ -0,0 +1,5 @@
+---
+description = "Convenient Administration for the Clan App"
+categories = ["Utility"]
+features = [ "inventory" ]
+---

clanModules/admin/default.nix

@@ -0,0 +1,6 @@
+# Dont import this file
+# It is only here for backwards compatibility.
+# Dont author new modules with this file.
+{
+  imports = [ ./roles/default.nix ];
+}

clanModules/admin/roles/default.nix

@@ -0,0 +1,22 @@
+{ lib, config, ... }:
+{
+  options.clan.admin = {
+    allowedKeys = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.str;
+      description = "The allowed public keys for ssh access to the admin user";
+      example = {
+        "key_1" = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD...";
+      };
+    };
+  };
+  # Bad practice.
+  # Should we add 'clanModules' to specialArgs?
+  imports = [
+    ../../sshd
+    ../../root-password
+  ];
+  config = {
+    users.users.root.openssh.authorizedKeys.keys = builtins.attrValues config.clan.admin.allowedKeys;
+  };
+}

clanModules/auto-upgrade/README.md

@@ -0,0 +1,8 @@
+---
+description = "Set up automatic upgrades"
+categories = ["System"]
+features = [ "inventory" ]
+---
+
+Whether to periodically upgrade NixOS to the latest version. If enabled, a
+systemd timer will run `nixos-rebuild switch --upgrade` once a day.

clanModules/auto-upgrade/roles/default.nix

@@ -0,0 +1,24 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.clan.autoUpgrade;
+in
+{
+  options.clan.autoUpgrade = {
+    flake = lib.mkOption {
+      type = lib.types.str;
+      description = "Flake reference";
+    };
+  };
+  config = {
+    system.autoUpgrade = {
+      inherit (cfg.clan.autoUpgrade) flake;
+      enable = true;
+      dates = "02:00";
+      randomizedDelaySec = "45min";
+    };
+  };
+}

clanModules/borgbackup-static/README.md

@@ -0,0 +1,16 @@
+---
+description = "Statically configure borgbackup with sane defaults."
+---
+!!! Danger "Deprecated"
+    Use [borgbackup](borgbackup.md) instead.
+
+    Don't use borgbackup-static through [inventory](../../manual/inventory.md).
+
+This module implements the `borgbackup` backend and implements sane defaults
+for backup management through `borgbackup` for members of the clan.
+
+Configure target machines where the backups should be sent to through `targets`.
+
+Configure machines that should be backuped either through `includeMachines`
+which will exclusively add the included machines to be backuped, or through
+`excludeMachines`, which will add every machine except the excluded machine to the backup.

clanModules/borgbackup-static/default.nix

@@ -0,0 +1,104 @@
+{ lib, config, ... }:
+let
+  dir = config.clan.core.settings.directory;
+  machineDir = dir + "/machines/";
+in
+{
+  imports = [ ../borgbackup ];
+
+  options.clan.borgbackup-static = {
+    excludeMachines = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [ config.clan.core.settings.machine.name ];
+      default = [ ];
+      description = ''
+        Machines that should not be backuped.
+        Mutually exclusive with includeMachines.
+        If this is not empty, every other machine except the targets in the clan will be backuped by this module.
+        If includeMachines is set, only the included machines will be backuped.
+      '';
+    };
+    includeMachines = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [ config.clan.core.settings.machine.name ];
+      default = [ ];
+      description = ''
+        Machines that should be backuped.
+        Mutually exclusive with excludeMachines.
+      '';
+    };
+    targets = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = ''
+        Machines that should act as target machines for backups.
+      '';
+    };
+  };
+
+  config.services.borgbackup.repos =
+    let
+      machines = builtins.readDir machineDir;
+      borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub";
+      filteredMachines =
+        if ((builtins.length config.clan.borgbackup-static.includeMachines) != 0) then
+          lib.filterAttrs (name: _: (lib.elem name config.clan.borgbackup-static.includeMachines)) machines
+        else
+          lib.filterAttrs (name: _: !(lib.elem name config.clan.borgbackup-static.excludeMachines)) machines;
+      machinesMaybeKey = lib.mapAttrsToList (
+        machine: _:
+        let
+          fullPath = borgbackupIpMachinePath machine;
+        in
+        if builtins.pathExists fullPath then machine else null
+      ) filteredMachines;
+      machinesWithKey = lib.filter (x: x != null) machinesMaybeKey;
+      hosts = builtins.map (machine: {
+        name = machine;
+        value = {
+          path = "/var/lib/borgbackup/${machine}";
+          authorizedKeys = [ (builtins.readFile (borgbackupIpMachinePath machine)) ];
+        };
+      }) machinesWithKey;
+    in
+    lib.mkIf
+      (builtins.any (
+        target: target == config.clan.core.settings.machine.name
+      ) config.clan.borgbackup-static.targets)
+      (if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { });
+
+  config.clan.borgbackup.destinations =
+    let
+      destinations = builtins.map (d: {
+        name = d;
+        value = {
+          repo = "borg@${d}:/var/lib/borgbackup/${config.clan.core.settings.machine.name}";
+        };
+      }) config.clan.borgbackup-static.targets;
+    in
+    lib.mkIf (builtins.any (
+      target: target == config.clan.core.settings.machine.name
+    ) config.clan.borgbackup-static.includeMachines) (builtins.listToAttrs destinations);
+
+  config.assertions = [
+    {
+      assertion =
+        !(
+          ((builtins.length config.clan.borgbackup-static.excludeMachines) != 0)
+          && ((builtins.length config.clan.borgbackup-static.includeMachines) != 0)
+        );
+      message = ''
+        The options:
+        config.clan.borgbackup-static.excludeMachines = [${builtins.toString config.clan.borgbackup-static.excludeMachines}]
+        and
+        config.clan.borgbackup-static.includeMachines = [${builtins.toString config.clan.borgbackup-static.includeMachines}]
+        are mutually exclusive.
+        Use excludeMachines to exclude certain machines and backup the other clan machines.
+        Use include machines to only backup certain machines.
+      '';
+    }
+  ];
+  config.warnings = lib.optional (
+    builtins.length config.clan.borgbackup-static.targets > 0
+  ) "The borgbackup-static module is deprecated use the service via the inventory interface instead.";
+}

clanModules/borgbackup/README.md

@@ -1,2 +1,14 @@
-Efficient, deduplicating backup program with optional compression and secure encryption.
----
+---
+description = "Efficient, deduplicating backup program with optional compression and secure encryption."
+categories = ["System"]
+features = [ "inventory" ]
+---
+BorgBackup (short: Borg) gives you:
+
+- Space efficient storage of backups.
+- Secure, authenticated encryption.
+- Compression: lz4, zstd, zlib, lzma or none.
+- Mountable backups with FUSE.
+- Easy installation on multiple platforms: Linux, macOS, BSD, …
+- Free software (BSD license).
+- Backed by a large and active open-source community.

clanModules/borgbackup/default.nix

@@ -1,131 +1,6 @@
+# Dont import this file
+# It is only here for backwards compatibility.
+# Dont author new modules with this file.
 {
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-let
-  cfg = config.clan.borgbackup;
-in
-{
-  options.clan.borgbackup.destinations = lib.mkOption {
-    type = lib.types.attrsOf (
-      lib.types.submodule (
-        { name, ... }:
-        {
-          options = {
-            name = lib.mkOption {
-              type = lib.types.strMatching "^[a-zA-Z0-9._-]+$";
-              default = name;
-              description = "the name of the backup job";
-            };
-            repo = lib.mkOption {
-              type = lib.types.str;
-              description = "the borgbackup repository to backup to";
-            };
-            rsh = lib.mkOption {
-              type = lib.types.str;
-              default = "ssh -i ${
-                config.clanCore.facts.services.borgbackup.secret."borgbackup.ssh".path
-              } -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
-              defaultText = "ssh -i \${config.clanCore.facts.services.borgbackup.secret.\"borgbackup.ssh\".path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
-              description = "the rsh to use for the backup";
-            };
-          };
-        }
-      )
-    );
-    default = { };
-    description = ''
-      destinations where the machine should be backuped to
-    '';
-  };
-
-  imports = [
-    (lib.mkRemovedOptionModule [
-      "clan"
-      "borgbackup"
-      "enable"
-    ] "Just define clan.borgbackup.destinations to enable it")
-  ];
-
-  config = lib.mkIf (cfg.destinations != { }) {
-    services.borgbackup.jobs = lib.mapAttrs (_: dest: {
-      paths = lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state));
-      exclude = [ "*.pyc" ];
-      repo = dest.repo;
-      environment.BORG_RSH = dest.rsh;
-      compression = "auto,zstd";
-      startAt = "*-*-* 01:00:00";
-      persistentTimer = true;
-      preHook = ''
-        set -x
-      '';
-
-      encryption = {
-        mode = "repokey";
-        passCommand = "cat ${config.clanCore.facts.services.borgbackup.secret."borgbackup.repokey".path}";
-      };
-
-      prune.keep = {
-        within = "1d"; # Keep all archives from the last day
-        daily = 7;
-        weekly = 4;
-        monthly = 0;
-      };
-    }) cfg.destinations;
-
-    clanCore.facts.services.borgbackup = {
-      public."borgbackup.ssh.pub" = { };
-      secret."borgbackup.ssh" = { };
-      secret."borgbackup.repokey" = { };
-      generator.path = [
-        pkgs.openssh
-        pkgs.coreutils
-        pkgs.xkcdpass
-      ];
-      generator.script = ''
-        ssh-keygen -t ed25519 -N "" -f "$secrets"/borgbackup.ssh
-        mv "$secrets"/borgbackup.ssh.pub "$facts"/borgbackup.ssh.pub
-        xkcdpass -n 4 -d - > "$secrets"/borgbackup.repokey
-      '';
-    };
-
-    environment.systemPackages = [
-      (pkgs.writeShellScriptBin "borgbackup-create" ''
-        set -efu -o pipefail
-        ${lib.concatMapStringsSep "\n" (dest: ''
-          systemctl start borgbackup-job-${dest.name}
-        '') (lib.attrValues cfg.destinations)}
-      '')
-      (pkgs.writeShellScriptBin "borgbackup-list" ''
-        set -efu
-        (${
-          lib.concatMapStringsSep "\n" (
-            dest:
-            # we need yes here to skip the changed url verification
-            ''yes y | borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.name}::${dest.repo}::" + .name)}]' ''
-          ) (lib.attrValues cfg.destinations)
-        }) | ${pkgs.jq}/bin/jq -s 'add'
-      '')
-      (pkgs.writeShellScriptBin "borgbackup-restore" ''
-        set -efux
-        cd /
-        IFS=';' read -ra FOLDER <<< "$FOLDERS"
-        job_name=$(echo "$NAME" | ${pkgs.gawk}/bin/awk -F'::' '{print $1}')
-        backup_name=''${NAME#"$job_name"::}
-        if ! command -v borg-job-"$job_name" &> /dev/null; then
-          echo "borg-job-$job_name not found: Backup name is invalid" >&2
-          exit 1
-        fi
-        yes y | borg-job-"$job_name" extract --list "$backup_name" "''${FOLDER[@]}"
-      '')
-    ];
-
-    clanCore.backups.providers.borgbackup = {
-      list = "borgbackup-list";
-      create = "borgbackup-create";
-      restore = "borgbackup-restore";
-    };
-  };
+  imports = [ ./roles/client.nix ];
 }

clanModules/borgbackup/roles/client.nix

@@ -0,0 +1,204 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  # Instances might be empty, if the module is not used via the inventory
+  instances = config.clan.inventory.services.borgbackup or { };
+  # roles = { ${role_name} :: { machines :: [string] } }
+  allServers = lib.foldlAttrs (
+    acc: _instanceName: instanceConfig:
+    acc
+    ++ (
+      if builtins.elem machineName instanceConfig.roles.client.machines then
+        instanceConfig.roles.server.machines
+      else
+        [ ]
+    )
+  ) [ ] instances;
+
+  machineName = config.clan.core.settings.machine.name;
+
+  cfg = config.clan.borgbackup;
+  preBackupScript = ''
+    declare -A preCommandErrors
+
+    ${lib.concatMapStringsSep "\n" (
+      state:
+      lib.optionalString (state.preBackupCommand != null) ''
+        echo "Running pre-backup command for ${state.name}"
+        if ! /run/current-system/sw/bin/${state.preBackupCommand}; then
+          preCommandErrors["${state.name}"]=1
+        fi
+      ''
+    ) (lib.attrValues config.clan.core.state)}
+
+    if [[ ''${#preCommandErrors[@]} -gt 0 ]]; then
+      echo "pre-backup commands failed for the following services:"
+      for state in "''${!preCommandErrors[@]}"; do
+        echo "  $state"
+      done
+      exit 1
+    fi
+  '';
+in
+{
+  options.clan.borgbackup.destinations = lib.mkOption {
+    type = lib.types.attrsOf (
+      lib.types.submodule (
+        { name, ... }:
+        {
+          options = {
+            name = lib.mkOption {
+              type = lib.types.strMatching "^[a-zA-Z0-9._-]+$";
+              default = name;
+              description = "the name of the backup job";
+            };
+            repo = lib.mkOption {
+              type = lib.types.str;
+              description = "the borgbackup repository to backup to";
+            };
+            rsh = lib.mkOption {
+              type = lib.types.str;
+              default = "ssh -i ${
+                config.clan.core.vars.generators.borgbackup.files."borgbackup.ssh".path
+              } -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=Yes";
+              defaultText = "ssh -i \${config.clan.core.vars.generators.borgbackup.files.\"borgbackup.ssh\".path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
+              description = "the rsh to use for the backup";
+            };
+          };
+        }
+      )
+    );
+    default = { };
+    description = ''
+      destinations where the machine should be backuped to
+    '';
+  };
+
+  options.clan.borgbackup.exclude = lib.mkOption {
+    type = lib.types.listOf lib.types.str;
+    example = [ "*.pyc" ];
+    default = [ ];
+    description = ''
+      Directories/Files to exclude from the backup.
+      Use * as a wildcard.
+    '';
+  };
+
+  config = {
+
+    # Destinations
+    clan.borgbackup.destinations =
+      let
+        destinations = builtins.map (serverName: {
+          name = serverName;
+          value = {
+            repo = "borg@${serverName}:/var/lib/borgbackup/${machineName}";
+          };
+        }) allServers;
+      in
+      (builtins.listToAttrs destinations);
+
+    # Derived from the destinations
+    systemd.services = lib.mapAttrs' (
+      _: dest:
+      lib.nameValuePair "borgbackup-job-${dest.name}" {
+        # since borgbackup mounts the system read-only, we need to run in a ExecStartPre script, so we can generate additional files.
+        serviceConfig.ExecStartPre = [
+          ''+${pkgs.writeShellScript "borgbackup-job-${dest.name}-pre-backup-commands" preBackupScript}''
+        ];
+      }
+    ) cfg.destinations;
+
+    services.borgbackup.jobs = lib.mapAttrs (_: dest: {
+      paths = lib.unique (
+        lib.flatten (map (state: state.folders) (lib.attrValues config.clan.core.state))
+      );
+      exclude = cfg.exclude;
+      repo = dest.repo;
+      environment.BORG_RSH = dest.rsh;
+      compression = "auto,zstd";
+      startAt = "*-*-* 01:00:00";
+      persistentTimer = true;
+
+      encryption = {
+        mode = "repokey";
+        passCommand = "cat ${config.clan.core.vars.generators.borgbackup.files."borgbackup.repokey".path}";
+      };
+
+      prune.keep = {
+        within = "1d"; # Keep all archives from the last day
+        daily = 7;
+        weekly = 4;
+        monthly = 0;
+      };
+    }) cfg.destinations;
+
+    environment.systemPackages = [
+      (pkgs.writeShellApplication {
+        name = "borgbackup-create";
+        runtimeInputs = [ config.systemd.package ];
+        text = ''
+          ${lib.concatMapStringsSep "\n" (dest: ''
+            systemctl start borgbackup-job-${dest.name}
+          '') (lib.attrValues cfg.destinations)}
+        '';
+      })
+      (pkgs.writeShellApplication {
+        name = "borgbackup-list";
+        runtimeInputs = [ pkgs.jq ];
+        text = ''
+          (${
+            lib.concatMapStringsSep "\n" (
+              dest:
+              # we need yes here to skip the changed url verification
+              ''echo y | /run/current-system/sw/bin/borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.name}::${dest.repo}::" + .name)}]' ''
+            ) (lib.attrValues cfg.destinations)
+          }) | jq -s 'add // []'
+        '';
+      })
+      (pkgs.writeShellApplication {
+        name = "borgbackup-restore";
+        runtimeInputs = [ pkgs.gawk ];
+        text = ''
+          cd /
+          IFS=':' read -ra FOLDER <<< "''${FOLDERS-}"
+          job_name=$(echo "$NAME" | awk -F'::' '{print $1}')
+          backup_name=''${NAME#"$job_name"::}
+          if [[ ! -x /run/current-system/sw/bin/borg-job-"$job_name" ]]; then
+            echo "borg-job-$job_name not found: Backup name is invalid" >&2
+            exit 1
+          fi
+          echo y | /run/current-system/sw/bin/borg-job-"$job_name" extract "$backup_name" "''${FOLDER[@]}"
+        '';
+      })
+    ];
+
+    clan.core.vars.generators.borgbackup = {
+
+      files."borgbackup.ssh.pub".secret = false;
+      files."borgbackup.ssh" = { };
+      files."borgbackup.repokey" = { };
+
+      migrateFact = "borgbackup";
+      runtimeInputs = [
+        pkgs.coreutils
+        pkgs.openssh
+        pkgs.xkcdpass
+      ];
+      script = ''
+        ssh-keygen -t ed25519 -N "" -f $out/borgbackup.ssh
+        xkcdpass -n 4 -d - > $out/borgbackup.repokey
+      '';
+    };
+
+    clan.core.backups.providers.borgbackup = {
+      list = "borgbackup-list";
+      create = "borgbackup-create";
+      restore = "borgbackup-restore";
+    };
+  };
+}

clanModules/borgbackup/roles/server.nix

@@ -0,0 +1,63 @@
+{ config, lib, ... }:
+let
+  dir = config.clan.core.settings.directory;
+  machineDir = dir + "/vars/per-machine/";
+  machineName = config.clan.core.settings.machine.name;
+
+  # Instances might be empty, if the module is not used via the inventory
+  #
+  # Type: { ${instanceName} :: { roles :: Roles } }
+  #   Roles :: { ${role_name} :: { machines :: [string] } }
+  instances = config.clan.inventory.services.borgbackup or { };
+
+  allClients = lib.foldlAttrs (
+    acc: _instanceName: instanceConfig:
+    acc
+    ++ (
+      if (builtins.elem machineName instanceConfig.roles.server.machines) then
+        instanceConfig.roles.client.machines
+      else
+        [ ]
+    )
+  ) [ ] instances;
+in
+{
+  options = {
+    clan.borgbackup.directory = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/lib/borgbackup";
+      description = ''
+        The directory where the borgbackup repositories are stored.
+      '';
+    };
+  };
+  config.services.borgbackup.repos =
+    let
+      borgbackupIpMachinePath = machine: machineDir + machine + "/borgbackup/borgbackup.ssh.pub/value";
+
+      machinesMaybeKey = builtins.map (
+        machine:
+        let
+          fullPath = borgbackupIpMachinePath machine;
+        in
+        if builtins.pathExists fullPath then
+          machine
+        else
+          lib.warn ''
+            Machine ${machine} does not have a borgbackup key at ${fullPath},
+            run `clan var generate ${machine}` to generate it.
+          '' null
+      ) allClients;
+
+      machinesWithKey = lib.filter (x: x != null) machinesMaybeKey;
+
+      hosts = builtins.map (machine: {
+        name = machine;
+        value = {
+          path = "${config.clan.borgbackup.directory}/${machine}";
+          authorizedKeys = [ (builtins.readFile (borgbackupIpMachinePath machine)) ];
+        };
+      }) machinesWithKey;
+    in
+    if (builtins.listToAttrs hosts) != [ ] then builtins.listToAttrs hosts else { };
+}

clanModules/deltachat/README.md

@@ -1,7 +1,6 @@
-Email-based instant messaging for Desktop.
 ---
-
-!!! warning "Under construction"
+description = "Email-based instant messaging for Desktop."
+---
 
 !!! info
     This module will automatically configure an email server on the machine for handling the e-mail messaging seamlessly.

clanModules/deltachat/default.nix

@@ -5,7 +5,7 @@
 
   services.maddy =
     let
-      domain = "${config.clanCore.machineName}.local";
+      domain = "${config.clan.core.settings.machine.name}.local";
     in
     {
       enable = true;

clanModules/disk-id/README.md

@@ -0,0 +1,5 @@
+---
+description = "Generates a uuid for use in disk device naming"
+features = [ "inventory" ]
+categories = [ "System" ]
+---

clanModules/disk-id/default.nix

@@ -0,0 +1,6 @@
+# Dont import this file
+# It is only here for backwards compatibility.
+# Dont author new modules with this file.
+{
+  imports = [ ./roles/default.nix ];
+}

clanModules/disk-id/roles/default.nix

@@ -0,0 +1,26 @@
+{
+  config,
+  pkgs,
+  ...
+}:
+
+{
+
+  config = {
+    clan.core.vars.generators.disk-id = {
+      files.diskId.secret = false;
+      runtimeInputs = [
+        pkgs.coreutils
+        pkgs.bash
+      ];
+      script = ''
+        uuid=$(bash ${./uuid4.sh})
+
+        # Remove the hyphens from the UUID
+        uuid_no_hyphens=$(echo -n "$uuid" | tr -d '-')
+
+        echo -n "$uuid_no_hyphens" > "$out/diskId"
+      '';
+    };
+  };
+}

clanModules/disk-id/roles/uuid4.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Read 16 bytes from /dev/urandom
+uuid=$(dd if=/dev/urandom bs=1 count=16 2>/dev/null | od -An -tx1 | tr -d ' \n')
+
+# Break the UUID into pieces and apply the required modifications
+byte6=${uuid:12:2}
+byte8=${uuid:16:2}
+
+# Construct the correct version and variant
+hex_byte6=$(printf "%x" $((0x$byte6 & 0x0F | 0x40)))
+hex_byte8=$(printf "%x" $((0x$byte8 & 0x3F | 0x80)))
+
+# Rebuild the UUID with the correct fields
+uuid_v4="${uuid:0:12}${hex_byte6}${uuid:14:2}${hex_byte8}${uuid:18:14}"
+
+# Format the UUID correctly 8-4-4-4-12
+uuid_formatted="${uuid_v4:0:8}-${uuid_v4:8:4}-${uuid_v4:12:4}-${uuid_v4:16:4}-${uuid_v4:20:12}"
+
+echo -n "$uuid_formatted"

clanModules/disk-layouts/README.md

@@ -1,2 +0,0 @@
-Automatically format a disk drive on clan installation
----

clanModules/dyndns/README.md

@@ -0,0 +1,6 @@
+---
+description = "A dynamic DNS service to update domain IPs"
+---
+
+To understand the possible options that can be set visit the documentation of [ddns-updater](https://github.com/qdm12/ddns-updater?tab=readme-ov-file#versioned-documentation)
+

clanModules/dyndns/default.nix

@@ -0,0 +1,253 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+
+let
+  name = "dyndns";
+  cfg = config.clan.${name};
+
+  # We dedup secrets if they have the same provider + base domain
+  secret_id = opt: "${name}-${opt.provider}-${opt.domain}";
+  secret_path =
+    opt: config.clan.core.vars.generators."${secret_id opt}".files."${secret_id opt}".path;
+
+  # We check that a secret has not been set in extraSettings.
+  extraSettingsSafe =
+    opt:
+    if (builtins.hasAttr opt.secret_field_name opt.extraSettings) then
+      throw "Please do not set ${opt.secret_field_name} in extraSettings, it is automatically set by the dyndns module."
+    else
+      opt.extraSettings;
+  /*
+    We go from:
+    {home.example.com:{value:{domain:example.com,host:home, provider:namecheap}}}
+    To:
+    {settings: [{domain: example.com, host: home, provider: namecheap, password: dyndns-namecheap-example.com}]}
+  */
+  service_config = {
+    settings = builtins.catAttrs "value" (
+      builtins.attrValues (
+        lib.mapAttrs (_: opt: {
+          value =
+            (extraSettingsSafe opt)
+            // {
+              domain = opt.domain;
+              provider = opt.provider;
+            }
+            // {
+              "${opt.secret_field_name}" = secret_id opt;
+            };
+        }) cfg.settings
+      )
+    );
+  };
+
+  secret_generator = _: opt: {
+    name = secret_id opt;
+    value = {
+      share = true;
+      migrateFact = "${secret_id opt}";
+      prompts.${secret_id opt} = {
+        type = "hidden";
+        persist = true;
+      };
+    };
+  };
+in
+{
+  options.clan.${name} = {
+    server = {
+      enable = lib.mkEnableOption "dyndns webserver";
+      domain = lib.mkOption {
+        type = lib.types.str;
+        description = "Domain to serve the webservice on";
+      };
+      port = lib.mkOption {
+        type = lib.types.int;
+        default = 54805;
+        description = "Port to listen on";
+      };
+    };
+
+    period = lib.mkOption {
+      type = lib.types.int;
+      default = 5;
+      description = "Domain update period in minutes";
+    };
+
+    settings = lib.mkOption {
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { ... }:
+          {
+            options = {
+              provider = lib.mkOption {
+                example = "namecheap";
+                type = lib.types.str;
+                description = "The dyndns provider to use";
+              };
+              domain = lib.mkOption {
+                type = lib.types.str;
+                example = "example.com";
+                description = "The top level domain to update.";
+              };
+              secret_field_name = lib.mkOption {
+                example = [
+                  "password"
+                  "api_key"
+                ];
+                type = lib.types.enum [
+                  "password"
+                  "token"
+                  "api_key"
+                  "secret_api_key"
+                ];
+                default = "password";
+                description = "The field name for the secret";
+              };
+              # TODO: Ideally we would create a gigantic list of all possible settings / types
+              # optimally we would have a way to generate the options from the source code
+              extraSettings = lib.mkOption {
+                type = lib.types.attrsOf lib.types.str;
+                default = { };
+                description = ''
+                  Extra settings for the provider.
+                  Provider specific settings: https://github.com/qdm12/ddns-updater#configuration
+                '';
+              };
+            };
+          }
+        )
+      );
+      default = [ ];
+      description = "Configuration for which domains to update";
+    };
+  };
+
+  imports = [
+    ../nginx
+  ];
+
+  config = lib.mkMerge [
+    (lib.mkIf (cfg.settings != { }) {
+      clan.core.vars.generators = lib.mapAttrs' secret_generator cfg.settings;
+
+      users.groups.${name} = { };
+      users.users.${name} = {
+        group = name;
+        isSystemUser = true;
+        description = "User for ${name} service";
+        home = "/var/lib/${name}";
+        createHome = true;
+      };
+
+      services.nginx = lib.mkIf cfg.server.enable {
+        enable = true;
+        virtualHosts = {
+          "${cfg.server.domain}" = {
+            forceSSL = true;
+            enableACME = true;
+            locations."/" = {
+              proxyPass = "http://localhost:${toString cfg.server.port}";
+            };
+          };
+        };
+      };
+
+      systemd.services.${name} = {
+        path = [ ];
+        description = "Dynamic DNS updater";
+        after = [ "network.target" ];
+        wantedBy = [ "multi-user.target" ];
+        environment = {
+          MYCONFIG = "${builtins.toJSON service_config}";
+          SERVER_ENABLED = if cfg.server.enable then "yes" else "no";
+          PERIOD = "${toString cfg.period}m";
+          LISTENING_ADDRESS = ":${toString cfg.server.port}";
+        };
+
+        serviceConfig =
+          let
+            pyscript =
+              pkgs.writers.writePython3Bin "generate_secret_config.py"
+                {
+                  libraries = [ ];
+                  doCheck = false;
+                }
+                ''
+                  import json
+                  from pathlib import Path
+                  import os
+
+                  cred_dir = Path(os.getenv("CREDENTIALS_DIRECTORY"))
+                  config_str = os.getenv("MYCONFIG")
+
+
+                  def get_credential(name):
+                      secret_p = cred_dir / name
+                      with open(secret_p, 'r') as f:
+                          return f.read().strip()
+
+
+                  config = json.loads(config_str)
+                  print(f"Config: {config}")
+                  for attrset in config["settings"]:
+                      if "password" in attrset:
+                          attrset['password'] = get_credential(attrset['password'])
+                      elif "token" in attrset:
+                          attrset['token'] = get_credential(attrset['token'])
+                      elif "secret_api_key" in attrset:
+                          attrset['secret_api_key'] = get_credential(attrset['secret_api_key'])
+                      elif "api_key" in attrset:
+                          attrset['api_key'] = get_credential(attrset['api_key'])
+                      else:
+                          raise ValueError(f"Missing secret field in {attrset}")
+
+                  # create directory data if it does not exist
+                  data_dir = Path('data')
+                  data_dir.mkdir(mode=0o770, exist_ok=True)
+
+                  # Write the config with secrets back
+                  config_path = data_dir / 'config.json'
+                  with open(config_path, 'w') as f:
+                      f.write(json.dumps(config, indent=4))
+
+                  # Set file permissions to read and write
+                  # only by the user and group
+                  config_path.chmod(0o660)
+
+                  # Set file permissions to read
+                  # and write only by the user and group
+                  for file in data_dir.iterdir():
+                      file.chmod(0o660)
+                '';
+          in
+          {
+            ExecStartPre = lib.getExe pyscript;
+            ExecStart = lib.getExe pkgs.ddns-updater;
+            LoadCredential = lib.mapAttrsToList (_: opt: "${secret_id opt}:${secret_path opt}") cfg.settings;
+            User = name;
+            Group = name;
+            NoNewPrivileges = true;
+            PrivateTmp = true;
+            ProtectSystem = "strict";
+            ReadOnlyPaths = "/";
+            PrivateDevices = "yes";
+            ProtectKernelModules = "yes";
+            ProtectKernelTunables = "yes";
+            WorkingDirectory = "/var/lib/${name}";
+            ReadWritePaths = [
+              "/proc/self"
+              "/var/lib/${name}"
+            ];
+
+            Restart = "always";
+            RestartSec = 60;
+          };
+      };
+    })
+  ];
+}

clanModules/ergochat/README.md

@@ -1,2 +1,3 @@
-A modern IRC server
+---
+description = "A modern IRC server"
 ---

clanModules/ergochat/default.nix

@@ -10,5 +10,5 @@ _: {
     };
   };
 
-  clanCore.state.ergochat.folders = [ "/var/lib/ergo" ];
+  clan.core.state.ergochat.folders = [ "/var/lib/ergo" ];
 }

clanModules/flake-module.nix

@@ -1,25 +1,51 @@
-{ ... }:
+{ lib, ... }:
+let
+  inherit (lib)
+    filterAttrs
+    pathExists
+    ;
+in
 {
-  flake.clanModules = {
-    disk-layouts = {
-      imports = [ ./disk-layouts ];
-    };
+  # only import available files, as this allows to filter the files for tests.
+  flake.clanModules = filterAttrs (_name: pathExists) {
+    admin = ./admin;
+    auto-upgrade = ./auto-upgrade;
     borgbackup = ./borgbackup;
+    borgbackup-static = ./borgbackup-static;
     deltachat = ./deltachat;
+    disk-id = ./disk-id;
+    dyndns = ./dyndns;
     ergochat = ./ergochat;
+    garage = ./garage;
+    golem-provider = ./golem-provider;
+    heisenbridge = ./heisenbridge;
+    importer = ./importer;
+    iwd = ./iwd;
     localbackup = ./localbackup;
     localsend = ./localsend;
+    machine-id = ./machine-id;
     matrix-synapse = ./matrix-synapse;
     moonlight = ./moonlight;
+    mumble = ./mumble;
+    mycelium = ./mycelium;
+    nginx = ./nginx;
+    packages = ./packages;
+    postgresql = ./postgresql;
     root-password = ./root-password;
+    single-disk = ./single-disk;
     sshd = ./sshd;
-    sunshine = ./sunshine;
+    state-version = ./state-version;
     static-hosts = ./static-hosts;
+    sunshine = ./sunshine;
     syncthing = ./syncthing;
+    syncthing-static-peers = ./syncthing-static-peers;
     thelounge = ./thelounge;
     trusted-nix-caches = ./trusted-nix-caches;
     user-password = ./user-password;
+    vaultwarden = ./vaultwarden;
+    wifi = ./wifi;
     xfce = ./xfce;
+    zerotier = ./zerotier;
     zerotier-static-peers = ./zerotier-static-peers;
     zt-tcp-relay = ./zt-tcp-relay;
   };

clanModules/garage/README.md

@@ -0,0 +1,9 @@
+---
+description = "S3-compatible object store for small self-hosted geo-distributed deployments"
+---
+
+This module generates garage specific keys automatically.
+Also shares the `rpc_secret` between instances.
+
+Options: [NixosModuleOptions](https://search.nixos.org/options?channel=unstable&size=50&sort=relevance&type=packages&query=garage)
+Documentation: https://garagehq.deuxfleurs.fr/

clanModules/garage/default.nix

@@ -0,0 +1,43 @@
+{ config, pkgs, ... }:
+{
+  systemd.services.garage.serviceConfig = {
+    LoadCredential = [
+      "rpc_secret_path:${config.clan.core.vars.generators.garage-shared.files.rpc_secret.path}"
+      "admin_token_path:${config.clan.core.vars.generators.garage.files.admin_token.path}"
+      "metrics_token_path:${config.clan.core.vars.generators.garage.files.metrics_token.path}"
+    ];
+    Environment = [
+      "GARAGE_ALLOW_WORLD_READABLE_SECRETS=true"
+      "GARAGE_RPC_SECRET_FILE=%d/rpc_secret_path"
+      "GARAGE_ADMIN_TOKEN_FILE=%d/admin_token_path"
+      "GARAGE_METRICS_TOKEN_FILE=%d/metrics_token_path"
+    ];
+  };
+
+  clan.core.vars.generators.garage = {
+    files.admin_token = { };
+    files.metrics_token = { };
+    runtimeInputs = [
+      pkgs.coreutils
+      pkgs.openssl
+    ];
+    script = ''
+      openssl rand -base64 -out $out/admin_token 32
+      openssl rand -base64 -out $out/metrics_token 32
+    '';
+  };
+
+  clan.core.vars.generators.garage-shared = {
+    share = true;
+    files.rpc_secret = { };
+    runtimeInputs = [
+      pkgs.coreutils
+      pkgs.openssl
+    ];
+    script = ''
+      openssl rand -hex -out $out/rpc_secret 32
+    '';
+  };
+
+  clan.core.state.garage.folders = [ config.services.garage.settings.metadata_dir ];
+}

clanModules/golem-provider/README.md

@@ -0,0 +1,7 @@
+---
+description = "Golem Provider for the Golem Network, an open-source and decentralized platform where everyone can use and share each other's computing power without relying on centralized entities like cloud computing corporations"
+---
+
+By running a golem provider your machine's compute resources are offered via the golem network which will allow other members to execute compute tasks on your machine. If this happens, you will be compensated with GLM, an ERC20 token.
+
+More about golem providers: https://docs.golem.network/docs/golem/overview

clanModules/golem-provider/default.nix

@@ -0,0 +1,34 @@
+{ config, pkgs, ... }:
+let
+  cfg = config.clan.golem-provider;
+  yagna = pkgs.callPackage ../../pkgs/yagna { };
+  accountFlag = if cfg.account != null then "--account ${cfg.account}" else "";
+in
+{
+  imports = [ ./interface.nix ];
+
+  users.users.golem = {
+    isSystemUser = true;
+    home = "/var/lib/golem";
+    group = "golem";
+    createHome = true;
+  };
+
+  users.groups.golem = { };
+
+  environment.systemPackages = [ yagna ];
+
+  systemd.services.golem-provider = {
+    description = "Golem Provider";
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network-online.target" ];
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${yagna}/bin/golemsp run --no-interactive ${accountFlag}";
+      Restart = "always";
+      RestartSec = "5";
+      User = "golem";
+      Group = "golem";
+    };
+  };
+}

clanModules/golem-provider/interface.nix

@@ -0,0 +1,20 @@
+{ lib, ... }:
+let
+  inherit (lib) mkOption;
+
+  inherit (lib.types) nullOr str;
+
+in
+{
+  options.clan.golem-provider = {
+    account = mkOption {
+      type = nullOr str;
+      description = ''
+        Ethereum address for payouts.
+
+        Leave empty to automatically generate a new address upon first start.
+      '';
+      default = null;
+    };
+  };
+}

clanModules/golem-provider/test/vm.nix

@@ -0,0 +1,4 @@
+{ ... }:
+{
+  imports = [ ../. ];
+}

clanModules/heisenbridge/README.md

@@ -0,0 +1,4 @@
+---
+description = "A matrix bridge to communicate with IRC"
+---
+

clanModules/heisenbridge/default.nix

@@ -0,0 +1,23 @@
+{
+  config,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.mkRemovedOptionModule [
+      "clan"
+      "heisenbridge"
+      "enable"
+    ] "Importing the module will already enable the service.")
+  ];
+  config = {
+    services.heisenbridge = {
+      enable = true;
+      homeserver = "http://localhost:8008"; # TODO: Sync with matrix-synapse
+    };
+    services.matrix-synapse.settings.app_service_config_files = [
+      "/var/lib/heisenbridge/registration.yml"
+    ];
+  };
+}

clanModules/importer/README.md

@@ -0,0 +1,27 @@
+---
+description = "Convenient, structured module imports for hosts."
+categories = ["Utility"]
+features = [ "inventory" ]
+---
+The importer module allows users to configure importing modules in a flexible and structured way.
+
+It exposes the `extraModules` functionality of the inventory, without any added configuration.
+
+## Usage:
+
+```nix
+inventory.services = {
+  importer.base = {
+    roles.default.tags = [ "all" ];
+    roles.default.extraModules = [ "modules/base.nix" ];
+  };
+  importer.zone1 = {
+    roles.default.tags = [ "zone1" ];
+    roles.default.extraModules = [ "modules/zone1.nix" ];
+  };
+};
+```
+
+This will import the module `modules/base.nix` to all machines that have the `all` tag,
+which by default is every machine managed by the clan.
+And also import for all machines tagged with `zone1` the module at `modules/zone1.nix`.

clanModules/importer/roles/default.nix

@@ -0,0 +1 @@
+{ }

clanModules/iwd/README.md

@@ -0,0 +1,9 @@
+---
+description = "Automatically provisions wifi credentials"
+features = [ "inventory" ]
+categories = [ "Network" ]
+---
+
+!!! Warning
+    If you've been using network manager + wpa_supplicant and now are switching to IWD read this migration guide:
+    https://archive.kernel.org/oldwiki/iwd.wiki.kernel.org/networkmanager.html#converting_network_profiles

clanModules/iwd/default.nix

@@ -0,0 +1,6 @@
+# Dont import this file
+# It is only here for backwards compatibility.
+# Dont author new modules with this file.
+{
+  imports = [ ./roles/default.nix ];
+}

clanModules/iwd/roles/default.nix

@@ -0,0 +1,99 @@
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.clan.iwd;
+  secret_path = ssid: config.clan.core.facts.services."iwd.${ssid}".secret."iwd.${ssid}".path;
+  secret_generator = name: value: {
+    name = "iwd.${value.ssid}";
+    value =
+      let
+        secret_name = "iwd.${value.ssid}";
+      in
+      {
+        secret.${secret_name} = { };
+        generator.prompt = "Wifi password for '${value.ssid}'";
+        # ref. man iwd.network
+        generator.script = ''
+          config="
+          [Settings]
+            AutoConnect=${if value.AutoConnect then "true" else "false"}
+          [Security]
+            Passphrase=$(echo -e "$prompt_value" | ${lib.getExe pkgs.gnused} "s=\\\=\\\\\\\=g;s=\t=\\\t=g;s=\r=\\\r=g;s=^ =\\\s=")
+          "
+          echo "$config" > "$secrets/${secret_name}"
+        '';
+      };
+  };
+in
+{
+  options.clan.iwd = {
+    networks = lib.mkOption {
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options = {
+              ssid = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "The name of the wifi network";
+              };
+              AutoConnect = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = "Automatically try to join this wifi network";
+              };
+            };
+          }
+        )
+      );
+      default = { };
+      description = "Wifi networks to predefine";
+    };
+  };
+
+  imports = [
+    (lib.mkRemovedOptionModule [
+      "clan"
+      "iwd"
+      "enable"
+    ] "Just define clan.iwd.networks to enable it")
+  ];
+
+  config = lib.mkMerge [
+    (lib.mkIf (cfg.networks != { }) {
+      # Systemd tmpfiles rule to create /var/lib/iwd/example.psk file
+      systemd.tmpfiles.rules = lib.mapAttrsToList (
+        _: value: "C /var/lib/iwd/${value.ssid}.psk 0600 root root - ${secret_path value.ssid}"
+      ) cfg.networks;
+
+      clan.core.facts.services = lib.mapAttrs' secret_generator cfg.networks;
+
+      # TODO: restart the iwd.service if something changes
+    })
+    {
+      # disable wpa supplicant
+      networking.wireless.enable = false;
+
+      # Set the network manager backend to iwd
+      networking.networkmanager.wifi.backend = "iwd";
+
+      # Use iwd instead of wpa_supplicant. It has a user friendly CLI
+      networking.wireless.iwd = {
+        enable = true;
+        settings = {
+          Network = {
+            EnableIPv6 = true;
+            RoutePriorityOffset = 300;
+          };
+          Settings.AutoConnect = true;
+        };
+      };
+    }
+  ];
+}

clanModules/localbackup/README.md

@@ -1,2 +1,3 @@
-Automatically backups current machine to local directory.
+---
+description = "Automatically backups current machine to local directory."
 ---

clanModules/localbackup/default.nix

@@ -6,7 +6,10 @@
 }:
 let
   cfg = config.clan.localbackup;
-  rsnapshotConfig = target: states: ''
+  uniqueFolders = lib.unique (
+    lib.flatten (lib.mapAttrsToList (_name: state: state.folders) config.clan.core.state)
+  );
+  rsnapshotConfig = target: ''
     config_version	1.2
     snapshot_root	${target.directory}
     sync_first	1
@@ -17,12 +20,6 @@ let
     cmd_logger	${pkgs.inetutils}/bin/logger
     cmd_du	${pkgs.coreutils}/bin/du
     cmd_rsnapshot_diff	${pkgs.rsnapshot}/bin/rsnapshot-diff
-    ${lib.optionalString (target.preBackupHook != null) ''
-      cmd_preexec	${pkgs.writeShellScript "preexec.sh" ''
-        set -efu -o pipefail
-        ${target.preBackupHook}
-      ''}
-    ''}
 
     ${lib.optionalString (target.postBackupHook != null) ''
       cmd_postexec	${pkgs.writeShellScript "postexec.sh" ''
@@ -31,11 +28,9 @@ let
       ''}
     ''}
     retain	snapshot	${builtins.toString config.clan.localbackup.snapshots}
-    ${lib.concatMapStringsSep "\n" (state: ''
-      ${lib.concatMapStringsSep "\n" (folder: ''
-        backup	${folder}	${config.networking.hostName}/
-      '') state.folders}
-    '') states}
+    ${lib.concatMapStringsSep "\n" (folder: ''
+      backup	${folder}	${config.networking.hostName}/
+    '') uniqueFolders}
   '';
 in
 {
@@ -129,14 +124,29 @@ in
               ]
             }
             ${lib.concatMapStringsSep "\n" (target: ''
-              (
-                ${mountHook target}
-                echo "Creating backup '${target.name}'"
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" sync
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" snapshot
-              )
-            '') (builtins.attrValues cfg.targets)}
-          '')
+              ${mountHook target}
+              echo "Creating backup '${target.name}'"
+
+              ${lib.optionalString (target.preBackupHook != null) ''
+                (
+                  ${target.preBackupHook}
+                )
+              ''}
+
+              declare -A preCommandErrors
+              ${lib.concatMapStringsSep "\n" (
+                state:
+                lib.optionalString (state.preBackupCommand != null) ''
+                  echo "Running pre-backup command for ${state.name}"
+                  if ! /run/current-system/sw/bin/${state.preBackupCommand}; then
+                    preCommandErrors["${state.name}"]=1
+                  fi
+                ''
+              ) (builtins.attrValues config.clan.core.state)}
+
+              rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
+              rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
+            '') (builtins.attrValues cfg.targets)}'')
           (pkgs.writeShellScriptBin "localbackup-list" ''
             set -efu -o pipefail
             export PATH=${
@@ -167,6 +177,14 @@ in
                 pkgs.gawk
               ]
             }
+            if [[ "''${NAME:-}" == "" ]]; then
+              echo "No backup name given via NAME environment variable"
+              exit 1
+            fi
+            if [[ "''${FOLDERS:-}" == "" ]]; then
+              echo "No folders given via FOLDERS environment variable"
+              exit 1
+            fi
             name=$(awk -F'::' '{print $1}' <<< $NAME)
             backupname=''${NAME#$name::}
 
@@ -182,8 +200,9 @@ in
               exit 1
             fi
 
-            IFS=';' read -ra FOLDER <<< "$FOLDERS"
+            IFS=':' read -ra FOLDER <<< "''$FOLDERS"
             for folder in "''${FOLDER[@]}"; do
+              mkdir -p "$folder"
               rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
             done
           '')
@@ -213,7 +232,7 @@ in
           ''
         ) cfg.targets;
 
-      clanCore.backups.providers.localbackup = {
+      clan.core.backups.providers.localbackup = {
         # TODO list needs to run locally or on the remote machine
         list = "localbackup-list";
         create = "localbackup-create";

clanModules/localsend/README.md

@@ -1,2 +1,3 @@
-Securely sharing files and messages over a local network without internet connectivity.
+---
+description = "Securely sharing files and messages over a local network without internet connectivity."
 ---

clanModules/localsend/default.nix

@@ -4,34 +4,56 @@
   lib,
   ...
 }:
+
+let
+  cfg = config.clan.localsend;
+in
 {
   # Integration can be improved, if the following issues get implemented:
   # - cli frontend: https://github.com/localsend/localsend/issues/11
   # - ipv6 support: https://github.com/localsend/localsend/issues/549
   options.clan.localsend = {
-    enable = lib.mkEnableOption "enable the localsend module";
-    defaultLocation = lib.mkOption {
-      type = lib.types.str;
-      description = "The default download location";
+
+    displayName = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "The name that localsend will use to display your instance.";
     };
+
     package = lib.mkPackageOption pkgs "localsend" { };
+
+    ipv4Addr = lib.mkOption {
+      type = lib.types.str;
+      example = "192.168.56.2/24";
+      description = "Every machine needs a unique ipv4 address";
+    };
   };
 
-  config = lib.mkIf config.clan.localsend.enable {
-    clanCore.state.localsend.folders = [
+  imports = [
+    (lib.mkRemovedOptionModule [
+      "clan"
+      "localsend"
+      "enable"
+    ] "Importing the module will already enable the service.")
+  ];
+  config = {
+    clan.core.state.localsend.folders = [
       "/var/localsend"
-      config.clan.localsend.defaultLocation
     ];
-    environment.systemPackages = [ config.clan.localsend.package ];
+    environment.systemPackages = [
+      (pkgs.callPackage ./localsend-ensure-config {
+        localsend = config.clan.localsend.package;
+        alias = config.clan.localsend.displayName;
+      })
+    ];
 
     networking.firewall.interfaces."zt+".allowedTCPPorts = [ 53317 ];
     networking.firewall.interfaces."zt+".allowedUDPPorts = [ 53317 ];
 
     #TODO: This is currently needed because there is no ipv6 multicasting support yet
-    #
     systemd.network.networks."09-zerotier" = {
       networkConfig = {
-        Address = "192.168.56.2/24";
+        Address = cfg.ipv4Addr;
       };
     };
   };

clanModules/localsend/localsend-ensure-config/default.nix

@@ -0,0 +1,22 @@
+{
+  lib,
+  writers,
+  writeShellScriptBin,
+  localsend,
+  alias ? null,
+}:
+let
+  localsend-ensure-config = writers.writePython3 "localsend-ensure-config" {
+    flakeIgnore = [
+      # We don't live in the dark ages anymore.
+      # Languages like Python that are whitespace heavy will overrun
+      # 79 characters..
+      "E501"
+    ];
+  } (builtins.readFile ./localsend-ensure-config.py);
+in
+writeShellScriptBin "localsend" ''
+  set -xeu
+  ${localsend-ensure-config} ${lib.optionalString (alias != null) alias}
+  ${lib.getExe localsend}
+''

clanModules/localsend/localsend-ensure-config/localsend-ensure-config.py

@@ -0,0 +1,64 @@
+import json
+import sys
+from pathlib import Path
+
+
+def load_json(file_path: Path) -> dict[str, any]:
+    try:
+        with file_path.open("r") as file:
+            return json.load(file)
+    except FileNotFoundError:
+        return {}
+
+
+def save_json(file_path: Path, data: dict[str, any]) -> None:
+    with file_path.open("w") as file:
+        json.dump(data, file, indent=4)
+
+
+def update_json(file_path: Path, updates: dict[str, any]) -> None:
+    data = load_json(file_path)
+    data.update(updates)
+    save_json(file_path, data)
+
+
+def config_location() -> str:
+    config_file = "shared_preferences.json"
+    config_directory = ".local/share/org.localsend.localsend_app"
+    config_path = Path.home() / Path(config_directory) / Path(config_file)
+    return config_path
+
+
+def ensure_config_directory() -> None:
+    config_directory = Path(config_location()).parent
+    config_directory.mkdir(parents=True, exist_ok=True)
+
+
+def load_config() -> dict[str, any]:
+    return load_json(config_location())
+
+
+def save_config(data: dict[str, any]) -> None:
+    save_json(config_location(), data)
+
+
+def update_username(username: str, data: dict[str, any]) -> dict[str, any]:
+    data["flutter.ls_alias"] = username
+    return data
+
+
+def main(argv: list[str]) -> None:
+    try:
+        display_name = argv[1]
+    except IndexError:
+        # This is not an error, just don't update the name
+        print("No display name provided.")
+        sys.exit(0)
+
+    ensure_config_directory()
+    updated_data = update_username(display_name, load_config())
+    save_config(updated_data)
+
+
+if __name__ == "__main__":
+    main(sys.argv[:2])

clanModules/machine-id/README.md

@@ -0,0 +1,4 @@
+---
+description = "Sets the /etc/machine-id and exposes it as a nix option"
+features = [ "inventory" ]
+---

clanModules/machine-id/default.nix

@@ -0,0 +1,6 @@
+# Dont import this file
+# It is only here for backwards compatibility.
+# Dont author new modules with this file.
+{
+  imports = [ ./roles/default.nix ];
+}

clanModules/machine-id/roles/default.nix

@@ -0,0 +1,45 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+
+let
+  var = config.clan.core.vars.generators.machine-id.files.machineId or { };
+in
+{
+  config = lib.mkMerge [
+    (lib.mkIf ((var.value or null) != null) {
+      assertions = [
+        {
+          assertion = lib.stringLength var.value == 32;
+          message = "machineId must be exactly 32 characters long.";
+        }
+      ];
+      boot.kernelParams = [
+        ''systemd.machine_id=${var.value}''
+      ];
+      environment.etc."machine-id" = {
+        text = var.value;
+      };
+    })
+    {
+      clan.core.vars.generators.machine-id = {
+        files.machineId.secret = false;
+        runtimeInputs = [
+          pkgs.coreutils
+          pkgs.bash
+        ];
+        script = ''
+          uuid=$(bash ${./uuid4.sh})
+
+          # Remove the hyphens from the UUID
+          uuid_no_hyphens=$(echo -n "$uuid" | tr -d '-')
+
+          echo -n "$uuid_no_hyphens" > "$out/machineId"
+        '';
+      };
+    }
+  ];
+}

clanModules/machine-id/roles/uuid4.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Read 16 bytes from /dev/urandom
+uuid=$(dd if=/dev/urandom bs=1 count=16 2>/dev/null | od -An -tx1 | tr -d ' \n')
+
+# Break the UUID into pieces and apply the required modifications
+byte6=${uuid:12:2}
+byte8=${uuid:16:2}
+
+# Construct the correct version and variant
+hex_byte6=$(printf "%x" $((0x$byte6 & 0x0F | 0x40)))
+hex_byte8=$(printf "%x" $((0x$byte8 & 0x3F | 0x80)))
+
+# Rebuild the UUID with the correct fields
+uuid_v4="${uuid:0:12}${hex_byte6}${uuid:14:2}${hex_byte8}${uuid:18:14}"
+
+# Format the UUID correctly 8-4-4-4-12
+uuid_formatted="${uuid_v4:0:8}-${uuid_v4:8:4}-${uuid_v4:12:4}-${uuid_v4:16:4}-${uuid_v4:20:12}"
+
+echo -n "$uuid_formatted"

clanModules/matrix-synapse/README.md

@@ -1,2 +1,3 @@
-A federated messaging server with end-to-end encryption.
+---
+description = "A federated messaging server with end-to-end encryption."
 ---

clanModules/matrix-synapse/default.nix

@@ -6,20 +6,74 @@
 }:
 let
   cfg = config.clan.matrix-synapse;
+  element-web =
+    pkgs.runCommand "element-web-with-config" { nativeBuildInputs = [ pkgs.buildPackages.jq ]; }
+      ''
+        cp -r ${pkgs.element-web} $out
+        chmod -R u+w $out
+        jq '."default_server_config"."m.homeserver" = { "base_url": "https://${cfg.app_domain}:443", "server_name": "${cfg.server_tld}" }' \
+          > $out/config.json < ${pkgs.element-web}/config.json
+        ln -s $out/config.json $out/config.${cfg.app_domain}.json
+      '';
 in
+# FIXME: This was taken from upstream. Drop this when our patch is upstream
 {
+  options.services.matrix-synapse.package = lib.mkOption { readOnly = false; };
   options.clan.matrix-synapse = {
-    enable = lib.mkEnableOption "Enable matrix-synapse";
-    domain = lib.mkOption {
+    server_tld = lib.mkOption {
       type = lib.types.str;
-      description = "The domain name of the matrix server";
+      description = "The address that is suffixed after your username i.e @alice:example.com";
+      example = "example.com";
+    };
+
+    app_domain = lib.mkOption {
+      type = lib.types.str;
+      description = "The matrix server hostname also serves the element client";
+      example = "matrix.example.com";
+    };
+
+    users = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options = {
+              name = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "The name of the user";
+              };
+
+              admin = lib.mkOption {
+                type = lib.types.bool;
+                default = false;
+                description = "Whether the user should be an admin";
+              };
+            };
+          }
+        )
+      );
+      description = "A list of users. Not that only new users will be created and existing ones are not modified.";
+      example.alice = {
+        admin = true;
+      };
     };
   };
-  config = lib.mkIf cfg.enable {
+  imports = [
+    ../postgresql
+    (lib.mkRemovedOptionModule [
+      "clan"
+      "matrix-synapse"
+      "enable"
+    ] "Importing the module will already enable the service.")
+    ../nginx
+  ];
+  config = {
     services.matrix-synapse = {
       enable = true;
       settings = {
-        server_name = cfg.domain;
+        server_name = cfg.server_tld;
         database = {
           args.user = "matrix-synapse";
           args.database = "matrix-synapse";
@@ -29,6 +83,7 @@ in
           "turn:turn.matrix.org?transport=udp"
           "turn:turn.matrix.org?transport=tcp"
         ];
+        registration_shared_secret_path = "/run/synapse-registration-shared-secret";
         listeners = [
           {
             port = 8008;
@@ -49,52 +104,80 @@ in
           }
         ];
       };
-      extraConfigFiles = [ "/var/lib/matrix-synapse/registration_shared_secret.yaml" ];
-    };
-    systemd.services.matrix-synapse.serviceConfig.ExecStartPre = [
-      "+${pkgs.writeScript "copy_registration_shared_secret" ''
-        #!/bin/sh
-        cp ${config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path} /var/lib/matrix-synapse/registration_shared_secret.yaml
-        chown matrix-synapse:matrix-synapse /var/lib/matrix-synapse/registration_shared_secret.yaml
-        chmod 600 /var/lib/matrix-synapse/registration_shared_secret.yaml
-      ''}"
-    ];
-
-    clanCore.facts.services."matrix-synapse" = {
-      secret."synapse-registration_shared_secret" = { };
-      generator.path = with pkgs; [
-        coreutils
-        pwgen
-      ];
-      generator.script = ''
-        echo "registration_shared_secret: $(pwgen -s 32 1)" > "$secrets"/synapse-registration_shared_secret
-      '';
     };
 
-    services.postgresql.enable = true;
-    # we need to use both ensusureDatabases and initialScript, because the former runs everytime but with the wrong collation
-    services.postgresql = {
-      ensureDatabases = [ "matrix-synapse" ];
-      ensureUsers = [
-        {
-          name = "matrix-synapse";
-          ensureDBOwnership = true;
+    clan.postgresql.users.matrix-synapse = { };
+    clan.postgresql.databases.matrix-synapse.create.options = {
+      TEMPLATE = "template0";
+      LC_COLLATE = "C";
+      LC_CTYPE = "C";
+      ENCODING = "UTF8";
+      OWNER = "matrix-synapse";
+    };
+    clan.postgresql.databases.matrix-synapse.restore.stopOnRestore = [ "matrix-synapse" ];
+
+    clan.core.vars.generators =
+      {
+        "matrix-synapse" = {
+          files."synapse-registration_shared_secret" = { };
+          runtimeInputs = with pkgs; [
+            coreutils
+            pwgen
+          ];
+          migrateFact = "matrix-synapse";
+          script = ''
+            echo -n "$(pwgen -s 32 1)" > "$out"/synapse-registration_shared_secret
+          '';
+        };
+      }
+      // lib.mapAttrs' (
+        name: user:
+        lib.nameValuePair "matrix-password-${user.name}" {
+          files."matrix-password-${user.name}" = { };
+          migrateFact = "matrix-password-${user.name}";
+          runtimeInputs = with pkgs; [ xkcdpass ];
+          script = ''
+            xkcdpass -n 4 -d - > "$out"/${lib.escapeShellArg "matrix-password-${user.name}"}
+          '';
         }
-      ];
-      initialScript = pkgs.writeText "synapse-init.sql" ''
-        CREATE DATABASE "matrix-synapse"
-          TEMPLATE template0
-          LC_COLLATE = "C"
-          LC_CTYPE = "C";
-      '';
-    };
+      ) cfg.users;
+
+    systemd.services.matrix-synapse =
+      let
+        usersScript =
+          ''
+            while ! ${pkgs.netcat}/bin/nc -z -v ::1 8008; do
+              if ! kill -0 "$MAINPID"; then exit 1; fi
+              sleep 1;
+            done
+          ''
+          + lib.concatMapStringsSep "\n" (user: ''
+            # only create user if it doesn't exist
+            /run/current-system/sw/bin/matrix-synapse-register_new_matrix_user --exists-ok --password-file ${
+              config.clan.core.vars.generators."matrix-password-${user.name}".files."matrix-password-${user.name}".path
+            } --user "${user.name}" ${if user.admin then "--admin" else "--no-admin"}
+          '') (lib.attrValues cfg.users);
+      in
+      {
+        path = [ pkgs.curl ];
+        serviceConfig.ExecStartPre = lib.mkBefore [
+          "+${pkgs.coreutils}/bin/install -o matrix-synapse -g matrix-synapse ${
+            lib.escapeShellArg
+              config.clan.core.vars.generators.matrix-synapse.files."synapse-registration_shared_secret".path
+          } /run/synapse-registration-shared-secret"
+        ];
+        serviceConfig.ExecStartPost = [
+          ''+${pkgs.writeShellScript "matrix-synapse-create-users" usersScript}''
+        ];
+      };
+
     services.nginx = {
       enable = true;
       virtualHosts = {
-        ${cfg.domain} = {
+        "${cfg.server_tld}" = {
           locations."= /.well-known/matrix/server".extraConfig = ''
             add_header Content-Type application/json;
-            return 200 '${builtins.toJSON { "m.server" = "matrix.${cfg.domain}:443"; }}';
+            return 200 '${builtins.toJSON { "m.server" = "${cfg.app_domain}:443"; }}';
           '';
           locations."= /.well-known/matrix/client".extraConfig = ''
             add_header Content-Type application/json;
@@ -102,7 +185,7 @@ in
             return 200 '${
               builtins.toJSON {
                 "m.homeserver" = {
-                  "base_url" = "https://matrix.${cfg.domain}";
+                  "base_url" = "https://${cfg.app_domain}";
                 };
                 "m.identity_server" = {
                   "base_url" = "https://vector.im";
@@ -110,16 +193,15 @@ in
               }
             }';
           '';
-        };
-        "matrix.${cfg.domain}" = {
           forceSSL = true;
           enableACME = true;
-          locations."/_matrix" = {
-            proxyPass = "http://localhost:8008";
-          };
-          locations."/test".extraConfig = ''
-            return 200 "Hello, world!";
-          '';
+        };
+        "${cfg.app_domain}" = {
+          forceSSL = true;
+          enableACME = true;
+          locations."/".root = element-web;
+          locations."/_matrix".proxyPass = "http://localhost:8008"; # TODO: We should make the port configurable
+          locations."/_synapse".proxyPass = "http://localhost:8008";
         };
       };
     };

clanModules/moonlight/README.md

@@ -1,2 +1,3 @@
-A desktop streaming client optimized for remote gaming and synchronized movie viewing.
+---
+description = "A desktop streaming client optimized for remote gaming and synchronized movie viewing."
 ---