modules/borgbackup: migrate to clanServices

Migrates the borgbackup module to clanServices, preserving it's original
configuration syntax and functionality

checks/borgbackup-legacy/default.nix

@@ -0,0 +1,51 @@
+(
+  { ... }:
+  {
+    name = "borgbackup";
+
+    nodes.machine =
+      { self, pkgs, ... }:
+      {
+        imports = [
+          self.clanModules.borgbackup
+          self.nixosModules.clanCore
+          {
+            services.openssh.enable = true;
+            services.borgbackup.repos.testrepo = {
+              authorizedKeys = [ (builtins.readFile ../assets/ssh/pubkey) ];
+            };
+          }
+          {
+            clan.core.settings.directory = ./.;
+            clan.core.state.testState.folders = [ "/etc/state" ];
+            environment.etc.state.text = "hello world";
+            systemd.tmpfiles.settings."vmsecrets" = {
+              "/etc/secrets/borgbackup/borgbackup.ssh" = {
+                C.argument = "${../assets/ssh/privkey}";
+                z = {
+                  mode = "0400";
+                  user = "root";
+                };
+              };
+              "/etc/secrets/borgbackup/borgbackup.repokey" = {
+                C.argument = builtins.toString (pkgs.writeText "repokey" "repokey12345");
+                z = {
+                  mode = "0400";
+                  user = "root";
+                };
+              };
+            };
+            # clan.core.facts.secretStore = "vm";
+            clan.core.vars.settings.secretStore = "vm";
+
+            clan.borgbackup.destinations.test.repo = "borg@localhost:.";
+          }
+        ];
+      };
+    testScript = ''
+      start_all()
+      machine.systemctl("start --wait borgbackup-job-test.service")
+      assert "machine-test" in machine.succeed("BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes /run/current-system/sw/bin/borg-job-test list")
+    '';
+  }
+)

checks/borgbackup/default.nix

@@ -1,51 +1,118 @@
-(
+{
-  { ... }:
+  pkgs,
-  {
+  self,
-    name = "borgbackup";
+  clanLib,
+  ...
+}:
 
-    nodes.machine =
+clanLib.test.makeTestClan {
-      { self, pkgs, ... }:
+  inherit pkgs self;
-      {
+  useContainers = true;
-        imports = [
-          self.clanModules.borgbackup
-          self.nixosModules.clanCore
-          {
-            services.openssh.enable = true;
-            services.borgbackup.repos.testrepo = {
-              authorizedKeys = [ (builtins.readFile ../assets/ssh/pubkey) ];
-            };
-          }
-          {
-            clan.core.settings.directory = ./.;
-            clan.core.state.testState.folders = [ "/etc/state" ];
-            environment.etc.state.text = "hello world";
-            systemd.tmpfiles.settings."vmsecrets" = {
-              "/etc/secrets/borgbackup/borgbackup.ssh" = {
-                C.argument = "${../assets/ssh/privkey}";
-                z = {
-                  mode = "0400";
-                  user = "root";
-                };
-              };
-              "/etc/secrets/borgbackup/borgbackup.repokey" = {
-                C.argument = builtins.toString (pkgs.writeText "repokey" "repokey12345");
-                z = {
-                  mode = "0400";
-                  user = "root";
-                };
-              };
-            };
-            # clan.core.facts.secretStore = "vm";
-            clan.core.vars.settings.secretStore = "vm";
 
-            clan.borgbackup.destinations.test.repo = "borg@localhost:.";
+  nixosTest = (
-          }
+    { ... }:
-        ];
+
+    {
+      name = "borgbackup";
+
+      clan = {
+        directory = ./.;
+        modules."@clan/borgbackup" = ../../clanServices/borgbackup/default.nix;
+        inventory = {
+
+          machines.clientone = { };
+          machines.serverone = { };
+
+          instances = {
+            borgone = {
+
+              module.name = "@clan/borgbackup";
+
+              roles.client.machines."clientone" = { };
+              roles.server.machines."serverone".settings.directory = "/tmp/borg-test";
+            };
+          };
+        };
       };
-    testScript = ''
+
-      start_all()
+      nodes = {
-      machine.systemctl("start --wait borgbackup-job-test.service")
+
-      assert "machine-test" in machine.succeed("BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes /run/current-system/sw/bin/borg-job-test list")
+        serverone = {
-    '';
+          services.openssh.enable = true;
-  }
+          # Needed so PAM doesn't see the user as locked
-)
+          users.users.borg.password = "borg";
+        };
+
+        clientone =
+          { config, pkgs, ... }:
+          let
+            dependencies = [
+              self
+              pkgs.stdenv.drvPath
+            ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
+            closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
+
+          in
+          {
+
+            services.openssh.enable = true;
+
+            users.users.root.openssh.authorizedKeys.keyFiles = [ ../assets/ssh/pubkey ];
+
+            clan.core.networking.targetHost = config.networking.hostName;
+
+            environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
+
+            environment.etc.install-closure.source = "${closureInfo}/store-paths";
+            nix.settings = {
+              substituters = pkgs.lib.mkForce [ ];
+              hashed-mirrors = null;
+              connect-timeout = pkgs.lib.mkForce 3;
+              flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
+            };
+            system.extraDependencies = dependencies;
+
+            clan.core.state.test-backups.folders = [ "/var/test-backups" ];
+          };
+
+      };
+
+      testScript = ''
+        import json
+        start_all()
+
+        machines = [clientone, serverone]
+
+        for m in machines:
+            m.systemctl("start network-online.target")
+
+        for m in machines:
+            m.wait_for_unit("network-online.target")
+
+        # dummy data
+        clientone.succeed("mkdir -p /var/test-backups /var/test-service")
+        clientone.succeed("echo testing > /var/test-backups/somefile")
+
+        clientone.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../assets/ssh/privkey} /root/.ssh/id_ed25519")
+        clientone.succeed("${pkgs.coreutils}/bin/touch /root/.ssh/known_hosts")
+        clientone.wait_until_succeeds("timeout 2 ssh -o StrictHostKeyChecking=accept-new localhost hostname")
+        clientone.wait_until_succeeds("timeout 2 ssh -o StrictHostKeyChecking=accept-new $(hostname) hostname")
+
+        # create
+        clientone.succeed("borgbackup-create >&2")
+        clientone.wait_until_succeeds("! systemctl is-active borgbackup-job-serverone >&2")
+
+        # list
+        backup_id = json.loads(clientone.succeed("borg-job-serverone list --json"))["archives"][0]["archive"]
+        out = clientone.succeed("borgbackup-list").strip()
+        print(out)
+        assert backup_id in out, f"backup {backup_id} not found in {out}"
+
+        # borgbackup restore
+        clientone.succeed("rm -f /var/test-backups/somefile")
+        clientone.succeed(f"NAME='serverone::borg@serverone:.::{backup_id}' borgbackup-restore >&2")
+        assert clientone.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
+      '';
+    }
+  );
+}

checks/borgbackup/sops/machines/clientone/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1tyyx2ratu8s9ugyre36xyksnquth9gxeh7wjdhvsk89rtf8yu5wq0pk04c",
+    "type": "age"
+  }
+]

checks/borgbackup/sops/secrets/clientone-age.key/secret

@@ -0,0 +1,15 @@
+{
+	"data": "ENC[AES256_GCM,data:wCKoKuJo4uXycfqEUYAXDlRRMGJaWgOFiaQa4Wigs0jx1eCI80lP3cEZ1QKyrU/9m9POoZz0JlaKHcuhziTKUqaevHvGfVq2y00=,iv:pH5a90bJbK9Ro6zndNJ18qd4/rU+Tdm+y+jJZtY7UGg=,tag:9lHZJ9C/zIfy8nFrYt9JBQ==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwUDhpd1ZqbWFqR0I3dVFI\nOHlyZnFUYXJnWElrRWhoUHVNMzdKd0VrcGdRCkphQVhuYzlJV0p1MG9MSW5ncWJ3\nREp1OEJxMzQzS2MxTk9aMkJ1a3B0Q0kKLS0tIENweVJ2Tk1yeXlFc2F5cTNIV3F3\nTkRFOVZ1amRIYmg1K3hGWUFSTTl4Wk0KHJRJ7756Msod7Bsmn9SgtwRo53B8Ilp3\nhsAPv+TtdmOD8He9MvGV+BElKEXCsLUwhp/Py6n6CJCczu0VIr8owg==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-05-20T13:33:56Z",
+		"mac": "ENC[AES256_GCM,data:FyfxXhnI6o4SVGJY2e1eMDnfkbMWiCkP4JL/G4PQvzz+c7OIuz8xaa03P3VW7b7o85NP2Tln4FMNTZ0FYtQwd0kKypLUnIxAHsixAHFCv4X8ul1gtZynzgbFbmc0GkfVWW8Lf+U+vvDwT+UrEVfcmksCjdvAOwP26PvlEhYEkSw=,iv:H+VrWYL+kLOLezCZrI8ZgeCsaUdpb7LxDMiLotezVPs=,tag:B/cbPdiEFumGKQHby5inCA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

checks/borgbackup/sops/secrets/clientone-age.key/users/admin

@@ -0,0 +1 @@
+../../../users/admin

checks/borgbackup/sops/users/admin/key.json

@@ -0,0 +1,4 @@
+{
+  "publickey": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+  "type": "age"
+}

checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/machines/clientone

@@ -0,0 +1 @@
+../../../../../../sops/machines/clientone

checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:52vY68gqbwiZRMUBKc9SeXR06fuKAhuAPciLpxXgEOxI,iv:Y34AVoHaZzRiFFTDbekXP1X3W8zSXJmzVCYODYkdxnY=,tag:8WQaGEHQKT/n+auHUZCE0w==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOdUFUZUZ2M00zTGlhNjF4\nL0VlMVY4Z2xMbWRWR29zZlFwdm1XRk12NGtBCnkrb3A4M3BkalMyeWdDaUdQdStt\nUWY3SXJROXdpRzN0NlBJNEpjTEZ0aFkKLS0tIGZkMGhsTXB2RnRqVHVrUFQwL2lw\nZnBreWhWa3Jrcm4yOXBiaUlPWFM1aDAKRE+Zzrja7KeANEJUbmFYuVoO3qGyi4iH\n0cfH0W8irRe9vsKMXz7YJxtByYLwRulrT8tXtElHvIEVJG0mwwaf0Q==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1tyyx2ratu8s9ugyre36xyksnquth9gxeh7wjdhvsk89rtf8yu5wq0pk04c",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsNEljUFdnQ0tTQ1IxZ2Zo\nYkc4V2dCaUk0YXh5SzlSazhsRTVKVzFvVXhFCkRyMlMxR3EyWEZIRzFQV3d2dVpz\na3NPbk9XdWR1NmtMQlZsNlBuU0NkQWMKLS0tIDlDYzMzOExVL1g5SVRHYlpUQlBV\na2lpdTUwaEd4OXhWUWxuV04xRVVKNHcK9coohAD1IoarLOXSGg3MIRXQ3BsTIA4y\nKrcS/PxITKJs7ihg93RZin70R79Qsij1RHZLKGfgGJ67i8ZCxc4N0g==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-05-20T13:33:59Z",
+		"mac": "ENC[AES256_GCM,data:eABMaIe07dwAMMlgrIUUpfpj73q1H5Keafql91MBQ5NN9Znr5lI/ennQsQsuLO8ZTCC34US/MJndliW34SqVM9y53p0jjPzqBxSKYq74iNcBz7+TxbjlY1aapgTRPr6Ta8I/5loohnxlHqjvLL70ZzfbChDN0/4jZsDVXYNfbIk=,iv:41Mz2u40JN0iE5zPUK6siaxo0rTtlk7fGWq7TF5NyUI=,tag:1A+h6XPH7DeQ6kxGDV3PgQ==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.repokey/users/admin

@@ -0,0 +1 @@
+../../../../../../sops/users/admin

checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh.pub/value

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE3clYF6BDZ0PxfDdprx7YYM4U4PKEZkWUuhpre0wb7w nixbld@kiwi

checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/machines/clientone

@@ -0,0 +1 @@
+../../../../../../sops/machines/clientone

checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:tAjfBW75XDS8lfJCf/+9rPYH3aMjRX1nmdN5dPMxnrlhuEPM3Smv9AM93Tz36k7BKk31bUWcV/99ax+KaIK1Rzgym/CwKGGxIUziuVOEOwrCOBeOw7amZ9YGsgiLUTLIhoeO6SjfdZ4q2JxGPw7KqNfUM9kiZT01vx5JTLa24JdvBKpizbtHRlL1lappTRVt0dG2WhT9/YhQUGu9ZFqPs8+bPOBclc78qjCm2DAPgsprK+JCBuq+r+qHgAx4Ee1QHI7FC39e5NeGBTBeZfZ5d95+0klKuTx9FCPs6QRBkQ0tN29OpwzkdSuRAXGGHpzPkZ+FupbETtSQWCmnjma6jPzEl8oDUTWooKK0mUEz8icvTQvRfyM3Qt3mQpkX3e0rTEbZzoLdWCwTufP/tRQNDCWvI/NV7OjIHpNPjymqE5uPmiBpA6y6hhCH7zL1eDo11ICSIX3hkyFJH2svvFQn6oLrPAoByvNutfetKhd8z7NFpVeIOWwtuPzO7wU5M7zESHww0JF78vjFwimQYYhQ,iv:fVjeVez4dTGSrANi5ZeP9PJhsSySqeqqJzBDbd0gFW4=,tag:Aa89+bWLljxV1tlSHtpddw==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVaW94M3VwcFJ2elcrRGlv\nUGdzVk9vU2ZweFpIVVlIRUEyRVlSMlEyeHpVCnJuV0xIS3hMLy9IbG92S0pvL2RP\nL0J0WkVuWVhQdldHekdYNTVXdFkrUlEKLS0tIFQzdGErZVBwQUFNMXErbDBQVURZ\naHlsY2hDa1Zud1E2dFh0ZHl4VEJ2S0kKVABqwRcCUTcsBInfo9CpFtoM3kl4KMyU\nGXDjHOSjlX5df7OKZAvYukgX7Q2penvq+Fq4fa4A1Cmkqga7cHdJ+A==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1tyyx2ratu8s9ugyre36xyksnquth9gxeh7wjdhvsk89rtf8yu5wq0pk04c",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnbHRSVEg3Vi9qTnAwWGF6\nbEdIR2gvZ2laZnJMbVF3NjcvN25OdXF3WXowCnVUODdEa1NWU3JISXlrNldOMjVi\ndUlMTVdBaWxvZHlwSTdJY3NCcll4SjAKLS0tIEp6ZVlDTklqVXdNYzJ2dElCR21o\nUWphMDdyVVppVnFHOVlHZTNtajZzOXMKRB61lUrAkUXSYl3ffOOK8k4QgLA4bFln\naQ7GOol8f8W5H68zXBMZrhjP6k4kZDfknc9jgyoWM7jaZNSWC5J19Q==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-05-20T13:33:59Z",
+		"mac": "ENC[AES256_GCM,data:NjVpDweqxTSQGt9VKR/CMfvbvHQJHCi8P7XbOuKLZKQ4GVoeZ5r4PsC6nxKHHikN6YL1oJCmaSxr0mJRk/sFZg/+wdW8L7F5aQeFRiWo9jCjH0MDMnfiu5a0xjRt21uPl/7LUJ9jNon5nyxPTlZMeYSvTP2Q9spnNuN8vqipP68=,iv:DPvbN9IvWiUfxiJk6mey/us8N1GGVJcSJrT8Bty4kB4=,tag:+emK8uSkfIGUXoYpaWeu3A==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

checks/borgbackup/vars/per-machine/clientone/borgbackup/borgbackup.ssh/users/admin

@@ -0,0 +1 @@
+../../../../../../sops/users/admin

checks/clan-core-for-checks.nix

@@ -1,6 +1,6 @@
 { fetchgit }:
 fetchgit {
   url = "https://git.clan.lol/clan/clan-core.git";
-  rev = "1523ac18c9c575d32033dcf1e769fccc324f248e";
+  rev = "cb4f2ab014aa01f249e852da9e1b92cdc44c2a66";
-  sha256 = "0nxhw5s9lva4g1rgx6pgczh3vxrskmmlxay48wvn2pnkrlvhr9j8";
+  sha256 = "sha256-8zlAW+iWKrxxozSNEZAVlDPOMLzetd36RL1GLnAvvjg=";
 }

checks/flake-module.nix

@@ -41,7 +41,7 @@ in
 
               # Base Tests
               secrets = self.clanLib.test.baseTest ./secrets nixosTestArgs;
-              borgbackup = self.clanLib.test.baseTest ./borgbackup nixosTestArgs;
+              borgbackup-legacy = self.clanLib.test.baseTest ./borgbackup-legacy nixosTestArgs;
               wayland-proxy-virtwl = self.clanLib.test.baseTest ./wayland-proxy-virtwl nixosTestArgs;
 
               # Container Tests
@@ -53,6 +53,7 @@ in
               # Clan Tests
               dummy-inventory-test = import ./dummy-inventory-test nixosTestArgs;
               admin = import ./admin nixosTestArgs;
+              borgbackup = import ./borgbackup nixosTestArgs;
               data-mesher = import ./data-mesher nixosTestArgs;
               syncthing = import ./syncthing nixosTestArgs;
             }
@@ -108,11 +109,13 @@ in
                 cat $schemaFile > $out/allSchemas.json
               '';
 
-          clan-core-for-checks = pkgs.runCommand "clan-core-for-checks" { } ''
+          clan-core-for-checks = self;
-            cp -r ${pkgs.callPackage ./clan-core-for-checks.nix { }} $out
+
-            chmod +w $out/flake.lock
+          #     pkgs.runCommand "clan-core-for-checks" { } ''
-            cp ${../flake.lock} $out/flake.lock
+          #   cp -r ${pkgs.callPackage ./clan-core-for-checks.nix { }} $out
-          '';
+          #   chmod +w $out/flake.lock
+          #   cp ${../flake.lock} $out/flake.lock
+          # '';
         };
       legacyPackages = {
         nixosTests =

clanModules/borgbackup/README.md

@@ -1,7 +1,7 @@
 ---
 description = "Efficient, deduplicating backup program with optional compression and secure encryption."
 categories = ["System"]
-features = [ "inventory" ]
+features = [ "inventory", "deprecated" ]
 ---
 BorgBackup (short: Borg) gives you:
 

clanModules/borgbackup/roles/client.nix

@@ -106,7 +106,8 @@ in
     systemd.services = lib.mapAttrs' (
       _: dest:
       lib.nameValuePair "borgbackup-job-${dest.name}" {
-        # since borgbackup mounts the system read-only, we need to run in a ExecStartPre script, so we can generate additional files.
+        # since borgbackup mounts the system read-only, we need to run in a
+        # ExecStartPre script, so we can generate additional files.
         serviceConfig.ExecStartPre = [
           ''+${pkgs.writeShellScript "borgbackup-job-${dest.name}-pre-backup-commands" preBackupScript}''
         ];

clanServices/borgbackup/README.md

@@ -0,0 +1,9 @@
+BorgBackup (short: Borg) gives you:
+
+- Space efficient storage of backups.
+- Secure, authenticated encryption.
+- Compression: lz4, zstd, zlib, lzma or none.
+- Mountable backups with FUSE.
+- Easy installation on multiple platforms: Linux, macOS, BSD, …
+- Free software (BSD license).
+- Backed by a large and active open-source community.

clanServices/borgbackup/default.nix

@@ -0,0 +1,313 @@
+{ ... }:
+{
+  _class = "clan.service";
+  manifest.name = "borgbackup";
+  manifest.description = "Efficient, deduplicating backup program with optional compression and secure encryption.";
+  manifest.categories = [ "System" ];
+  manifest.readme = builtins.readFile ./README.md;
+
+  # TODO: a client can only be in one instance, add constraint
+
+  roles.server = {
+
+    interface =
+      { lib, ... }:
+      {
+
+        options.directory = lib.mkOption {
+          type = lib.types.str;
+          default = "/var/lib/borgbackup";
+          description = ''
+            The directory where the borgbackup repositories are stored.
+          '';
+        };
+
+      };
+
+    perInstance =
+      {
+        roles,
+        settings,
+        ...
+      }:
+      {
+        nixosModule =
+          {
+            config,
+            ...
+          }:
+          {
+
+            config.services.openssh.enable = true;
+
+            config.services.borgbackup.repos =
+              let
+                borgbackupIpMachinePath =
+                  machine:
+                  config.clan.core.settings.directory
+                  + "/vars/per-machine/${machine}/borgbackup/borgbackup.ssh.pub/value";
+
+                hosts = builtins.mapAttrs (machineName: _machineSettings: {
+                  # name = "${instanceName}-${machineName}";
+                  # value = {
+                  path = "${settings.directory}/${machineName}";
+                  authorizedKeys = [ (builtins.readFile (borgbackupIpMachinePath machineName)) ];
+                  # };
+                  # }) machinesWithKey;
+                }) roles.client.machines;
+              in
+              hosts;
+          };
+      };
+  };
+
+  roles.client = {
+    interface =
+      {
+        lib,
+        ...
+      }:
+      {
+
+        options.destinations = lib.mkOption {
+          type = lib.types.attrsOf (
+            lib.types.submodule (
+              { name, ... }:
+              {
+                options = {
+                  name = lib.mkOption {
+                    type = lib.types.strMatching "^[a-zA-Z0-9._-]+$";
+                    default = name;
+                    description = "the name of the backup job";
+                  };
+                  repo = lib.mkOption {
+                    type = lib.types.str;
+                    description = "the borgbackup repository to backup to";
+                  };
+                  rsh = lib.mkOption {
+                    type = lib.types.str;
+                    defaultText = "ssh -i \${config.clan.core.vars.generators.borgbackup.files.\"borgbackup.ssh\".path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
+                    description = "the rsh to use for the backup";
+                  };
+                };
+              }
+            )
+          );
+          default = { };
+          description = ''
+            external destinations where the machine should be backuped to
+          '';
+        };
+
+        options.exclude = lib.mkOption {
+          type = lib.types.listOf lib.types.str;
+          example = [ "*.pyc" ];
+          default = [ ];
+          description = ''
+            Directories/Files to exclude from the backup.
+            Use * as a wildcard.
+          '';
+        };
+      };
+
+    perInstance =
+      {
+        extendSettings,
+        roles,
+        ...
+      }:
+      {
+        nixosModule =
+          {
+            config,
+            lib,
+            pkgs,
+            ...
+          }:
+          let
+            settings = extendSettings {
+
+              # Adding default value with option merging, because it depends on
+              # generators, which we can reference here.
+              options.destinations = lib.mkOption {
+                type = lib.types.attrsOf (
+                  lib.types.submodule {
+                    options = {
+                      rsh = lib.mkOption {
+                        default = "ssh -i ${
+                          config.clan.core.vars.generators.borgbackup.files."borgbackup.ssh".path
+                        } -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=Yes -o PasswordAuthentication=no";
+                      };
+                    };
+                  }
+                );
+              };
+            };
+
+          in
+
+          {
+            config =
+              let
+                preBackupScript = ''
+                  declare -A preCommandErrors
+
+                  ${lib.concatMapStringsSep "\n" (
+                    state:
+                    lib.optionalString (state.preBackupCommand != null) ''
+                      echo "Running pre-backup command for ${state.name}"
+                      if ! /run/current-system/sw/bin/${state.preBackupCommand}; then
+                        preCommandErrors["${state.name}"]=1
+                      fi
+                    ''
+                  ) (lib.attrValues config.clan.core.state)}
+
+                  if [[ ''${#preCommandErrors[@]} -gt 0 ]]; then
+                    echo "pre-backup commands failed for the following services:"
+                    for state in "''${!preCommandErrors[@]}"; do
+                      echo "  $state"
+                    done
+                    exit 1
+                  fi
+                '';
+
+                # The destinations from server.roles.machines.*
+                # name is the server, machine can only be in one instance
+                internalDestinations =
+                  let
+                    destinations = builtins.map (serverName: {
+                      name = "${serverName}";
+                      value = {
+                        # inherit name;
+                        name = "${serverName}";
+                        repo = "borg@${serverName}:.";
+                        # rsh = "";
+
+                        rsh = "ssh -i ${
+                          config.clan.core.vars.generators.borgbackup.files."borgbackup.ssh".path
+                        } -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=Yes";
+                      };
+                    }) (builtins.attrNames roles.server.machines);
+                  in
+                  (builtins.listToAttrs destinations);
+
+                # The destinations specified via roles.client.machines.*.settings.destinations.<name>
+                # name is the <name>
+                externalDestinations = lib.mapAttrs' (
+                  name: dest: lib.nameValuePair name dest
+                ) settings.destinations;
+
+                allDestinations =
+                  lib.warnIf ((builtins.intersectAttrs externalDestinations internalDestinations) != { })
+                    "You are overwriting an internalDestinations through an externalDestination configuration."
+                    (internalDestinations // externalDestinations);
+
+              in
+              {
+                services.openssh.enable = true;
+
+                # Derived from the destinations
+                systemd.services = lib.mapAttrs' (
+                  destName: _dest:
+                  lib.nameValuePair "borgbackup-job-${destName}" {
+                    # since borgbackup mounts the system read-only, we need to
+                    # run in a ExecStartPre script, so we can generate
+                    # additional files.
+                    serviceConfig.ExecStartPre = [
+                      ''+${pkgs.writeShellScript "borgbackup-job-${destName}-pre-backup-commands" preBackupScript}''
+                    ];
+                  }
+                ) allDestinations;
+
+                services.borgbackup.jobs = lib.mapAttrs (_: dest: {
+                  paths = lib.unique (
+                    lib.flatten (map (state: state.folders) (lib.attrValues config.clan.core.state))
+                  );
+                  exclude = settings.exclude;
+                  repo = dest.repo;
+                  environment.BORG_RSH = dest.rsh;
+                  compression = "auto,zstd";
+                  startAt = "*-*-* 01:00:00";
+                  persistentTimer = true;
+
+                  encryption = {
+                    mode = "repokey";
+                    passCommand = "cat ${config.clan.core.vars.generators.borgbackup.files."borgbackup.repokey".path}";
+                  };
+
+                  prune.keep = {
+                    within = "1d"; # Keep all archives from the last day
+                    daily = 7;
+                    weekly = 4;
+                    monthly = 0;
+                  };
+                }) allDestinations;
+
+                clan.core.vars.generators.borgbackup = {
+
+                  files."borgbackup.ssh.pub".secret = false;
+                  files."borgbackup.ssh" = { };
+                  files."borgbackup.repokey" = { };
+
+                  migrateFact = "borgbackup";
+                  runtimeInputs = [
+                    pkgs.coreutils
+                    pkgs.openssh
+                    pkgs.xkcdpass
+                  ];
+                  script = ''
+                    ssh-keygen -t ed25519 -N "" -f "$out"/borgbackup.ssh
+                    xkcdpass -n 4 -d - > "$out"/borgbackup.repokey
+                  '';
+                };
+
+                clan.core.backups.providers.borgbackup = {
+                  list = "borgbackup-list";
+                  create = "borgbackup-create";
+                  restore = "borgbackup-restore";
+                };
+
+                environment.systemPackages = [
+                  (pkgs.writeShellApplication {
+                    name = "borgbackup-create";
+                    runtimeInputs = [ config.systemd.package ];
+                    text = ''
+                      ${lib.concatMapStringsSep "\n" (dest: ''
+                        systemctl start borgbackup-job-${dest}
+                      '') (lib.attrNames allDestinations)}
+                    '';
+                  })
+                  (pkgs.writeShellApplication {
+                    name = "borgbackup-list";
+                    runtimeInputs = [ pkgs.jq ];
+                    text = ''
+                      (${
+                        lib.concatMapStringsSep "\n" (
+                          dest:
+                          # we need yes here to skip the changed url verification
+                          ''echo y | /run/current-system/sw/bin/borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.name}::${dest.repo}::" + .name)}]' ''
+                        ) (lib.attrValues allDestinations)
+                      }) | jq -s 'add // []'
+                    '';
+                  })
+                  (pkgs.writeShellApplication {
+                    name = "borgbackup-restore";
+                    runtimeInputs = [ pkgs.gawk ];
+                    text = ''
+                      cd /
+                      IFS=':' read -ra FOLDER <<< "''${FOLDERS-}"
+                      job_name=$(echo "$NAME" | awk -F'::' '{print $1}')
+                      backup_name=''${NAME#"$job_name"::}
+                      if [[ ! -x /run/current-system/sw/bin/borg-job-"$job_name" ]]; then
+                        echo "borg-job-$job_name not found: Backup name is invalid" >&2
+                        exit 1
+                      fi
+                      echo y | /run/current-system/sw/bin/borg-job-"$job_name" extract "$backup_name" "''${FOLDER[@]}"
+                    '';
+                  })
+                ];
+              };
+          };
+      };
+  };
+}

clanServices/borgbackup/flake-module.nix

@@ -0,0 +1,6 @@
+{ lib, ... }:
+{
+  clan.modules = {
+    borgbackup = lib.modules.importApply ./default.nix { };
+  };
+}

clanServices/flake-module.nix

@@ -4,5 +4,6 @@
     ./admin/flake-module.nix
     ./hello-world/flake-module.nix
     ./wifi/flake-module.nix
+    ./borgbackup/flake-module.nix
   ];
 }

docs/mkdocs.yml

@@ -83,6 +83,7 @@ nav:
       - Clan Services:
           - Overview: reference/clanServices/index.md
           - reference/clanServices/admin.md
+          - reference/clanServices/borgbackup.md
           - reference/clanServices/hello-world.md
           - reference/clanServices/wifi.md
       - Clan Modules:

docs/nix/render_options/__init__.py

@@ -494,6 +494,9 @@ Learn how to use `clanServices` in practice in the [Using clanServices guide](..
         output += render_categories(
             module_info["manifest"]["categories"], fm.categories_info
         )
+
+        output += f"{module_info['manifest']['readme']}\n"
+
         output += "\n---\n\n## Roles\n"
 
         output += f"The {module_name} module has the following roles:\n\n"