Merge pull request 'localbackup' (#1020) from localbackup into main

checks/backups/flake-module.nix

@@ -76,7 +76,15 @@
         clanCore.secretStore = "vm";
         clanCore.clanDir = ../..;
 
-        environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
+        environment.systemPackages = [
+          self.packages.${pkgs.system}.clan-cli
+          (pkgs.writeShellScriptBin "pre-restore-command" ''
+            touch /var/test-service/pre-restore-command
+          '')
+          (pkgs.writeShellScriptBin "post-restore-command" ''
+            touch /var/test-service/post-restore-command
+          '')
+        ];
         environment.etc.install-closure.source = "${closureInfo}/store-paths";
         nix.settings = {
           substituters = lib.mkForce [ ];
@@ -86,6 +94,12 @@
         };
         system.extraDependencies = dependencies;
         clanCore.state.test-backups.folders = [ "/var/test-backups" ];
+
+        clanCore.state.test-service = {
+          preRestoreCommand = "pre-restore-command";
+          postRestoreCommand = "post-restore-command";
+          folders = [ "/var/test-service" ];
+        };
         clan.borgbackup.destinations.test-backup.repo = "borg@machine:.";
 
         services.borgbackup.repos.test-backups = {
@@ -110,7 +124,7 @@
             start_all()
 
             # dummy data
-            machine.succeed("mkdir -p /var/test-backups")
+            machine.succeed("mkdir -p /var/test-backups /var/test-service")
             machine.succeed("echo testing > /var/test-backups/somefile")
 
             # create
@@ -119,14 +133,16 @@
 
             # list
             backup_id = json.loads(machine.succeed("borg-job-test-backup list --json"))["archives"][0]["archive"]
-            out = machine.succeed("clan --debug --flake ${self} backups list test-backup")
+            out = machine.succeed("clan --debug --flake ${self} backups list test-backup").strip()
             print(out)
             assert backup_id in out, f"backup {backup_id} not found in {out}"
 
             # restore
             machine.succeed("rm -f /var/test-backups/somefile")
-            machine.succeed(f"clan --debug --flake ${self} backups restore test-backup borgbackup borg@machine:.::{backup_id} >&2")
+            machine.succeed(f"clan --debug --flake ${self} backups restore test-backup borgbackup {out} >&2")
             assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
+            machine.succeed("test -f /var/test-service/pre-restore-command")
+            machine.succeed("test -f /var/test-service/post-restore-command")
           '';
         } { inherit pkgs self; };
       };

clanModules/borgbackup.nix

@@ -15,7 +15,7 @@ in
         {
           options = {
             name = lib.mkOption {
-              type = lib.types.str;
+              type = lib.types.strMatching "^[a-zA-Z0-9._-]+$";
               default = name;
               description = "the name of the backup job";
             };
@@ -90,32 +90,41 @@ in
       '';
     };
 
-    environment.systemPackages = [ pkgs.jq ];
+    environment.systemPackages = [
-
+      (pkgs.writeShellScriptBin "borgbackup-create" ''
-    clanCore.backups.providers.borgbackup = {
+        set -efu -o pipefail
-      # TODO list needs to run locally or on the remote machine
-      list = ''
-        set -efu
-        # we need yes here to skip the changed url verification
-        ${
-          lib.concatMapStringsSep "\\\n" (
-            dest:
-            ''yes y | borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.repo}::" + .name), "job_name": "${dest.name}"}]' ''
-          ) (lib.attrValues cfg.destinations)
-        } | jq -s 'add'
-      '';
-      create = ''
         ${lib.concatMapStringsSep "\n" (dest: ''
           systemctl start borgbackup-job-${dest.name}
         '') (lib.attrValues cfg.destinations)}
-      '';
+      '')
-
+      (pkgs.writeShellScriptBin "borgbackup-list" ''
-      restore = ''
         set -efu
+        (${
+          lib.concatMapStringsSep "\n" (
+            dest:
+            # we need yes here to skip the changed url verification
+            ''yes y | borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.name}::${dest.repo}::" + .name)}]' ''
+          ) (lib.attrValues cfg.destinations)
+        }) | ${pkgs.jq}/bin/jq -s 'add'
+      '')
+      (pkgs.writeShellScriptBin "borgbackup-restore" ''
+        set -efux
         cd /
         IFS=';' read -ra FOLDER <<< "$FOLDERS"
-        yes y | borg-job-"$JOB_NAME" extract --list "$NAME" "''${FOLDER[@]}"
+        job_name=$(echo "$NAME" | ${pkgs.gawk}/bin/awk -F'::' '{print $1}')
-      '';
+        backup_name=''${NAME#"$job_name"::}
+        if ! command -v borg-job-"$job_name" &> /dev/null; then
+          echo "borg-job-$job_name not found: Backup name is invalid" >&2
+          exit 1
+        fi
+        yes y | borg-job-"$job_name" extract --list "$backup_name" "''${FOLDER[@]}"
+      '')
+    ];
+
+    clanCore.backups.providers.borgbackup = {
+      list = "borgbackup-list";
+      create = "borgbackup-create";
+      restore = "borgbackup-restore";
     };
   };
 }

clanModules/flake-module.nix

@@ -8,6 +8,7 @@
       ];
     };
     borgbackup = ./borgbackup.nix;
+    localbackup = ./localbackup.nix;
     deltachat = ./deltachat.nix;
     moonlight = ./moonlight.nix;
     sunshine = ./sunshine.nix;

clanModules/localbackup.nix

@@ -0,0 +1,151 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.clan.localbackup;
+  rsnapshotConfig = target: states: ''
+    config_version	1.2
+    snapshot_root	${target}
+    sync_first	1
+    cmd_cp	${pkgs.coreutils}/bin/cp
+    cmd_rm	${pkgs.coreutils}/bin/rm
+    cmd_rsync	${pkgs.rsync}/bin/rsync
+    cmd_ssh	${pkgs.openssh}/bin/ssh
+    cmd_logger	${pkgs.inetutils}/bin/logger
+    cmd_du	${pkgs.coreutils}/bin/du
+    cmd_rsnapshot_diff	${pkgs.rsnapshot}/bin/rsnapshot-diff
+    retain	snapshot	${builtins.toString config.clan.localbackup.snapshots}
+    ${lib.concatMapStringsSep "\n" (state: ''
+      ${lib.concatMapStringsSep "\n" (folder: ''
+        backup	${folder}	${config.networking.hostName}/
+      '') state.folders}
+    '') states}
+  '';
+in
+{
+  options.clan.localbackup = {
+    targets = lib.mkOption {
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options = {
+              name = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "the name of the backup job";
+              };
+              directory = lib.mkOption {
+                type = lib.types.str;
+                description = "the directory to backup";
+              };
+              mountpoint = lib.mkOption {
+                type = lib.types.nullOr (lib.types.strMatching "^[a-zA-Z0-9./_-]+$");
+                default = null;
+                description = "mountpoint of the directory to backup. If set, the directory will be mounted before the backup and unmounted afterwards";
+              };
+            };
+          }
+        )
+      );
+      description = "List of directories where backups are stored";
+    };
+
+    snapshots = lib.mkOption {
+      type = lib.types.int;
+      default = 20;
+      description = "Number of snapshots to keep";
+    };
+  };
+
+  config =
+    let
+      setupMount =
+        mountpoint:
+        lib.optionalString (mountpoint != null) ''
+          mkdir -p ${lib.escapeShellArg mountpoint}
+          if mountpoint -q ${lib.escapeShellArg mountpoint}; then
+            umount ${lib.escapeShellArg mountpoint}
+          fi
+          mount ${lib.escapeShellArg mountpoint}
+          trap "umount ${lib.escapeShellArg mountpoint}" EXIT
+        '';
+    in
+    lib.mkIf (cfg.targets != [ ]) {
+      environment.systemPackages = [
+        (pkgs.writeShellScriptBin "localbackup-create" ''
+          set -efu -o pipefail
+          export PATH=${
+            lib.makeBinPath [
+              pkgs.rsnapshot
+              pkgs.coreutils
+              pkgs.util-linux
+            ]
+          }
+          ${lib.concatMapStringsSep "\n" (target: ''
+            (
+              echo "Creating backup '${target.name}'"
+              ${setupMount target.mountpoint}
+              rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target.directory (lib.attrValues config.clanCore.state))}" sync
+              rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target.directory (lib.attrValues config.clanCore.state))}" snapshot
+            )
+          '') (builtins.attrValues cfg.targets)}
+        '')
+        (pkgs.writeShellScriptBin "localbackup-list" ''
+          set -efu -o pipefail
+          export PATH=${
+            lib.makeBinPath [
+              pkgs.jq
+              pkgs.findutils
+              pkgs.coreutils
+              pkgs.util-linux
+            ]
+          }
+          (${
+            lib.concatMapStringsSep "\n" (target: ''
+              (
+                ${setupMount target.mountpoint}
+                find ${lib.escapeShellArg target.directory} -mindepth 1 -maxdepth 1 -name "snapshot.*" -print0 -type d \
+                  | jq -Rs 'split("\u0000") | .[] | select(. != "") | { "name": ("${target.mountpoint}::" + .)}'
+              )
+            '') (builtins.attrValues cfg.targets)
+          }) | jq -s .
+        '')
+        (pkgs.writeShellScriptBin "localbackup-restore" ''
+          set -efu -o pipefail
+          export PATH=${
+            lib.makeBinPath [
+              pkgs.rsync
+              pkgs.coreutils
+              pkgs.util-linux
+              pkgs.gawk
+            ]
+          }
+          mountpoint=$(awk -F'::' '{print $1}' <<< $NAME)
+          backupname=''${NAME#$mountpoint::}
+
+          mkdir -p "$mountpoint"
+          if mountpoint -q "$mountpoint"; then
+            umount "$mountpoint"
+          fi
+          mount "$mountpoint"
+          trap "umount $mountpoint" EXIT
+
+          IFS=';' read -ra FOLDER <<< "$FOLDERS"
+          for folder in "''${FOLDER[@]}"; do
+            rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
+          done
+        '')
+      ];
+
+      clanCore.backups.providers.localbackup = {
+        # TODO list needs to run locally or on the remote machine
+        list = "localbackup-list";
+        create = "localbackup-create";
+        restore = "localbackup-restore";
+      };
+    };
+}

nixosModules/clanCore/backups.nix

@@ -18,25 +18,21 @@
               list = lib.mkOption {
                 type = lib.types.str;
                 description = ''
-                  script to list backups
+                  Command to list backups.
                 '';
               };
               restore = lib.mkOption {
                 type = lib.types.str;
                 description = ''
-                  script to restore a backup
+                  Command to restore a backup.
-                  should take an optional service name as argument
+                  The name of the backup and the folders to restore will be
-                  gets ARCHIVE_ID, LOCATION, JOB and FOLDERS as environment variables
+                  set as environment variables NAME and FOLDERS respectively.
-                  ARCHIVE_ID is the id of the backup
-                  LOCATION is the remote identifier of the backup
-                  JOB is the job name of the backup
-                  FOLDERS is a colon separated list of folders to restore
                 '';
               };
               create = lib.mkOption {
                 type = lib.types.str;
                 description = ''
-                  script to start a backup
+                  Command to start a backup
                 '';
               };
             };

nixosModules/clanCore/state.nix

@@ -17,18 +17,18 @@
                 Folder where state resides in
               '';
             };
-            preRestoreScript = lib.mkOption {
+            preRestoreCommand = lib.mkOption {
-              type = lib.types.str;
+              type = lib.types.nullOr lib.types.str;
-              default = ":";
+              default = null;
               description = ''
                 script to run before restoring the state dir from a backup
 
                 Utilize this to stop services which currently access these folders
               '';
             };
-            postRestoreScript = lib.mkOption {
+            postRestoreCommand = lib.mkOption {
-              type = lib.types.str;
+              type = lib.types.nullOr lib.types.str;
-              default = ":";
+              default = null;
               description = ''
                 script to restore the service after the state dir was restored from a backup
 

pkgs/clan-cli/clan_cli/backups/create.py

@@ -14,7 +14,7 @@ def create_backup(machine: Machine, provider: str | None = None) -> None:
     if provider is None:
         for provider in backup_scripts["providers"]:
             proc = machine.target_host.run(
-                ["bash", "-c", backup_scripts["providers"][provider]["create"]],
+                [backup_scripts["providers"][provider]["create"]],
             )
             if proc.returncode != 0:
                 raise ClanError("failed to start backup")
@@ -24,7 +24,7 @@ def create_backup(machine: Machine, provider: str | None = None) -> None:
         if provider not in backup_scripts["providers"]:
             raise ClanError(f"provider {provider} not found")
         proc = machine.target_host.run(
-            ["bash", "-c", backup_scripts["providers"][provider]["create"]],
+            [backup_scripts["providers"][provider]["create"]],
         )
         if proc.returncode != 0:
             raise ClanError("failed to start backup")

pkgs/clan-cli/clan_cli/backups/list.py

@@ -17,7 +17,7 @@ def list_provider(machine: Machine, provider: str) -> list[Backup]:
     results = []
     backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
     proc = machine.target_host.run(
-        ["bash", "-c", backup_metadata["providers"][provider]["list"]],
+        [backup_metadata["providers"][provider]["list"]],
         stdout=subprocess.PIPE,
         check=False,
     )

pkgs/clan-cli/clan_cli/backups/restore.py

@@ -4,42 +4,29 @@ import subprocess
 
 from ..errors import ClanError
 from ..machines.machines import Machine
-from .list import Backup, list_backups
 
 
-def restore_service(
+def restore_service(machine: Machine, name: str, provider: str, service: str) -> None:
-    machine: Machine, backup: Backup, provider: str, service: str
-) -> None:
     backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
     backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
     folders = backup_folders[service]["folders"]
     env = {}
-    env["NAME"] = backup.name
+    env["NAME"] = name
     env["FOLDERS"] = ":".join(folders)
 
-    if backup.job_name is not None:
+    if pre_restore := backup_folders[service]["preRestoreCommand"]:
-        env["JOB_NAME"] = backup.job_name
+        proc = machine.target_host.run(
-
+            [pre_restore],
-    proc = machine.target_host.run(
+            stdout=subprocess.PIPE,
-        [
+            extra_env=env,
-            "bash",
-            "-c",
-            backup_folders[service]["preRestoreScript"],
-        ],
-        stdout=subprocess.PIPE,
-        extra_env=env,
-    )
-    if proc.returncode != 0:
-        raise ClanError(
-            f"failed to run preRestoreScript: {backup_folders[service]['preRestoreScript']}, error was: {proc.stdout}"
         )
+        if proc.returncode != 0:
+            raise ClanError(
+                f"failed to run preRestoreCommand: {pre_restore}, error was: {proc.stdout}"
+            )
 
     proc = machine.target_host.run(
-        [
+        [backup_metadata["providers"][provider]["restore"]],
-            "bash",
-            "-c",
-            backup_metadata["providers"][provider]["restore"],
-        ],
         stdout=subprocess.PIPE,
         extra_env=env,
     )
@@ -48,52 +35,36 @@ def restore_service(
             f"failed to restore backup: {backup_metadata['providers'][provider]['restore']}"
         )
 
-    proc = machine.target_host.run(
+    if post_restore := backup_folders[service]["postRestoreCommand"]:
-        [
+        proc = machine.target_host.run(
-            "bash",
+            [post_restore],
-            "-c",
+            stdout=subprocess.PIPE,
-            backup_folders[service]["postRestoreScript"],
+            extra_env=env,
-        ],
-        stdout=subprocess.PIPE,
-        extra_env=env,
-    )
-    if proc.returncode != 0:
-        raise ClanError(
-            f"failed to run postRestoreScript: {backup_folders[service]['postRestoreScript']}, error was: {proc.stdout}"
         )
+        if proc.returncode != 0:
+            raise ClanError(
+                f"failed to run postRestoreCommand: {post_restore}, error was: {proc.stdout}"
+            )
 
 
 def restore_backup(
     machine: Machine,
-    backups: list[Backup],
     provider: str,
     name: str,
     service: str | None = None,
 ) -> None:
     if service is None:
-        for backup in backups:
+        backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
-            if backup.name == name:
+        for _service in backup_folders:
-                backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
+            restore_service(machine, name, provider, _service)
-                for _service in backup_folders:
-                    restore_service(machine, backup, provider, _service)
-                break
-        else:
-            raise ClanError(f"backup {name} not found")
     else:
-        for backup in backups:
+        restore_service(machine, name, provider, service)
-            if backup.name == name:
-                restore_service(machine, backup, provider, service)
-                break
-        else:
-            raise ClanError(f"backup {name} not found")
 
 
 def restore_command(args: argparse.Namespace) -> None:
     machine = Machine(name=args.machine, flake=args.flake)
-    backups = list_backups(machine=machine, provider=args.provider)
     restore_backup(
         machine=machine,
-        backups=backups,
         provider=args.provider,
         name=args.name,
         service=args.service,