From 7c37bddeea9aa60ba40bcc39e677d828045841eb Mon Sep 17 00:00:00 2001 From: pinpox Date: Sat, 9 Aug 2025 19:36:49 +0200 Subject: [PATCH] Add localbackup clan service --- checks/backups/flake-module.nix | 7 +- clanServices/localbackup/README.md | 35 +++ clanServices/localbackup/default.nix | 267 ++++++++++++++++++ clanServices/localbackup/flake-module.nix | 16 ++ clanServices/localbackup/tests/vm/default.nix | 62 ++++ docs/mkdocs.yml | 2 +- .../postgresql/tests/flake-module.nix | 2 +- 7 files changed, 383 insertions(+), 8 deletions(-) create mode 100644 clanServices/localbackup/README.md create mode 100644 clanServices/localbackup/default.nix create mode 100644 clanServices/localbackup/flake-module.nix create mode 100644 clanServices/localbackup/tests/vm/default.nix diff --git a/checks/backups/flake-module.nix b/checks/backups/flake-module.nix index 70af8a3fa..979317860 100644 --- a/checks/backups/flake-module.nix +++ b/checks/backups/flake-module.nix @@ -5,6 +5,7 @@ fileSystems."/".device = "/dev/null"; boot.loader.grub.device = "/dev/null"; }; + clan.inventory.services = { borgbackup.test-backup = { roles.client.machines = [ "test-backup" ]; @@ -26,12 +27,6 @@ closureInfo = pkgs.closureInfo { rootPaths = dependencies; }; in { - imports = [ - # Do not import inventory modules. They should be configured via 'clan.inventory' - # - # TODO: Configure localbackup via inventory - self.clanModules.localbackup - ]; # Borgbackup overrides services.borgbackup.repos.test-backups = { path = "/var/lib/borgbackup/test-backups"; diff --git a/clanServices/localbackup/README.md b/clanServices/localbackup/README.md new file mode 100644 index 000000000..e58aba538 --- /dev/null +++ b/clanServices/localbackup/README.md @@ -0,0 +1,35 @@ +## Features + +- Creates incremental snapshots using rsnapshot +- Supports multiple backup targets +- Mount/unmount hooks for external storage +- Pre/post backup hooks for custom scripts +- Configurable snapshot retention +- Automatic state folder detection + +## Usage + +Enable the localbackup service and configure backup targets: + +```nix +instances = { + localbackup = { + module.name = "@clan/localbackup"; + module.input = "self"; + roles.default.machines."machine".settings = { + targets.external= { + directory = "/mnt/backup"; + mountpoint = "/mnt/backup"; + }; + }; + }; +}; +``` + +## Commands + +The service provides these commands: + +- `localbackup-create`: Create a new backup +- `localbackup-list`: List available backups +- `localbackup-restore`: Restore from backup (requires NAME and FOLDERS environment variables) diff --git a/clanServices/localbackup/default.nix b/clanServices/localbackup/default.nix new file mode 100644 index 000000000..88c4ca1d8 --- /dev/null +++ b/clanServices/localbackup/default.nix @@ -0,0 +1,267 @@ +{ ... }: +{ + _class = "clan.service"; + manifest.name = "localbackup"; + manifest.description = "Automatically backups current machine to local directory."; + manifest.categories = [ "System" ]; + manifest.readme = builtins.readFile ./README.md; + + roles.default = { + interface = + { lib, ... }: + { + + options = { + + targets = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, ... }: + { + options = { + name = lib.mkOption { + type = lib.types.strMatching "^[a-zA-Z0-9._-]+$"; + default = name; + description = "the name of the backup job"; + }; + directory = lib.mkOption { + type = lib.types.str; + description = "the directory to backup"; + }; + mountpoint = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "mountpoint of the directory to backup. If set, the directory will be mounted before the backup and unmounted afterwards"; + }; + preMountHook = lib.mkOption { + type = lib.types.nullOr lib.types.lines; + default = null; + description = "Shell commands to run before the directory is mounted"; + }; + postMountHook = lib.mkOption { + type = lib.types.nullOr lib.types.lines; + default = null; + description = "Shell commands to run after the directory is mounted"; + }; + preUnmountHook = lib.mkOption { + type = lib.types.nullOr lib.types.lines; + default = null; + description = "Shell commands to run before the directory is unmounted"; + }; + postUnmountHook = lib.mkOption { + type = lib.types.nullOr lib.types.lines; + default = null; + description = "Shell commands to run after the directory is unmounted"; + }; + preBackupHook = lib.mkOption { + type = lib.types.nullOr lib.types.lines; + default = null; + description = "Shell commands to run before the backup"; + }; + postBackupHook = lib.mkOption { + type = lib.types.nullOr lib.types.lines; + default = null; + description = "Shell commands to run after the backup"; + }; + }; + } + ) + ); + # default = { }; + description = "List of directories where backups are stored"; + }; + + snapshots = lib.mkOption { + type = lib.types.int; + default = 20; + description = "Number of snapshots to keep"; + }; + }; + + }; + + perInstance = + { + settings, + ... + }: + { + nixosModule = + { + config, + lib, + pkgs, + ... + }: + + let + mountHook = target: '' + if [[ -x /run/current-system/sw/bin/localbackup-mount-${target.name} ]]; then + /run/current-system/sw/bin/localbackup-mount-${target.name} + fi + if [[ -x /run/current-system/sw/bin/localbackup-unmount-${target.name} ]]; then + trap "/run/current-system/sw/bin/localbackup-unmount-${target.name}" EXIT + fi + ''; + + uniqueFolders = lib.unique ( + lib.flatten (lib.mapAttrsToList (_name: state: state.folders) config.clan.core.state) + ); + + rsnapshotConfig = target: '' + config_version 1.2 + snapshot_root ${target.directory} + sync_first 1 + cmd_cp ${pkgs.coreutils}/bin/cp + cmd_rm ${pkgs.coreutils}/bin/rm + cmd_rsync ${pkgs.rsync}/bin/rsync + cmd_ssh ${pkgs.openssh}/bin/ssh + cmd_logger ${pkgs.inetutils}/bin/logger + cmd_du ${pkgs.coreutils}/bin/du + cmd_rsnapshot_diff ${pkgs.rsnapshot}/bin/rsnapshot-diff + + ${lib.optionalString (target.postBackupHook != null) '' + cmd_postexec ${pkgs.writeShellScript "postexec.sh" '' + set -efu -o pipefail + ${target.postBackupHook} + ''} + ''} + retain snapshot ${builtins.toString settings.snapshots} + ${lib.concatMapStringsSep "\n" (folder: '' + backup ${folder} ${config.networking.hostName}/ + '') uniqueFolders} + ''; + in + + { + + environment.systemPackages = [ + (pkgs.writeShellScriptBin "localbackup-create" '' + set -efu -o pipefail + export PATH=${ + lib.makeBinPath [ + pkgs.rsnapshot + pkgs.coreutils + pkgs.util-linux + ] + } + ${lib.concatMapStringsSep "\n" (target: '' + ${mountHook target} + echo "Creating backup '${target.name}'" + + ${lib.optionalString (target.preBackupHook != null) '' + ( + ${target.preBackupHook} + ) + ''} + + declare -A preCommandErrors + ${lib.concatMapStringsSep "\n" ( + state: + lib.optionalString (state.preBackupCommand != null) '' + echo "Running pre-backup command for ${state.name}" + if ! /run/current-system/sw/bin/${state.preBackupCommand}; then + preCommandErrors["${state.name}"]=1 + fi + '' + ) (builtins.attrValues config.clan.core.state)} + + rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync + rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot + '') (builtins.attrValues settings.targets)}'') + (pkgs.writeShellScriptBin "localbackup-list" '' + set -efu -o pipefail + export PATH=${ + lib.makeBinPath [ + pkgs.jq + pkgs.findutils + pkgs.coreutils + pkgs.util-linux + ] + } + (${ + lib.concatMapStringsSep "\n" (target: '' + ( + ${mountHook target} + find ${lib.escapeShellArg target.directory} -mindepth 1 -maxdepth 1 -name "snapshot.*" -print0 -type d \ + | jq -Rs 'split("\u0000") | .[] | select(. != "") | { "name": ("${target.name}::" + .)}' + ) + '') (builtins.attrValues settings.targets) + }) | jq -s . + '') + (pkgs.writeShellScriptBin "localbackup-restore" '' + set -efu -o pipefail + export PATH=${ + lib.makeBinPath [ + pkgs.rsync + pkgs.coreutils + pkgs.util-linux + pkgs.gawk + ] + } + if [[ "''${NAME:-}" == "" ]]; then + echo "No backup name given via NAME environment variable" + exit 1 + fi + if [[ "''${FOLDERS:-}" == "" ]]; then + echo "No folders given via FOLDERS environment variable" + exit 1 + fi + name=$(awk -F'::' '{print $1}' <<< $NAME) + backupname=''${NAME#$name::} + + if command -v localbackup-mount-$name; then + localbackup-mount-$name + fi + if command -v localbackup-unmount-$name; then + trap "localbackup-unmount-$name" EXIT + fi + + if [[ ! -d $backupname ]]; then + echo "No backup found $backupname" + exit 1 + fi + + IFS=':' read -ra FOLDER <<< "''$FOLDERS" + for folder in "''${FOLDER[@]}"; do + mkdir -p "$folder" + rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder" + done + '') + ] + ++ (lib.mapAttrsToList ( + name: target: + pkgs.writeShellScriptBin ("localbackup-mount-" + name) '' + set -efu -o pipefail + ${lib.optionalString (target.preMountHook != null) target.preMountHook} + ${lib.optionalString (target.mountpoint != null) '' + if ! ${pkgs.util-linux}/bin/mountpoint -q ${lib.escapeShellArg target.mountpoint}; then + ${pkgs.util-linux}/bin/mount -o X-mount.mkdir ${lib.escapeShellArg target.mountpoint} + fi + ''} + ${lib.optionalString (target.postMountHook != null) target.postMountHook} + '' + ) settings.targets) + ++ lib.mapAttrsToList ( + name: target: + pkgs.writeShellScriptBin ("localbackup-unmount-" + name) '' + set -efu -o pipefail + ${lib.optionalString (target.preUnmountHook != null) target.preUnmountHook} + ${lib.optionalString ( + target.mountpoint != null + ) "${pkgs.util-linux}/bin/umount ${lib.escapeShellArg target.mountpoint}"} + ${lib.optionalString (target.postUnmountHook != null) target.postUnmountHook} + '' + ) settings.targets; + + clan.core.backups.providers.localbackup = { + # TODO list needs to run locally or on the remote machine + list = "localbackup-list"; + create = "localbackup-create"; + restore = "localbackup-restore"; + }; + + }; + }; + }; +} diff --git a/clanServices/localbackup/flake-module.nix b/clanServices/localbackup/flake-module.nix new file mode 100644 index 000000000..9cd9eb626 --- /dev/null +++ b/clanServices/localbackup/flake-module.nix @@ -0,0 +1,16 @@ +{ lib, ... }: +let + module = lib.modules.importApply ./default.nix { }; +in +{ + clan.modules.localbackup = module; + perSystem = + { ... }: + { + clan.nixosTests.localbackup = { + imports = [ ./tests/vm/default.nix ]; + + clan.modules."@clan/localbackup" = module; + }; + }; +} diff --git a/clanServices/localbackup/tests/vm/default.nix b/clanServices/localbackup/tests/vm/default.nix new file mode 100644 index 000000000..2a266303e --- /dev/null +++ b/clanServices/localbackup/tests/vm/default.nix @@ -0,0 +1,62 @@ +{ ... }: +{ + name = "service-localbackup"; + + clan = { + directory = ./.; + test.useContainers = true; + inventory = { + + machines.machine = { }; + + instances = { + localbackup = { + module.name = "@clan/localbackup"; + module.input = "self"; + roles.default.machines."machine".settings = { + + targets.hdd = { + directory = "/mnt/external-disk"; + preMountHook = '' + touch /run/mount-external-disk + ''; + postUnmountHook = '' + touch /run/unmount-external-disk + ''; + }; + }; + }; + }; + }; + }; + + nodes.machine = { + clan.core.state.test-backups.folders = [ "/var/test-backups" ]; + }; + + testScript = '' + import json + start_all() + + machine.systemctl("start network-online.target") + machine.wait_for_unit("network-online.target") + + # dummy data + machine.succeed("mkdir -p /var/test-backups") + machine.succeed("echo testing > /var/test-backups/somefile") + + # create + machine.succeed("localbackup-create >&2") + machine.wait_until_succeeds("! systemctl is-active localbackup-job-serverone >&2") + + # list + snapshot_list = machine.succeed("localbackup-list").strip() + assert json.loads(snapshot_list)[0]["name"].strip() == "hdd::/mnt/external-disk/snapshot.0" + + # borgbackup restore + machine.succeed("rm -f /var/test-backups/somefile") + + machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/test-backups /run/current-system/sw/bin/localbackup-restore >&2") + assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed" + ''; +} diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 5a8109c00..869ed12d1 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -144,7 +144,7 @@ nav: - reference/clanModules/heisenbridge.md - reference/clanModules/importer.md - reference/clanModules/iwd.md - - reference/clanModules/localbackup.md + - reference/clanServices/localbackup.md - reference/clanModules/localsend.md - reference/clanModules/matrix-synapse.md - reference/clanModules/moonlight.md diff --git a/nixosModules/clanCore/postgresql/tests/flake-module.nix b/nixosModules/clanCore/postgresql/tests/flake-module.nix index 7f04bb585..602aaa184 100644 --- a/nixosModules/clanCore/postgresql/tests/flake-module.nix +++ b/nixosModules/clanCore/postgresql/tests/flake-module.nix @@ -24,7 +24,7 @@ imports = [ # self.nixosModules.clanCore - self.clanModules.localbackup + self.clanServices.localbackup ]; clan.core.postgresql.enable = true;