yadunut/clan-core
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request 'localbackup' (#1020) from localbackup into main

Browse Source
This commit is contained in:
clan-bot
2024-03-20 08:49:07 +00:00
parent 691d2ca3e9 350593ccde
commit 1b8ae090b8
9 changed files with 241 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
checks/backups/flake-module.nix
View File

@@ -76,7 +76,15 @@
clanCore.secretStore = "vm";
clanCore.clanDir = ../..;
environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
environment.systemPackages = [
self.packages.${pkgs.system}.clan-cli
(pkgs.writeShellScriptBin "pre-restore-command" ''
touch /var/test-service/pre-restore-command
'')
(pkgs.writeShellScriptBin "post-restore-command" ''
touch /var/test-service/post-restore-command
'')
];
environment.etc.install-closure.source = "${closureInfo}/store-paths";
nix.settings = {
substituters = lib.mkForce [ ];
@@ -86,6 +94,12 @@
};
system.extraDependencies = dependencies;
clanCore.state.test-backups.folders = [ "/var/test-backups" ];
clanCore.state.test-service = {
preRestoreCommand = "pre-restore-command";
postRestoreCommand = "post-restore-command";
folders = [ "/var/test-service" ];
};
clan.borgbackup.destinations.test-backup.repo = "borg@machine:.";
services.borgbackup.repos.test-backups = {
@@ -110,7 +124,7 @@
start_all()
# dummy data
machine.succeed("mkdir -p /var/test-backups")
machine.succeed("mkdir -p /var/test-backups /var/test-service")
machine.succeed("echo testing > /var/test-backups/somefile")
# create
@@ -119,14 +133,16 @@
# list
backup_id = json.loads(machine.succeed("borg-job-test-backup list --json"))["archives"][0]["archive"]
out = machine.succeed("clan --debug --flake ${self} backups list test-backup")
out = machine.succeed("clan --debug --flake ${self} backups list test-backup").strip()
print(out)
assert backup_id in out, f"backup {backup_id} not found in {out}"
# restore
machine.succeed("rm -f /var/test-backups/somefile")
machine.succeed(f"clan --debug --flake ${self} backups restore test-backup borgbackup borg@machine:.::{backup_id} >&2")
machine.succeed(f"clan --debug --flake ${self} backups restore test-backup borgbackup {out} >&2")
assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
machine.succeed("test -f /var/test-service/pre-restore-command")
machine.succeed("test -f /var/test-service/post-restore-command")
'';
} { inherit pkgs self; };
};

51
clanModules/borgbackup.nix
View File

@@ -15,7 +15,7 @@ in
{
options = {
name = lib.mkOption {
type = lib.types.str;
type = lib.types.strMatching "^[a-zA-Z0-9._-]+$";
default = name;
description = "the name of the backup job";
};
@@ -90,32 +90,41 @@ in
'';
};
environment.systemPackages = [ pkgs.jq ];
clanCore.backups.providers.borgbackup = {
# TODO list needs to run locally or on the remote machine
list = ''
set -efu
# we need yes here to skip the changed url verification
${
lib.concatMapStringsSep "\\\n" (
dest:
''yes y | borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.repo}::" + .name), "job_name": "${dest.name}"}]' ''
) (lib.attrValues cfg.destinations)
} | jq -s 'add'
'';
create = ''
environment.systemPackages = [
(pkgs.writeShellScriptBin "borgbackup-create" ''
set -efu -o pipefail
${lib.concatMapStringsSep "\n" (dest: ''
systemctl start borgbackup-job-${dest.name}
'') (lib.attrValues cfg.destinations)}
'';
restore = ''
'')
(pkgs.writeShellScriptBin "borgbackup-list" ''
set -efu
(${
lib.concatMapStringsSep "\n" (
dest:
# we need yes here to skip the changed url verification
''yes y | borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.name}::${dest.repo}::" + .name)}]' ''
) (lib.attrValues cfg.destinations)
}) | ${pkgs.jq}/bin/jq -s 'add'
'')
(pkgs.writeShellScriptBin "borgbackup-restore" ''
set -efux
cd /
IFS=';' read -ra FOLDER <<< "$FOLDERS"
yes y | borg-job-"$JOB_NAME" extract --list "$NAME" "''${FOLDER[@]}"
'';
job_name=$(echo "$NAME" | ${pkgs.gawk}/bin/awk -F'::' '{print $1}')
backup_name=''${NAME#"$job_name"::}
if ! command -v borg-job-"$job_name" &> /dev/null; then
echo "borg-job-$job_name not found: Backup name is invalid" >&2
exit 1
fi
yes y | borg-job-"$job_name" extract --list "$backup_name" "''${FOLDER[@]}"
'')
];
clanCore.backups.providers.borgbackup = {
list = "borgbackup-list";
create = "borgbackup-create";
restore = "borgbackup-restore";
};
};
}

1
clanModules/flake-module.nix
View File

@@ -8,6 +8,7 @@
];
};
borgbackup = ./borgbackup.nix;
localbackup = ./localbackup.nix;
deltachat = ./deltachat.nix;
moonlight = ./moonlight.nix;
sunshine = ./sunshine.nix;

151
clanModules/localbackup.nix Normal file
View File

@@ -0,0 +1,151 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.clan.localbackup;
rsnapshotConfig = target: states: ''
config_version 1.2
snapshot_root ${target}
sync_first 1
cmd_cp ${pkgs.coreutils}/bin/cp
cmd_rm ${pkgs.coreutils}/bin/rm
cmd_rsync ${pkgs.rsync}/bin/rsync
cmd_ssh ${pkgs.openssh}/bin/ssh
cmd_logger ${pkgs.inetutils}/bin/logger
cmd_du ${pkgs.coreutils}/bin/du
cmd_rsnapshot_diff ${pkgs.rsnapshot}/bin/rsnapshot-diff
retain snapshot ${builtins.toString config.clan.localbackup.snapshots}
${lib.concatMapStringsSep "\n" (state: ''
${lib.concatMapStringsSep "\n" (folder: ''
backup ${folder} ${config.networking.hostName}/
'') state.folders}
'') states}
'';
in
{
options.clan.localbackup = {
targets = lib.mkOption {
type = lib.types.attrsOf (
lib.types.submodule (
{ name, ... }:
{
options = {
name = lib.mkOption {
type = lib.types.str;
default = name;
description = "the name of the backup job";
};
directory = lib.mkOption {
type = lib.types.str;
description = "the directory to backup";
};
mountpoint = lib.mkOption {
type = lib.types.nullOr (lib.types.strMatching "^[a-zA-Z0-9./_-]+$");
default = null;
description = "mountpoint of the directory to backup. If set, the directory will be mounted before the backup and unmounted afterwards";
};
};
}
)
);
description = "List of directories where backups are stored";
};
snapshots = lib.mkOption {
type = lib.types.int;
default = 20;
description = "Number of snapshots to keep";
};
};
config =
let
setupMount =
mountpoint:
lib.optionalString (mountpoint != null) ''
mkdir -p ${lib.escapeShellArg mountpoint}
if mountpoint -q ${lib.escapeShellArg mountpoint}; then
umount ${lib.escapeShellArg mountpoint}
fi
mount ${lib.escapeShellArg mountpoint}
trap "umount ${lib.escapeShellArg mountpoint}" EXIT
'';
in
lib.mkIf (cfg.targets != [ ]) {
environment.systemPackages = [
(pkgs.writeShellScriptBin "localbackup-create" ''
set -efu -o pipefail
export PATH=${
lib.makeBinPath [
pkgs.rsnapshot
pkgs.coreutils
pkgs.util-linux
]
}
${lib.concatMapStringsSep "\n" (target: ''
(
echo "Creating backup '${target.name}'"
${setupMount target.mountpoint}
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target.directory (lib.attrValues config.clanCore.state))}" sync
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target.directory (lib.attrValues config.clanCore.state))}" snapshot
)
'') (builtins.attrValues cfg.targets)}
'')
(pkgs.writeShellScriptBin "localbackup-list" ''
set -efu -o pipefail
export PATH=${
lib.makeBinPath [
pkgs.jq
pkgs.findutils
pkgs.coreutils
pkgs.util-linux
]
}
(${
lib.concatMapStringsSep "\n" (target: ''
(
${setupMount target.mountpoint}
find ${lib.escapeShellArg target.directory} -mindepth 1 -maxdepth 1 -name "snapshot.*" -print0 -type d \
| jq -Rs 'split("\u0000") | .[] | select(. != "") | { "name": ("${target.mountpoint}::" + .)}'
)
'') (builtins.attrValues cfg.targets)
}) | jq -s .
'')
(pkgs.writeShellScriptBin "localbackup-restore" ''
set -efu -o pipefail
export PATH=${
lib.makeBinPath [
pkgs.rsync
pkgs.coreutils
pkgs.util-linux
pkgs.gawk
]
}
mountpoint=$(awk -F'::' '{print $1}' <<< $NAME)
backupname=''${NAME#$mountpoint::}
mkdir -p "$mountpoint"
if mountpoint -q "$mountpoint"; then
umount "$mountpoint"
fi
mount "$mountpoint"
trap "umount $mountpoint" EXIT
IFS=';' read -ra FOLDER <<< "$FOLDERS"
for folder in "''${FOLDER[@]}"; do
rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
done
'')
];
clanCore.backups.providers.localbackup = {
# TODO list needs to run locally or on the remote machine
list = "localbackup-list";
create = "localbackup-create";
restore = "localbackup-restore";
};
};
}

14
nixosModules/clanCore/backups.nix
View File

@@ -18,25 +18,21 @@
list = lib.mkOption {
type = lib.types.str;
description = ''
script to list backups
Command to list backups.
'';
};
restore = lib.mkOption {
type = lib.types.str;
description = ''
script to restore a backup
should take an optional service name as argument
gets ARCHIVE_ID, LOCATION, JOB and FOLDERS as environment variables
ARCHIVE_ID is the id of the backup
LOCATION is the remote identifier of the backup
JOB is the job name of the backup
FOLDERS is a colon separated list of folders to restore
Command to restore a backup.
The name of the backup and the folders to restore will be
set as environment variables NAME and FOLDERS respectively.
'';
};
create = lib.mkOption {
type = lib.types.str;
description = ''
script to start a backup
Command to start a backup
'';
};
};

12
nixosModules/clanCore/state.nix
View File

@@ -17,18 +17,18 @@
Folder where state resides in
'';
};
preRestoreScript = lib.mkOption {
type = lib.types.str;
default = ":";
preRestoreCommand = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = ''
script to run before restoring the state dir from a backup
Utilize this to stop services which currently access these folders
'';
};
postRestoreScript = lib.mkOption {
type = lib.types.str;
default = ":";
postRestoreCommand = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = ''
script to restore the service after the state dir was restored from a backup

4
pkgs/clan-cli/clan_cli/backups/create.py
View File

@@ -14,7 +14,7 @@ def create_backup(machine: Machine, provider: str | None = None) -> None:
if provider is None:
for provider in backup_scripts["providers"]:
proc = machine.target_host.run(
["bash", "-c", backup_scripts["providers"][provider]["create"]],
[backup_scripts["providers"][provider]["create"]],
)
if proc.returncode != 0:
raise ClanError("failed to start backup")
@@ -24,7 +24,7 @@ def create_backup(machine: Machine, provider: str | None = None) -> None:
if provider not in backup_scripts["providers"]:
raise ClanError(f"provider {provider} not found")
proc = machine.target_host.run(
["bash", "-c", backup_scripts["providers"][provider]["create"]],
[backup_scripts["providers"][provider]["create"]],
)
if proc.returncode != 0:
raise ClanError("failed to start backup")

2
pkgs/clan-cli/clan_cli/backups/list.py
View File

@@ -17,7 +17,7 @@ def list_provider(machine: Machine, provider: str) -> list[Backup]:
results = []
backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
proc = machine.target_host.run(
["bash", "-c", backup_metadata["providers"][provider]["list"]],
[backup_metadata["providers"][provider]["list"]],
stdout=subprocess.PIPE,
check=False,
)

79
pkgs/clan-cli/clan_cli/backups/restore.py
View File

@@ -4,42 +4,29 @@ import subprocess
from ..errors import ClanError
from ..machines.machines import Machine
from .list import Backup, list_backups
def restore_service(
machine: Machine, backup: Backup, provider: str, service: str
) -> None:
def restore_service(machine: Machine, name: str, provider: str, service: str) -> None:
backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
folders = backup_folders[service]["folders"]
env = {}
env["NAME"] = backup.name
env["NAME"] = name
env["FOLDERS"] = ":".join(folders)
if backup.job_name is not None:
env["JOB_NAME"] = backup.job_name
proc = machine.target_host.run(
[
"bash",
"-c",
backup_folders[service]["preRestoreScript"],
],
stdout=subprocess.PIPE,
extra_env=env,
)
if proc.returncode != 0:
raise ClanError(
f"failed to run preRestoreScript: {backup_folders[service]['preRestoreScript']}, error was: {proc.stdout}"
if pre_restore := backup_folders[service]["preRestoreCommand"]:
proc = machine.target_host.run(
[pre_restore],
stdout=subprocess.PIPE,
extra_env=env,
)
if proc.returncode != 0:
raise ClanError(
f"failed to run preRestoreCommand: {pre_restore}, error was: {proc.stdout}"
)
proc = machine.target_host.run(
[
"bash",
"-c",
backup_metadata["providers"][provider]["restore"],
],
[backup_metadata["providers"][provider]["restore"]],
stdout=subprocess.PIPE,
extra_env=env,
)
@@ -48,52 +35,36 @@ def restore_service(
f"failed to restore backup: {backup_metadata['providers'][provider]['restore']}"
)
proc = machine.target_host.run(
[
"bash",
"-c",
backup_folders[service]["postRestoreScript"],
],
stdout=subprocess.PIPE,
extra_env=env,
)
if proc.returncode != 0:
raise ClanError(
f"failed to run postRestoreScript: {backup_folders[service]['postRestoreScript']}, error was: {proc.stdout}"
if post_restore := backup_folders[service]["postRestoreCommand"]:
proc = machine.target_host.run(
[post_restore],
stdout=subprocess.PIPE,
extra_env=env,
)
if proc.returncode != 0:
raise ClanError(
f"failed to run postRestoreCommand: {post_restore}, error was: {proc.stdout}"
)
def restore_backup(
machine: Machine,
backups: list[Backup],
provider: str,
name: str,
service: str | None = None,
) -> None:
if service is None:
for backup in backups:
if backup.name == name:
backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
for _service in backup_folders:
restore_service(machine, backup, provider, _service)
break
else:
raise ClanError(f"backup {name} not found")
backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
for _service in backup_folders:
restore_service(machine, name, provider, _service)
else:
for backup in backups:
if backup.name == name:
restore_service(machine, backup, provider, service)
break
else:
raise ClanError(f"backup {name} not found")
restore_service(machine, name, provider, service)
def restore_command(args: argparse.Namespace) -> None:
machine = Machine(name=args.machine, flake=args.flake)
backups = list_backups(machine=machine, provider=args.provider)
restore_backup(
machine=machine,
backups=backups,
provider=args.provider,
name=args.name,
service=args.service,