From 87fa99f90097edf29ce2580d23e8dde07563ddba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 23 Dec 2024 14:01:05 +0100 Subject: [PATCH 1/7] borgbackup-list: fix empty backup case --- clanModules/borgbackup/roles/client.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clanModules/borgbackup/roles/client.nix b/clanModules/borgbackup/roles/client.nix index 73a29f84a..105f717c7 100644 --- a/clanModules/borgbackup/roles/client.nix +++ b/clanModules/borgbackup/roles/client.nix @@ -152,7 +152,7 @@ in # we need yes here to skip the changed url verification ''yes y | borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.name}::${dest.repo}::" + .name)}]' '' ) (lib.attrValues cfg.destinations) - }) | ${pkgs.jq}/bin/jq -s 'add' + }) | ${pkgs.jq}/bin/jq -s 'add // []' '') (pkgs.writeShellScriptBin "borgbackup-restore" '' set -efux From 71b3351061d03d90aa356d1a5ee8140f39a0536a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 23 Dec 2024 14:16:20 +0100 Subject: [PATCH 2/7] borgbackup-list: use pipefail --- clanModules/borgbackup/roles/client.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clanModules/borgbackup/roles/client.nix b/clanModules/borgbackup/roles/client.nix index 105f717c7..ef8e19496 100644 --- a/clanModules/borgbackup/roles/client.nix +++ b/clanModules/borgbackup/roles/client.nix @@ -145,7 +145,7 @@ in '') (lib.attrValues cfg.destinations)} '') (pkgs.writeShellScriptBin "borgbackup-list" '' - set -efu + set -efu -o pipefail (${ lib.concatMapStringsSep "\n" ( dest: From cc6f344001af345cc07f6db78bf7be30f4d3a07a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 23 Dec 2024 14:20:32 +0100 Subject: [PATCH 3/7] backup/list: improve error reporting --- pkgs/clan-cli/clan_cli/backups/list.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/backups/list.py b/pkgs/clan-cli/clan_cli/backups/list.py index 1db697d14..cfe5cc575 100644 --- a/pkgs/clan-cli/clan_cli/backups/list.py +++ b/pkgs/clan-cli/clan_cli/backups/list.py @@ -21,15 +21,25 @@ class Backup: def list_provider(machine: Machine, provider: str) -> list[Backup]: results = [] backup_metadata = json.loads(machine.eval_nix("config.clan.core.backups")) + list_command = backup_metadata["providers"][provider]["list"] proc = machine.target_host.run( - [backup_metadata["providers"][provider]["list"]], - RunOpts(log=Log.STDERR, check=False), + [list_command], + RunOpts(log=Log.NONE, check=False), ) if proc.returncode != 0: # TODO this should be a warning, only raise exception if no providers succeed - msg = f"failed to list backups for provider {provider}: {proc.stdout}" + msg = f"Failed to list backups for provider {provider}:" + msg += f"\n{list_command} exited with {proc.returncode}" + if proc.stderr: + msg += f"\nerror output: {proc.stderr}" raise ClanError(msg) - parsed_json = json.loads(proc.stdout) + + try: + parsed_json = json.loads(proc.stdout) + except json.JSONDecodeError as e: + msg = f"Failed to parse json output from provider {provider}:\n{proc.stdout}" + raise ClanError(msg) from e + for archive in parsed_json: results.append(Backup(name=archive["name"], job_name=archive.get("job_name"))) return results From ed72d63c57cfb03d3d7ef4c1630fe1332b21dbc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 23 Dec 2024 14:24:41 +0100 Subject: [PATCH 4/7] deltachat: fix test name --- checks/deltachat/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checks/deltachat/default.nix b/checks/deltachat/default.nix index efee1e417..04ad3ad64 100644 --- a/checks/deltachat/default.nix +++ b/checks/deltachat/default.nix @@ -1,7 +1,7 @@ (import ../lib/container-test.nix) ( { pkgs, ... }: { - name = "secrets"; + name = "deltachat"; nodes.machine = { self, ... }: From 6880d58344bcb27bb6f4efd18643ddd160c28f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 23 Dec 2024 14:43:46 +0100 Subject: [PATCH 5/7] borgbackup: avoid broken pipe --- clanModules/borgbackup/roles/client.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clanModules/borgbackup/roles/client.nix b/clanModules/borgbackup/roles/client.nix index ef8e19496..d1f342e83 100644 --- a/clanModules/borgbackup/roles/client.nix +++ b/clanModules/borgbackup/roles/client.nix @@ -150,12 +150,12 @@ in lib.concatMapStringsSep "\n" ( dest: # we need yes here to skip the changed url verification - ''yes y | borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.name}::${dest.repo}::" + .name)}]' '' + ''echo y | borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.name}::${dest.repo}::" + .name)}]' '' ) (lib.attrValues cfg.destinations) }) | ${pkgs.jq}/bin/jq -s 'add // []' '') (pkgs.writeShellScriptBin "borgbackup-restore" '' - set -efux + set -efu -o pipefail cd / IFS=':' read -ra FOLDER <<< "$FOLDERS" job_name=$(echo "$NAME" | ${pkgs.gawk}/bin/awk -F'::' '{print $1}') @@ -164,7 +164,7 @@ in echo "borg-job-$job_name not found: Backup name is invalid" >&2 exit 1 fi - yes y | borg-job-"$job_name" extract --list "$backup_name" "''${FOLDER[@]}" + echo y | borg-job-"$job_name" extract --list "$backup_name" "''${FOLDER[@]}" '') ]; From bdbad62cac32477f757c583d32d5f00085017812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 23 Dec 2024 15:41:21 +0100 Subject: [PATCH 6/7] borgbackup: make restore less verbose --- clanModules/borgbackup/roles/client.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clanModules/borgbackup/roles/client.nix b/clanModules/borgbackup/roles/client.nix index d1f342e83..eb5582b10 100644 --- a/clanModules/borgbackup/roles/client.nix +++ b/clanModules/borgbackup/roles/client.nix @@ -164,7 +164,7 @@ in echo "borg-job-$job_name not found: Backup name is invalid" >&2 exit 1 fi - echo y | borg-job-"$job_name" extract --list "$backup_name" "''${FOLDER[@]}" + echo y | borg-job-"$job_name" extract "$backup_name" "''${FOLDER[@]}" '') ]; From 7029ceda1140cc36e5d6245d3e7db08029e0062b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 24 Dec 2024 07:21:05 +0100 Subject: [PATCH 7/7] borgbackup: use writeShellApplication --- clanModules/borgbackup/roles/client.nix | 65 ++++++++++++++----------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/clanModules/borgbackup/roles/client.nix b/clanModules/borgbackup/roles/client.nix index eb5582b10..bb4439d84 100644 --- a/clanModules/borgbackup/roles/client.nix +++ b/clanModules/borgbackup/roles/client.nix @@ -138,34 +138,43 @@ in }) cfg.destinations; environment.systemPackages = [ - (pkgs.writeShellScriptBin "borgbackup-create" '' - set -efu -o pipefail - ${lib.concatMapStringsSep "\n" (dest: '' - systemctl start borgbackup-job-${dest.name} - '') (lib.attrValues cfg.destinations)} - '') - (pkgs.writeShellScriptBin "borgbackup-list" '' - set -efu -o pipefail - (${ - lib.concatMapStringsSep "\n" ( - dest: - # we need yes here to skip the changed url verification - ''echo y | borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.name}::${dest.repo}::" + .name)}]' '' - ) (lib.attrValues cfg.destinations) - }) | ${pkgs.jq}/bin/jq -s 'add // []' - '') - (pkgs.writeShellScriptBin "borgbackup-restore" '' - set -efu -o pipefail - cd / - IFS=':' read -ra FOLDER <<< "$FOLDERS" - job_name=$(echo "$NAME" | ${pkgs.gawk}/bin/awk -F'::' '{print $1}') - backup_name=''${NAME#"$job_name"::} - if ! command -v borg-job-"$job_name" &> /dev/null; then - echo "borg-job-$job_name not found: Backup name is invalid" >&2 - exit 1 - fi - echo y | borg-job-"$job_name" extract "$backup_name" "''${FOLDER[@]}" - '') + (pkgs.writeShellApplication { + name = "borgbackup-create"; + runtimeInputs = [ config.systemd.package ]; + text = '' + ${lib.concatMapStringsSep "\n" (dest: '' + systemctl start borgbackup-job-${dest.name} + '') (lib.attrValues cfg.destinations)} + ''; + }) + (pkgs.writeShellApplication { + name = "borgbackup-list"; + runtimeInputs = [ pkgs.jq ]; + text = '' + (${ + lib.concatMapStringsSep "\n" ( + dest: + # we need yes here to skip the changed url verification + ''echo y | /run/current-system/sw/bin/borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.name}::${dest.repo}::" + .name)}]' '' + ) (lib.attrValues cfg.destinations) + }) | jq -s 'add // []' + ''; + }) + (pkgs.writeShellApplication { + name = "borgbackup-restore"; + runtimeInputs = [ pkgs.gawk ]; + text = '' + cd / + IFS=':' read -ra FOLDER <<< "''${FOLDERS-}" + job_name=$(echo "$NAME" | awk -F'::' '{print $1}') + backup_name=''${NAME#"$job_name"::} + if [[ ! -x /run/current-system/sw/bin/borg-job-"$job_name" ]]; then + echo "borg-job-$job_name not found: Backup name is invalid" >&2 + exit 1 + fi + echo y | /run/current-system/sw/bin/borg-job-"$job_name" extract "$backup_name" "''${FOLDER[@]}" + ''; + }) ]; # Facts generation. So the client can authenticate to the server