Merge pull request 'make backup provider more generic' (#1019) from backup into main

Reviewed-on: https://git.clan.lol/clan/clan-core/pulls/1019

checks/backups/flake-module.nix

@@ -114,20 +114,18 @@
             machine.succeed("echo testing > /var/test-backups/somefile")
 
             # create
-            machine.succeed("ping -c1 machine >&2")
-            machine.succeed("ssh -i /etc/secrets/borgbackup.ssh -v machine hostname >&2")
-            machine.succeed("systemctl status >&2")
-            machine.succeed("systemctl start borgbackup-job-test-backup")
             machine.succeed("clan --debug --flake ${self} backups create test-backup")
             machine.wait_until_succeeds("! systemctl is-active borgbackup-job-test-backup >&2")
 
             # list
             backup_id = json.loads(machine.succeed("borg-job-test-backup list --json"))["archives"][0]["archive"]
-            assert backup_id in machine.succeed("clan --debug --flake ${self} backups list test-backup"), "backup not listed"
+            out = machine.succeed("clan --debug --flake ${self} backups list test-backup")
+            print(out)
+            assert backup_id in out, f"backup {backup_id} not found in {out}"
 
             # restore
             machine.succeed("rm -f /var/test-backups/somefile")
-            machine.succeed(f"clan --debug --flake ${self} backups restore test-backup borgbackup {backup_id}")
+            machine.succeed(f"clan --debug --flake ${self} backups restore test-backup borgbackup borg@machine:.::{backup_id} >&2")
             assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
           '';
         } { inherit pkgs self; };

clanModules/borgbackup.nix

@@ -90,13 +90,19 @@ in
       '';
     };
 
+    environment.systemPackages = [ pkgs.jq ];
+
     clanCore.backups.providers.borgbackup = {
       # TODO list needs to run locally or on the remote machine
       list = ''
+        set -efu
         # we need yes here to skip the changed url verification
-        ${lib.concatMapStringsSep "\n" (
+        ${
-          dest: ''yes y | borg-job-${dest.name} list --json | jq -r '. + {"job-name": "${dest.name}"}' ''
+          lib.concatMapStringsSep "\\\n" (
-        ) (lib.attrValues cfg.destinations)}
+            dest:
+            ''yes y | borg-job-${dest.name} list --json | jq '[.archives[] | {"name": ("${dest.repo}::" + .name), "job_name": "${dest.name}"}]' ''
+          ) (lib.attrValues cfg.destinations)
+        } | jq -s 'add'
       '';
       create = ''
         ${lib.concatMapStringsSep "\n" (dest: ''
@@ -108,7 +114,7 @@ in
         set -efu
         cd /
         IFS=';' read -ra FOLDER <<< "$FOLDERS"
-        yes y | borg-job-"$JOB" extract --list "$LOCATION"::"$ARCHIVE_ID" "''${FOLDER[@]}"
+        yes y | borg-job-"$JOB_NAME" extract --list "$NAME" "''${FOLDER[@]}"
       '';
     };
   };

pkgs/clan-cli/clan_cli/backups/list.py

@@ -9,11 +9,8 @@ from ..machines.machines import Machine
 
 @dataclass
 class Backup:
-    archive_id: str
+    name: str
-    date: str
+    job_name: str | None = None
-    provider: str
-    remote_path: str
-    job_name: str
 
 
 def list_provider(machine: Machine, provider: str) -> list[Backup]:
@@ -26,19 +23,14 @@ def list_provider(machine: Machine, provider: str) -> list[Backup]:
     )
     if proc.returncode != 0:
         # TODO this should be a warning, only raise exception if no providers succeed
-        ClanError(f"failed to list backups for provider {provider}")
+        msg = f"failed to list backups for provider {provider}: {proc.stdout}"
+        raise ClanError(msg)
     else:
         parsed_json = json.loads(proc.stdout)
-        # TODO move borg specific code to borgbackup.nix
+        for archive in parsed_json:
-        for archive in parsed_json["archives"]:
+            results.append(
-            backup = Backup(
+                Backup(name=archive["name"], job_name=archive.get("job_name"))
-                archive_id=archive["archive"],
-                date=archive["time"],
-                provider=provider,
-                remote_path=parsed_json["repository"]["location"],
-                job_name=parsed_json["job-name"],
             )
-            results.append(backup)
     return results
 
 
@@ -59,7 +51,7 @@ def list_command(args: argparse.Namespace) -> None:
     machine = Machine(name=args.machine, flake=args.flake)
     backups = list_backups(machine=machine, provider=args.provider)
     for backup in backups:
-        print(backup.archive_id)
+        print(backup.name)
 
 
 def register_list_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/backups/restore.py

@@ -1,6 +1,5 @@
 import argparse
 import json
-import os
 import subprocess
 
 from ..errors import ClanError
@@ -14,12 +13,13 @@ def restore_service(
     backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
     backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
     folders = backup_folders[service]["folders"]
-    env = os.environ.copy()
+    env = {}
-    env["ARCHIVE_ID"] = backup.archive_id
+    env["NAME"] = backup.name
-    env["LOCATION"] = backup.remote_path
-    env["JOB"] = backup.job_name
     env["FOLDERS"] = ":".join(folders)
 
+    if backup.job_name is not None:
+        env["JOB_NAME"] = backup.job_name
+
     proc = machine.target_host.run(
         [
             "bash",
@@ -67,19 +67,25 @@ def restore_backup(
     machine: Machine,
     backups: list[Backup],
     provider: str,
-    archive_id: str,
+    name: str,
     service: str | None = None,
 ) -> None:
     if service is None:
         for backup in backups:
-            if backup.archive_id == archive_id:
+            if backup.name == name:
                 backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
                 for _service in backup_folders:
                     restore_service(machine, backup, provider, _service)
+                break
+        else:
+            raise ClanError(f"backup {name} not found")
     else:
         for backup in backups:
-            if backup.archive_id == archive_id:
+            if backup.name == name:
                 restore_service(machine, backup, provider, service)
+                break
+        else:
+            raise ClanError(f"backup {name} not found")
 
 
 def restore_command(args: argparse.Namespace) -> None:
@@ -89,7 +95,7 @@ def restore_command(args: argparse.Namespace) -> None:
         machine=machine,
         backups=backups,
         provider=args.provider,
-        archive_id=args.archive_id,
+        name=args.name,
         service=args.service,
     )
 
@@ -99,6 +105,6 @@ def register_restore_parser(parser: argparse.ArgumentParser) -> None:
         "machine", type=str, help="machine in the flake to create backups of"
     )
     parser.add_argument("provider", type=str, help="backup provider to use")
-    parser.add_argument("archive_id", type=str, help="id of the backup to restore")
+    parser.add_argument("name", type=str, help="Name of the backup to restore")
     parser.add_argument("--service", type=str, help="name of the service to restore")
     parser.set_defaults(func=restore_command)