backups: support services for restore

clanModules/borgbackup.nix

@@ -66,27 +66,24 @@ in
     };
 
     clanCore.backups.providers.borgbackup = {
+      # TODO list needs to run locally or on the remote machine
       list = ''
-        ssh ${config.clan.networking.deploymentAddress} <<EOF
         ${lib.concatMapStringsSep "\n" (dest: ''
-            borg-job-${dest.name} list --json | jq -r '. + {"job-name": "${dest.name}"}'
+          # we need yes here to skip the changed url verification
+          yes y | borg-job-${dest.name} list --json | jq -r '. + {"job-name": "${dest.name}"}'
         '') (lib.attrValues cfg.destinations)}
-        EOF
       '';
-      start = ''
+      create = ''
-        ssh ${config.clan.networking.deploymentAddress} -- '
         ${lib.concatMapStringsSep "\n" (dest: ''
           systemctl start borgbackup-job-${dest.name}
         '') (lib.attrValues cfg.destinations)}
-        '
       '';
 
       restore = ''
-        ssh ${config.clan.networking.deploymentAddress} -- LOCATION="$LOCATION" ARCHIVE="$ARCHIVE_ID" JOB="$JOB" '
+        set -efu
-          set -efux
         cd /
-          borg-job-"$JOB" extract --list --dry-run "$LOCATION"::"$ARCHIVE"
+        IFS=';' read -ra FOLDER <<< "$FOLDERS"
-        '
+        yes y | borg-job-"$JOB" extract --list --dry-run "$LOCATION"::"$ARCHIVE_ID" "''${FOLDER[@]}"
       '';
     };
   };

nixosModules/clanCore/backups.nix

@@ -50,9 +50,14 @@
             description = ''
               script to restore a backup
               should take an optional service name as argument
+              gets ARCHIVE_ID, LOCATION, JOB and FOLDERS as environment variables
+              ARCHIVE_ID is the id of the backup
+              LOCATION is the remote identifier of the backup
+              JOB is the job name of the backup
+              FOLDERS is a colon separated list of folders to restore
             '';
           };
-          start = lib.mkOption {
+          create = lib.mkOption {
             type = lib.types.str;
             description = ''
               script to start a backup

pkgs/clan-cli/clan_cli/backups/create.py

@@ -1,6 +1,5 @@
 import argparse
 import json
-import subprocess
 
 from ..errors import ClanError
 from ..machines.machines import Machine
@@ -12,8 +11,8 @@ def create_backup(machine: Machine, provider: str | None = None) -> None:
     )
     if provider is None:
         for provider in backup_scripts["providers"]:
-            proc = subprocess.run(
+            proc = machine.host.run(
-                ["bash", "-c", backup_scripts["providers"][provider]["start"]],
+                ["bash", "-c", backup_scripts["providers"][provider]["create"]],
             )
             if proc.returncode != 0:
                 raise ClanError("failed to start backup")
@@ -22,8 +21,8 @@ def create_backup(machine: Machine, provider: str | None = None) -> None:
     else:
         if provider not in backup_scripts["providers"]:
             raise ClanError(f"provider {provider} not found")
-        proc = subprocess.run(
+        proc = machine.host.run(
-            ["bash", "-c", backup_scripts["providers"][provider]["start"]],
+            ["bash", "-c", backup_scripts["providers"][provider]["create"]],
         )
         if proc.returncode != 0:
             raise ClanError("failed to start backup")

pkgs/clan-cli/clan_cli/backups/list.py

@@ -1,47 +1,68 @@
 import argparse
 import json
 import subprocess
-from typing import Any
+from dataclasses import dataclass
 
 from ..errors import ClanError
 from ..machines.machines import Machine
 
 
-def list_backups(machine: Machine, provider: str | None = None) -> list[dict[str, Any]]:
+@dataclass
-    backup_scripts = json.loads(
+class Backup:
+    archive_id: str
+    date: str
+    provider: str
+    remote_path: str
+    job_name: str
+
+
+def list_provider(machine: Machine, provider: str) -> list[Backup]:
+    results = []
+    backup_metadata = json.loads(
+        machine.eval_nix(f"nixosConfigurations.{machine.name}.config.clanCore.backups")
+    )
+    proc = machine.host.run(
+        ["bash", "-c", backup_metadata["providers"][provider]["list"]],
+        stdout=subprocess.PIPE,
+        check=False,
+    )
+    if proc.returncode != 0:
+        # TODO this should be a warning, only raise exception if no providers succeed
+        ClanError(f"failed to list backups for provider {provider}")
+    else:
+        parsed_json = json.loads(proc.stdout)
+        # TODO move borg specific code to borgbackup.nix
+        for archive in parsed_json["archives"]:
+            backup = Backup(
+                archive_id=archive["archive"],
+                date=archive["time"],
+                provider=provider,
+                remote_path=parsed_json["repository"]["location"],
+                job_name=parsed_json["job-name"],
+            )
+            results.append(backup)
+    return results
+
+
+def list_backups(machine: Machine, provider: str | None = None) -> list[Backup]:
+    backup_metadata = json.loads(
         machine.eval_nix(f"nixosConfigurations.{machine.name}.config.clanCore.backups")
     )
     results = []
     if provider is None:
-        for provider in backup_scripts["providers"]:
+        for _provider in backup_metadata["providers"]:
-            proc = subprocess.run(
+            results += list_provider(machine, _provider)
-                ["bash", "-c", backup_scripts["providers"][provider]["list"]],
-                stdout=subprocess.PIPE,
-            )
-            if proc.returncode != 0:
-                # TODO this should be a warning, only raise exception if no providers succeed
-                raise ClanError("failed to list backups")
-            else:
-                results.append(proc.stdout)
-    else:
-        if provider not in backup_scripts["providers"]:
-            raise ClanError(f"provider {provider} not found")
-        proc = subprocess.run(
-            ["bash", "-c", backup_scripts["providers"][provider]["list"]],
-            stdout=subprocess.PIPE,
-        )
-        if proc.returncode != 0:
-            raise ClanError("failed to list backup")
-        else:
-            results.append(proc.stdout)
 
-    return list(map(json.loads, results))
+    else:
+        results += list_provider(machine, provider)
+
+    return results
 
 
 def list_command(args: argparse.Namespace) -> None:
     machine = Machine(name=args.machine, flake_dir=args.flake)
-    backups_data = list_backups(machine=machine, provider=args.provider)
+    backups = list_backups(machine=machine, provider=args.provider)
-    print(json.dumps(list(backups_data)))
+    print(backups)
 
 
 def register_list_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/backups/restore.py

@@ -2,64 +2,100 @@ import argparse
 import json
 import os
 import subprocess
-from typing import Any
 
 from ..errors import ClanError
 from ..machines.machines import Machine
-from .list import list_backups
+from .list import Backup, list_backups
 
 
-def restore_backup(
+def restore_service(
-    backup_data: list[dict[str, Any]],
+    machine: Machine, backup: Backup, provider: str, service: str
-    machine: Machine,
-    provider: str,
-    archive_id: str,
-    service: str | None = None,
 ) -> None:
-    backup_scripts = json.loads(
+    backup_metadata = json.loads(
         machine.eval_nix(f"nixosConfigurations.{machine.name}.config.clanCore.backups")
     )
     backup_folders = json.loads(
         machine.eval_nix(f"nixosConfigurations.{machine.name}.config.clanCore.state")
     )
-    if service is None:
+    folders = backup_folders[service]["folders"]
-        for backup in backup_data:
-            for archive in backup["archives"]:
-                if archive["archive"] == archive_id:
     env = os.environ.copy()
-                    env["ARCHIVE_ID"] = archive_id
+    env["ARCHIVE_ID"] = backup.archive_id
-                    env["LOCATION"] = backup["repository"]["location"]
+    env["LOCATION"] = backup.remote_path
-                    env["JOB"] = backup["job-name"]
+    env["JOB"] = backup.job_name
-                    proc = subprocess.run(
+    env["FOLDERS"] = ":".join(folders)
+
+    proc = machine.host.run(
         [
             "bash",
             "-c",
-                            backup_scripts["providers"][provider]["restore"],
+            backup_folders[service]["preRestoreScript"],
         ],
         stdout=subprocess.PIPE,
-                        env=env,
+        extra_env=env,
     )
     if proc.returncode != 0:
-                        # TODO this should be a warning, only raise exception if no providers succeed
+        raise ClanError(
-                        raise ClanError("failed to restore backup")
+            f"failed to run preRestoreScript: {backup_folders[service]['preRestoreScript']}, error was: {proc.stdout}"
-    else:
-        print(
-            "would restore backup",
-            machine,
-            provider,
-            archive_id,
-            "of service:",
-            service,
         )
-        print(backup_folders)
+
+    proc = machine.host.run(
+        [
+            "bash",
+            "-c",
+            backup_metadata["providers"][provider]["restore"],
+        ],
+        stdout=subprocess.PIPE,
+        extra_env=env,
+    )
+    if proc.returncode != 0:
+        raise ClanError(
+            f"failed to restore backup: {backup_metadata['providers'][provider]['restore']}"
+        )
+
+    proc = machine.host.run(
+        [
+            "bash",
+            "-c",
+            backup_folders[service]["postRestoreScript"],
+        ],
+        stdout=subprocess.PIPE,
+        extra_env=env,
+    )
+    if proc.returncode != 0:
+        raise ClanError(
+            f"failed to run postRestoreScript: {backup_folders[service]['postRestoreScript']}, error was: {proc.stdout}"
+        )
+
+
+def restore_backup(
+    machine: Machine,
+    backups: list[Backup],
+    provider: str,
+    archive_id: str,
+    service: str | None = None,
+) -> None:
+    if service is None:
+        for backup in backups:
+            if backup.archive_id == archive_id:
+                backup_folders = json.loads(
+                    machine.eval_nix(
+                        f"nixosConfigurations.{machine.name}.config.clanCore.state"
+                    )
+                )
+                for _service in backup_folders:
+                    restore_service(machine, backup, provider, _service)
+    else:
+        for backup in backups:
+            if backup.archive_id == archive_id:
+                restore_service(machine, backup, provider, service)
 
 
 def restore_command(args: argparse.Namespace) -> None:
     machine = Machine(name=args.machine, flake_dir=args.flake)
-    backup_data = list_backups(machine=machine, provider=args.provider)
+    backups = list_backups(machine=machine, provider=args.provider)
     restore_backup(
-        backup_data=backup_data,
         machine=machine,
+        backups=backups,
         provider=args.provider,
         archive_id=args.archive_id,
         service=args.service,