clan.core.state: wrap all commands in shell scripts

Otherwise we cannot execute them via ssh and also have nix store
dependencies.

clanModules/borgbackup/default.nix

@@ -13,14 +13,14 @@ let
       state:
       lib.optionalString (state.preBackupCommand != null) ''
         echo "Running pre-backup command for ${state.name}"
-        if ! ( ${state.preBackupCommand} ) then
+        if ! /run/current-system/sw/bin/${state.preBackupCommand}; then
           preCommandErrors["${state.name}"]=1
         fi
       ''
     ) (lib.attrValues config.clan.core.state)}
 
     if [[ ''${#preCommandErrors[@]} -gt 0 ]]; then
-      echo "PreBackupCommand failed for the following services:"
+      echo "pre-backup commands failed for the following services:"
       for state in "''${!preCommandErrors[@]}"; do
         echo "  $state"
       done

clanModules/localbackup/default.nix

@@ -125,7 +125,6 @@ in
             }
             ${lib.concatMapStringsSep "\n" (target: ''
               ${mountHook target}
-              set -x
               echo "Creating backup '${target.name}'"
 
               ${lib.optionalString (target.preBackupHook != null) ''
@@ -139,7 +138,7 @@ in
                 state:
                 lib.optionalString (state.preBackupCommand != null) ''
                   echo "Running pre-backup command for ${state.name}"
-                  if ! ( ${state.preBackupCommand} ) then
+                  if ! /run/current-system/sw/bin/${state.preBackupCommand}; then
                     preCommandErrors["${state.name}"]=1
                   fi
                 ''

clanModules/postgresql/default.nix

@@ -14,7 +14,7 @@ let
     in
     {
       folders = [ folder ];
-      preBackupCommand = ''
+      preBackupScript = ''
         export PATH=${
           lib.makeBinPath [
             config.services.postgresql.package
@@ -32,7 +32,38 @@ let
         runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
         mv "${current}.tmp" ${current}
       '';
-      postRestoreCommand = "postgres-db-restore-command-${db.name}";
+      postRestoreScript = ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          (
+            systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
+            trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
+
+            mkdir -p "${folder}"
+            runuser -u postgres -- dropdb "${db.name}"
+            runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          )
+        else
+          echo No database backup found, skipping restore
+        fi
+      '';
     };
 
   createDatabase = db: ''