From 8fd73e09dea2c0b1fcccfb6820950e98e55ac317 Mon Sep 17 00:00:00 2001
From: Franziska Kunsmann <hi@kunsmann.eu>
Date: Sat, 7 Aug 2021 06:59:33 +0200
Subject: [PATCH] bundles/backup-client: retry backups

rsync just fails too often in the last couple of days. I don't want to
babysit this every morning.
---
 bundles/backup-client/files/generate-backup | 36 +++++++++++++++------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/bundles/backup-client/files/generate-backup b/bundles/backup-client/files/generate-backup
index b1351bf..b95bab4 100644
--- a/bundles/backup-client/files/generate-backup
+++ b/bundles/backup-client/files/generate-backup
@@ -20,19 +20,35 @@ then
 fi
 
 do_backup() {
-    # Compress level 1 is a good compromise between speed and cpu usage.
-    rsync --compress-level=1 -aAP --numeric-ids --delete --relative \
-        --rsync-path="/usr/bin/rsync --fake-super" \
-        -e "ssh -o IdentityFile=/etc/backup.priv -o StrictHostKeyChecking=accept-new -p ${port}" \
-        "$1" "$ssh_login":backups/
+    rsync_errorcodes_for_this_path=""
+    backup_has_successfully_run="no"
 
-    # Exit code 24 means some files have vanished during rsync.
-    # I don't know why, but this is very common, apparently?
-    exitcode=$?
-    if [[ $exitcode != 0 ]] && [[ $exitcode != 24 ]]
+    for try in {1..5}
+    do
+        echo "Running Backup for $1, try $try ..." >&2
+
+        # Compress level 1 is a good compromise between speed and cpu usage.
+        rsync --compress-level=1 -aAP --numeric-ids --delete --relative \
+            --rsync-path="/usr/bin/rsync --fake-super" \
+            -e "ssh -o IdentityFile=/etc/backup.priv -o StrictHostKeyChecking=accept-new -p ${port}" \
+            "$1" "$ssh_login":backups/
+
+        # Exit code 24 means some files have vanished during rsync.
+        # I don't know why, but this is very common, apparently?
+        exitcode=$?
+        if [[ $exitcode != 0 ]] && [[ $exitcode != 24 ]]
+        then
+            rsync_errorcodes_for_this_path+=" $exitcode"
+        else
+            backup_has_successfully_run="yes"
+            break
+        fi
+    done
+
+    if [[ "$backup_has_successfully_run" != "yes" ]]
     then
 <%text>
-        rsync_errors+="${NL}${1} ${exitcode}"
+        rsync_errors+="${NL}${1}${rsync_errors_for_this_path}"
 </%text>
     fi
 }