bundles/backup-client: retry backups

rsync just fails too often in the last couple of days. I don't want to
babysit this every morning.

bundles/backup-client/files/generate-backup

@@ -20,19 +20,35 @@ then
 fi
 
 do_backup() {
-    # Compress level 1 is a good compromise between speed and cpu usage.
-    rsync --compress-level=1 -aAP --numeric-ids --delete --relative \
-        --rsync-path="/usr/bin/rsync --fake-super" \
-        -e "ssh -o IdentityFile=/etc/backup.priv -o StrictHostKeyChecking=accept-new -p ${port}" \
-        "$1" "$ssh_login":backups/
+    rsync_errorcodes_for_this_path=""
+    backup_has_successfully_run="no"
 
-    # Exit code 24 means some files have vanished during rsync.
-    # I don't know why, but this is very common, apparently?
-    exitcode=$?
-    if [[ $exitcode != 0 ]] && [[ $exitcode != 24 ]]
+    for try in {1..5}
+    do
+        echo "Running Backup for $1, try $try ..." >&2
+
+        # Compress level 1 is a good compromise between speed and cpu usage.
+        rsync --compress-level=1 -aAP --numeric-ids --delete --relative \
+            --rsync-path="/usr/bin/rsync --fake-super" \
+            -e "ssh -o IdentityFile=/etc/backup.priv -o StrictHostKeyChecking=accept-new -p ${port}" \
+            "$1" "$ssh_login":backups/
+
+        # Exit code 24 means some files have vanished during rsync.
+        # I don't know why, but this is very common, apparently?
+        exitcode=$?
+        if [[ $exitcode != 0 ]] && [[ $exitcode != 24 ]]
+        then
+            rsync_errorcodes_for_this_path+=" $exitcode"
+        else
+            backup_has_successfully_run="yes"
+            break
+        fi
+    done
+
+    if [[ "$backup_has_successfully_run" != "yes" ]]
     then
 <%text>
-        rsync_errors+="${NL}${1} ${exitcode}"
+        rsync_errors+="${NL}${1}${rsync_errors_for_this_path}"
 </%text>
     fi
 }