bundles/backup-client: retry backups

rsync just fails too often in the last couple of days. I don't want to
babysit this every morning.

bundles/backup-client/files/generate-backup

@@ -20,6 +20,13 @@ then
 fi
 
 do_backup() {
+    rsync_errorcodes_for_this_path=""
+    backup_has_successfully_run="no"
+
+    for try in {1..5}
+    do
+        echo "Running Backup for $1, try $try ..." >&2
+
         # Compress level 1 is a good compromise between speed and cpu usage.
         rsync --compress-level=1 -aAP --numeric-ids --delete --relative \
             --rsync-path="/usr/bin/rsync --fake-super" \
@@ -30,9 +37,18 @@ do_backup() {
         # I don't know why, but this is very common, apparently?
         exitcode=$?
         if [[ $exitcode != 0 ]] && [[ $exitcode != 24 ]]
+        then
+            rsync_errorcodes_for_this_path+=" $exitcode"
+        else
+            backup_has_successfully_run="yes"
+            break
+        fi
+    done
+
+    if [[ "$backup_has_successfully_run" != "yes" ]]
     then
 <%text>
-        rsync_errors+="${NL}${1} ${exitcode}"
+        rsync_errors+="${NL}${1}${rsync_errors_for_this_path}"
 </%text>
     fi
 }