bundles/backup-client: rework backup generation

bundles/backup-client/files/generate-backup-with-retries

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Try generating a backup multiple times. If one attempt succeeds, we're
+# done. If not, there will be logs for every attempt, plus monitoring
+# will read the result of the last backup.
+for try in {1..3}
+do
+    generate-backup "$try"
+    exitcode=$?
+
+    if [[ $exitcode -eq 100 ]]
+    then
+        # fatal error, cannot recover
+        exit 1
+    elif [[ $exitcode -eq 0 ]]
+    then
+        # successful backup
+        exit 0
+    else
+        sleep 60
+    fi
+done