bundles/backup-client: add monitoring for backups

bundles/backup-client/files/check_backup_last_run

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+from os.path import getmtime, isfile
+from sys import exit
+from time import time
+
+statusfile = '/var/tmp/backup.monitoring'
+if not isfile(statusfile):
+    print('Status file not found')
+    exit(3)
+
+mtime = getmtime(statusfile)
+now = time()
+
+if now-mtime > 60*60*24*2:
+    print('Status file is older than 2 days!')
+    exit(3)
+
+with open(statusfile, 'r') as f:
+    status = f.read().split()
+
+if status[0] == 'ok':
+    print('OK')
+    exit(0)
+elif status[0] == 'rsync_error':
+    print('rsync produced some errors, exit codes were: {}'.format(
+        ', '.join(status[1:])
+    ))
+    exit(2)
+else:
+    # garbage in file
+    print(' '.join(status))
+    exit(3)

bundles/backup-client/items.py

@@ -24,6 +24,9 @@ else:
             },
             'mode': '0700',
         },
+        '/usr/local/share/icinga/plugins/check_backup_last_run': {
+            'mode': '0755',
+        },
         '/etc/backup.priv': {
             'content': repo.vault.decrypt_file(join('backup', 'keys', f'{node.name}.key.vault')),
             'mode': '0400',

bundles/backup-client/metadata.py

@@ -20,4 +20,13 @@ def cron(metadata):
                 (node.magic_number % 60),
             ),
         },
+        'icinga2_api': {
+            'backup-client': {
+                'services': {
+                    'BACKUP LAST RUN': {
+                        'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_backup_last_run',
+                    },
+                },
+            },
+        },
     }