bundles/backup-client: add check for last backup of specific client

bundles/backup-server/files/check_backup_for_node

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+from datetime import datetime
+from json import load
+from subprocess import check_output
+from sys import argv, exit
+from time import time
+
+NODE = argv[1]
+
+NOW = int(time())
+DAY_SECONDS = 60 * 60 * 24
+
+snaps = set()
+
+try:
+    with open(f'/etc/backup-server/config.json', 'r') as f:
+        server_settings = load(f)
+
+    # get all existing snapshots for NODE
+    for line in check_output('LC_ALL=C zfs list -H -t snapshot -o name', shell=True).splitlines():
+        line = line.decode('UTF-8')
+
+        if line.startswith('{}/{}@'.format(server_settings['zfs-base'], NODE)):
+            _, snapname = line.split('@', 1)
+
+            if 'zfs-auto-snap' in snapname:
+                # migration from auto-snapshots, ignore
+                continue
+
+            ts, bucket = snapname.split('-', 1)
+            snaps.add(int(ts))
+
+    if not snaps:
+        print('No backups found!')
+        exit(2)
+
+    last_snap = sorted(snaps)[-1]
+    delta = NOW - last_snap
+
+    print('Last backup was on {}'.format(
+        datetime.fromtimestamp(last_snap).strftime('%Y-%m-%d %H:%M:%S'),
+    ))
+
+    if delta > (DAY_SECONDS * 2):
+        exit(2)
+    elif delta > DAY_SECONDS:
+        exit(1)
+    else:
+        exit(0)
+except Exception as e:
+    print(repr(e))
+    exit(3)

bundles/backup-server/items.py

@@ -14,6 +14,9 @@ files = {
     '/usr/local/bin/rotate-single-backup-client': {
         'mode': '0755',
     },
+    '/usr/local/share/icinga/plugins/check_backup_for_node': {
+        'mode': '0755',
+    },
 }
 
 directories['/etc/backup-server/clients'] = {

bundles/backup-server/metadata.py

@@ -104,7 +104,6 @@ def zfs_pool(metadata):
     }
 
 
-
 @metadata_reactor.provides(
     'zfs/datasets',
     'zfs/snapshots/snapshot_never',
@@ -131,3 +130,23 @@ def zfs_datasets_and_snapshots(metadata):
             },
         },
     }
+
+
+@metadata_reactor.provides(
+    'icinga2_api/backup-server/services',
+)
+def monitoring(metadata):
+    services = {}
+
+    for client in metadata.get('backup-server/clients', {}).keys():
+        services[f'BACKUPS FOR NODE {client}'] = {
+            'command_on_monitored_host': f'sudo /usr/local/share/icinga/plugins/check_backup_for_node {client}',
+        }
+
+    return {
+        'icinga2_api': {
+            'backup-server': {
+                'services': services,
+            },
+        },
+    }