diff --git a/bundles/backup-server/files/check_backup_for_node b/bundles/backup-server/files/check_backup_for_node new file mode 100644 index 0000000..5d37d88 --- /dev/null +++ b/bundles/backup-server/files/check_backup_for_node @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +from datetime import datetime +from json import load +from subprocess import check_output +from sys import argv, exit +from time import time + +NODE = argv[1] + +NOW = int(time()) +DAY_SECONDS = 60 * 60 * 24 + +snaps = set() + +try: + with open(f'/etc/backup-server/config.json', 'r') as f: + server_settings = load(f) + + # get all existing snapshots for NODE + for line in check_output('LC_ALL=C zfs list -H -t snapshot -o name', shell=True).splitlines(): + line = line.decode('UTF-8') + + if line.startswith('{}/{}@'.format(server_settings['zfs-base'], NODE)): + _, snapname = line.split('@', 1) + + if 'zfs-auto-snap' in snapname: + # migration from auto-snapshots, ignore + continue + + ts, bucket = snapname.split('-', 1) + snaps.add(int(ts)) + + if not snaps: + print('No backups found!') + exit(2) + + last_snap = sorted(snaps)[-1] + delta = NOW - last_snap + + print('Last backup was on {}'.format( + datetime.fromtimestamp(last_snap).strftime('%Y-%m-%d %H:%M:%S'), + )) + + if delta > (DAY_SECONDS * 2): + exit(2) + elif delta > DAY_SECONDS: + exit(1) + else: + exit(0) +except Exception as e: + print(repr(e)) + exit(3) diff --git a/bundles/backup-server/items.py b/bundles/backup-server/items.py index 0b43f62..c70512c 100644 --- a/bundles/backup-server/items.py +++ b/bundles/backup-server/items.py @@ -14,6 +14,9 @@ files = { '/usr/local/bin/rotate-single-backup-client': { 'mode': '0755', }, + '/usr/local/share/icinga/plugins/check_backup_for_node': { + 'mode': '0755', + }, } directories['/etc/backup-server/clients'] = { diff --git a/bundles/backup-server/metadata.py b/bundles/backup-server/metadata.py index 3a8b89e..d94c9f3 100644 --- a/bundles/backup-server/metadata.py +++ b/bundles/backup-server/metadata.py @@ -104,7 +104,6 @@ def zfs_pool(metadata): } - @metadata_reactor.provides( 'zfs/datasets', 'zfs/snapshots/snapshot_never', @@ -131,3 +130,23 @@ def zfs_datasets_and_snapshots(metadata): }, }, } + + +@metadata_reactor.provides( + 'icinga2_api/backup-server/services', +) +def monitoring(metadata): + services = {} + + for client in metadata.get('backup-server/clients', {}).keys(): + services[f'BACKUPS FOR NODE {client}'] = { + 'command_on_monitored_host': f'sudo /usr/local/share/icinga/plugins/check_backup_for_node {client}', + } + + return { + 'icinga2_api': { + 'backup-server': { + 'services': services, + }, + }, + }