bundles/backups-server: read backup snapshot info from file instead of asking zfs every time

bundles/backup-server/files/check_backup_for_node

@@ -2,7 +2,6 @@
 
 from datetime import datetime
 from json import load
-from subprocess import check_output
 from sys import argv, exit
 from time import time
 
@@ -18,29 +17,17 @@ try:
     with open(f'/etc/backup-server/config.json', 'r') as f:
         server_settings = load(f)
 
-    # get all existing snapshots for NODE
-    for line in check_output('LC_ALL=C zfs list -H -t snapshot -o name', shell=True).splitlines():
-        line = line.decode('UTF-8')
+    with open(f'/etc/backup-server/backups.json', 'r') as f:
+        backups = load(f)
 
-        if line.startswith('{}/{}@'.format(server_settings['zfs-base'], NODE)):
-            _, snapname = line.split('@', 1)
-
-            if 'zfs-auto-snap' in snapname:
-                # migration from auto-snapshots, ignore
-                continue
-
-            ts, bucket = snapname.split('-', 1)
-            snaps.add(int(ts))
-
-    if not snaps:
+    if NODE not in backups:
         print('No backups found!')
         exit(2)
 
-    last_snap = sorted(snaps)[-1]
-    delta = NOW - last_snap
+    delta = NOW - backups[NODE]
 
     print('Last backup was on {} UTC'.format(
-        datetime.fromtimestamp(last_snap).strftime('%Y-%m-%d %H:%M:%S'),
+        datetime.fromtimestamp(backups[NODE]).strftime('%Y-%m-%d %H:%M:%S'),
     ))
 
     # One day without backups is still okay. There may be fluctuations

bundles/backup-server/files/check_backup_for_node-cron

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+from json import load, dump
+from subprocess import check_output
+from shutil import move
+from os import remove
+from collections import defaultdict
+
+with open('/etc/backup-server/config.json', 'r') as f:
+    server_settings = load(f)
+
+snapshots = defaultdict(set)
+
+for line in check_output('LC_ALL=C zfs list -H -t snapshot -o name', shell=True).splitlines():
+    line = line.decode('UTF-8')
+
+    if line.startswith('{}/'.format(server_settings['zfs-base'])):
+        dataset, snapname = line.split('@', 1)
+
+        dataset = dataset.split('/')[-1]
+        ts, bucket = snapname.split('-', 1)
+
+        if not ts.isdigit():
+            # garbage, ignore
+            continue
+
+        snapshots[dataset].add(int(ts))
+
+backups = {}
+for dataset, snaps in snapshots.items():
+    backups[dataset] = sorted(snaps)[-1]
+
+with open('/etc/backup-server/backups.tmp.json', 'w') as f:
+    dump(backups, f)
+
+move(
+    '/etc/backup-server/backups.tmp.json',
+    '/etc/backup-server/backups.json',
+)

bundles/backup-server/items.py

@@ -18,6 +18,9 @@ files = {
     '/usr/local/share/icinga/plugins/check_backup_for_node': {
         'mode': '0755',
     },
+    '/usr/local/share/icinga/plugins/check_backup_for_node-cron': {
+        'mode': '0755',
+    },
 }
 
 directories['/etc/backup-server/clients'] = {

bundles/backup-server/metadata.py

@@ -10,6 +10,14 @@ defaults = {
             'c-*',
         },
     },
+    'systemd-timers': {
+        'timers': {
+            'check_backup_for_node-cron': {
+                'command': '/usr/local/share/icinga/plugins/check_backup_for_node-cron',
+                'when': '*-*-* *:00/5:00', # every five minutes
+            }
+        },
+    },
     'zfs': {
         # The whole point of doing backups is to keep them for a long
         # time, which eliminates the need for this check.
@@ -183,7 +191,7 @@ def monitoring(metadata):
             continue
 
         services[f'BACKUPS FOR NODE {client}'] = {
-            'command_on_monitored_host': 'sudo /usr/local/share/icinga/plugins/check_backup_for_node {} {}'.format(
+            'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_backup_for_node {} {}'.format(
                 client,
                 config['one_backup_every_hours'],
             ),