From a7a59fd690fa08bd0c1e4bc36826dd9ae64eddaf Mon Sep 17 00:00:00 2001
From: Franziska Kunsmann <hi@kunsmann.eu>
Date: Fri, 14 Feb 2025 21:25:10 +0100
Subject: [PATCH] bundles/backups-server: read backup snapshot info from file
 instead of asking zfs every time

---
 .../backup-server/files/check_backup_for_node | 23 +++--------
 .../files/check_backup_for_node-cron          | 39 +++++++++++++++++++
 bundles/backup-server/items.py                |  3 ++
 bundles/backup-server/metadata.py             | 10 ++++-
 4 files changed, 56 insertions(+), 19 deletions(-)
 create mode 100644 bundles/backup-server/files/check_backup_for_node-cron

diff --git a/bundles/backup-server/files/check_backup_for_node b/bundles/backup-server/files/check_backup_for_node
index b7866f8..bf57012 100644
--- a/bundles/backup-server/files/check_backup_for_node
+++ b/bundles/backup-server/files/check_backup_for_node
@@ -2,7 +2,6 @@
 
 from datetime import datetime
 from json import load
-from subprocess import check_output
 from sys import argv, exit
 from time import time
 
@@ -18,29 +17,17 @@ try:
     with open(f'/etc/backup-server/config.json', 'r') as f:
         server_settings = load(f)
 
-    # get all existing snapshots for NODE
-    for line in check_output('LC_ALL=C zfs list -H -t snapshot -o name', shell=True).splitlines():
-        line = line.decode('UTF-8')
+    with open(f'/etc/backup-server/backups.json', 'r') as f:
+        backups = load(f)
 
-        if line.startswith('{}/{}@'.format(server_settings['zfs-base'], NODE)):
-            _, snapname = line.split('@', 1)
-
-            if 'zfs-auto-snap' in snapname:
-                # migration from auto-snapshots, ignore
-                continue
-
-            ts, bucket = snapname.split('-', 1)
-            snaps.add(int(ts))
-
-    if not snaps:
+    if NODE not in backups:
         print('No backups found!')
         exit(2)
 
-    last_snap = sorted(snaps)[-1]
-    delta = NOW - last_snap
+    delta = NOW - backups[NODE]
 
     print('Last backup was on {} UTC'.format(
-        datetime.fromtimestamp(last_snap).strftime('%Y-%m-%d %H:%M:%S'),
+        datetime.fromtimestamp(backups[NODE]).strftime('%Y-%m-%d %H:%M:%S'),
     ))
 
     # One day without backups is still okay. There may be fluctuations
diff --git a/bundles/backup-server/files/check_backup_for_node-cron b/bundles/backup-server/files/check_backup_for_node-cron
new file mode 100644
index 0000000..b82217d
--- /dev/null
+++ b/bundles/backup-server/files/check_backup_for_node-cron
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+from json import load, dump
+from subprocess import check_output
+from shutil import move
+from os import remove
+from collections import defaultdict
+
+with open('/etc/backup-server/config.json', 'r') as f:
+    server_settings = load(f)
+
+snapshots = defaultdict(set)
+
+for line in check_output('LC_ALL=C zfs list -H -t snapshot -o name', shell=True).splitlines():
+    line = line.decode('UTF-8')
+
+    if line.startswith('{}/'.format(server_settings['zfs-base'])):
+        dataset, snapname = line.split('@', 1)
+
+        dataset = dataset.split('/')[-1]
+        ts, bucket = snapname.split('-', 1)
+
+        if not ts.isdigit():
+            # garbage, ignore
+            continue
+
+        snapshots[dataset].add(int(ts))
+
+backups = {}
+for dataset, snaps in snapshots.items():
+    backups[dataset] = sorted(snaps)[-1]
+
+with open('/etc/backup-server/backups.tmp.json', 'w') as f:
+    dump(backups, f)
+
+move(
+    '/etc/backup-server/backups.tmp.json',
+    '/etc/backup-server/backups.json',
+)
diff --git a/bundles/backup-server/items.py b/bundles/backup-server/items.py
index bd4d12f..e872231 100644
--- a/bundles/backup-server/items.py
+++ b/bundles/backup-server/items.py
@@ -18,6 +18,9 @@ files = {
     '/usr/local/share/icinga/plugins/check_backup_for_node': {
         'mode': '0755',
     },
+    '/usr/local/share/icinga/plugins/check_backup_for_node-cron': {
+        'mode': '0755',
+    },
 }
 
 directories['/etc/backup-server/clients'] = {
diff --git a/bundles/backup-server/metadata.py b/bundles/backup-server/metadata.py
index 4be6390..aace61b 100644
--- a/bundles/backup-server/metadata.py
+++ b/bundles/backup-server/metadata.py
@@ -10,6 +10,14 @@ defaults = {
             'c-*',
         },
     },
+    'systemd-timers': {
+        'timers': {
+            'check_backup_for_node-cron': {
+                'command': '/usr/local/share/icinga/plugins/check_backup_for_node-cron',
+                'when': '*-*-* *:00/5:00', # every five minutes
+            }
+        },
+    },
     'zfs': {
         # The whole point of doing backups is to keep them for a long
         # time, which eliminates the need for this check.
@@ -183,7 +191,7 @@ def monitoring(metadata):
             continue
 
         services[f'BACKUPS FOR NODE {client}'] = {
-            'command_on_monitored_host': 'sudo /usr/local/share/icinga/plugins/check_backup_for_node {} {}'.format(
+            'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_backup_for_node {} {}'.format(
                 client,
                 config['one_backup_every_hours'],
             ),