bundles/zfs: introduce zfs-snapshot-backups, enable backups for nodes which need zfs-snapshot-backups

bundles/backup-client/items.py

@@ -1,5 +1,22 @@
 from os.path import join
 
+if node.has_bundle('zfs'):
+    wanted_paths = node.metadata.get('backups/paths', set())
+    snapshot_paths = node.metadata.get('zfs/filesystems_with_backup_snapshots', {})
+    backup_paths = set()
+
+    for path in wanted_paths:
+        path_found = False
+        for zfs_paths in snapshot_paths.values():
+            if path in zfs_paths:
+                backup_paths.add(f'/mnt/backup-snapshot{path}')
+                path_found = True
+
+        if not path_found:
+            backup_paths.add(path)
+else:
+    backup_paths = node.metadata.get('backups/paths', set())
+
 if node.metadata.get('backups/exclude_from_backups', False):
     files['/etc/backup.priv'] = {
         'delete': True,
@@ -17,7 +34,7 @@ else:
             'username': node.metadata['backup-client']['user-name'],
             'server': server,
             'port': port,
-            'paths': node.metadata.get('backups/paths', {}),
+            'paths': backup_paths,
         },
         'mode': '0700',
     }

bundles/influxdb2/items.py

@@ -11,6 +11,17 @@ directories = {
     },
 }
 
+files = {
+    '/etc/zfs-snapshot-backup-pre.d/50-influxdb': {
+        'content': '#!/bin/sh\nsystemctl stop influxdb',
+        'mode': '0755',
+    },
+    '/etc/zfs-snapshot-backup-post.d/50-influxdb': {
+        'content': '#!/bin/sh\nsystemctl start influxdb',
+        'mode': '0755',
+    },
+}
+
 
 svc_systemd = {
     'influxdb': {

bundles/influxdb2/metadata.py

@@ -15,6 +15,11 @@ defaults = {
             },
         },
     },
+    'backups': {
+        'paths': {
+            '/var/lib/influxdb',
+        },
+    },
     'icinga2_api': {
         'telegraf': {
             'services': {

bundles/postgresql/items.py

@@ -60,9 +60,7 @@ files = {
     },
 }
 
-# FIXME currently we do not have a mechanism to use snapshot-backups of
-# zfs datasets.
-if node.has_bundle('backup-client'): # and not node.has_bundle('zfs'):
+if node.has_bundle('backup-client') and not node.has_bundle('zfs'):
     files['/etc/backup-pre-hooks.d/90-postgresql-dump-all'] = {
         'source': 'backup-pre-hook',
         'content_type': 'mako',
@@ -72,6 +70,10 @@ if node.has_bundle('backup-client'): # and not node.has_bundle('zfs'):
         'mode': '0700',
     }
     directories['/var/tmp/postgresdumps'] = {}
+else:
+    files['/var/tmp/postgresdumps'] = {
+        'delete': True,
+    }
 
 postgres_roles = {
     'root': {

bundles/postgresql/metadata.py

@@ -2,9 +2,6 @@ defaults = {
     'backups': {
         'paths': {
             '/var/lib/postgresql',
-
-            # FIXME
-            '/var/tmp/postgresdumps',
         },
     },
     'bash_functions': {
@@ -66,9 +63,8 @@ if node.has_bundle('zfs'):
             },
         },
     }
-# FIXME
-#else:
-#    defaults['backups']['paths'].add('/var/tmp/postgresdumps')
+else:
+    defaults['backups']['paths'].add('/var/tmp/postgresdumps')
 
 
 @metadata_reactor.provides(

bundles/zfs/files/backup-pre-hook

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -euo pipefail
+
+run-parts --exit-on-error -- /etc/zfs-snapshot-backup-pre.d
+
+
+% for dataset in sorted(node.metadata.get('zfs/filesystems_with_backup_snapshots')):
+if zfs get type ${dataset}@snapshot-backup >/dev/null 2>&1
+then
+    zfs destroy -f ${dataset}@snapshot-backup
+fi
+
+zfs snapshot ${dataset}@snapshot-backup
+
+mkdir -p /mnt/backup-snapshot${node.metadata['zfs']['datasets'][dataset]['mountpoint']}
+mount -t zfs ${dataset}@snapshot-backup /mnt/backup-snapshot${node.metadata['zfs']['datasets'][dataset]['mountpoint']}
+
+
+% endfor
+run-parts --exit-on-error -- /etc/zfs-snapshot-backup-post.d

bundles/zfs/items.py

@@ -95,7 +95,11 @@ directories = {
     "/etc/zfs-snapshot-backup-post.d": {
         'purge': True,
     },
-    "/etc/zfs-snapshot-backup-final.d": {
-        'purge': True,
-    },
 }
+
+if node.metadata.get('zfs/filesystems_with_backup_snapshots', {}) and node.has_bundle('backup-client'):
+    files['/etc/backup-pre-hooks.d/99-zfs-snapshots'] = {
+        'content_type': 'mako',
+        'source': 'backup-pre-hook',
+        'mode': '0755',
+    }

bundles/zfs/metadata.py

@@ -104,7 +104,9 @@ def zfs_scrub_cronjob(metadata):
     }
 
 
-@metadata_reactor
+@metadata_reactor.provides(
+    'icinga2_api/zfs/services',
+)
 def monitoring(metadata):
     if not node.has_bundle('sshmon'):
         raise DoNotRunAgain
@@ -129,3 +131,25 @@ def monitoring(metadata):
             },
         },
     }
+
+
+@metadata_reactor.provides(
+    'zfs/filesystems_with_backup_snapshots',
+)
+def backups_with_snapshot(metadata):
+    backups = metadata.get('backups/paths', set())
+    datasets = metadata.get('zfs/datasets', {})
+
+    backups_in_zfs_datasets = {}
+
+    for path in backups:
+        for dname, dconfig in datasets.items():
+            if 'mountpoint' in dconfig:
+                if path[:len(dconfig['mountpoint'])] == dconfig['mountpoint']:
+                    backups_in_zfs_datasets.setdefault(dname, set()).add(path)
+
+    return {
+        'zfs': {
+            'filesystems_with_backup_snapshots': backups_in_zfs_datasets,
+        },
+    }

data/backup/keys/htz-cloud.influxdb.key.vault

@@ -0,0 +1 @@
+encrypt$gAAAAABgoBM4n7eEwMxfav4200d5LoGyLsM67Ps9CCPBgjZjdsOpyQulbp2l4LIhWj1kljgrXZ_LnOaRUf3S8q7-drv3yEfG1d-cyK8vP7r8wkTcawaazM5XdhR8VkgxCyBCuOZmM1vvAmOwIMi1JiQcgiJ4G32ThS085onN3T9HvEu2a9sWYuOlk-yVUBpelqP97vbO6r2n3hn-62AC7Ww-Q_EQ_kcdDdJLOawNe1anJOsOeLb1XOlMIJWI74LZXfszsRi9LmxUpzaB4Gd_nzDLO1AZHD_GOf9UOeeab8PujwhQ4UhbEHCdB-uVH88LGCw25-6eiv0yA_kRulj7InA9sKRyBZ1okSF4Xhl-htez6XwBD6BuA_ly6ulSxWuoOV_qsCNiXhJYGTuKPJS0-wpLyeLb_PV1tlYOKZv3VK5a_EpCRa4fCdX7oj9pcA8ZQkQeFAx0P4b3oYkz8YkDiBINFdOLE177lC6Kuk33sLfsZBuoR7MSjHUtZHPOXUHu8pV0o0_YxxF5fBs6hyReXvCbZB18NikeH4Ki-RR5IE4ofTngf6dIQhCxp9u-cNs-mNP0GRKiYIBUInr9Udpr-ymXRq-7OvyTM92950ePm-qoqjbpVFA8HN057WfkW6N6DASRRa2HjqQJ

data/backup/keys/htz-cloud.influxdb.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFpAUytQ0ncucltODEr3MgcKF5U/6TS4zZG1OYJDIEQp kunsi@kunsi-t470.kunbox.net

nodes/htz-cloud/influxdb.py

@@ -31,10 +31,6 @@ nodes['htz-cloud.influxdb'] = {
                 },
             },
         },
-        'backups': {
-            # TODO enable this once we have zfs-snapshot-backups
-            'exclude_from_backups': True,
-        },
         'grafana': {
             'domain': 'grafana.kunsmann.eu',
             'login_max_duration': '30d',