backups: do backup rotation ourselves instead of relying on zfs-auto-snapshot

bundles/backup-client/files/generate-backup

@@ -3,8 +3,10 @@
 statusfile=/var/tmp/backup.monitoring
 ssh_login="${username}@${server}"
 ssh_cmnd="ssh -o IdentityFile=/etc/backup.priv -o StrictHostKeyChecking=accept-new -p ${port}"
+nodename="${node.name}"
 
 <%text>
+[[ -n "$DEBUG" ]] && set -x
 NL=$'\n'
 
 if ! [[ -f /etc/backup.priv ]]
@@ -59,6 +61,8 @@ do_backup() {
 }
 
 rsync_errors=""
+
+$ssh_cmnd $ssh_login "sudo /usr/local/bin/rotate-single-backup-client $nodename"
 </%text>
 
 % for path in sorted(paths):

bundles/backup-server/files/rotate-single-backup-client

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+from json import load
+from subprocess import check_call, check_output
+from sys import argv
+from time import time
+
+NODE = argv[1]
+
+NOW = int(time())
+DAY_SECONDS = 60 * 60 * 24
+INTERVALS = {
+    'daily': DAY_SECONDS,
+    'weekly': 7 * DAY_SECONDS,
+    'monthly': 30 * DAY_SECONDS,
+}
+
+buckets = {}
+
+def syslog(msg):
+    check_output(['logger', '-t', f'backup-{NODE}', msg])
+
+
+with open(f'/etc/backup-server/config.json', 'r') as f:
+    server_settings = load(f)
+
+with open(f'/etc/backup-server/clients/{NODE}', 'r') as f:
+    client_settings = load(f)
+
+# get all existing snapshots for NODE
+for line in check_output('LC_ALL=C zfs list -H -t snapshot -o name', shell=True).splitlines():
+    line = line.decode('UTF-8')
+
+    if line.startswith('{}/{}@'.format(server_settings['zfs-base'], NODE)):
+        _, snapname = line.split('@', 1)
+
+        if 'zfs-auto-snap' in snapname:
+            # migration from auto-snapshots, ignore
+            continue
+
+        ts, bucket = snapname.split('-', 1)
+        buckets.setdefault(bucket, set()).add(int(ts))
+        syslog(f'classified {line} as {bucket} from {ts}')
+
+# determine if we need to create a new snapshot
+for bucket in INTERVALS.keys():
+    snapshots = sorted(buckets.get(bucket, set()))
+
+    if snapshots:
+        last_snap = snapshots[-1]
+        delta = NOW - last_snap
+        fresh_age = INTERVALS[bucket] - DAY_SECONDS
+
+        if delta > fresh_age:
+            # last snapshot is older than what we want. create a new one.
+            check_call(
+                'zfs snapshot {}/{}@{}-{}'.format(
+                    server_settings['zfs-base'],
+                    NODE,
+                    NOW,
+                    bucket,
+                ),
+                shell=True,
+            )
+            buckets.setdefault(bucket, set()).add(NOW)
+            syslog(f'created new snapshot {NOW}-{bucket}')
+        else:
+            syslog(f'existing snapshot {last_snap}-{bucket} is fresh enough')
+    else:
+        check_call(
+            'zfs snapshot {}/{}@{}-{}'.format(
+                server_settings['zfs-base'],
+                NODE,
+                NOW,
+                bucket,
+            ),
+            shell=True,
+        )
+        buckets.setdefault(bucket, set()).add(NOW)
+        syslog(f'created initial snapshot {NOW}-{bucket}')
+
+# finally, see if we can delete any snapshots, because they are old enough
+for bucket in INTERVALS.keys():
+    snapshots = sorted(buckets.get(bucket, set()))
+
+    if not snapshots:
+        syslog(f'something is wrong, there are no snapshots for {bucket}')
+        continue
+
+    keep_age = INTERVALS[bucket] * client_settings[bucket]
+
+    # oldest snapshots come first
+    for ts in snapshots[:-int(client_settings[bucket])]:
+        delta = NOW - ts
+
+        if delta >= keep_age:
+            check_call(
+                'zfs destroy {}/{}@{}-{}'.format(
+                    server_settings['zfs-base'],
+                    NODE,
+                    ts,
+                    bucket,
+                ),
+                shell=True,
+            )
+            syslog(f'removing snapshot {ts}-{bucket}, age {delta}, keep_age {keep_age}')
+        else:
+            syslog(f'keeping snapshot {ts}-{bucket}, age not reached')
+
+    for ts in snapshots[int(client_settings[bucket]):]:
+        syslog(f'keeping snapshot {ts}-{bucket}, count')

bundles/backup-server/files/sudoers

@@ -0,0 +1,3 @@
+% for username, nodename in sorted(clients.items()):
+${username} ALL=NOPASSWD:/usr/local/bin/rotate-single-backup-client ${nodename}
+% endfor

bundles/backup-server/items.py

@@ -1,17 +1,41 @@
 repo.libs.tools.require_bundle(node, 'zfs')
 
 from os.path import join
+from bundlewrap.metadata import metadata_to_json
 
 dataset = node.metadata.get('backup-server/zfs-base')
 
+files = {
+    '/etc/backup-server/config.json': {
+        'content': metadata_to_json({
+            'zfs-base': dataset,
+        }),
+    },
+    '/usr/local/bin/rotate-single-backup-client': {
+        'mode': '0755',
+    },
+}
+
+directories['/etc/backup-server/clients'] = {
+    'purge': True,
+}
+
+sudoers = {}
+
 for nodename, config in node.metadata.get('backup-server/clients', {}).items():
     with open(join(repo.path, 'data', 'backup', 'keys', f'{nodename}.pub'), 'r') as f:
         pubkey = f.read().strip()
 
+    sudoers[config['user']] = nodename
+
     users[config['user']] = {
         'home': f'/srv/backups/{nodename}',
     }
 
+    files[f'/etc/backup-server/clients/{nodename}'] = {
+        'content': metadata_to_json(config['retain']),
+    }
+
     files[f'/srv/backups/{nodename}/.ssh/authorized_keys'] = {
         'content': pubkey,
         'owner': config['user'],
@@ -28,3 +52,11 @@ for nodename, config in node.metadata.get('backup-server/clients', {}).items():
             f'zfs_dataset:{dataset}/{nodename}',
         },
     }
+
+files['/etc/sudoers.d/backup-server'] = {
+    'source': 'sudoers',
+    'content_type': 'mako',
+    'context': {
+        'clients': sudoers,
+    },
+}

bundles/backup-server/metadata.py

@@ -16,6 +16,11 @@ defaults = {
 )
 def get_my_clients(metadata):
     my_clients = {}
+    retain_defaults = {
+        'daily': 14,
+        'weekly': 4,
+        'monthly': 6,
+    }
 
     for rnode in repo.nodes:
         if not rnode.has_bundle('backup-client') or rnode.metadata.get('backups/exclude_from_backups', False):
@@ -26,6 +31,11 @@ def get_my_clients(metadata):
 
         my_clients[rnode.name] = {
             'user': rnode.metadata.get('backup-client/user-name'),
+            'retain': {
+                'daily': rnode.metadata.get('backups/retain/daily', retain_defaults['daily']),
+                'weekly': rnode.metadata.get('backups/retain/weekly', retain_defaults['weekly']),
+                'monthly': rnode.metadata.get('backups/retain/monthly', retain_defaults['monthly']),
+            },
         }
 
     return {
@@ -97,15 +107,10 @@ def zfs_pool(metadata):
 
 @metadata_reactor.provides(
     'zfs/datasets',
-    'zfs/snapshots/retain_per_dataset',
+    'zfs/snapshots/snapshot_never',
 )
 def zfs_datasets_and_snapshots(metadata):
     zfs_datasets = {}
-    zfs_retains = {}
-    retain_defaults = {
-        'weekly': 4,
-        'monthly': 6,
-    }
 
     for client in metadata.get('backup-server/clients', {}).keys():
         dataset = '{}/{}'.format(metadata.get('backup-server/zfs-base'), client)
@@ -115,13 +120,14 @@ def zfs_datasets_and_snapshots(metadata):
             'compression': 'on',
         }
 
-        zfs_retains[dataset] = retain_defaults.copy()
 
     return {
         'zfs': {
             'datasets': zfs_datasets,
             'snapshots': {
-                'retain_per_dataset': zfs_retains,
+                'snapshot_never': {
+                    metadata.get('backup-server/zfs-base'),
+                },
             },
         },
     }

nodes/home/nas.py

@@ -50,11 +50,6 @@ nodes['home.nas'] = {
             'exclude_from_backups': True,
         },
         'backup-server': {
-            'clients': {
-                'kunsi-t470': {
-                    'user': 'kunsi-t470',
-                },
-            },
             'my_hostname': 'franzi-home.kunbox.net',
             'my_ssh_port': 2022,
             'zfs-base': 'storage/backups',

nodes/htz-hel/backup-kunsi.py

@@ -35,6 +35,11 @@ nodes['htz-hel.backup-kunsi'] = {
             'clients': {
                 'kunsi-t470': {
                     'user': 'kunsi-t470',
+                    'retain': {
+                        'daily': 30,
+                        'weekly': 6,
+                        'monthly': 12,
+                    },
                 },
             },
         },