From c9054a243afbade5f75e7820eae43b762827dd74 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Wed, 5 Jan 2022 09:53:18 +0100 Subject: [PATCH] backups: do backup rotation ourselves instead of relying on zfs-auto-snapshot --- bundles/backup-client/files/generate-backup | 4 + .../files/rotate-single-backup-client | 111 ++++++++++++++++++ bundles/backup-server/files/sudoers | 3 + bundles/backup-server/items.py | 32 +++++ bundles/backup-server/metadata.py | 22 ++-- nodes/home/nas.py | 5 - nodes/htz-hel/backup-kunsi.py | 5 + 7 files changed, 169 insertions(+), 13 deletions(-) create mode 100644 bundles/backup-server/files/rotate-single-backup-client create mode 100644 bundles/backup-server/files/sudoers diff --git a/bundles/backup-client/files/generate-backup b/bundles/backup-client/files/generate-backup index fa5ad60..ef648f4 100644 --- a/bundles/backup-client/files/generate-backup +++ b/bundles/backup-client/files/generate-backup @@ -3,8 +3,10 @@ statusfile=/var/tmp/backup.monitoring ssh_login="${username}@${server}" ssh_cmnd="ssh -o IdentityFile=/etc/backup.priv -o StrictHostKeyChecking=accept-new -p ${port}" +nodename="${node.name}" <%text> +[[ -n "$DEBUG" ]] && set -x NL=$'\n' if ! [[ -f /etc/backup.priv ]] @@ -59,6 +61,8 @@ do_backup() { } rsync_errors="" + +$ssh_cmnd $ssh_login "sudo /usr/local/bin/rotate-single-backup-client $nodename" % for path in sorted(paths): diff --git a/bundles/backup-server/files/rotate-single-backup-client b/bundles/backup-server/files/rotate-single-backup-client new file mode 100644 index 0000000..b031866 --- /dev/null +++ b/bundles/backup-server/files/rotate-single-backup-client @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +from json import load +from subprocess import check_call, check_output +from sys import argv +from time import time + +NODE = argv[1] + +NOW = int(time()) +DAY_SECONDS = 60 * 60 * 24 +INTERVALS = { + 'daily': DAY_SECONDS, + 'weekly': 7 * DAY_SECONDS, + 'monthly': 30 * DAY_SECONDS, +} + +buckets = {} + +def syslog(msg): + check_output(['logger', '-t', f'backup-{NODE}', msg]) + + +with open(f'/etc/backup-server/config.json', 'r') as f: + server_settings = load(f) + +with open(f'/etc/backup-server/clients/{NODE}', 'r') as f: + client_settings = load(f) + +# get all existing snapshots for NODE +for line in check_output('LC_ALL=C zfs list -H -t snapshot -o name', shell=True).splitlines(): + line = line.decode('UTF-8') + + if line.startswith('{}/{}@'.format(server_settings['zfs-base'], NODE)): + _, snapname = line.split('@', 1) + + if 'zfs-auto-snap' in snapname: + # migration from auto-snapshots, ignore + continue + + ts, bucket = snapname.split('-', 1) + buckets.setdefault(bucket, set()).add(int(ts)) + syslog(f'classified {line} as {bucket} from {ts}') + +# determine if we need to create a new snapshot +for bucket in INTERVALS.keys(): + snapshots = sorted(buckets.get(bucket, set())) + + if snapshots: + last_snap = snapshots[-1] + delta = NOW - last_snap + fresh_age = INTERVALS[bucket] - DAY_SECONDS + + if delta > fresh_age: + # last snapshot is older than what we want. create a new one. + check_call( + 'zfs snapshot {}/{}@{}-{}'.format( + server_settings['zfs-base'], + NODE, + NOW, + bucket, + ), + shell=True, + ) + buckets.setdefault(bucket, set()).add(NOW) + syslog(f'created new snapshot {NOW}-{bucket}') + else: + syslog(f'existing snapshot {last_snap}-{bucket} is fresh enough') + else: + check_call( + 'zfs snapshot {}/{}@{}-{}'.format( + server_settings['zfs-base'], + NODE, + NOW, + bucket, + ), + shell=True, + ) + buckets.setdefault(bucket, set()).add(NOW) + syslog(f'created initial snapshot {NOW}-{bucket}') + +# finally, see if we can delete any snapshots, because they are old enough +for bucket in INTERVALS.keys(): + snapshots = sorted(buckets.get(bucket, set())) + + if not snapshots: + syslog(f'something is wrong, there are no snapshots for {bucket}') + continue + + keep_age = INTERVALS[bucket] * client_settings[bucket] + + # oldest snapshots come first + for ts in snapshots[:-int(client_settings[bucket])]: + delta = NOW - ts + + if delta >= keep_age: + check_call( + 'zfs destroy {}/{}@{}-{}'.format( + server_settings['zfs-base'], + NODE, + ts, + bucket, + ), + shell=True, + ) + syslog(f'removing snapshot {ts}-{bucket}, age {delta}, keep_age {keep_age}') + else: + syslog(f'keeping snapshot {ts}-{bucket}, age not reached') + + for ts in snapshots[int(client_settings[bucket]):]: + syslog(f'keeping snapshot {ts}-{bucket}, count') diff --git a/bundles/backup-server/files/sudoers b/bundles/backup-server/files/sudoers new file mode 100644 index 0000000..a29e702 --- /dev/null +++ b/bundles/backup-server/files/sudoers @@ -0,0 +1,3 @@ +% for username, nodename in sorted(clients.items()): +${username} ALL=NOPASSWD:/usr/local/bin/rotate-single-backup-client ${nodename} +% endfor diff --git a/bundles/backup-server/items.py b/bundles/backup-server/items.py index bae34a7..0b43f62 100644 --- a/bundles/backup-server/items.py +++ b/bundles/backup-server/items.py @@ -1,17 +1,41 @@ repo.libs.tools.require_bundle(node, 'zfs') from os.path import join +from bundlewrap.metadata import metadata_to_json dataset = node.metadata.get('backup-server/zfs-base') +files = { + '/etc/backup-server/config.json': { + 'content': metadata_to_json({ + 'zfs-base': dataset, + }), + }, + '/usr/local/bin/rotate-single-backup-client': { + 'mode': '0755', + }, +} + +directories['/etc/backup-server/clients'] = { + 'purge': True, +} + +sudoers = {} + for nodename, config in node.metadata.get('backup-server/clients', {}).items(): with open(join(repo.path, 'data', 'backup', 'keys', f'{nodename}.pub'), 'r') as f: pubkey = f.read().strip() + sudoers[config['user']] = nodename + users[config['user']] = { 'home': f'/srv/backups/{nodename}', } + files[f'/etc/backup-server/clients/{nodename}'] = { + 'content': metadata_to_json(config['retain']), + } + files[f'/srv/backups/{nodename}/.ssh/authorized_keys'] = { 'content': pubkey, 'owner': config['user'], @@ -28,3 +52,11 @@ for nodename, config in node.metadata.get('backup-server/clients', {}).items(): f'zfs_dataset:{dataset}/{nodename}', }, } + +files['/etc/sudoers.d/backup-server'] = { + 'source': 'sudoers', + 'content_type': 'mako', + 'context': { + 'clients': sudoers, + }, +} diff --git a/bundles/backup-server/metadata.py b/bundles/backup-server/metadata.py index 990c39f..3a8b89e 100644 --- a/bundles/backup-server/metadata.py +++ b/bundles/backup-server/metadata.py @@ -16,6 +16,11 @@ defaults = { ) def get_my_clients(metadata): my_clients = {} + retain_defaults = { + 'daily': 14, + 'weekly': 4, + 'monthly': 6, + } for rnode in repo.nodes: if not rnode.has_bundle('backup-client') or rnode.metadata.get('backups/exclude_from_backups', False): @@ -26,6 +31,11 @@ def get_my_clients(metadata): my_clients[rnode.name] = { 'user': rnode.metadata.get('backup-client/user-name'), + 'retain': { + 'daily': rnode.metadata.get('backups/retain/daily', retain_defaults['daily']), + 'weekly': rnode.metadata.get('backups/retain/weekly', retain_defaults['weekly']), + 'monthly': rnode.metadata.get('backups/retain/monthly', retain_defaults['monthly']), + }, } return { @@ -97,15 +107,10 @@ def zfs_pool(metadata): @metadata_reactor.provides( 'zfs/datasets', - 'zfs/snapshots/retain_per_dataset', + 'zfs/snapshots/snapshot_never', ) def zfs_datasets_and_snapshots(metadata): zfs_datasets = {} - zfs_retains = {} - retain_defaults = { - 'weekly': 4, - 'monthly': 6, - } for client in metadata.get('backup-server/clients', {}).keys(): dataset = '{}/{}'.format(metadata.get('backup-server/zfs-base'), client) @@ -115,13 +120,14 @@ def zfs_datasets_and_snapshots(metadata): 'compression': 'on', } - zfs_retains[dataset] = retain_defaults.copy() return { 'zfs': { 'datasets': zfs_datasets, 'snapshots': { - 'retain_per_dataset': zfs_retains, + 'snapshot_never': { + metadata.get('backup-server/zfs-base'), + }, }, }, } diff --git a/nodes/home/nas.py b/nodes/home/nas.py index 2e11900..fa87ce1 100644 --- a/nodes/home/nas.py +++ b/nodes/home/nas.py @@ -50,11 +50,6 @@ nodes['home.nas'] = { 'exclude_from_backups': True, }, 'backup-server': { - 'clients': { - 'kunsi-t470': { - 'user': 'kunsi-t470', - }, - }, 'my_hostname': 'franzi-home.kunbox.net', 'my_ssh_port': 2022, 'zfs-base': 'storage/backups', diff --git a/nodes/htz-hel/backup-kunsi.py b/nodes/htz-hel/backup-kunsi.py index 1fe796d..5c8dad8 100644 --- a/nodes/htz-hel/backup-kunsi.py +++ b/nodes/htz-hel/backup-kunsi.py @@ -35,6 +35,11 @@ nodes['htz-hel.backup-kunsi'] = { 'clients': { 'kunsi-t470': { 'user': 'kunsi-t470', + 'retain': { + 'daily': 30, + 'weekly': 6, + 'monthly': 12, + }, }, }, },