backups: do backup rotation ourselves instead of relying on zfs-auto-snapshot

This commit is contained in:
Franzi 2022-01-05 09:53:18 +01:00
parent b6eb12ed90
commit c9054a243a
Signed by: kunsi
GPG key ID: 12E3D2136B818350
7 changed files with 169 additions and 13 deletions

View file

@ -3,8 +3,10 @@
statusfile=/var/tmp/backup.monitoring statusfile=/var/tmp/backup.monitoring
ssh_login="${username}@${server}" ssh_login="${username}@${server}"
ssh_cmnd="ssh -o IdentityFile=/etc/backup.priv -o StrictHostKeyChecking=accept-new -p ${port}" ssh_cmnd="ssh -o IdentityFile=/etc/backup.priv -o StrictHostKeyChecking=accept-new -p ${port}"
nodename="${node.name}"
<%text> <%text>
[[ -n "$DEBUG" ]] && set -x
NL=$'\n' NL=$'\n'
if ! [[ -f /etc/backup.priv ]] if ! [[ -f /etc/backup.priv ]]
@ -59,6 +61,8 @@ do_backup() {
} }
rsync_errors="" rsync_errors=""
$ssh_cmnd $ssh_login "sudo /usr/local/bin/rotate-single-backup-client $nodename"
</%text> </%text>
% for path in sorted(paths): % for path in sorted(paths):

View file

@ -0,0 +1,111 @@
#!/usr/bin/env python3
from json import load
from subprocess import check_call, check_output
from sys import argv
from time import time
NODE = argv[1]
NOW = int(time())
DAY_SECONDS = 60 * 60 * 24
INTERVALS = {
'daily': DAY_SECONDS,
'weekly': 7 * DAY_SECONDS,
'monthly': 30 * DAY_SECONDS,
}
buckets = {}
def syslog(msg):
check_output(['logger', '-t', f'backup-{NODE}', msg])
with open(f'/etc/backup-server/config.json', 'r') as f:
server_settings = load(f)
with open(f'/etc/backup-server/clients/{NODE}', 'r') as f:
client_settings = load(f)
# get all existing snapshots for NODE
for line in check_output('LC_ALL=C zfs list -H -t snapshot -o name', shell=True).splitlines():
line = line.decode('UTF-8')
if line.startswith('{}/{}@'.format(server_settings['zfs-base'], NODE)):
_, snapname = line.split('@', 1)
if 'zfs-auto-snap' in snapname:
# migration from auto-snapshots, ignore
continue
ts, bucket = snapname.split('-', 1)
buckets.setdefault(bucket, set()).add(int(ts))
syslog(f'classified {line} as {bucket} from {ts}')
# determine if we need to create a new snapshot
for bucket in INTERVALS.keys():
snapshots = sorted(buckets.get(bucket, set()))
if snapshots:
last_snap = snapshots[-1]
delta = NOW - last_snap
fresh_age = INTERVALS[bucket] - DAY_SECONDS
if delta > fresh_age:
# last snapshot is older than what we want. create a new one.
check_call(
'zfs snapshot {}/{}@{}-{}'.format(
server_settings['zfs-base'],
NODE,
NOW,
bucket,
),
shell=True,
)
buckets.setdefault(bucket, set()).add(NOW)
syslog(f'created new snapshot {NOW}-{bucket}')
else:
syslog(f'existing snapshot {last_snap}-{bucket} is fresh enough')
else:
check_call(
'zfs snapshot {}/{}@{}-{}'.format(
server_settings['zfs-base'],
NODE,
NOW,
bucket,
),
shell=True,
)
buckets.setdefault(bucket, set()).add(NOW)
syslog(f'created initial snapshot {NOW}-{bucket}')
# finally, see if we can delete any snapshots, because they are old enough
for bucket in INTERVALS.keys():
snapshots = sorted(buckets.get(bucket, set()))
if not snapshots:
syslog(f'something is wrong, there are no snapshots for {bucket}')
continue
keep_age = INTERVALS[bucket] * client_settings[bucket]
# oldest snapshots come first
for ts in snapshots[:-int(client_settings[bucket])]:
delta = NOW - ts
if delta >= keep_age:
check_call(
'zfs destroy {}/{}@{}-{}'.format(
server_settings['zfs-base'],
NODE,
ts,
bucket,
),
shell=True,
)
syslog(f'removing snapshot {ts}-{bucket}, age {delta}, keep_age {keep_age}')
else:
syslog(f'keeping snapshot {ts}-{bucket}, age not reached')
for ts in snapshots[int(client_settings[bucket]):]:
syslog(f'keeping snapshot {ts}-{bucket}, count')

View file

@ -0,0 +1,3 @@
% for username, nodename in sorted(clients.items()):
${username} ALL=NOPASSWD:/usr/local/bin/rotate-single-backup-client ${nodename}
% endfor

View file

@ -1,17 +1,41 @@
repo.libs.tools.require_bundle(node, 'zfs') repo.libs.tools.require_bundle(node, 'zfs')
from os.path import join from os.path import join
from bundlewrap.metadata import metadata_to_json
dataset = node.metadata.get('backup-server/zfs-base') dataset = node.metadata.get('backup-server/zfs-base')
files = {
'/etc/backup-server/config.json': {
'content': metadata_to_json({
'zfs-base': dataset,
}),
},
'/usr/local/bin/rotate-single-backup-client': {
'mode': '0755',
},
}
directories['/etc/backup-server/clients'] = {
'purge': True,
}
sudoers = {}
for nodename, config in node.metadata.get('backup-server/clients', {}).items(): for nodename, config in node.metadata.get('backup-server/clients', {}).items():
with open(join(repo.path, 'data', 'backup', 'keys', f'{nodename}.pub'), 'r') as f: with open(join(repo.path, 'data', 'backup', 'keys', f'{nodename}.pub'), 'r') as f:
pubkey = f.read().strip() pubkey = f.read().strip()
sudoers[config['user']] = nodename
users[config['user']] = { users[config['user']] = {
'home': f'/srv/backups/{nodename}', 'home': f'/srv/backups/{nodename}',
} }
files[f'/etc/backup-server/clients/{nodename}'] = {
'content': metadata_to_json(config['retain']),
}
files[f'/srv/backups/{nodename}/.ssh/authorized_keys'] = { files[f'/srv/backups/{nodename}/.ssh/authorized_keys'] = {
'content': pubkey, 'content': pubkey,
'owner': config['user'], 'owner': config['user'],
@ -28,3 +52,11 @@ for nodename, config in node.metadata.get('backup-server/clients', {}).items():
f'zfs_dataset:{dataset}/{nodename}', f'zfs_dataset:{dataset}/{nodename}',
}, },
} }
files['/etc/sudoers.d/backup-server'] = {
'source': 'sudoers',
'content_type': 'mako',
'context': {
'clients': sudoers,
},
}

View file

@ -16,6 +16,11 @@ defaults = {
) )
def get_my_clients(metadata): def get_my_clients(metadata):
my_clients = {} my_clients = {}
retain_defaults = {
'daily': 14,
'weekly': 4,
'monthly': 6,
}
for rnode in repo.nodes: for rnode in repo.nodes:
if not rnode.has_bundle('backup-client') or rnode.metadata.get('backups/exclude_from_backups', False): if not rnode.has_bundle('backup-client') or rnode.metadata.get('backups/exclude_from_backups', False):
@ -26,6 +31,11 @@ def get_my_clients(metadata):
my_clients[rnode.name] = { my_clients[rnode.name] = {
'user': rnode.metadata.get('backup-client/user-name'), 'user': rnode.metadata.get('backup-client/user-name'),
'retain': {
'daily': rnode.metadata.get('backups/retain/daily', retain_defaults['daily']),
'weekly': rnode.metadata.get('backups/retain/weekly', retain_defaults['weekly']),
'monthly': rnode.metadata.get('backups/retain/monthly', retain_defaults['monthly']),
},
} }
return { return {
@ -97,15 +107,10 @@ def zfs_pool(metadata):
@metadata_reactor.provides( @metadata_reactor.provides(
'zfs/datasets', 'zfs/datasets',
'zfs/snapshots/retain_per_dataset', 'zfs/snapshots/snapshot_never',
) )
def zfs_datasets_and_snapshots(metadata): def zfs_datasets_and_snapshots(metadata):
zfs_datasets = {} zfs_datasets = {}
zfs_retains = {}
retain_defaults = {
'weekly': 4,
'monthly': 6,
}
for client in metadata.get('backup-server/clients', {}).keys(): for client in metadata.get('backup-server/clients', {}).keys():
dataset = '{}/{}'.format(metadata.get('backup-server/zfs-base'), client) dataset = '{}/{}'.format(metadata.get('backup-server/zfs-base'), client)
@ -115,13 +120,14 @@ def zfs_datasets_and_snapshots(metadata):
'compression': 'on', 'compression': 'on',
} }
zfs_retains[dataset] = retain_defaults.copy()
return { return {
'zfs': { 'zfs': {
'datasets': zfs_datasets, 'datasets': zfs_datasets,
'snapshots': { 'snapshots': {
'retain_per_dataset': zfs_retains, 'snapshot_never': {
metadata.get('backup-server/zfs-base'),
},
}, },
}, },
} }

View file

@ -50,11 +50,6 @@ nodes['home.nas'] = {
'exclude_from_backups': True, 'exclude_from_backups': True,
}, },
'backup-server': { 'backup-server': {
'clients': {
'kunsi-t470': {
'user': 'kunsi-t470',
},
},
'my_hostname': 'franzi-home.kunbox.net', 'my_hostname': 'franzi-home.kunbox.net',
'my_ssh_port': 2022, 'my_ssh_port': 2022,
'zfs-base': 'storage/backups', 'zfs-base': 'storage/backups',

View file

@ -35,6 +35,11 @@ nodes['htz-hel.backup-kunsi'] = {
'clients': { 'clients': {
'kunsi-t470': { 'kunsi-t470': {
'user': 'kunsi-t470', 'user': 'kunsi-t470',
'retain': {
'daily': 30,
'weekly': 6,
'monthly': 12,
},
}, },
}, },
}, },