bundles/backup-{client,server}: introduce

bundles/backup-client/files/generate-backup

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+statusfile=/var/tmp/backup.monitoring
+ssh_login="${username}@${server}"
+
+if ! [[ -f /etc/backup.priv ]]
+then
+    echo "abort_no_key" > "$statusfile"
+    exit 1
+fi
+
+rsync_errors=""
+% for path in sorted(paths):
+rsync -zaAP --numeric-ids --delete --relative \
+    --rsync-path="/usr/bin/rsync --fake-super" \
+    -e "ssh -o IdentityFile=/etc/backup.priv -o StrictHostKeyChecking=accept-new" \
+    "${path}" "$ssh_login":backups/
+
+exitcode=$?
+if (( exitcode != 0 )) && (( exitcode != 24 ))
+then
+    rsync_errors+=" $ret"
+fi
+% endfor
+
+if [[ -n "$rsync_errors" ]]
+then
+    echo "rsync_error$rsync_errors" > "$statusfile"
+fi
+
+echo "ok" > "$statusfile"

bundles/backup-client/items.py

@@ -0,0 +1,25 @@
+from os.path import join
+
+if node.metadata['backups'].get('exclude_from_backups', False):
+    files = {
+        '/etc/backup.priv': {
+            'delete': True,
+        },
+    }
+else:
+    files = {
+        '/usr/local/bin/generate-backup': {
+            'content_type': 'mako',
+            'context': {
+                'username': node.metadata['backup-client']['user-name'],
+                'server': node.metadata['backup-client']['server'],
+                'paths': node.metadata.get('backups', {}).get('paths', {}),
+            },
+            'mode': '0700',
+        },
+        '/etc/backup.priv': {
+            'content': repo.vault.decrypt_file(join('backup', 'keys', f'{node.name}.key.vault')),
+            'mode': '0400',
+        },
+    }
+

bundles/backup-client/metadata.py

@@ -0,0 +1,21 @@
+from hashlib import md5
+
+defaults = {
+    'backup-client': {
+        # unix user names cannot be longer than 32 characters.
+        # bundlewrap raises an error if the name is longer than 30 chars.
+        'user-name': 'c-' + md5(node.name.encode('UTF-8')).hexdigest()[:28],
+    },
+}
+
+
+@metadata_reactor
+def cron(metadata):
+    if metadata.get('backups/exclude_from_backups', False):
+        return {}
+
+    return {
+        'cron': {
+            'backup': '{} 1 * * *    root    /usr/local/bin/generate-backup',
+        },
+    }

bundles/backup-server/items.py

@@ -0,0 +1,28 @@
+assert node.has_bundle('zfs')
+
+from os.path import join
+
+for nodename, config in node.metadata.get('backup-server', {}).get('clients', {}).items():
+    with open(join(repo.path, 'data', 'backup', 'keys', f'{nodename}.pub'), 'r') as f:
+        pubkey = f.read().strip()
+
+    users[config['user']] = {
+        'home': f'/srv/backups/{nodename}',
+    }
+
+    files[f'/srv/backups/{nodename}/.ssh/authorized_keys'] = {
+        'content': pubkey,
+        'owner': config['user'],
+        'mode': '0400',
+        'needs': {
+            'bundle:zfs',
+        },
+    }
+
+    directories[f'/srv/backups/{nodename}/backups'] = {
+        'owner': config['user'],
+        'mode': '0700',
+        'needs': {
+            'bundle:zfs',
+        },
+    }

bundles/backup-server/metadata.py

@@ -0,0 +1,45 @@
+@metadata_reactor
+def get_my_clients(metadata):
+    my_clients = {}
+
+    for rnode in repo.nodes:
+        if rnode.metadata.get('backups/exclude_from_backups', False):
+            continue
+
+        my_clients[rnode.name] = {
+            'user': rnode.metadata.get('backup-client/user-name'),
+        }
+
+    return {
+        'backup-server': {
+            'clients': my_clients,
+        },
+    }
+
+
+@metadata_reactor
+def zfs(metadata):
+    zfs_datasets = {}
+    zfs_retains = {}
+    retain_defaults = {
+        'weekly': 4,
+        'monthly': 6,
+    }
+
+    for client in metadata.get('backup-server/clients', {}).keys():
+        dataset = '{}/{}'.format(metadata.get('backup-server/zfs-base'), client)
+
+        zfs_datasets[dataset] = {
+            'mountpoint': '/srv/backups/{}'.format(client),
+        }
+
+        zfs_retains[dataset] = retain_defaults.copy()
+
+    return {
+        'zfs': {
+            'datasets': zfs_datasets,
+            'snapshots': {
+                'retain_per_dataset': zfs_retains,
+            },
+        },
+    }

data/backup/keys/gce.bind01.key.vault

@@ -0,0 +1 @@
+encrypt$gAAAAABfrlWfFm8-vDqO9So0Ru3QCA_kWvO2bbIcYtnq3VnJfq0QxPKW4TTuUbS8gloq19TbQRTZeZ0-22H-CdIeiXv_SGtKzK7ijbV3pUfNppy5I9c1Kcn--6YnLEBRx9DxhOh3n3i3gxyF8dA9izjp_-XS3XjjPcdw6WAp1z55a6p6ggTDEyXn1MGEUl8405ri8kpe9AtPIBZV7GND8GmH8jG8jrMJGTta_TJlrW_FcsYqcEKf5f1N1ShOCWCxUijlTwLVZzufZCR3-IJpcdKR8L2ifTggT04meHRzd_4HkC3X-3wdfqnoNCo7ln63SeerseN0Gnz_Psk0L9CnwQwWlfTbCMVdn2oiRUc8wLZ06R-GVhdIs9C4jGnQJZeStOFYYtHWgqZcToNx_Bq5zIK4aMa5vZ8cmKgCDWBMfjcaWJ8SKK8_zRZwRbsPOzuzSfvGoAmcnhQbnDmmhtSaka4POk-aH-8ZV_1dNq0JK5g7xcC6vUb1GSvfFPqXhx9ypo48NueHC9seJt7Pp05hP91z8yBT9-CHtMH91G4iBkyJf-DfG65YfDFVmXTU4ikV5UV6leXFkzmIzGshKAwuDuRVWA5tXEHAyoluTaX2nZXziz_wNszj2Fc=

data/backup/keys/gce.bind01.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF4wrVPICvYitHaR0Qp6K8LzlPaHothuw0BI3XGiyAmN kunsi@kunsi-t470

data/backup/keys/home.kodi-wohnzimmer.key.vault

@@ -0,0 +1 @@
+encrypt$gAAAAABfrlWfJTYZVIlllCPefCyzG16p3-JLyr1vST3xWvkc5rB9jvCNw-7LwP7CSh62YTchvyJBk5NfrDCnZnYhW44rn4I2YWr-LfHkVNIsq_b1Kv7rL_xvgcHt1iww_0Fa0nUmK5gGbbedz0uJtTO_9IS8P7KJUWziW3Ugsajt0NKIAB__-M7d461E6coLKmbkD9EnTGkXGp14U1vA0oyR8xsfHasWtQ8ntNu3it4_SFmu_xMbeEXOV1RZACKkCr-nS7ctjQ4LNgIIdfLWs-KKM1cmCjwDQqPWRIoPD5YJJ8EtBxvUyNc0KT8ySMS7m2TNfw158U2QdO4KQUdbuwPTpDWuhOMRp5nzliEkiw2QHhKbZGbHrliw1AD9naQWUh-R1XtMx3gKRp-vser4RFQk83bhcL63j7dSjzKHpANa3HB0f2GEoek9VOwZIHpXWu1OkJNMVk5a_F8f75Iggmj5xiz_O_nRRhYRA3MzXgfV_QKTvPHEKFvkoh_-esb33qgiJ8tL-5uJ2ADWFbBqy-KoUJDKFeoyDNlJKwFpgFq0Kbd5eJbbVp95D9LCCkdiwQ34_SopMqbVBfauHdloygsgs35ifAvEW1VDyHtz-cpmRDYc0jV-iyI=

data/backup/keys/home.kodi-wohnzimmer.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE/WEgalbJUsr2q3DurqR7NkY9RXnuMs7BmmBgVmW3tj kunsi@kunsi-t470

data/backup/keys/home.octoprint-vielschichtigkeit.key.vault

@@ -0,0 +1 @@
+encrypt$gAAAAABfrlWgqqfc6tymm4tFnl4qy6rodRU7ZCMsPXw-bLOBoPdxxPdQEVVvWWK6WhqidBtGRGEvyp32W3rltA_lTZFhSEy5y-xOHK7waviQi2wahK4B3zBYPc-nKOREzbSKqOaaNlpOsReAfyZgizeKb_XGump--sOwLn120k9ImMGmVhhQSx0BdJpi2Z23aqV6TvRgDM2utCR2aRFXyFbFG_TR8exI9tQLg80qaotXv9O5I2pnIyTyanXEm4pZBmN88kGSW-ZPTVO2SpWjfGO46XtPirsFAp7pya-0O8EeXApEGjtQiVUw_JrlQmMTJ14j8AV4m_lNsiu_6bKPawaNJCcSfOF9C_49LMj-0mupyss2Py3qtF-KTxU0TvODg2DnLIMlcxtv_zheYFeY90nPBpQ3Dh8L2qAOd_eDu4gFvQLQvWQyB6aAlChC9ufTrhDFNyNI3Am5oWh32iFcv8Ie7UNtIB0Jc2bHfApJl8LJhizpObLgHtuxK127m2D5jEXRYwjLYDGGDwyL-qfKpxQoKaBPBP8JNxT0LbtsecveAyLIknyqtX3fxvZpon1DbJ0UTwvlUeoRcmOThmtx_hlGS7As12Ds60EnDbuhMddFGnZyo4GObqA=

data/backup/keys/home.octoprint-vielschichtigkeit.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII6c5QTi50obr8Eh3pCCy+y8E4HXb/5YwRIVD1WZneO9 kunsi@kunsi-t470

data/backup/keys/htz-cloud.luther.key.vault

@@ -0,0 +1 @@
+encrypt$gAAAAABfrlWgO_QYaKznjoFXGPw4PTjv92niiHE_61_Tp7dOnKiqWHJc06MOxmFdZaf9i8wLH1H0R8MNgt5gGuGDk6frQh1cz_EMyQh9Vo9iBbHI6q5OXYHmTX3mEeeoudhGUNaXwaea0SaErcql_jJDRGrbvxGJH2-wmiJaeZ5oyIkedp6F7_Y1SuRw-c2YG8hEBtjALlLhz3bL25_2V8hzrKZ1OtK1TyoWvdbA7yo2PdE1RfxfzJG5MMCNBTb9ngVvCXz6bZuq6EQidBONOvR7mWncMKeuB_pd87DvxIvahhhI4roHp8H7rbH_6eqRQLKiNBvmJxmqtNlb_wJPFtzTGZUNNOzZLuddJsBZggx6R3CsDigQVK6MKBi6qGlZTsn3nBNhEtX9jmRWU2Xx9IaVNuc9a3qlkMN2qTXg4B6ijMYa6emIva5Y-2ByK4dBwZd9nQSqk_QNcaLA_EVGBah1yIXCTRWqF5A3VrIlPnpVxTZZoLqnyjtWRh23L0-K47V3NuvXS7R1sZGGAapadVenRUH-iRs8493v07aJlH2DHNSuINEw15sPWALWpOiGJ6UdVsZ5FYtXcCTBX87PfmCp6OKChmkRqVXS_j3LTH48HzknZvYP-YY=

data/backup/keys/htz-cloud.luther.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO6bxLAHdTe9gQIwFhFWRBKq9BSMeds3lyDK9iud4kpL kunsi@kunsi-t470

data/backup/keys/htz-cloud.pirmasens.key.vault

@@ -0,0 +1 @@
+encrypt$gAAAAABfrlWgKPszezQgjWLwmoRjEF1k_AedpCj3sYAIrzKDfdSEVuYjj_8tbzflMRuCax9FPCwffQdy2Y-79SuIoHgjVDfrV3jRPTegFhSWudIcbSMrlc-3Ypob_BD2pGyz8XbdYXdaPTz8R7wTLpYgqxWPkBAnHj_AsJeXO3qtbFkuMwqiUST9fFDbNfmUxunbmjk-WYYr6pBNy91dydWVR7Th_XxJNPtTucK1qRJgzaA5aA1UsiMXoc07jkDMVJAvs7Qy1ynofz0hh8DEb8SHo2htPQyKEWljU6vdYQ4PgYIWdP746m4fuDvTVKU2EkMmMxTtBF-lSHpg_AxVt3krB3Geo9MHTzodBmKkwHRRD49ZjY6E1QXQqjsrJ9T8eudokyaLuOZz9AwzgBZfWKNMh1D8BqaJVOoGgK0S1nLvRiONqX0sLq6XmQqEHalR1puMwugOBDNmrt2dBH283Jr9p_zbxe8fnNK0hgOeVJCe9tAr68Cn_dcWJgLsL-KUnhORXLjZXP44k8-k8ovj15cbW8fUobf8VyK6XqyUt119hXMCrjDed0RbjRBYjwm0A9Zv_DjsH6dFVKfyC1mu1nWTeOK0km3H6CnzWEPPhD3bA8YvXAoQfFY=

data/backup/keys/htz-cloud.pirmasens.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIILD1lwc1lNzVWQDxBxs9//5OUioFGOA7qGZjDykSIRa kunsi@kunsi-t470

data/backup/keys/htz-cloud.sewfile.key.vault

@@ -0,0 +1 @@
+encrypt$gAAAAABfrlWhKNaNm-FvjJIB97rK-RoXRnLHHZ63k8Y-beVmSwuYnXYEjZLSD5yrvE3TtkodVikztjx4Cuck6lTECQR2MwSSlZ76L_uJOCmwyCKbDmjCRAfJsr8ni0WIIDa6GBWeqy1KDqZsaqEJwaH_zF6Ps-JHsUB7NCpqcDLGCTGOQLUrgH_Qzyi4Jme0LAnH2DeY7bSyzOGdLezwGUd2nhv7eKet0NeJwWWTnN3HSd6KGJybZLR2I2FsqiNutqGNLnJgeuTHHsUVUxroJepmE4bC0sK9kd_yWWQDNTVc2MRsJA2XkfgfeWzusmjQyho-9iOucX66E2DnSLOSVfvV1ZQ5iTUx_iYQstDs_V48-Za8OEh0wtMvWJlw4fIZvT2CTbFMjv-Z3ID9O8zu-MOZTKJlvGgQzaCJlvI1xMAd9UcfwKvoejGrZNZdHdadyjB_hbNZk1e2KLYEXsnSzIyTU544K4yWOaKpA70di_rQHfrgUosdRn-CtZkWJolUzKVILNmaaI1gZYI3jr3SYqWWfHjVIlQIt8z5qmR527bvEKhJhDIkJ-RnvTNHeXx9Kqw3VPIWxHaYSS1Fv-M47e2rXsM5eDfPWRsQvYOiQO3g0EbMQhjho8o=

data/backup/keys/htz-cloud.sewfile.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN9fWj0+jwIxDSvwyP6sOiFeI6huU/kQ+N5nIBAj+2eT kunsi@kunsi-t470

data/backup/keys/htz.ex42-1048908.key.vault

@@ -0,0 +1 @@
+encrypt$gAAAAABfrlWhxrcCNO5KHOsPsxD5ZymUgHG3my4qIJxfr3nJQyBVMRjFjxx2akQubpWWG9UC8xKthSvADXUZxjcGTAvqcELX60W2cdvJ90bY8RGHQ-Dua6-PsThurLVJeWZSmxo3vCneTj0lYEdAwK1aiZsRvxLjWhkyRELXvgPhC3GeNnS0SeC1AHGuRJVnC3ZMbRfSps1_LBxRvIES_acDJSfsRpcLg0oPmdpkDV9wdB0ZBuimngUYCl6jZ-syofI9yRU9q9lpNtZbahKAIBKNHeFzkgC6I5oi8e2-mZqnh_BSLomvRXPdZRvlHQWsSPNX2_25IlZiXyXrBIsN5rXAAwl16PNZjuG703WWiV2RxifGRux7cbJVE-LREBKCADgLduOZPe2voXo3jRq8v4NZfTtk8CKSm4QxS6Q7fRb7_0dAWZadd_dap8HigCUkr_5l-CotSKhiBNTAwyClLSMpDW1oUAaeLgM2YLF7V8TlWAwtxKi1lqmDchoWk221CQ97njVfhUOCNrdiGXOtFeiB-JRsXK8eAgthDzrR2F78s2w9NGZf0SdRxxNNil1f7ikDZfXbsdagaP_HZvjmq8oeTcwibcjsGl7HDVw0xV_7SvOJ3iDhLdQ=

data/backup/keys/htz.ex42-1048908.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFn5SDyIV+GtSqSDvKPsfkxRgmUKdu8eOOyEWUo5ZtJl kunsi@kunsi-t470

data/backup/keys/ovh.icinga2.key.vault

@@ -0,0 +1 @@
+encrypt$gAAAAABfrlWizt5Lp7t_CRYxVrlPmXFKSKKpNzWIKaM8X6l-7eUD-vDMN9G1tj6nucjQ3sHOll7WXM367HIcqIlOQfUDM2Qat3_4MstsKnEHUvoPh1xyjrui74ZQvLrdedYjtQ_YlsLJnoHkqLThJQ1D3pazifMYouF0CO9MMz4pVxTNiSGYPzVaixUN_LcMm9-u0vWaVh3UqDa3mLxufI36C5lKR6p7jEhB3vTpxtahquDxSjMmCYQv1AiEbPfoh0-8mFlZ5QZ9ZPxno5q_5SnZViv4jDuLLcW1VeK4ocOP2vjh8QuN2uc2-AuQRzykOAHBjprKcjgrp_M9sejy4W5I40wgpMliPtgc8z_tdBhU5uLwKR50l0xjCW9oR7mPQIzrs8Y6b-KPO3Hy9v2iCKYT0XOLiY9fCF_hmIk-hN7ekS2zUlU4TzRC9nDD-YBX28mqXU7n1-0QciDjVkpmcxvBFzBbNt5XXJJ7jLdfj6fx2keErnmSLWAnMv-ztJX93sfxYfnQejqhYIc_H81xF4Nm3P3V7lf8PeR_FsfqvQujR9ECBWQ6vo8-5KnAiYnMSyPapirY8b4FPUjKhEgel5goSZ4DhbmUBKPVecByUTYSBAXP76IyXRE=

data/backup/keys/ovh.icinga2.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE5hLAqBz7Vm6oVv+oye5hQCsRI3cPA9q5B8KCWYCYUw kunsi@kunsi-t470

hooks/test_backup_metadata.py

@@ -0,0 +1,3 @@
+def test_node(repo, node, **kwargs):
+    if not node.metadata.get('backups', {}).get('exclude_from_backups', False):
+        assert len(node.metadata.get('backups', {}).get('paths', set())) > 0, f'{node.name} has backups configured, but no backup paths defined!'

hooks/test_nodename_follows_convention.py

@@ -0,0 +1,2 @@
+def test_node(repo, node, **kwargs):
+    assert node.name == node.name.lower(), f'{node.name} must be all lowercase!'