From b6cedd2bce6c49fa24b76a59621353c0b91c52f1 Mon Sep 17 00:00:00 2001 From: Sophie Schiller Date: Wed, 16 Apr 2025 17:36:45 +0200 Subject: [PATCH 01/27] navidrome: initial bundle --- bundles/navidrome/files/navidrome.service | 44 +++++++++++++ bundles/navidrome/items.py | 45 +++++++++++++ bundles/navidrome/metadata.py | 77 +++++++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 bundles/navidrome/files/navidrome.service create mode 100644 bundles/navidrome/items.py create mode 100644 bundles/navidrome/metadata.py diff --git a/bundles/navidrome/files/navidrome.service b/bundles/navidrome/files/navidrome.service new file mode 100644 index 0000000..90a1816 --- /dev/null +++ b/bundles/navidrome/files/navidrome.service @@ -0,0 +1,44 @@ +[Unit] +Description=Navidrome Music Server and Streamer compatible with Subsonic/Airsonic +After=remote-fs.target network.target +AssertPathExists=/var/opt/navidrome + +[Install] +WantedBy=multi-user.target + +[Service] +User=navidrome +Group=navidrome +Type=simple +ExecStart=/opt/navidrome/navidrome --configfile "/opt/navidrome/config.toml" +WorkingDirectory=/var/opt/navidrome +TimeoutStopSec=20 +KillMode=process +Restart=on-failure + +# See https://www.freedesktop.org/software/systemd/man/systemd.exec.html +DevicePolicy=closed +NoNewPrivileges=yes +PrivateTmp=yes +PrivateUsers=yes +ProtectControlGroups=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +RestrictNamespaces=yes +RestrictRealtime=yes +SystemCallFilter=~@clock @debug @module @mount @obsolete @reboot @setuid @swap +ReadWritePaths=/var/opt/navidrome + +# You can uncomment the following line if you're not using the jukebox This +# will prevent navidrome from accessing any real (physical) devices +PrivateDevices=yes + +# You can change the following line to `strict` instead of `full` if you don't +# want navidrome to be able to write anything on your filesystem outside of +# /var/lib/navidrome. +ProtectSystem=full + +# You can uncomment the following line if you don't have any media in /home/*. +# This will prevent navidrome from ever reading/writing anything there. +ProtectHome=true diff --git a/bundles/navidrome/items.py b/bundles/navidrome/items.py new file mode 100644 index 0000000..84e4e29 --- /dev/null +++ b/bundles/navidrome/items.py @@ -0,0 +1,45 @@ +users = { + 'navidrome': { + 'home': '/opt/navidrome', + }, +} + +directories = { + '/opt/navidrome': {}, + '/var/opt/navidrome': { + 'owner': 'navidrome', + }, +} +svc_systemd = { + 'navidrome': { + 'needs': { + 'file:/etc/systemd/system/navidrome.service', + 'file:/opt/navidrome/config.toml', + 'file:/opt/navidrome/navidrome', + }, + }, +} + +files = { + '/opt/navidrome/config.toml': { + 'content': repo.libs.faults.dict_as_toml(node.metadata.get('navidrome/config')), + 'triggers': { + 'svc_systemd:navidrome:restart', + }, + }, + '/etc/systemd/system/navidrome.service': { + 'triggers': { + 'action:systemd-reload', + 'svc_systemd:navidrome:restart', + }, + }, + '/opt/navidrome/navidrome': { + 'content_hash': node.metadata.get('navidrome/sha1', None), + 'content_type': 'download', + 'mode': '0755', + 'source': f'https://github.com/navidrome/navidrome/releases/download/v{node.metadata.get('navidrome/version')}/navidrome_{node.metadata.get('navidrome/version')}_linux_amd64.tar.gz', + 'triggers': { + 'svc_systemd:navidrome:restart', + }, + }, +} diff --git a/bundles/navidrome/metadata.py b/bundles/navidrome/metadata.py new file mode 100644 index 0000000..4eeebb5 --- /dev/null +++ b/bundles/navidrome/metadata.py @@ -0,0 +1,77 @@ +defaults = { + 'apt': { + 'packages': { + 'ffmpeg': {}, + 'mpv': {}, + + }, + }, + 'navidrome': { + 'config': { + 'DataFolder': '/var/opt/navidrome', + 'Address': '127.0.0.1', + 'MusicFolder': '/mnt/music', + 'EnableExternalServices': False, + 'LastFM.Enabled': False, + 'ListenBrainz.Enabled': False, + 'PasswordEncryptionKey': repo.vault.password_for('{} encryption navidrome'.format(node.name)), + 'Scanner.Schedule': '@every 72h', + 'Port': 4533, + }, + }, + 'zfs': { + 'datasets': { + 'tank/navidrome': {}, + 'tank/navidrome/install': { + 'mountpoint': '/opt/navidrome', + 'needed_by': { + 'directory:/opt/navidrome', + }, + }, + 'tank/navidrome/home': { + 'mountpoint': '/var/opt/navidrome', + 'needed_by': { + 'directory:/var/opt/navidrome', + }, + }, + }, + }, +} + + +@metadata_reactor.provides( + 'navidrome/config/baseurl', +) +def baseurl(metadata): + return { + 'navidrome': { + 'config': { + 'BaseUrl': f'https://{metadata.get('navidrome/domain')}', + }, + }, + } + + +@metadata_reactor.provides( + 'nginx/vhosts/navidrome', +) +def nginx(metadata): + if not node.has_bundle('nginx'): + raise DoNotRunAgain + + return { + 'nginx': { + 'vhosts': { + 'navidrome': { + 'domain': metadata.get('navidrome/domain'), + 'locations': { + '/': { + 'target': f'http://127.0.0.1:{metadata.get('navidrome/config/port')}', + }, + }, + 'website_check_path': '/user/login', + 'website_check_string': 'Sign in', + }, + }, + }, + } From 3fa4e780fd1b3c2804bbc87ef2c9034038de23e1 Mon Sep 17 00:00:00 2001 From: Sophie Schiller Date: Wed, 16 Apr 2025 17:37:11 +0200 Subject: [PATCH 02/27] sophie.navidrome: initial nodefile --- nodes/sophie/sophie.navidrome.toml | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 nodes/sophie/sophie.navidrome.toml diff --git a/nodes/sophie/sophie.navidrome.toml b/nodes/sophie/sophie.navidrome.toml new file mode 100644 index 0000000..93df587 --- /dev/null +++ b/nodes/sophie/sophie.navidrome.toml @@ -0,0 +1,44 @@ +hostname = "172.19.164.5" +bundles = [ + 'navidrome', + 'nginx', + 'nfs-client', +] +groups = [ + "debian-bookworm", +] + +[metadata.interfaces.enp7s0] +ips = [ + "172.19.164.5/24", +] +gateway4 = "172.19.164.1" +ipv6_accept_ra = true + +[metadata.vm] +cpu = 2 +ram = 4 + +[metadata.navidrome] +domain = 'navidrome.home.sophies-kitchen.eu' +version = '0.55.2' +sha1 = '97661ab81438a521d2b149a417a7282a9bf4c2dc79c5148c4fa32a102e416940' + +[metadata.navidrome.config] +MusicFolder = "/mnt/media/Musik" + +[metadata.nfs-client.mounts.media] +mountpoint = '/mnt/media' +serverpath = '172.19.164.2:/srv/nas' +mount_options =[ + 'retry=0', + 'ro', +] + +[metadata.nginx] +restrict-to = [ + '172.19.164.0/22', +] + +[metadata.nginx.vhosts.navidrome] +ssl = '_.home.sophies-kitchen.eu' From 53c658c5d8dfc2b01549a38f753820cc95e6f0ff Mon Sep 17 00:00:00 2001 From: Sophie Schiller Date: Wed, 16 Apr 2025 18:10:26 +0200 Subject: [PATCH 03/27] navidrome: unpack action --- bundles/navidrome/items.py | 18 ++++++++++++++++-- bundles/navidrome/metadata.py | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/bundles/navidrome/items.py b/bundles/navidrome/items.py index 84e4e29..60e4c04 100644 --- a/bundles/navidrome/items.py +++ b/bundles/navidrome/items.py @@ -15,11 +15,24 @@ svc_systemd = { 'needs': { 'file:/etc/systemd/system/navidrome.service', 'file:/opt/navidrome/config.toml', - 'file:/opt/navidrome/navidrome', + 'action:navidrome_install', }, }, } +actions['navidrome_install'] = { + 'command': ' && '.join([ + 'tar -C /opt/navidrome -xf /opt/navidrome/navidrome.tar.gz', + ]), + 'after': { + 'pkg_apt:', + }, + 'triggered': True, + 'triggers': { + 'svc_systemd:navidrome:restart', + }, +} + files = { '/opt/navidrome/config.toml': { 'content': repo.libs.faults.dict_as_toml(node.metadata.get('navidrome/config')), @@ -33,12 +46,13 @@ files = { 'svc_systemd:navidrome:restart', }, }, - '/opt/navidrome/navidrome': { + '/opt/navidrome/navidrome.tar.gz': { 'content_hash': node.metadata.get('navidrome/sha1', None), 'content_type': 'download', 'mode': '0755', 'source': f'https://github.com/navidrome/navidrome/releases/download/v{node.metadata.get('navidrome/version')}/navidrome_{node.metadata.get('navidrome/version')}_linux_amd64.tar.gz', 'triggers': { + 'action:navidrome_install', 'svc_systemd:navidrome:restart', }, }, diff --git a/bundles/navidrome/metadata.py b/bundles/navidrome/metadata.py index 4eeebb5..4151c51 100644 --- a/bundles/navidrome/metadata.py +++ b/bundles/navidrome/metadata.py @@ -66,7 +66,7 @@ def nginx(metadata): 'domain': metadata.get('navidrome/domain'), 'locations': { '/': { - 'target': f'http://127.0.0.1:{metadata.get('navidrome/config/port')}', + 'target': f'http://127.0.0.1:{metadata.get('navidrome/config/Port')}', }, }, 'website_check_path': '/user/login', From 0c52fe7dd89a366f33fd9cf179f2c55e4075e58e Mon Sep 17 00:00:00 2001 From: Sophie Schiller Date: Wed, 16 Apr 2025 18:10:41 +0200 Subject: [PATCH 04/27] sophie.navidrome: use correct hash --- nodes/sophie/sophie.navidrome.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nodes/sophie/sophie.navidrome.toml b/nodes/sophie/sophie.navidrome.toml index 93df587..e7d0315 100644 --- a/nodes/sophie/sophie.navidrome.toml +++ b/nodes/sophie/sophie.navidrome.toml @@ -8,7 +8,7 @@ groups = [ "debian-bookworm", ] -[metadata.interfaces.enp7s0] +[metadata.interfaces.enp1s0] ips = [ "172.19.164.5/24", ] @@ -22,7 +22,7 @@ ram = 4 [metadata.navidrome] domain = 'navidrome.home.sophies-kitchen.eu' version = '0.55.2' -sha1 = '97661ab81438a521d2b149a417a7282a9bf4c2dc79c5148c4fa32a102e416940' +sha1 = 'c5e513fb830f40bea33537ef0c649a3621bd443c' [metadata.navidrome.config] MusicFolder = "/mnt/media/Musik" From 63c3e35e79e27b581c76a4eb2dab8118ed90180b Mon Sep 17 00:00:00 2001 From: Sophie Schiller Date: Thu, 17 Apr 2025 21:53:40 +0200 Subject: [PATCH 05/27] sophie.home additional routes --- nodes/sophie/sophie.homeassistant.toml | 2 ++ nodes/sophie/sophie.navidrome.toml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/nodes/sophie/sophie.homeassistant.toml b/nodes/sophie/sophie.homeassistant.toml index 42e25d0..8e2c4da 100644 --- a/nodes/sophie/sophie.homeassistant.toml +++ b/nodes/sophie/sophie.homeassistant.toml @@ -14,6 +14,8 @@ ips = [ ] gateway4 = "172.19.164.1" ipv6_accept_ra = true +[metadata.interfaces.enp7s0.routes."172.19.165.0/24"] +via = "172.19.164.2" [metadata.vm] cpu = 2 diff --git a/nodes/sophie/sophie.navidrome.toml b/nodes/sophie/sophie.navidrome.toml index e7d0315..d1df8eb 100644 --- a/nodes/sophie/sophie.navidrome.toml +++ b/nodes/sophie/sophie.navidrome.toml @@ -14,6 +14,8 @@ ips = [ ] gateway4 = "172.19.164.1" ipv6_accept_ra = true +[metadata.interfaces.enp1s0.routes."172.19.165.0/24"] +via = "172.19.164.2" [metadata.vm] cpu = 2 From a15740c89972723879463b4be67f6055e390ee13 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Wed, 16 Apr 2025 09:02:18 +0200 Subject: [PATCH 06/27] bundles/backup-server: improve --- .../files/check_backup_for_node-cron | 15 +++--- bundles/backup-server/metadata.py | 53 ++++++------------- 2 files changed, 22 insertions(+), 46 deletions(-) diff --git a/bundles/backup-server/files/check_backup_for_node-cron b/bundles/backup-server/files/check_backup_for_node-cron index b82217d..ff1a368 100644 --- a/bundles/backup-server/files/check_backup_for_node-cron +++ b/bundles/backup-server/files/check_backup_for_node-cron @@ -15,16 +15,15 @@ for line in check_output('LC_ALL=C zfs list -H -t snapshot -o name', shell=True) line = line.decode('UTF-8') if line.startswith('{}/'.format(server_settings['zfs-base'])): - dataset, snapname = line.split('@', 1) + try: + dataset, snapname = line.split('@', 1) - dataset = dataset.split('/')[-1] - ts, bucket = snapname.split('-', 1) + dataset = dataset.split('/')[-1] + ts, bucket = snapname.split('-', 1) - if not ts.isdigit(): - # garbage, ignore - continue - - snapshots[dataset].add(int(ts)) + snapshots[dataset].add(int(ts)) + except Exception as e: + print(f"Exception while parsing snapshot name {line!r}: {e!r}") backups = {} for dataset, snaps in snapshots.items(): diff --git a/bundles/backup-server/metadata.py b/bundles/backup-server/metadata.py index aace61b..6714288 100644 --- a/bundles/backup-server/metadata.py +++ b/bundles/backup-server/metadata.py @@ -83,47 +83,24 @@ def zfs_pool(metadata): devices = metadata.get('backup-server/encrypted-devices') - # TODO remove this once we have migrated all systems - if isinstance(devices, dict): - pool_devices = set() + pool_devices = set() - for number, (device, passphrase) in enumerate(sorted(devices.items())): - crypt_devices[device] = { - 'dm-name': f'backup{number}', - 'passphrase': passphrase, - } - pool_devices.add(f'/dev/mapper/backup{number}') - unlock_actions.add(f'action:dm-crypt_open_backup{number}') + for device, dconfig in devices.items(): + crypt_devices[dconfig['device']] = { + 'dm-name': f'backup-{device}', + 'passphrase': dconfig['passphrase'], + } + pool_devices.add(f'/dev/mapper/backup-{device}') + unlock_actions.add(f'action:dm-crypt_open_backup-{device}') - pool_config = [{ - 'devices': pool_devices, - }] + pool_config = [{ + 'devices': pool_devices, + }] - if len(pool_devices) > 2: - pool_config[0]['type'] = 'raidz' - elif len(pool_devices) > 1: - pool_config[0]['type'] = 'mirror' - - elif isinstance(devices, list): - pool_config = [] - - for idx, intended_pool in enumerate(devices): - pool_devices = set() - - for number, (device, passphrase) in enumerate(sorted(intended_pool.items())): - crypt_devices[device] = { - 'dm-name': f'backup{idx}-{number}', - 'passphrase': passphrase, - } - pool_devices.add(f'/dev/mapper/backup{idx}-{number}') - unlock_actions.add(f'action:dm-crypt_open_backup{idx}-{number}') - - pool_config.append({ - 'devices': pool_devices, - 'type': 'raidz', - }) - else: - raise BundleError(f'{node.name}: unsupported configuration for backup-server/encrypted-devices') + if len(pool_devices) > 2: + pool_config[0]['type'] = 'raidz' + elif len(pool_devices) > 1: + pool_config[0]['type'] = 'mirror' return { 'backup-server': { From a34f3a8d980b8b69e8a3577a5d27c93e7a7eaf50 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Wed, 16 Apr 2025 09:03:23 +0200 Subject: [PATCH 07/27] backup-kunsi: new disks --- nodes/backup-kunsi.toml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/nodes/backup-kunsi.toml b/nodes/backup-kunsi.toml index 3e17bd7..4a47ae4 100644 --- a/nodes/backup-kunsi.toml +++ b/nodes/backup-kunsi.toml @@ -22,15 +22,17 @@ exclude_from_backups = true [metadata.backup-server.zpool_create_options] ashift = 12 -[[metadata.backup-server.encrypted-devices]] -"/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1-part1" = "!bwpass:bw/backup-kunsi/ata-ST18000NM0092-3CX103_ZVV06SLR-part1" -"/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi2-part1" = "!bwpass:bw/backup-kunsi/ata-ST18000NM0092-3CX103_ZVV0686W-part1" -"/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi3-part1" = "!bwpass:bw/backup-kunsi/ata-ST18000NM0092-3CX103_ZVV06JV7-part1" +[metadata.backup-server.encrypted-devices.WVT0RNKF] +device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi4" +passphrase = "!bwpass:bw/backup-kunsi/ata-ST20000NM007D-3DJ103_WVT0RNKF" -[[metadata.backup-server.encrypted-devices]] -"/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi1-part2" = "!bwpass:bw/backup-kunsi/ata-ST18000NM0092-3CX103_ZVV06SLR-part2" -"/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi2-part2" = "!bwpass:bw/backup-kunsi/ata-ST18000NM0092-3CX103_ZVV0686W-part2" -"/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi3-part2" = "!bwpass:bw/backup-kunsi/ata-ST18000NM0092-3CX103_ZVV06JV7-part2" +[metadata.backup-server.encrypted-devices.WVT0V0NQ] +device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi5" +passphrase = "!bwpass:bw/backup-kunsi/ata-ST20000NM007D-3DJ103_WVT0V0NQ" + +[metadata.backup-server.encrypted-devices.WVT0W64H] +device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi6" +passphrase = "!bwpass:bw/backup-kunsi/ata-ST20000NM007D-3DJ103_WVT0W64H" [metadata.zfs] scrub_when = "Wed 08:00 Europe/Berlin" From af5a75e0656793e7f250c25f64f14f4963278842 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Wed, 16 Apr 2025 09:03:53 +0200 Subject: [PATCH 08/27] home.nas: change storage layout --- nodes/home/downloadhelper.py | 2 +- nodes/home/nas.py | 99 ++++++++++++++---------------------- 2 files changed, 38 insertions(+), 63 deletions(-) diff --git a/nodes/home/downloadhelper.py b/nodes/home/downloadhelper.py index 4874561..4bd2f10 100644 --- a/nodes/home/downloadhelper.py +++ b/nodes/home/downloadhelper.py @@ -42,7 +42,7 @@ nodes['home.downloadhelper'] = { 'mounts': { 'storage': { 'mountpoint': '/mnt/nas', - 'serverpath': '172.19.138.20:/storage/download', + 'serverpath': '172.19.138.20:/mnt/download', 'mount_options': { 'retry=0', 'rw', diff --git a/nodes/home/nas.py b/nodes/home/nas.py index 13694e6..2f210d6 100644 --- a/nodes/home/nas.py +++ b/nodes/home/nas.py @@ -5,7 +5,6 @@ nodes['home.nas'] = { 'bundles': { 'avahi-daemon', 'backup-client', - 'dm-crypt', 'jellyfin', 'lm-sensors', 'mixcloud-downloader', @@ -69,22 +68,6 @@ nodes['home.nas'] = { 'avahi-aruba-fixup': '17,47 * * * * root /usr/bin/systemctl restart avahi-daemon.service', }, }, - 'dm-crypt': { - 'encrypted-devices': { - '/dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNJ0X409404K': { - 'dm-name': 'sam-S5SSNJ0X409404K', - 'passphrase': bwpass.password('bw/home.nas/dmcrypt/S5SSNJ0X409404K'), - }, - '/dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNJ0X409845F': { - 'dm-name': 'sam-S5SSNJ0X409845F', - 'passphrase': bwpass.password('bw/home.nas/dmcrypt/S5SSNJ0X409845F'), - }, - '/dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNJ0X409870J': { - 'dm-name': 'sam-S5SSNJ0X409870J', - 'passphrase': bwpass.password('bw/home.nas/dmcrypt/S5SSNJ0X409870J'), - }, - }, - }, 'groups': { 'nas': {}, }, @@ -154,7 +137,7 @@ nodes['home.nas'] = { }, 'nfs-server': { 'shares': { - '/storage/download': { + '/mnt/download': { 'home.downloadhelper': 'rw,all_squash,anonuid=65534,anongid=1012,no_subtree_check', }, '/storage/nas': { @@ -192,7 +175,7 @@ nodes['home.nas'] = { 'disks': { '/dev/nvme0', - # old nas disks + # nas/timemachine disks '/dev/disk/by-id/ata-WDC_WD6003FFBX-68MU3N0_V8GE15GR', '/dev/disk/by-id/ata-WDC_WD6003FFBX-68MU3N0_V8HJ406R', '/dev/disk/by-id/ata-WDC_WD6003FFBX-68MU3N0_V8HJBTLR', @@ -200,10 +183,9 @@ nodes['home.nas'] = { '/dev/disk/by-id/ata-WDC_WD6003FFBX-68MU3N0_V8J8ZKRR', '/dev/disk/by-id/ata-WDC_WD6003FFBX-68MU3N0_V9JS5UYL', - # encrypted disks - '/dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNJ0X409404K', - '/dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNJ0X409845F', - '/dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNJ0X409870J', + # ssdpool disks + '/dev/disk/by-id/ata-INTEL_SSDSC2KB960G8_PHYF244001QU960CGN', + '/dev/disk/by-id/ata-INTEL_SSDSC2KB960G8_PHYF244002AS960CGN', }, }, 'systemd-networkd': { @@ -258,6 +240,20 @@ nodes['home.nas'] = { 'zfs_arc_max_gb': 8, }, 'pools': { + 'ssdpool': { + 'when_creating': { + 'config': [ + { + 'type': 'mirror', + 'devices': { + '/dev/disk/by-id/ata-INTEL_SSDSC2KB960G8_PHYF244001QU960CGN', + '/dev/disk/by-id/ata-INTEL_SSDSC2KB960G8_PHYF244002AS960CGN', + }, + }, + ], + 'ashift': 12, + }, + }, 'tank': { 'when_creating': { 'config': [ @@ -276,67 +272,46 @@ nodes['home.nas'] = { 'ashift': 12, }, }, - 'encrypted': { - 'when_creating': { - 'config': [ - { - 'type': 'raidz', - 'devices': { - '/dev/mapper/sam-S5SSNJ0X409404K', - '/dev/mapper/sam-S5SSNJ0X409845F', - '/dev/mapper/sam-S5SSNJ0X409870J', - }, - }, - ], - 'ashift': 12, - }, - 'needs': { - 'action:dm-crypt_open_sam-S5SSNJ0X409404K', - 'action:dm-crypt_open_sam-S5SSNJ0X409845F', - 'action:dm-crypt_open_sam-S5SSNJ0X409870J', - }, - # see comment in bundle:backup-server - 'unless': 'zpool import encrypted', - }, }, 'datasets': { - 'encrypted': { + 'ssdpool': { 'primarycache': 'metadata', }, - 'encrypted/nas': { + 'ssdpool/yate': { + 'mountpoint': '/opt/yate', + }, + 'ssdpool/download': { + 'mountpoint': '/mnt/download', + 'quota': '858993459200', # 800 GB + }, + 'ssdpool/paperless': { + 'mountpoint': '/srv/paperless', + }, + 'tank': { + 'primarycache': 'metadata', + }, + 'tank/nas': { 'acltype': 'off', 'atime': 'off', 'compression': 'off', 'mountpoint': '/storage/nas', }, - 'tank': { - 'primarycache': 'metadata', - }, - 'tank/opt-yate': { - 'mountpoint': '/opt/yate', - }, - 'tank/download': { - 'mountpoint': '/storage/download', - }, - 'tank/paperless': { - 'mountpoint': '/srv/paperless', - }, }, 'snapshots': { 'retain_per_dataset': { - 'encrypted/nas': { + 'tank/nas': { # juuuuuuuust to be sure. 'daily': 14, 'weekly': 6, 'monthly': 12, }, - 'tank/download': { + 'ssdpool/download': { 'hourly': 48, 'daily': 0, 'weekly': 0, 'monthly': 0, }, - 'tank/paperless': { + 'ssdpool/paperless': { 'daily': 14, 'weekly': 6, 'monthly': 24, From 6f902c5c7b6c1e1852e086626b538964d59fac4d Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Wed, 16 Apr 2025 09:04:17 +0200 Subject: [PATCH 09/27] proxmox-backupstorage: more disks --- nodes/proxmox-backupstorage.toml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nodes/proxmox-backupstorage.toml b/nodes/proxmox-backupstorage.toml index eee0256..7f10946 100644 --- a/nodes/proxmox-backupstorage.toml +++ b/nodes/proxmox-backupstorage.toml @@ -14,6 +14,18 @@ check_command = "sshmon" check_command = "sshmon" "vars.sshmon_command" = "CT480BX500SSD1_2314E6C5C6C8" +[metadata.icinga2_api.smartd.services."SMART STATUS ST20000NM007D-3DJ103_WVT0RNKF"] +check_command = "sshmon" +"vars.sshmon_command" = "ST20000NM007D-3DJ103_WVT0RNKF" + +[metadata.icinga2_api.smartd.services."SMART STATUS ST20000NM007D-3DJ103_WVT0V0NQ"] +check_command = "sshmon" +"vars.sshmon_command" = "ST20000NM007D-3DJ103_WVT0V0NQ" + +[metadata.icinga2_api.smartd.services."SMART STATUS ST20000NM007D-3DJ103_WVT0W64H"] +check_command = "sshmon" +"vars.sshmon_command" = "ST20000NM007D-3DJ103_WVT0W64H" + [metadata.icinga2_api.smartd.services."SMART STATUS ST18000NM0092-3CX103_ZVV0686W"] check_command = "sshmon" "vars.sshmon_command" = "ST18000NM0092-3CX103_ZVV0686W" From 80a5d3563a70bc8d423b82c48fc3ad48bd5c05db Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Wed, 16 Apr 2025 09:07:18 +0200 Subject: [PATCH 10/27] htz-cloud.wireguard: also announce ip we're routing --- nodes/htz-cloud/wireguard.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nodes/htz-cloud/wireguard.py b/nodes/htz-cloud/wireguard.py index 3ceaf2d..e560667 100644 --- a/nodes/htz-cloud/wireguard.py +++ b/nodes/htz-cloud/wireguard.py @@ -37,6 +37,7 @@ nodes['htz-cloud.wireguard'] = { '172.19.137.0/24', '172.19.136.62/31', '172.19.136.64/31', + '172.19.136.66/31', '192.168.100.0/24', }, }, From 4bc94987a761aaba6cf778ce989c8242696edc59 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Wed, 16 Apr 2025 09:07:45 +0200 Subject: [PATCH 11/27] carlene: add 42c3 topic timer --- nodes/carlene.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nodes/carlene.toml b/nodes/carlene.toml index fb6d22a..3457ef6 100644 --- a/nodes/carlene.toml +++ b/nodes/carlene.toml @@ -244,6 +244,11 @@ disks = [ "/dev/disk/by-id/nvme-SAMSUNG_MZVL22T0HBLB-00B00_S677NX0W114380", ] +[metadata.systemd-timers.timers.42c3-topic] +command = "/home/kunsi/42c3-topic.sh" +user = "kunsi" +when = "04:00:00 Europe/Berlin" + [metadata.travelynx] version = "2.11.13" mail_from = "travelynx@franzi.business" From 1af04b684657fae6e50e68c468540d9b10f777fe Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Wed, 23 Apr 2025 10:56:17 +0200 Subject: [PATCH 12/27] update forgejo to 11.0.0 --- nodes/carlene.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nodes/carlene.toml b/nodes/carlene.toml index 3457ef6..ed051fb 100644 --- a/nodes/carlene.toml +++ b/nodes/carlene.toml @@ -37,8 +37,8 @@ imap_host = "secureimap.t-online.de" imap_pass = "!bwpass_attr:t-online.de/franzi.kunsmann@t-online.de:imap" [metadata.forgejo] -version = "10.0.3" -sha1 = "d1199c43de9e69f6bb8058c15290e79862913413" +version = "11.0.0" +sha1 = "3a12529ab21ca04f2b3e6cf7a6c91af18f00ee5d" domain = "git.franzi.business" enable_git_hooks = true install_ssh_key = true From a999071cca1c9c1a8d88dceb812af019979117a5 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Wed, 23 Apr 2025 10:56:34 +0200 Subject: [PATCH 13/27] update mautrix-whatsapp to 0.12.0 --- nodes/carlene.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nodes/carlene.toml b/nodes/carlene.toml index ed051fb..f98154c 100644 --- a/nodes/carlene.toml +++ b/nodes/carlene.toml @@ -98,8 +98,8 @@ provisioning.shared_secret = "!decrypt:encrypt$gAAAAABfVKflEMAi07C_QGP8cy97hF-4g "'@kunsi:franzi.business'" = "admin" [metadata.mautrix-whatsapp] -version = "v0.11.4" -sha1 = "71a064b82072d2cec3d655c8848af418c1f54c77" +version = "v0.12.0" +sha1 = "02094da0a164099d4d35e5edb4b87875ad694833" permissions."'@kunsi:franzi.business'" = "admin" [metadata.mautrix-whatsapp.homeserver] domain = "franzi.business" From 19d80513915cfc0810c3bbe60fd6216b5e8485e0 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Wed, 23 Apr 2025 10:57:03 +0200 Subject: [PATCH 14/27] update netbox to 4.2.8 --- bundles/netbox/items.py | 4 ++-- nodes/carlene.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bundles/netbox/items.py b/bundles/netbox/items.py index f261641..9edbf0b 100644 --- a/bundles/netbox/items.py +++ b/bundles/netbox/items.py @@ -38,8 +38,8 @@ actions['netbox_install'] = { 'triggered': True, 'command': ' && '.join([ 'cd /opt/netbox/src', - '/opt/netbox/venv/bin/pip install --upgrade pip wheel setuptools django-auth-ldap gunicorn', - '/opt/netbox/venv/bin/pip install --upgrade -r requirements.txt', + '/opt/netbox/venv/bin/pip install --upgrade --upgrade-strategy=eager pip wheel setuptools django-auth-ldap gunicorn', + '/opt/netbox/venv/bin/pip install --upgrade --upgrade-strategy=eager -r requirements.txt', ]), 'needs': { 'pkg_apt:build-essential', diff --git a/nodes/carlene.toml b/nodes/carlene.toml index f98154c..9c79e23 100644 --- a/nodes/carlene.toml +++ b/nodes/carlene.toml @@ -110,7 +110,7 @@ domain = "rss.franzi.business" [metadata.netbox] domain = "netbox.franzi.business" -version = "v4.2.6" +version = "v4.2.8" admins.kunsi = "hostmaster@kunbox.net" [metadata.nextcloud] From f72f701a5a92e1b24d8303132599320740f6ead0 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Wed, 23 Apr 2025 10:57:24 +0200 Subject: [PATCH 15/27] update paperless-ngx to 2.15.3 --- bundles/paperless-ng/files/paperless-webserver.service | 5 ++++- bundles/paperless-ng/metadata.py | 2 +- nodes/home/paperless.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bundles/paperless-ng/files/paperless-webserver.service b/bundles/paperless-ng/files/paperless-webserver.service index 5d7f806..7c41aa7 100644 --- a/bundles/paperless-ng/files/paperless-webserver.service +++ b/bundles/paperless-ng/files/paperless-webserver.service @@ -8,8 +8,11 @@ Requires=redis.service User=paperless Group=paperless Environment=PAPERLESS_CONFIGURATION_PATH=/opt/paperless/paperless.conf +Environment=GRANIAN_PORT=22070 +Environment=GRANIAN_WORKERS=4 +Environment=GRANIAN_HOST=::1 WorkingDirectory=/opt/paperless/src/paperless-ngx/src -ExecStart=/opt/paperless/venv/bin/gunicorn -c /opt/paperless/src/paperless-ngx/gunicorn.conf.py -b 127.0.0.1:22070 paperless.asgi:application +ExecStart=/opt/paperless/venv/bin/granian --interface asginl --ws "paperless.asgi:application" Restart=always RestartSec=10 SyslogIdentifier=paperless-webserver diff --git a/bundles/paperless-ng/metadata.py b/bundles/paperless-ng/metadata.py index 6746616..8db5342 100644 --- a/bundles/paperless-ng/metadata.py +++ b/bundles/paperless-ng/metadata.py @@ -99,7 +99,7 @@ def nginx(metadata): 'domain': metadata.get('paperless/domain'), 'locations': { '/': { - 'target': 'http://127.0.0.1:22070', + 'target': 'http://[::1]:22070', 'websockets': True, 'proxy_set_header': { 'X-Forwarded-Host': '$server_name', diff --git a/nodes/home/paperless.py b/nodes/home/paperless.py index caffb73..f7035a5 100644 --- a/nodes/home/paperless.py +++ b/nodes/home/paperless.py @@ -49,7 +49,7 @@ nodes['home.paperless'] = { }, 'paperless': { 'domain': 'paperless.home.kunbox.net', - 'version': 'v2.14.7', + 'version': 'v2.15.3', 'timezone': 'Europe/Berlin', }, 'postgresql': { From 3ec701b2b6dc0bc73174f76207247dcb6e90520c Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Wed, 23 Apr 2025 10:58:10 +0200 Subject: [PATCH 16/27] add rottenraptor vpn --- libs/s2s.py | 1 + nodes/htz-cloud/wireguard.py | 8 ++++++++ nodes/rottenraptor-vpn.toml | 27 +++++++++++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 nodes/rottenraptor-vpn.toml diff --git a/libs/s2s.py b/libs/s2s.py index fe0fc4e..8372ec2 100644 --- a/libs/s2s.py +++ b/libs/s2s.py @@ -6,6 +6,7 @@ AS_NUMBERS = { 'htz-cloud': 4290000137, 'ionos': 4290000002, 'revision': 4290000078, + 'rottenraptor': 4290000030, } WG_AUTOGEN_NODES = [ diff --git a/nodes/htz-cloud/wireguard.py b/nodes/htz-cloud/wireguard.py index e560667..1139390 100644 --- a/nodes/htz-cloud/wireguard.py +++ b/nodes/htz-cloud/wireguard.py @@ -53,6 +53,7 @@ nodes['htz-cloud.wireguard'] = { 'udp dport 1194 accept', 'udp dport 51800 accept', 'udp dport 51804 accept', + 'udp dport 51805 accept', # wg.c3voc.de 'udp dport 51801 ip saddr 185.106.84.42 accept', @@ -126,6 +127,13 @@ nodes['htz-cloud.wireguard'] = { 'my_ip': '172.19.136.66', 'their_ip': '172.19.136.67', }, + 'rottenraptor-vpn': { + 'endpoint': None, + 'exclude_from_monitoring': True, + 'my_port': 51805, + 'my_ip': '172.19.136.68', + 'their_ip': '172.19.136.69', + }, }, }, }, diff --git a/nodes/rottenraptor-vpn.toml b/nodes/rottenraptor-vpn.toml new file mode 100644 index 0000000..342ce1c --- /dev/null +++ b/nodes/rottenraptor-vpn.toml @@ -0,0 +1,27 @@ +hostname = "172.30.17.53" +bundles = ["bird", "wireguard"] +groups = ["debian-bookworm"] + +[metadata] +location = "rottenraptor" +backups.exclude_from_backups = true +icinga_options.exclude_from_monitoring = true + +[metadata.bird] +static_routes = [ + "172.30.17.0/24", +] + +[metadata.interfaces.ens18] +ips = ["172.30.17.53/24"] +gateway4 = "172.30.17.1" + +[metadata.nftables.postrouting] +"50-router" = [ + "oifname ens18 masquerade", +] + +[metadata.wireguard.peers."htz-cloud.wireguard"] +my_port = 51804 +my_ip = "172.19.136.69" +their_ip = "172.19.136.68" From 3d643efe0fb7b455f1ab7dff1a26895a7c7957f1 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Wed, 23 Apr 2025 11:05:48 +0200 Subject: [PATCH 17/27] bundles/zfs: fix dependencies --- bundles/zfs/items.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bundles/zfs/items.py b/bundles/zfs/items.py index c63250e..530d27f 100644 --- a/bundles/zfs/items.py +++ b/bundles/zfs/items.py @@ -67,6 +67,7 @@ svc_systemd = { 'file:/etc/systemd/system/zfs-import-scan.service.d/bundlewrap.conf', }, 'after': { + 'bundle:dm-crypt', # might unlock disks 'pkg_apt:', }, 'before': { @@ -83,6 +84,7 @@ svc_systemd = { }, 'zfs-mount.service': { 'after': { + 'bundle:dm-crypt', # might unlock disks 'pkg_apt:', }, }, From f9e87bde9e660bf818c922b8950b5a9da494afa4 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Thu, 24 Apr 2025 11:12:49 +0200 Subject: [PATCH 18/27] update travelynx to 2.11.23 --- nodes/carlene.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes/carlene.toml b/nodes/carlene.toml index 9c79e23..3b53d0f 100644 --- a/nodes/carlene.toml +++ b/nodes/carlene.toml @@ -250,7 +250,7 @@ user = "kunsi" when = "04:00:00 Europe/Berlin" [metadata.travelynx] -version = "2.11.13" +version = "2.11.23" mail_from = "travelynx@franzi.business" domain = "travelynx.franzi.business" From 57c1eb26056694b7ca1b25db256708fae337044f Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Tue, 6 May 2025 18:32:20 +0200 Subject: [PATCH 19/27] bundles/docker-immich: database not existing should not error out the script after all, we have monitoring to ensure the database container runs --- .../files/immich-auto-album-share.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/bundles/docker-immich/files/immich-auto-album-share.py b/bundles/docker-immich/files/immich-auto-album-share.py index 863f8b2..2cac6c2 100644 --- a/bundles/docker-immich/files/immich-auto-album-share.py +++ b/bundles/docker-immich/files/immich-auto-album-share.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import logging from json import loads from os import environ from subprocess import check_output @@ -12,6 +13,8 @@ PSQL_USER = environ['DB_USERNAME'] PSQL_PASS = environ['DB_PASSWORD'] PSQL_DB = environ['DB_DATABASE_NAME'] +logging.basicConfig(level=logging.INFO) + docker_networks = loads(check_output(['docker', 'network', 'inspect', 'aaarghhh'])) container_ip = None @@ -26,9 +29,9 @@ for network in docker_networks: container_ip = container['IPv4Address'].split('/')[0] if not container_ip: - print(f'could not find ip address for container {PSQL_HOST=} in json') - print(docker_networks) - exit(1) + logging.error(f'could not find ip address for container {PSQL_HOST=} in json') + logging.debug(f'{docker_networks=}') + exit(0) print(f'{PSQL_HOST=} {container_ip=}') @@ -49,6 +52,7 @@ with conn: } for i in cur.fetchall() } + logging.debug(f'{albums=}') with conn.cursor() as cur: cur.execute('SELECT "id","name" FROM users;') @@ -56,25 +60,27 @@ with conn: i[0]: i[1] for i in cur.fetchall() } + logging.debug(f'{users=}') for album_id, album in albums.items(): - print(f'----- working on album: {album["name"]}') + log = logging.getLogger(album["name"]) with conn: with conn.cursor() as cur: cur.execute('SELECT "usersId" FROM albums_shared_users_users WHERE "albumsId" = %s;', (album_id,)) album_shares = [i[0] for i in cur.fetchall()] - print(f' album is shared with {len(album_shares)} users: {album_shares}') + log.info(f'album is shared with {len(album_shares)} users: {album_shares}') for user_id, user_name in users.items(): if user_id == album['owner'] or user_id in album_shares: continue - print(f' sharing album with user {user_name} ... ', end='') - with conn.cursor() as cur: - cur.execute( - 'INSERT INTO albums_shared_users_users ("albumsId","usersId","role") VALUES (%s, %s, %s);', - (album_id, user_id, 'viewer'), - ) - print('done') - print() + log.info(f'sharing album with user {user_name}') + try: + with conn.cursor() as cur: + cur.execute( + 'INSERT INTO albums_shared_users_users ("albumsId","usersId","role") VALUES (%s, %s, %s);', + (album_id, user_id, 'viewer'), + ) + except Exception: + log.exception('failure while creating share') conn.close() From 29799a1d339a3e5d1a01446c79d795cf5ef284f0 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Tue, 6 May 2025 18:33:49 +0200 Subject: [PATCH 20/27] bundles/docker-immich; do not log all those user ids if we don't need them --- bundles/docker-immich/files/immich-auto-album-share.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bundles/docker-immich/files/immich-auto-album-share.py b/bundles/docker-immich/files/immich-auto-album-share.py index 2cac6c2..ad9aac7 100644 --- a/bundles/docker-immich/files/immich-auto-album-share.py +++ b/bundles/docker-immich/files/immich-auto-album-share.py @@ -68,7 +68,8 @@ for album_id, album in albums.items(): with conn.cursor() as cur: cur.execute('SELECT "usersId" FROM albums_shared_users_users WHERE "albumsId" = %s;', (album_id,)) album_shares = [i[0] for i in cur.fetchall()] - log.info(f'album is shared with {len(album_shares)} users: {album_shares}') + log.info(f'album is shared with {len(album_shares)} users') + log.debug(f'{album_shares=}') for user_id, user_name in users.items(): if user_id == album['owner'] or user_id in album_shares: continue From 2b0e559f6ce507c1d52840b1abbd5d411f5ab626 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Tue, 6 May 2025 18:35:31 +0200 Subject: [PATCH 21/27] bundles/docker-immich: remove leftover print statement --- bundles/docker-immich/files/immich-auto-album-share.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundles/docker-immich/files/immich-auto-album-share.py b/bundles/docker-immich/files/immich-auto-album-share.py index ad9aac7..cafd32c 100644 --- a/bundles/docker-immich/files/immich-auto-album-share.py +++ b/bundles/docker-immich/files/immich-auto-album-share.py @@ -33,7 +33,7 @@ if not container_ip: logging.debug(f'{docker_networks=}') exit(0) -print(f'{PSQL_HOST=} {container_ip=}') +logging.debug(f'{PSQL_HOST=} {container_ip=}') conn = psycopg2.connect( dbname=PSQL_DB, From ae079764395f3b770e52134eaf5610a3693f52e4 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Tue, 6 May 2025 20:57:41 +0200 Subject: [PATCH 22/27] bundles/nfs-server: add avahi config --- bundles/nfs-server/files/avahi.service | 10 +++++ bundles/nfs-server/files/exports | 2 +- bundles/nfs-server/items.py | 51 +++++++++++++++++--------- nodes/home/nas.py | 2 +- 4 files changed, 45 insertions(+), 20 deletions(-) create mode 100644 bundles/nfs-server/files/avahi.service diff --git a/bundles/nfs-server/files/avahi.service b/bundles/nfs-server/files/avahi.service new file mode 100644 index 0000000..394cdca --- /dev/null +++ b/bundles/nfs-server/files/avahi.service @@ -0,0 +1,10 @@ + + + + NFS ${path} on %h + + _nfs._tcp + 2049 + path=${path} + + diff --git a/bundles/nfs-server/files/exports b/bundles/nfs-server/files/exports index ad2ca4c..ac9c8f8 100644 --- a/bundles/nfs-server/files/exports +++ b/bundles/nfs-server/files/exports @@ -1,4 +1,4 @@ -% for path, shares in sorted(node.metadata['nfs-server']['shares'].items()): +% for path, shares in sorted(node.metadata.get('nfs-server/shares', {}).items()): % for share_target, share_options in sorted(shares.items()): % for ip_list in repo.libs.tools.resolve_identifier(repo, share_target).values(): % for ip in sorted(ip_list): diff --git a/bundles/nfs-server/items.py b/bundles/nfs-server/items.py index dacbc48..ce025cf 100644 --- a/bundles/nfs-server/items.py +++ b/bundles/nfs-server/items.py @@ -1,25 +1,40 @@ -files = { - '/etc/exports': { - 'content_type': 'mako', - 'triggers': { - 'action:nfs_reload_shares', - }, - }, - '/etc/default/nfs-kernel-server': { - 'source': 'etc-default', - 'triggers': { - 'svc_systemd:nfs-server:restart', - }, +from re import sub + +files['/etc/exports'] = { + 'content_type': 'mako', + 'triggers': { + 'action:nfs_reload_shares', }, } -actions = { - 'nfs_reload_shares': { - 'command': 'exportfs -a', - 'triggered': True, +files['/etc/default/nfs-kernel-server'] = { + 'source': 'etc-default', + 'triggers': { + 'svc_systemd:nfs-server:restart', }, } -svc_systemd = { - 'nfs-server': {}, +actions['nfs_reload_shares'] = { + 'command': 'exportfs -a', + 'triggered': True, } + +svc_systemd['nfs-server'] = {} + +if node.has_bundle('avahi-daemon'): + for path, shares in node.metadata.get('nfs-server/shares', {}).items(): + create_avahi_file = False + for share_target, share_options in shares.items(): + if ',insecure,' in f',{share_options},': + create_avahi_file = True + + if create_avahi_file: + share_name_normalized = sub('[^a-z0-9-_]+', '_', path) + + files[f'/etc/avahi/services/nfs{share_name_normalized}.service'] = { + 'source': 'avahi.service', + 'content_type': 'mako', + 'context': { + 'path': path, + }, + } diff --git a/nodes/home/nas.py b/nodes/home/nas.py index 2f210d6..e98955c 100644 --- a/nodes/home/nas.py +++ b/nodes/home/nas.py @@ -141,7 +141,7 @@ nodes['home.nas'] = { 'home.downloadhelper': 'rw,all_squash,anonuid=65534,anongid=1012,no_subtree_check', }, '/storage/nas': { - '172.19.138.0/24': 'ro,all_squash,anonuid=65534,anongid=65534,no_subtree_check', + '172.19.138.0/24': 'ro,all_squash,anonuid=65534,anongid=65534,no_subtree_check,insecure', }, '/srv/paperless': { 'home.paperless': 'rw,all_squash,anonuid=65534,anongid=65534,no_subtree_check', From cc94f10c2da1e8943246e6f3424697113f77d20e Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Tue, 6 May 2025 20:58:33 +0200 Subject: [PATCH 23/27] remove mitel rfp35 --- nodes/home.mitel-rfp35.toml | 4 ---- nodes/home/nas.py | 3 --- 2 files changed, 7 deletions(-) delete mode 100644 nodes/home.mitel-rfp35.toml diff --git a/nodes/home.mitel-rfp35.toml b/nodes/home.mitel-rfp35.toml deleted file mode 100644 index 414658a..0000000 --- a/nodes/home.mitel-rfp35.toml +++ /dev/null @@ -1,4 +0,0 @@ -dummy = true - -[metadata.interfaces.default] -ips = ["172.19.138.41"] diff --git a/nodes/home/nas.py b/nodes/home/nas.py index e98955c..ebfdc2c 100644 --- a/nodes/home/nas.py +++ b/nodes/home/nas.py @@ -79,11 +79,9 @@ nodes['home.nas'] = { }, '5060/tcp': { # yate SIP 'home.snom-wohnzimmer', - 'home.mitel-rfp35', }, '5061/tcp': { # yate SIPS 'home.snom-wohnzimmer', - 'home.mitel-rfp35', }, # yate RTP uses some random UDP port. We cannot firewall # it, because for incoming calls the other side decides @@ -93,7 +91,6 @@ nodes['home.nas'] = { # to deal with randomly changing IPs here. '*/udp': { 'home.snom-wohnzimmer', - 'home.mitel-rfp35', }, }, }, From 2b69953d961e94e27821096eeff606b28a0ed419 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Sat, 10 May 2025 10:05:50 +0200 Subject: [PATCH 24/27] update travelynx to 2.11.24 --- nodes/carlene.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes/carlene.toml b/nodes/carlene.toml index 3b53d0f..d5625d9 100644 --- a/nodes/carlene.toml +++ b/nodes/carlene.toml @@ -250,7 +250,7 @@ user = "kunsi" when = "04:00:00 Europe/Berlin" [metadata.travelynx] -version = "2.11.23" +version = "2.11.24" mail_from = "travelynx@franzi.business" domain = "travelynx.franzi.business" From afb6d21326ea053b04a0a7a0e8d5c91cb19e0a6b Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Sat, 10 May 2025 11:19:45 +0200 Subject: [PATCH 25/27] home.nas: backup /home/kunsi --- nodes/home/nas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nodes/home/nas.py b/nodes/home/nas.py index ebfdc2c..d4a4211 100644 --- a/nodes/home/nas.py +++ b/nodes/home/nas.py @@ -60,6 +60,7 @@ nodes['home.nas'] = { }, 'backups': { 'paths': { + '/home/kunsi/', '/storage/nas/', }, }, From e47c8ce341125f67f07edcd1473b83571b7fd474 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Sat, 10 May 2025 11:20:07 +0200 Subject: [PATCH 26/27] bundles/travelynx: disable registration by default --- bundles/travelynx/files/travelynx.conf | 6 ++++++ bundles/travelynx/metadata.py | 9 +++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/bundles/travelynx/files/travelynx.conf b/bundles/travelynx/files/travelynx.conf index 46883cf..f73e85f 100644 --- a/bundles/travelynx/files/travelynx.conf +++ b/bundles/travelynx/files/travelynx.conf @@ -33,6 +33,12 @@ from => '${mail_from}', }, +% if not enable_registration: + registration => { + disabled => 1, + }, +% endif + ref => { issues => 'https://github.com/derf/travelynx/issues', source => 'https://github.com/derf/travelynx', diff --git a/bundles/travelynx/metadata.py b/bundles/travelynx/metadata.py index b7dadd6..630fd27 100644 --- a/bundles/travelynx/metadata.py +++ b/bundles/travelynx/metadata.py @@ -10,11 +10,12 @@ defaults = { 'password': repo.vault.password_for('{} postgresql travelynx'.format(node.name)), 'database': 'travelynx', }, - 'workers': 4, - 'spare_workers': 2, - 'mail_from': 'travelynx@{}'.format(node.hostname), - 'cookie_secret': repo.vault.password_for('{} travelynx cookie_secret'.format(node.name)), 'additional_cookie_secrets': set(), + 'cookie_secret': repo.vault.password_for('{} travelynx cookie_secret'.format(node.name)), + 'enable_registration': False, + 'mail_from': 'travelynx@{}'.format(node.hostname), + 'spare_workers': 2, + 'workers': 4, }, 'postgresql': { 'roles': { From 9c41d73f93ac0a84f3f1687eecfe407461b1293f Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Wed, 14 May 2025 10:00:58 +0100 Subject: [PATCH 27/27] move dns back to zone files --- bundles/powerdns/items.py | 43 ++++++++++++++----- data/powerdns/files/bind-zones/_mail_NULL | 2 + data/powerdns/files/bind-zones/_mail_carlene | 11 +++++ data/powerdns/files/bind-zones/_parked | 3 ++ data/powerdns/files/bind-zones/afra.berlin | 6 +++ .../bind-zones/die-brontosaurier-waren-es.org | 1 + data/powerdns/files/bind-zones/emails.sexy | 1 + .../files/bind-zones/eskalation.jetzt | 3 ++ .../files/bind-zones/felix-kunsmann.de | 3 ++ .../files/bind-zones/flauschehorn.sexy | 8 ++++ .../powerdns/files/bind-zones/franzi.business | 29 +++++++++++++ data/powerdns/files/bind-zones/kunbox.net | 14 +----- data/powerdns/files/bind-zones/kunsi.scot | 1 + .../powerdns/files/bind-zones/kunsitracker.de | 6 +++ data/powerdns/files/bind-zones/kunsmann.eu | 14 ++++++ data/powerdns/files/bind-zones/raptor.events | 1 + .../powerdns/files/bind-zones/trans-agenda.de | 1 + .../powerdns/files/bind-zones/trans-agenda.eu | 1 + data/powerdns/files/bind-zones/warnochwas.de | 6 +++ .../powerdns/files/bind-zones/winkeeinhorn.de | 1 + 20 files changed, 131 insertions(+), 24 deletions(-) create mode 100644 data/powerdns/files/bind-zones/_mail_NULL create mode 100644 data/powerdns/files/bind-zones/_mail_carlene create mode 100644 data/powerdns/files/bind-zones/_parked create mode 100644 data/powerdns/files/bind-zones/afra.berlin create mode 120000 data/powerdns/files/bind-zones/die-brontosaurier-waren-es.org create mode 120000 data/powerdns/files/bind-zones/emails.sexy create mode 100644 data/powerdns/files/bind-zones/eskalation.jetzt create mode 100644 data/powerdns/files/bind-zones/felix-kunsmann.de create mode 100644 data/powerdns/files/bind-zones/flauschehorn.sexy create mode 100644 data/powerdns/files/bind-zones/franzi.business create mode 120000 data/powerdns/files/bind-zones/kunsi.scot create mode 100644 data/powerdns/files/bind-zones/kunsitracker.de create mode 100644 data/powerdns/files/bind-zones/kunsmann.eu create mode 120000 data/powerdns/files/bind-zones/raptor.events create mode 120000 data/powerdns/files/bind-zones/trans-agenda.de create mode 120000 data/powerdns/files/bind-zones/trans-agenda.eu create mode 100644 data/powerdns/files/bind-zones/warnochwas.de create mode 120000 data/powerdns/files/bind-zones/winkeeinhorn.de diff --git a/bundles/powerdns/items.py b/bundles/powerdns/items.py index b6a5e8f..c972f90 100644 --- a/bundles/powerdns/items.py +++ b/bundles/powerdns/items.py @@ -2,13 +2,14 @@ from datetime import datetime from os import listdir from os.path import isfile, join from subprocess import check_output +from textwrap import dedent from bundlewrap.utils.ui import io zone_path = join(repo.path, 'data', 'powerdns', 'files', 'bind-zones') nameservers = set() -for rnode in sorted(repo.nodes_in_group('dns')): +for rnode in repo.nodes_in_group('dns'): nameservers.add(rnode.metadata.get('powerdns/my_hostname', rnode.metadata.get('hostname'))) my_primary_servers = set() @@ -75,25 +76,45 @@ actions = { } if node.metadata.get('powerdns/features/bind', False): + try: + output = check_output(['git', 'log', '-1', '--pretty=%ci']).decode('utf-8').strip() + serial = datetime.strptime(output, '%Y-%m-%d %H:%M:%S %z').strftime('%y%m%d%H%M') + except Exception as e: + io.stderr(f"{node.name} Error while parsing commit time for powerdns zone serial: {e!r}") + serial = datetime.now().strftime('%y%m%d0000') + + HEADER = dedent(f""" + $TTL 60 + @ IN SOA ns-mephisto.kunbox.net. hostmaster.kunbox.net. ( + {serial} + 3600 + 600 + 86400 + 300 + ) + """).strip() + + for ns in sorted(nameservers): + HEADER += f"\n@ IN NS {ns}." + primary_zones = set() for zone in listdir(zone_path): - if not isfile(join(zone_path, zone)) or zone.startswith(".") or zone.startswith("_"): + if ( + not ( + isfile(join(zone_path, zone)) + or islink(join(zone_path, zone)) + ) + or zone.startswith(".") + or zone.startswith("_") + ): continue - try: - output = check_output(['git', 'log', '-1', '--pretty=%ci']).decode('utf-8').strip() - serial = datetime.strptime(output, '%Y-%m-%d %H:%M:%S %z').strftime('%y%m%d%H%M') - except Exception as e: - io.stderr(f"Error while parsing commit time for {zone} serial: {e!r}") - serial = datetime.now().strftime('%y%m%d0000') - primary_zones.add(zone) files[f'/var/lib/powerdns/zones/{zone}'] = { 'content_type': 'mako', 'context': { - 'NAMESERVERS': '\n'.join(sorted({f'@ IN NS {ns}.' for ns in nameservers})), - 'SERIAL': serial, + 'HEADER': HEADER + f"\n$ORIGIN {zone}.", 'metadata_records': node.metadata.get(f'powerdns/bind-zones/{zone}/records', []), }, 'source': f'bind-zones/{zone}', diff --git a/data/powerdns/files/bind-zones/_mail_NULL b/data/powerdns/files/bind-zones/_mail_NULL new file mode 100644 index 0000000..907abc8 --- /dev/null +++ b/data/powerdns/files/bind-zones/_mail_NULL @@ -0,0 +1,2 @@ +@ IN TXT "v=spf1 -all" +_dmarc IN TXT "v=DMARC1; p=reject" diff --git a/data/powerdns/files/bind-zones/_mail_carlene b/data/powerdns/files/bind-zones/_mail_carlene new file mode 100644 index 0000000..7a8e210 --- /dev/null +++ b/data/powerdns/files/bind-zones/_mail_carlene @@ -0,0 +1,11 @@ +@ IN TXT "v=spf1 mx -all" +@ IN MX 10 mail.franzi.business. +_dmarc IN TXT "v=DMARC1; p=quarantine; rua=mailto:dmarc@kunbox.net; ruf=mailto:dmarc@kunbox.net; fo=0:d:s; adkim=s; aspf=s" +_mta-sts IN TXT "v=STSv1;id=20201111;" +_smtp._tls IN TXT "v=TLSRPTv1;rua=mailto:tlsrpt@kunbox.net" + +mta-sts IN CNAME carlene.kunbox.net. + +2019._domainkey IN TXT "v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwkg6UAcu3V98hal1UVf6yB0WT1CKDS0AK83CUlSP8bUwraPxkxK1nkQOUsmjbQs6a3FhdsKprMi32GeUaTVvZg81JIybPk3jNugfNWfSjs2TXPomYu+XD2pmmbR3cZlzC5NGR2nmBFt/P/S2ihPHj35KziiBIwK1TdvOi1M2+upCjK33Icco0ByCm0gJpD2O0cbqcBcUKqd6X440vYhNXH1ygp0e91P0iRnvS9sg6yD0xjD8kD6j/8GfxBY+9bpU3EvDoBgyJSbjw5b6PUVJbKMXzw1NIRNj0SXKs5BakjS8+7u62vR11IPCYRwy+yr0rDT0tNegM7gStIIgoTpOoQIDAQAB" + +uo4anejdvvdw8bkne3kjiqavcqmj0416._domainkey IN TXT "v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnh5Ym9PO7r+wdOIKfopvHzn3KU3qT6IlCG/gvvbmIqoeFQfRbAe3gQmcG6RcLue55cJQGhI6y2r0lm59ZeoHR40aM+VabAOlplekM7xWmoXb/9vG2OZLIqAyF4I+7GQmTN6B9keBHp9SWtDUkI0B0G9neZ5MkXJP705M0duxritqQlb4YvCZwteHiyckKcg9aE9j+GF2EEawBoVDpoveoB3+wgde3lWEUjjwKFtXNXxuN354o6jgXgPNWtIEdPMLfK/o0CaCjZNlzaLTsTegY/+67hdHFqDmm8zXO9s+Xiyfq7CVq21t7wDhQ2W1agj+up6lH82FMh5rZNxJ6XB0yQIDAQAB" diff --git a/data/powerdns/files/bind-zones/_parked b/data/powerdns/files/bind-zones/_parked new file mode 100644 index 0000000..8331fc4 --- /dev/null +++ b/data/powerdns/files/bind-zones/_parked @@ -0,0 +1,3 @@ +${HEADER} + +<%include file="bind-zones/_mail_NULL" /> diff --git a/data/powerdns/files/bind-zones/afra.berlin b/data/powerdns/files/bind-zones/afra.berlin new file mode 100644 index 0000000..93ffc96 --- /dev/null +++ b/data/powerdns/files/bind-zones/afra.berlin @@ -0,0 +1,6 @@ +${HEADER} + +@ IN AAAA 2a0a:51c0:0:225::2 +@ IN A 193.135.9.29 + +<%include file="bind-zones/_mail_NULL" /> diff --git a/data/powerdns/files/bind-zones/die-brontosaurier-waren-es.org b/data/powerdns/files/bind-zones/die-brontosaurier-waren-es.org new file mode 120000 index 0000000..e0f69f8 --- /dev/null +++ b/data/powerdns/files/bind-zones/die-brontosaurier-waren-es.org @@ -0,0 +1 @@ +_parked \ No newline at end of file diff --git a/data/powerdns/files/bind-zones/emails.sexy b/data/powerdns/files/bind-zones/emails.sexy new file mode 120000 index 0000000..e0f69f8 --- /dev/null +++ b/data/powerdns/files/bind-zones/emails.sexy @@ -0,0 +1 @@ +_parked \ No newline at end of file diff --git a/data/powerdns/files/bind-zones/eskalation.jetzt b/data/powerdns/files/bind-zones/eskalation.jetzt new file mode 100644 index 0000000..8331fc4 --- /dev/null +++ b/data/powerdns/files/bind-zones/eskalation.jetzt @@ -0,0 +1,3 @@ +${HEADER} + +<%include file="bind-zones/_mail_NULL" /> diff --git a/data/powerdns/files/bind-zones/felix-kunsmann.de b/data/powerdns/files/bind-zones/felix-kunsmann.de new file mode 100644 index 0000000..42bac92 --- /dev/null +++ b/data/powerdns/files/bind-zones/felix-kunsmann.de @@ -0,0 +1,3 @@ +${HEADER} + +<%include file="bind-zones/_mail_carlene" /> diff --git a/data/powerdns/files/bind-zones/flauschehorn.sexy b/data/powerdns/files/bind-zones/flauschehorn.sexy new file mode 100644 index 0000000..4779fe4 --- /dev/null +++ b/data/powerdns/files/bind-zones/flauschehorn.sexy @@ -0,0 +1,8 @@ +${HEADER} + +@ IN AAAA 2a03:4000:4d:5e::1 +@ IN A 194.36.145.49 + +<%include file="bind-zones/_mail_carlene" /> + +_acme-challenge IN CNAME 63bc37c61bda3c1f4fa1f270f8890c7f89c24353.acme.ctu.cx. diff --git a/data/powerdns/files/bind-zones/franzi.business b/data/powerdns/files/bind-zones/franzi.business new file mode 100644 index 0000000..ce864a7 --- /dev/null +++ b/data/powerdns/files/bind-zones/franzi.business @@ -0,0 +1,29 @@ +${HEADER} + +@ IN AAAA 2a0a:51c0:0:225::2 +@ IN A 193.135.9.29 + +<%include file="bind-zones/_mail_carlene" /> + +_atproto IN TXT "did=did:plc:d762mg6wvvmpeu66zojntlof" +_token._dnswl IN TXT "gg3mbwjx9bbuo5osvh7oz6bc881wcmc" +_matrix._tcp IN SRV 10 10 443 matrix.franzi.business. + +; carlene +git IN CNAME carlene.kunbox.net. +irc IN CNAME carlene.kunbox.net. +mail IN CNAME carlene.kunbox.net. +matrix IN CNAME carlene.kunbox.net. +matrix-stickers IN CNAME carlene.kunbox.net. +netbox IN CNAME carlene.kunbox.net. +ntfy IN CNAME carlene.kunbox.net. +postfixadmin IN CNAME carlene.kunbox.net. +rss IN CNAME carlene.kunbox.net. +travelynx IN CNAME carlene.kunbox.net. + +; icinga2 +icinga IN CNAME icinga2.kunbox.net. +status IN CNAME icinga2.kunbox.net. + +; pretix +tickets IN CNAME franzi-business.cname.pretix.eu. diff --git a/data/powerdns/files/bind-zones/kunbox.net b/data/powerdns/files/bind-zones/kunbox.net index bb45655..2292b7d 100644 --- a/data/powerdns/files/bind-zones/kunbox.net +++ b/data/powerdns/files/bind-zones/kunbox.net @@ -1,16 +1,4 @@ -$TTL 60 -@ IN SOA ns-mephisto.kunbox.net. hostmaster.kunbox.net. ( - ${SERIAL} - 3600 - 600 - 86400 - 300 - ) - - -${NAMESERVERS} - -$ORIGIN kunbox.net. +${HEADER} ; ends up on carlene.kunbox.net @ IN A 193.135.9.29 diff --git a/data/powerdns/files/bind-zones/kunsi.scot b/data/powerdns/files/bind-zones/kunsi.scot new file mode 120000 index 0000000..e0f69f8 --- /dev/null +++ b/data/powerdns/files/bind-zones/kunsi.scot @@ -0,0 +1 @@ +_parked \ No newline at end of file diff --git a/data/powerdns/files/bind-zones/kunsitracker.de b/data/powerdns/files/bind-zones/kunsitracker.de new file mode 100644 index 0000000..9c641b6 --- /dev/null +++ b/data/powerdns/files/bind-zones/kunsitracker.de @@ -0,0 +1,6 @@ +${HEADER} + +@ IN AAAA 2a0a:51c0:0:225::2 +@ IN A 193.135.9.29 + +<%include file="bind-zones/_mail_carlene" /> diff --git a/data/powerdns/files/bind-zones/kunsmann.eu b/data/powerdns/files/bind-zones/kunsmann.eu new file mode 100644 index 0000000..f5b8acf --- /dev/null +++ b/data/powerdns/files/bind-zones/kunsmann.eu @@ -0,0 +1,14 @@ +${HEADER} + +@ IN AAAA 2a0a:51c0:0:225::2 +@ IN A 193.135.9.29 + +<%include file="bind-zones/_mail_carlene" /> + +@ IN TXT "google-site-verification=Xl-OBZpTL1maD2Qr8QmQ2aKRXZLnCmvddpFdrTT8L34" + +_token._dnswl IN TXT "5mx0rv9ru8s1zz4tf4xlt48osh09czmg" + +git IN CNAME git.franzi.business. +grafana IN CNAME influxdb.htz-cloud.kunbox.net. +influxdb IN CNAME influxdb.htz-cloud.kunbox.net. diff --git a/data/powerdns/files/bind-zones/raptor.events b/data/powerdns/files/bind-zones/raptor.events new file mode 120000 index 0000000..e0f69f8 --- /dev/null +++ b/data/powerdns/files/bind-zones/raptor.events @@ -0,0 +1 @@ +_parked \ No newline at end of file diff --git a/data/powerdns/files/bind-zones/trans-agenda.de b/data/powerdns/files/bind-zones/trans-agenda.de new file mode 120000 index 0000000..e0f69f8 --- /dev/null +++ b/data/powerdns/files/bind-zones/trans-agenda.de @@ -0,0 +1 @@ +_parked \ No newline at end of file diff --git a/data/powerdns/files/bind-zones/trans-agenda.eu b/data/powerdns/files/bind-zones/trans-agenda.eu new file mode 120000 index 0000000..e0f69f8 --- /dev/null +++ b/data/powerdns/files/bind-zones/trans-agenda.eu @@ -0,0 +1 @@ +_parked \ No newline at end of file diff --git a/data/powerdns/files/bind-zones/warnochwas.de b/data/powerdns/files/bind-zones/warnochwas.de new file mode 100644 index 0000000..9c641b6 --- /dev/null +++ b/data/powerdns/files/bind-zones/warnochwas.de @@ -0,0 +1,6 @@ +${HEADER} + +@ IN AAAA 2a0a:51c0:0:225::2 +@ IN A 193.135.9.29 + +<%include file="bind-zones/_mail_carlene" /> diff --git a/data/powerdns/files/bind-zones/winkeeinhorn.de b/data/powerdns/files/bind-zones/winkeeinhorn.de new file mode 120000 index 0000000..e0f69f8 --- /dev/null +++ b/data/powerdns/files/bind-zones/winkeeinhorn.de @@ -0,0 +1 @@ +_parked \ No newline at end of file