bundles/netbox: reindex must be lazy

This reverts commit 493dc91e0d.

Jenkinsfile

@@ -25,15 +25,6 @@ pipeline {
                         """
                     }
                 }
-                stage('syntax checking using isort') {
-                    steps {
-                        sh """
-                            . venv/bin/activate
-
-                            isort --check .
-                        """
-                    }
-                }
                 stage('config and metadata determinism') {
                     steps {
                         sh """

PORT_MAP.md

@@ -30,13 +30,13 @@ Rule of thumb: keep ports below 10000 free for stuff that reserves ports.
 | 20010       | mautrix-telegram     | Bridge |
 | 20020       | mautrix-whatsapp     | Bridge |
 | 20030       | matrix-dimension     | Matrix Integrations Manager|
+| 20070       | matrix-synapse       | sliding-sync |
 | 20080       | matrix-synapse       | client, federation |
 | 20081       | matrix-synapse       | prometheus metrics |
 | 20090       | matrix-media-repo    | media_repo |
 | 20090       | matrix-media-repo    | prometheus metrics |
-| 21000       | pleroma              | pleroma |
 | 21010       | grafana              | grafana |
-| 22000       | gitea                | forgejo |
+| 22000       | forgejo              | forgejo |
 | 22010       | jenkins-ci           | Jenkins CI |
 | 22020       | travelynx            | Travelynx Web |
 | 22030       | octoprint            | OctoPrint Web Interface |
@@ -45,6 +45,9 @@ Rule of thumb: keep ports below 10000 free for stuff that reserves ports.
 | 22060       | pretalx              | gunicorn |
 | 22070       | paperless-ng         | gunicorn |
 | 22080       | netbox               | gunicorn |
+| 22090       | jugendhackt_tools    | gunicorn |
+| 22100       | powerdnsadmin        | gunicorn |
+| 22110       | icinga2-statuspage   | gunicorn |
 | 22999       | nginx                | stub_status |
 | 22100       | ntfy                 | http |
 

README.md

@@ -7,3 +7,16 @@ onto shared webhosting.
 
 `bw test` runs according to Jenkinsfile after every commit.
 [![Build Status](https://jenkins.franzi.business/buildStatus/icon?job=kunsi%2Fbundlewrap%2Fmain)](https://jenkins.franzi.business/job/kunsi/job/bundlewrap/job/main/)
+
+## automatix
+
+Ensure you set `bundlewrap: true` in your `~/.automatix.cfg.yaml`.
+
+## system naming
+
+All systems should be named after their location and use.
+
+For example, influxdb hosted at hetzner cloud will be `htz-cloud.influxdb`.
+
+The only exception to this are name servers, they are named after [demons
+in fiction](https://en.wikipedia.org/wiki/List_of_demons_in_fiction).

automatix/upgrade_debian_bullseye.yaml

@@ -0,0 +1,45 @@
+name: Upgrade to debian bullseye
+systems:
+    node: foonode
+
+always:
+    - has_zfs=python: NODES.node.has_bundle('zfs')
+
+pipeline:
+    - manual: "set icinga2 downtime: https://icinga.franzi.business/monitoring/host/schedule-downtime?host={SYSTEMS.node}"
+
+    # apply first so we only see the upgrade changes later
+    - local: bw apply {SYSTEMS.node}
+    - manual: update debian version in node groups
+    - local: "bw apply -o bundle:apt -s symlink:/usr/bin/python pkg_apt: -- {SYSTEMS.node}"
+
+    # double time!
+    - remote@node: DEBIAN_FRONTEND=noninteractive apt-get -y -q -o Dpkg::Options::=--force-confold dist-upgrade
+    - remote@node: DEBIAN_FRONTEND=noninteractive apt-get -y -q -o Dpkg::Options::=--force-confold dist-upgrade
+
+    # reboot into bullseye
+    - remote@node: systemctl reboot
+    - local: |
+        exit=1
+        while [[ $exit -ne 0 ]];
+        do
+            sleep 1
+            ssh {SYSTEMS.node} true
+            exit=$?
+        done
+
+    # fix zfs and reboot again
+    - has_zfs?remote@node: zpool import tank -f
+    - has_zfs?remote@node: zpool upgrade -a
+    - has_zfs?remote@node: systemctl reboot
+    - has_zfs?local: |
+        exit=1
+        while [[ $exit -ne 0 ]];
+        do
+            sleep 1
+            ssh {SYSTEMS.node} true
+            exit=$?
+        done
+
+    # final apply
+    - local: bw apply {SYSTEMS.node}

bundles/apt/files/deb822-sources

@@ -0,0 +1,9 @@
+% for uri in sorted(uris):
+Types: ${' '.join(sorted(data.get('types', {'deb'})))}
+URIs: ${uri}
+Suites: ${os_release}
+Components: ${' '.join(sorted(data.get('components', {'main'})))}
+Architectures: ${' '.join(sorted(data.get('architectures', {'amd64'})))}
+Signed-By: /etc/apt/trusted.gpg.d/${name}.list.asc
+
+% endfor

bundles/apt/files/do-unattended-upgrades

@@ -6,10 +6,10 @@ apt-get update
 
 DEBIAN_FRONTEND=noninteractive apt-get -y -q -o Dpkg::Options::=--force-confold dist-upgrade
 
-DEBIAN_FRONTEND=noninteractive apt-get -y -q autoclean
-
 DEBIAN_FRONTEND=noninteractive apt-get -y -q autoremove
 
+DEBIAN_FRONTEND=noninteractive apt-get -y -q clean
+
 % if clean_old_kernels:
 existing=$(dpkg --get-selections | grep -E '^linux-(image|headers)-[0-9]' || true)
 

bundles/apt/files/sources.list-debian-bookworm

@@ -0,0 +1,3 @@
+deb http://deb.debian.org/debian/ bookworm main non-free contrib non-free-firmware
+deb http://security.debian.org/debian-security bookworm-security main contrib non-free
+deb http://deb.debian.org/debian/ bookworm-updates main contrib non-free

bundles/apt/files/upgrade-and-reboot

@@ -46,10 +46,6 @@ fi
 
 if [[ -f /var/run/reboot-required ]] && [[ "$auto_reboot_enabled" == "True" ]]
 then
-    if [[ -n "$reboot_mail_to" ]]
-    then
-        date | mail -s "SYSREBOOTNOW $nodename" "$reboot_mail_to"
-    fi
     systemctl reboot
 fi
 

bundles/apt/files/upgrade-and-reboot.conf

@@ -1,3 +1,2 @@
 nodename="${node.name}"
-reboot_mail_to="${node.metadata.get('apt/unattended-upgrades/reboot_mail_to', '')}"
 auto_reboot_enabled="${node.metadata.get('apt/unattended-upgrades/reboot_enabled', True)}"

bundles/apt/items.py

@@ -4,6 +4,7 @@ supported_os = {
     'debian': {
         10: 'buster',
         11: 'bullseye',
+        12: 'bookworm',
         99: 'unstable',
     },
     'raspbian': {
@@ -113,7 +114,7 @@ pkg_apt = {
     'mtr': {},
     'ncdu': {},
     'ncurses-term': {},
-    'netcat': {},
+    'netcat-openbsd': {},
     'nmap': {},
     'python3': {},
     'python3-dev': {},
@@ -152,6 +153,9 @@ pkg_apt = {
     'popularity-contest': {
         'installed': False,
     },
+    'python3-packaging': {
+        'installed': False,
+    },
     'unattended-upgrades': {
         'installed': False,
     },
@@ -168,21 +172,44 @@ if node.os_version[0] >= 11:
     }
 
 for name, data in node.metadata.get('apt/repos', {}).items():
-    files['/etc/apt/sources.list.d/{}.list'.format(name)] = {
-        'content_type': 'mako',
-        'content': ("\n".join(sorted(data['items']))).format(
-            os=node.os,
-            os_release=supported_os[node.os][node.os_version[0]],
-        ),
-        'triggers': {
-            'action:apt_update',
-        },
-    }
+    if 'items' in data:
+        files['/etc/apt/sources.list.d/{}.list'.format(name)] = {
+            'content_type': 'mako',
+            'content': ("\n".join(sorted(data['items']))).format(
+                os=node.os,
+                os_release=supported_os[node.os][node.os_version[0]],
+            ),
+            'triggers': {
+                'action:apt_update',
+            },
+        }
+    elif 'uris' in data:
+        uris = {
+            x.format(
+                os=node.os,
+                os_release=supported_os[node.os][node.os_version[0]],
+            ) for x in data['uris']
+        }
+
+        files['/etc/apt/sources.list.d/{}.sources'.format(name)] = {
+            'source': 'deb822-sources',
+            'content_type': 'mako',
+            'context': {
+                'data': data,
+                'name': name,
+                'os_release': supported_os[node.os][node.os_version[0]],
+                'uris': uris,
+            },
+            'triggers': {
+                'action:apt_update',
+            },
+        }
 
     if data.get('install_gpg_key', True):
-        files['/etc/apt/sources.list.d/{}.list'.format(name)]['needs'] = {
-            'file:/etc/apt/trusted.gpg.d/{}.list.asc'.format(name),
-        }
+        if 'items' in data:
+            files['/etc/apt/sources.list.d/{}.list'.format(name)]['needs'] = {
+                'file:/etc/apt/trusted.gpg.d/{}.list.asc'.format(name),
+            }
 
         files['/etc/apt/trusted.gpg.d/{}.list.asc'.format(name)] = {
             'source': 'gpg-keys/{}.asc'.format(name),

bundles/apt/metadata.py

@@ -24,13 +24,18 @@ def patchday(metadata):
     day = metadata.get('apt/unattended-upgrades/day')
     hour = metadata.get('apt/unattended-upgrades/hour')
 
+    spread = metadata.get('apt/unattended-upgrades/spread_in_group', None)
+    if spread is not None:
+        spread_nodes = sorted(repo.nodes_in_group(spread))
+        day += spread_nodes.index(node)
+
     return {
         'cron': {
             'jobs': {
                 'upgrade-and-reboot': '{minute} {hour} * * {day}    root    /usr/local/sbin/upgrade-and-reboot'.format(
                     minute=node.magic_number % 30,
                     hour=hour,
-                    day=day,
+                    day=day%7,
                 ),
             },
         },

bundles/arch-with-gui/files/50-network.conf

@@ -0,0 +1,5 @@
+context.exec = [
+    { path = "pactl" args = "load-module module-native-protocol-tcp" }
+    { path = "pactl" args = "load-module module-zeroconf-discover" }
+    { path = "pactl" args = "load-module module-zeroconf-publish" }
+]

bundles/arch-with-gui/items.py

@@ -44,6 +44,11 @@ directories = {
 }
 
 svc_systemd = {
+    'avahi-daemon': {
+        'needs': {
+            'pkg_pacman:avahi',
+        },
+    },
     'sddm': {
         'needs': {
             'pkg_pacman:sddm',
@@ -61,6 +66,8 @@ git_deploy = {
     },
 }
 
+files['/etc/pipewire/pipewire-pulse.conf.d/50-network.conf'] = {}
+
 for filename in listdir(join(repo.path, 'data', 'arch-with-gui', 'files', 'fonts')):
     if filename.startswith('.'):
         continue

bundles/arch-with-gui/metadata.py

@@ -9,6 +9,14 @@ defaults = {
     'icinga_options': {
         'exclude_from_monitoring': True,
     },
+    'nftables': {
+        'input': {
+            '50-avahi': {
+                'udp dport 5353 accept',
+                'udp sport 5353 accept',
+            },
+        },
+    },
     'pacman': {
         'packages': {
             # fonts
@@ -23,6 +31,7 @@ defaults = {
             'sddm': {},
 
             # networking
+            'avahi': {},
             'netctl': {},
             'rfkill': {},
             'wpa_supplicant': {},
@@ -45,6 +54,7 @@ defaults = {
             'pipewire': {},
             'pipewire-jack': {},
             'pipewire-pulse': {},
+            'pipewire-zeroconf': {},
             'qpwgraph': {},
 
             # window management

bundles/backup-server/metadata.py

@@ -35,8 +35,15 @@ def get_my_clients(metadata):
             continue
 
         my_clients[rnode.name] = {
-            'user': rnode.metadata.get('backup-client/user-name'),
+            'exclude_from_monitoring': rnode.metadata.get(
+                'backup-client/exclude_from_monitoring',
+                rnode.metadata.get(
+                    'icinga_options/exclude_from_monitoring',
+                    False,
+                ),
+            ),
             'one_backup_every_hours': rnode.metadata.get('backup-client/one_backup_every_hours', 24),
+            'user': rnode.metadata.get('backup-client/user-name'),
             'retain': {
                 'daily': rnode.metadata.get('backups/retain/daily', retain_defaults['daily']),
                 'weekly': rnode.metadata.get('backups/retain/weekly', retain_defaults['weekly']),

bundles/basic/files/htoprc

@@ -32,8 +32,8 @@ account_guest_in_cpu_meter=0
 color_scheme=0
 enable_mouse=0
 delay=10
-left_meters=Tasks LoadAverage Uptime Memory CPU LeftCPUs CPU
+left_meters=Tasks LoadAverage Uptime Memory CPU LeftCPUs2 CPU
 left_meter_modes=2 2 2 1 1 1 2
-right_meters=Hostname CPU RightCPUs
+right_meters=Hostname CPU RightCPUs2
 right_meter_modes=2 3 1
 hide_function_bar=0

bundles/basic/items.py

@@ -29,8 +29,19 @@ files = {
     },
 }
 
+if node.has_any_bundle([
+    'dovecot',
+    'nginx',
+    'postfix',
+]):
+    actions['generate-dhparam'] = {
+        'command': 'openssl dhparam -out /etc/ssl/certs/dhparam.pem 2048',
+        'unless': 'test -f /etc/ssl/certs/dhparam.pem',
+    }
+
+
 locale_needs = set()
-for locale in sorted(node.metadata['locale']['installed']):
+for locale in sorted(node.metadata.get('locale/installed')):
     actions[f'ensure_locale_{locale}_is_enabled'] = {
         'command': f"sed -i '/{locale}/s/^# *//g' /etc/locale.gen",
         'unless': f"grep -e '^{locale}' /etc/locale.gen",
@@ -41,17 +52,15 @@ for locale in sorted(node.metadata['locale']['installed']):
     }
     locale_needs = {f'action:ensure_locale_{locale}_is_enabled'}
 
-actions = {
-    'locale-gen': {
-        'triggered': True,
-        'command': 'locale-gen',
-    },
+actions['locale-gen'] = {
+    'triggered': True,
+    'command': 'locale-gen',
 }
 
 description = []
 
 if not node.metadata.get('icinga_options/exclude_from_monitoring', False):
-    description.append('icingaweb2: https://icinga.kunsmann.eu/monitoring/host/show?host={}'.format(node.name))
+    description.append('icingaweb2: https://icinga.franzi.business/monitoring/host/show?host={}'.format(node.name))
 
 if node.has_bundle('telegraf'):
     description.append('Grafana: https://grafana.kunsmann.eu/d/{}'.format(UUID(int=node.magic_number).hex[:10]))

bundles/bird/files/bird.conf

@@ -19,7 +19,9 @@ protocol static {
     ipv4;
 
 % for route in sorted(node.metadata.get('bird/static_routes', set())):
-    route ${route} via ${node.metadata.get('bird/my_ip')};
+%  for name, config in sorted(node.metadata.get('bird/bgp_neighbors', {}).items()):
+    route ${route} via ${config['local_ip']};
+%  endfor
 % endfor
 }
 % endif

bundles/bird/metadata.py

@@ -24,7 +24,7 @@ defaults = {
     },
     'sysctl': {
         'options': {
-            'net.ipv4.ip_forward': '1',
+            'net.ipv4.conf.all.forwarding': '1',
             'net.ipv6.conf.all.forwarding': '1',
         },
     },
@@ -43,6 +43,9 @@ def neighbor_info_from_wireguard(metadata):
         except NoSuchNode:
             continue
 
+        if not rnode.has_bundle('bird'):
+            continue
+
         neighbors[name] = {
             'local_ip': config['my_ip'],
             'local_as': my_as,
@@ -62,7 +65,10 @@ def neighbor_info_from_wireguard(metadata):
 )
 def my_ip(metadata):
     if node.has_bundle('wireguard'):
-        my_ip = sorted(metadata.get('interfaces/wg0/ips'))[0].split('/')[0]
+        wg_ifaces = sorted({iface for iface in metadata.get('interfaces').keys() if iface.startswith('wg_')})
+        if not wg_ifaces:
+            return {}
+        my_ip = sorted(metadata.get(f'interfaces/{wg_ifaces[0]}/ips'))[0].split('/')[0]
     else:
         my_ip = str(sorted(repo.libs.tools.resolve_identifier(repo, node.name))[0])
 
@@ -84,7 +90,7 @@ def firewall(metadata):
     return {
         'firewall': {
             'port_rules': {
-                '179': atomic(sources),
+                '179/tcp': atomic(sources),
             },
         },
     }

bundles/c3voc-addons/items.py

@@ -1,5 +1,22 @@
 from bundlewrap.exceptions import BundleError
 
+supported_os = {
+    'debian': {
+        10: 'buster',
+        11: 'bullseye',
+        12: 'bookworm',
+        99: 'unstable',
+    },
+    'raspbian': {
+        10: 'buster',
+    },
+}
+
+try:
+    supported_os[node.os][node.os_version[0]]
+except (KeyError, IndexError):
+    raise BundleError(f'{node.name}: OS {node.os} {node.os_version} is not supported by bundle:apt')
+
 CONFLICTING_BUNDLES = {
     'apt',
     'nginx',
@@ -57,6 +74,14 @@ actions = {
             'svc_systemd:',
         },
     },
+    'apt_update': {
+        'command': 'apt-get update',
+        'needed_by': {
+            'pkg_apt:',
+        },
+        'triggered': True,
+        'cascade_skip': False,
+    },
 }
 
 directories = {
@@ -92,6 +117,30 @@ files = {
     },
 }
 
+for name, data in node.metadata.get('apt/repos', {}).items():
+    files['/etc/apt/sources.list.d/{}.list'.format(name)] = {
+        'content_type': 'mako',
+        'content': ("\n".join(sorted(data['items']))).format(
+            os=node.os,
+            os_release=supported_os[node.os][node.os_version[0]],
+        ),
+        'triggers': {
+            'action:apt_update',
+        },
+    }
+
+    if data.get('install_gpg_key', True):
+        files['/etc/apt/sources.list.d/{}.list'.format(name)]['needs'] = {
+            'file:/etc/apt/trusted.gpg.d/{}.list.asc'.format(name),
+        }
+
+        files['/etc/apt/trusted.gpg.d/{}.list.asc'.format(name)] = {
+            'source': 'gpg-keys/{}.asc'.format(name),
+            'triggers': {
+                'action:apt_update',
+            },
+        }
+
 for crontab, content in node.metadata.get('cron/jobs', {}).items():
     files['/etc/cron.d/{}'.format(crontab)] = {
         'source': 'cron_template',

bundles/cron/items.py

@@ -17,7 +17,7 @@ files = {
 directories = {
     '/etc/cron.d': {
         'purge': True,
-        'needs': {
+        'after': {
             'pkg_apt:',
         },
     },

bundles/dhcpd/files/dhcpd.conf

@@ -1,36 +0,0 @@
-<%
-    import re
-    from ipaddress import ip_network
-%>
-ddns-update-style none;
-
-authoritative;
-
-% for interface, subnet in sorted(dhcp_config.get('subnets', {}).items()):
-<%
-    network = ip_network(subnet['subnet'])
-%>
-# interface ${interface} provides ${subnet['subnet']}
-subnet ${network.network_address} netmask ${network.netmask} {
-%   if subnet.get('range_lower', None) and subnet.get('range_higher', None):
-    range ${subnet['range_lower']} ${subnet['range_higher']};
-%   endif
-    interface "${interface}";
-    default-lease-time ${subnet.get('default-lease-time', 600)};
-    max-lease-time ${subnet.get('max-lease-time', 3600)};
-%   for option, value in sorted(subnet.get('options', {}).items()):
-%     if re.match('([^0-9\.,\ ])', value):
-    option ${option} "${value}";
-%     else:
-    option ${option} ${value};
-%     endif
-%   endfor
-}
-% endfor
-
-% for identifier, allocation in dhcp_config.get('fixed_allocations', {}).items():
-host ${identifier} {
-    hardware ethernet ${allocation['mac']};
-    fixed-address ${allocation['ipv4']};
-}
-% endfor

bundles/dhcpd/files/isc-dhcp-server

@@ -1,18 +0,0 @@
-# Defaults for isc-dhcp-server (sourced by /etc/init.d/isc-dhcp-server)
-
-# Path to dhcpd's config file (default: /etc/dhcp/dhcpd.conf).
-#DHCPDv4_CONF=/etc/dhcp/dhcpd.conf
-#DHCPDv6_CONF=/etc/dhcp/dhcpd6.conf
-
-# Path to dhcpd's PID file (default: /var/run/dhcpd.pid).
-#DHCPDv4_PID=/var/run/dhcpd.pid
-#DHCPDv6_PID=/var/run/dhcpd6.pid
-
-# Additional options to start dhcpd with.
-#	Don't use options -cf or -pf here; use DHCPD_CONF/ DHCPD_PID instead
-#OPTIONS=""
-
-# On what interfaces should the DHCP server (dhcpd) serve DHCP requests?
-#	Separate multiple interfaces with spaces, e.g. "eth0 eth1".
-INTERFACESv4="${' '.join(sorted(node.metadata.get('dhcpd/subnets', {})))}"
-INTERFACESv6=""

bundles/dhcpd/items.py

@@ -1,41 +0,0 @@
-files = {
-    '/etc/dhcp/dhcpd.conf': {
-        'content_type': 'mako',
-        'context': {
-            'dhcp_config': node.metadata['dhcpd'],
-        },
-        'needs': {
-            'pkg_apt:isc-dhcp-server'
-        },
-        'triggers': {
-            'svc_systemd:isc-dhcp-server:restart',
-        },
-    },
-    '/etc/default/isc-dhcp-server': {
-        'content_type': 'mako',
-        'needs': {
-            'pkg_apt:isc-dhcp-server'
-        },
-        'triggers': {
-            'svc_systemd:isc-dhcp-server:restart',
-        },
-    },
-}
-
-actions = {
-    # needed for dhcp-lease-list
-    'dhcpd_download_oui.txt': {
-        'command': 'wget http://standards-oui.ieee.org/oui.txt -O /usr/local/etc/oui.txt',
-        'unless': 'test -f /usr/local/etc/oui.txt',
-    },
-}
-
-svc_systemd = {
-    'isc-dhcp-server': {
-        'needs': {
-            'pkg_apt:isc-dhcp-server',
-            'file:/etc/dhcp/dhcpd.conf',
-            'file:/etc/default/isc-dhcp-server',
-        },
-    },
-}

bundles/dhcpd/metadata.py

@@ -1,54 +0,0 @@
-defaults = {
-    'apt': {
-        'packages': {
-            'isc-dhcp-server': {},
-        },
-    },
-    'bash_aliases': {
-        'leases': 'sudo dhcp-lease-list | tail -n +4 | sort -k 2,2',
-    },
-}
-
-
-@metadata_reactor.provides(
-    'dhcpd/fixed_allocations',
-)
-def get_static_allocations(metadata):
-    allocations = {}
-    for rnode in repo.nodes:
-        if rnode.metadata.get('location', '') != metadata.get('location', ''):
-            continue
-
-        for iface_name, iface_config in rnode.metadata.get('interfaces', {}).items():
-            if iface_config.get('dhcp', False):
-                try:
-                    allocations[f'{rnode.name}_{iface_name}'] = {
-                        'ipv4': sorted(iface_config['ips'])[0],
-                        'mac': iface_config['mac'],
-                    }
-                except KeyError:
-                    pass
-
-    return {
-        'dhcpd': {
-            'fixed_allocations': allocations,
-        }
-    }
-
-
-@metadata_reactor.provides(
-    'nftables/rules/10-dhcpd',
-)
-def nftables(metadata):
-    rules = set()
-    for iface in node.metadata.get('dhcpd/subnets', {}):
-        rules.add(f'inet filter input udp dport {{ 67, 68 }} iif {iface} accept')
-
-    return {
-        'nftables': {
-            'rules': {
-                # can't use port_rules here, because we're generating interface based rules.
-                '10-dhcpd': sorted(rules),
-            },
-        }
-    }

bundles/dovecot/files/dovecot-sql.conf

@@ -3,3 +3,4 @@ driver = pgsql
 default_pass_scheme = MD5-CRYPT
 password_query = SELECT username as user, password FROM mailbox WHERE username = '%u' AND active = true
 user_query = SELECT '/var/mail/vmail/' || maildir as home, 65534 as uid, 65534 as gid FROM mailbox WHERE username = '%u' AND active = true
+iterate_query = SELECT username as user FROM mailbox WHERE active = true

bundles/dovecot/files/dovecot.conf

@@ -28,19 +28,19 @@ namespace inbox {
 mail_location = maildir:/var/mail/vmail/%d/%n
 protocols = imap lmtp sieve
 
-ssl = yes
+ssl = required
 ssl_cert = </var/lib/dehydrated/certs/${node.metadata.get('postfix/myhostname', node.metadata['hostname'])}/fullchain.pem
 ssl_key =  </var/lib/dehydrated/certs/${node.metadata.get('postfix/myhostname', node.metadata['hostname'])}/privkey.pem
-ssl_dh = </etc/dovecot/ssl/dhparam.pem
+ssl_dh = </etc/ssl/certs/dhparam.pem
 ssl_min_protocol = TLSv1.2
-ssl_cipher_list = EECDH+AESGCM:EDH+AESGCM
-ssl_prefer_server_ciphers = yes
+ssl_cipher_list = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305
+ssl_prefer_server_ciphers = no
 
 login_greeting = IMAPd ready
 auth_mechanisms = plain login
 first_valid_uid = 65534
 disable_plaintext_auth = yes
-mail_plugins = $mail_plugins zlib old_stats
+mail_plugins = $mail_plugins zlib old_stats fts fts_xapian
 
 plugin {
     zlib_save_level = 6
@@ -56,6 +56,15 @@ plugin {
     old_stats_refresh = 30 secs
     old_stats_track_cmds = yes
 
+    fts = xapian
+    fts_xapian = partial=3 full=20
+
+    fts_autoindex = yes
+    fts_enforced = yes
+
+    # Index attachements
+    fts_decoder = decode2text
+
 % if node.has_bundle('rspamd'):
     sieve_before = /var/mail/vmail/sieve/global/spam-global.sieve
 
@@ -86,14 +95,19 @@ service auth {
     }
 }
 
-service lmtp {
-    unix_listener /var/spool/postfix/private/dovecot-lmtp {
-        group = postfix
-        mode = 0600
-        user = postfix
+service decode2text {
+    executable = script /usr/lib/dovecot/decode2text.sh
+    user = dovecot
+    unix_listener decode2text {
+        mode = 0666
     }
 }
 
+service indexer-worker {
+    vsz_limit = 0
+    process_limit = 0
+}
+
 service imap {
     executable = imap
 }
@@ -104,6 +118,14 @@ service imap-login {
     vsz_limit = 64M
 }
 
+service lmtp {
+    unix_listener /var/spool/postfix/private/dovecot-lmtp {
+        group = postfix
+        mode = 0600
+        user = postfix
+    }
+}
+
 service managesieve-login {
     inet_listener sieve {
         port = 4190

bundles/dovecot/items.py

@@ -2,10 +2,6 @@
 # by this bundle
 repo.libs.tools.require_bundle(node, 'postfix')
 
-directories = {
-    '/etc/dovecot/ssl': {},
-}
-
 files = {
     '/etc/dovecot/dovecot.conf': {
         'content_type': 'mako',
@@ -49,25 +45,17 @@ files = {
     },
 }
 
-actions = {
-    'dovecot_generate_dhparam': {
-        'command': 'openssl dhparam -out /etc/dovecot/ssl/dhparam.pem 2048',
-        'unless': 'test -f /etc/dovecot/ssl/dhparam.pem',
-        'cascade_skip': False,
-        'needs': {
-            'directory:/etc/dovecot/ssl',
-            'pkg_apt:'
-        },
-        'triggers': {
-            'svc_systemd:dovecot:restart',
-        },
+symlinks['/usr/lib/dovecot/decode2text.sh'] = {
+    'target': '/usr/share/doc/dovecot-core/examples/decode2text.sh',
+    'before': {
+        'svc_systemd:dovecot',
     },
 }
 
 svc_systemd = {
     'dovecot': {
         'needs': {
-            'action:dovecot_generate_dhparam',
+            'action:generate-dhparam',
             'file:/etc/dovecot/dovecot.conf',
             'file:/etc/dovecot/dovecot-sql.conf',
         },

bundles/dovecot/metadata.py

@@ -3,6 +3,7 @@ from bundlewrap.metadata import atomic
 defaults = {
     'apt': {
         'packages': {
+            'dovecot-fts-xapian': {},
             'dovecot-imapd': {},
             'dovecot-lmtpd': {},
             'dovecot-managesieved': {},
@@ -35,6 +36,16 @@ defaults = {
             'dovecot',
         },
     },
+    'systemd-timers': {
+        'timers': {
+            'dovecot_fts_optimize': {
+                'command': [
+                    '/usr/bin/doveadm fts optimize -A',
+                ],
+                'when': '02:{}:00'.format(node.magic_number % 60),
+            },
+        },
+    },
 }
 
 if node.has_bundle('postfixadmin'):
@@ -76,19 +87,19 @@ def import_database_settings_from_postfixadmin(metadata):
 
 
 @metadata_reactor.provides(
-    'firewall/port_rules/143',
-    'firewall/port_rules/993',
-    'firewall/port_rules/4190',
+    'firewall/port_rules',
+    'firewall/port_rules',
+    'firewall/port_rules',
 )
 def firewall(metadata):
     return {
         'firewall': {
             'port_rules': {
                 # imap(s)
-                '143': atomic(metadata.get('dovecot/restrict-to', {'*'})),
-                '993': atomic(metadata.get('dovecot/restrict-to', {'*'})),
+                '143/tcp': atomic(metadata.get('dovecot/restrict-to', {'*'})),
+                '993/tcp': atomic(metadata.get('dovecot/restrict-to', {'*'})),
                 # managesieve
-                '4190': atomic(metadata.get('dovecot/restrict-to', {'*'})),
+                '4190/tcp': atomic(metadata.get('dovecot/restrict-to', {'*'})),
             },
         },
     }

bundles/element-web/items.py

@@ -8,7 +8,7 @@ directories = {
 
 git_deploy = {
     '/opt/element-web': {
-        'rev': node.metadata['element-web']['version'],
+        'rev': node.metadata.get('element-web/version'),
         'repo': 'https://github.com/vector-im/element-web.git',
         'triggers': {
             'action:element-web_yarn',
@@ -18,22 +18,16 @@ git_deploy = {
 
 files = {
     '/opt/element-web/webapp/config.json': {
-        'content': metadata_to_json(node.metadata['element-web']['config']),
+        'content': metadata_to_json(node.metadata.get('element-web/config')),
         'needs': {
             'action:element-web_yarn',
         },
     },
 }
 
-extra_install_cmds = []
-if node.metadata.get('nodejs/version') >= 17:
-    # TODO verify this is still needed when upgrading to 1.12
-    extra_install_cmds.append('export NODE_OPTIONS=--openssl-legacy-provider')
-
 actions = {
     'element-web_yarn': {
         'command': ' && '.join([
-            *extra_install_cmds,
             'cd /opt/element-web',
             'yarn install --pure-lockfile --ignore-scripts',
             'yarn build',

bundles/forgejo/files/app.ini

@@ -1,9 +1,10 @@
 APP_NAME = ${app_name}
 RUN_USER = git
 RUN_MODE = prod
+WORK_PATH = /var/lib/forgejo
 
 [repository]
-ROOT = /home/git/gitea-repositories
+ROOT = /var/lib/forgejo/repositories
 MAX_CREATION_LIMIT = 0
 DEFAULT_BRANCH = main
 

bundles/forgejo/files/forgejo.service

@@ -5,14 +5,13 @@ After=network.target
 Requires=postgresql.service
 
 [Service]
-RestartSec=2s
+RestartSec=10
 Type=simple
 User=git
 Group=git
-WorkingDirectory=/var/lib/gitea/
-ExecStart=/usr/local/bin/gitea web -c /etc/gitea/app.ini
+WorkingDirectory=/var/lib/forgejo
+ExecStart=/usr/local/bin/forgejo web -c /etc/forgejo/app.ini
 Restart=always
-Environment=USER=git HOME=/home/git GITEA_WORK_DIR=/var/lib/gitea
 
 [Install]
 WantedBy=multi-user.target

bundles/forgejo/items.py

@@ -0,0 +1,65 @@
+users = {
+    'git': {
+        'home': '/var/lib/forgejo',
+    },
+}
+
+directories = {
+    '/var/lib/forgejo/.ssh': {
+        'mode': '0700',
+        'owner': 'git',
+        'group': 'git',
+    },
+    '/var/lib/forgejo': {
+        'owner': 'git',
+        'mode': '0700',
+        'triggers': {
+            'svc_systemd:forgejo:restart',
+        },
+    },
+}
+
+files = {
+    '/usr/local/lib/systemd/system/forgejo.service': {
+        'content_type': 'mako',
+        'context': node.metadata.get('forgejo'),
+        'triggers': {
+            'action:systemd-reload',
+            'svc_systemd:forgejo:restart',
+        },
+    },
+    '/etc/forgejo/app.ini': {
+        'content_type': 'mako',
+        'context': node.metadata.get('forgejo'),
+        'triggers': {
+            'svc_systemd:forgejo:restart',
+        },
+    },
+    '/usr/local/bin/forgejo': {
+        'content_type': 'download',
+        'source': 'https://codeberg.org/forgejo/forgejo/releases/download/v{0}/forgejo-{0}-linux-amd64'.format(node.metadata.get('forgejo/version')),
+        'content_hash': node.metadata.get('forgejo/sha1', None),
+        'mode': '0755',
+        'triggers': {
+            'svc_systemd:forgejo:restart',
+        },
+    },
+}
+
+if node.metadata.get('forgejo/install_ssh_key', False):
+    files['/var/lib/forgejo/.ssh/id_ed25519'] = {
+        'content': repo.vault.decrypt_file(f'forgejo/files/ssh-keys/{node.name}.key.vault'),
+        'mode': '0600',
+        'owner': 'git',
+        'group': 'git',
+    }
+
+svc_systemd = {
+    'forgejo': {
+        'needs': {
+            'file:/etc/forgejo/app.ini',
+            'file:/usr/local/bin/forgejo',
+            'file:/usr/local/lib/systemd/system/forgejo.service',
+        },
+    },
+}

bundles/forgejo/metadata.py

@@ -1,33 +1,31 @@
 defaults = {
     'backups': {
         'paths': {
-            '/home/git',
-            '/var/lib/gitea',
+            '/var/lib/forgejo',
         },
     },
-    'gitea': {
+    'forgejo': {
         'app_name': 'Forgejo',
         'database': {
-            'username': 'gitea',
-            'password': repo.vault.password_for('{} postgresql gitea'.format(node.name)),
-            'database': 'gitea',
+            'username': 'forgejo',
+            'password': repo.vault.password_for('{} postgresql forgejo'.format(node.name)),
+            'database': 'forgejo',
         },
         'disable_registration': True,
         'email_domain_blocklist': set(),
         'enable_git_hooks': False,
-        'internal_token': repo.vault.password_for('{} gitea internal_token'.format(node.name)),
-        'lfs_secret_key': repo.vault.password_for('{} gitea lfs_secret_key'.format(node.name)),
-        'oauth_secret_key': repo.vault.password_for('{} gitea oauth_secret_key'.format(node.name)),
-        'security_secret_key': repo.vault.password_for('{} gitea security_secret_key'.format(node.name)),
+        'internal_token': repo.vault.password_for('{} forgejo internal_token'.format(node.name)),
+        'lfs_secret_key': repo.vault.password_for('{} forgejo lfs_secret_key'.format(node.name)),
+        'oauth_secret_key': repo.vault.password_for('{} forgejo oauth_secret_key'.format(node.name)),
+        'security_secret_key': repo.vault.password_for('{} forgejo security_secret_key'.format(node.name)),
     },
     'icinga2_api': {
-        'gitea': {
+        'forgejo': {
             'services': {
                 'FORGEJO PROCESS': {
-                    'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_systemd_unit gitea',
+                    'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_systemd_unit forgejo',
                 },
                 'FORGEJO UPDATE': {
-                    'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_forgejo_for_new_release codeberg.org forgejo/forgejo v$(gitea --version | cut -d" " -f3)',
                     'vars.notification.mail': True,
                     'check_interval': '60m',
                 },
@@ -41,29 +39,22 @@ defaults = {
     },
     'postgresql': {
         'roles': {
-            'gitea': {
-                'password': repo.vault.password_for('{} postgresql gitea'.format(node.name)),
+            'forgejo': {
+                'password': repo.vault.password_for('{} postgresql forgejo'.format(node.name)),
             },
         },
         'databases': {
-            'gitea': {
-                'owner': 'gitea',
+            'forgejo': {
+                'owner': 'forgejo',
             },
         },
     },
     'zfs': {
         'datasets': {
-            'tank/gitea': {},
-            'tank/gitea/home': {
-                'mountpoint': '/home/git',
+            'tank/forgejo': {
+                'mountpoint': '/var/lib/forgejo',
                 'needed_by': {
-                    'directory:/home/git',
-                },
-            },
-            'tank/gitea/var': {
-                'mountpoint': '/var/lib/gitea',
-                'needed_by': {
-                    'directory:/var/lib/gitea',
+                    'directory:/var/lib/forgejo',
                 },
             },
         },
@@ -71,6 +62,23 @@ defaults = {
 }
 
 
+@metadata_reactor.provides(
+    'icinga2_api/forgejo',
+)
+def update_monitoring(metadata):
+    return {
+        'icinga2_api': {
+            'forgejo': {
+                'services': {
+                    'FORGEJO UPDATE': {
+                        'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_forgejo_for_new_release codeberg.org forgejo/forgejo v{}'.format(metadata.get('forgejo/version')),
+                    },
+                },
+            },
+        },
+    }
+
+
 @metadata_reactor.provides(
     'nginx/vhosts/forgejo',
 )
@@ -82,7 +90,7 @@ def nginx(metadata):
         'nginx': {
             'vhosts': {
                 'forgejo': {
-                    'domain': metadata.get('gitea/domain'),
+                    'domain': metadata.get('forgejo/domain'),
                     'locations': {
                         '/': {
                             'target': 'http://127.0.0.1:22000',
@@ -97,11 +105,3 @@ def nginx(metadata):
             },
         },
     }
-
-
-@metadata_reactor.provides(
-    'icinga2_api/gitea/services',
-)
-def icinga_check_for_new_release(metadata):
-    return {
-    }

bundles/gce-workaround/items.py

@@ -1,33 +0,0 @@
-svc_systemd = {}
-pkg_apt = {}
-
-for i in {
-    'gce-disk-expand',
-    'google-cloud-packages-archive-keyring',
-    'google-cloud-sdk',
-    'google-compute-engine',
-    'google-compute-engine-oslogin',
-    'google-guest-agent',
-    'google-osconfig-agent',
-}:
-    pkg_apt[i] = {
-        'installed': False,
-    }
-
-for i in {
-    'google-accounts-daemon.service',
-    'google-accounts-manager.service',
-    'google-clock-skew-daemon.service',
-    'google-clock-sync-manager.service',
-    'google-guest-agent.service',
-    'google-osconfig-agent.service',
-    'google-shutdown-scripts.service',
-    'google-startup-scripts.service',
-    'sshguard.service',
-
-    'google-oslogin-cache.timer',
-}:
-    svc_systemd[i] = {
-        'enabled': False,
-        'running': False,
-    }

bundles/gitea/items.py

@@ -1,68 +0,0 @@
-users = {
-    'git': {},
-}
-
-directories = {
-    '/home/git': {
-        'mode': '0755',
-        'owner': 'git',
-        'group': 'git',
-    },
-    '/home/git/.ssh': {
-        'mode': '0755',
-        'owner': 'git',
-        'group': 'git',
-    },
-    '/var/lib/gitea': {
-        'owner': 'git',
-        'mode': '0700',
-        'triggers': {
-            'svc_systemd:gitea:restart',
-        },
-    },
-}
-
-files = {
-    '/etc/systemd/system/gitea.service': {
-        'content_type': 'mako',
-        'context': node.metadata.get('gitea'),
-        'triggers': {
-            'action:systemd-reload',
-            'svc_systemd:gitea:restart',
-        },
-    },
-    '/etc/gitea/app.ini': {
-        'content_type': 'mako',
-        'context': node.metadata.get('gitea'),
-        'triggers': {
-            'svc_systemd:gitea:restart',
-        },
-    },
-    '/usr/local/bin/gitea': {
-        'content_type': 'download',
-        'source': node.metadata.get('gitea/url'),
-        'content_hash': node.metadata.get('gitea/sha1', None),
-        'mode': '0755',
-        'triggers': {
-            'svc_systemd:gitea:restart',
-        },
-    },
-}
-
-if node.metadata['gitea'].get('install_ssh_key', False):
-    files['/home/git/.ssh/id_ed25519'] = {
-        'content': repo.vault.decrypt_file(f'gitea/files/ssh-keys/{node.name}.key.vault'),
-        'mode': '0600',
-        'owner': 'git',
-        'group': 'git',
-    }
-
-svc_systemd = {
-    'gitea': {
-        'needs': {
-            'file:/etc/gitea/app.ini',
-            'file:/etc/systemd/system/gitea.service',
-            'file:/usr/local/bin/gitea',
-        },
-    },
-}

bundles/grafana/dashboard-rows/smartd.py

@@ -47,7 +47,7 @@ def dashboard_row_smartd(panel_id, node):
                 'renderer': 'flot',
                 'seriesOverrides': [],
                 'spaceLength': 10,
-                'span': 8,
+                'span': 12,
                 'stack': False,
                 'steppedLine': False,
                 'targets': [
@@ -114,115 +114,5 @@ def dashboard_row_smartd(panel_id, node):
                     'alignLevel': None
                 }
             },
-            {
-                'aliasColors': {},
-                'bars': False,
-                'dashLength': 10,
-                'dashes': False,
-                'datasource': None,
-                'fieldConfig': {
-                    'defaults': {
-                        'displayName': '${__field.labels.device}'
-                    },
-                    'overrides': []
-                },
-                'fill': 0,
-                'fillGradient': 0,
-                'hiddenSeries': False,
-                'id': next(panel_id),
-                'legend': {
-                    'alignAsTable': False,
-                    'avg': False,
-                    'current': False,
-                    'hideEmpty': True,
-                    'hideZero': True,
-                    'max': False,
-                    'min': False,
-                    'rightSide': False,
-                    'show': True,
-                    'total': False,
-                    'values': False
-                },
-                'lines': True,
-                'linewidth': 1,
-                'NonePointMode': 'None',
-                'options': {
-                    'alertThreshold': True
-                },
-                'percentage': False,
-                'pluginVersion': '7.5.5',
-                'pointradius': 2,
-                'points': False,
-                'renderer': 'flot',
-                'seriesOverrides': [],
-                'spaceLength': 10,
-                'span': 4,
-                'stack': False,
-                'steppedLine': False,
-                'targets': [
-                    {
-                        'groupBy': [
-                            {'type': 'time', 'params': ['$__interval']},
-                            {'type': 'fill', 'params': ['linear']},
-                        ],
-                        'orderByTime': "ASC",
-                        'policy': "default",
-                        'query': f"""from(bucket: "telegraf")
-    |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
-    |> filter(fn: (r) =>
-        r["_measurement"] == "smartd_stats" and
-        r["_field"] == "power_on_hours" and
-        r["host"] == "{node.name}"
-    )
-    |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)
-    |> yield(name: "fan")""",
-                        'resultFormat': 'time_series',
-                        'select': [[
-                            {'type': 'field', 'params': ['value']},
-                            {'type': 'mean', 'params': []},
-                        ]],
-                        "tags": []
-                    },
-                ],
-                'thresholds': [],
-                'timeRegions': [],
-                'title': 'fans',
-                'tooltip': {
-                    'shared': True,
-                    'sort': 0,
-                    'value_type': 'individual'
-                },
-                'type': 'graph',
-                'xaxis': {
-                    'buckets': None,
-                    'mode': 'time',
-                    'name': None,
-                    'show': True,
-                    'values': []
-                },
-                'yaxes': [
-                    {
-                        'format': 'hours',
-                        'label': None,
-                        'logBase': 1,
-                        'max': None,
-                        'min': None,
-                        'show': True,
-                        'decimals': 0,
-                    },
-                    {
-                        'format': 'short',
-                        'label': None,
-                        'logBase': 1,
-                        'max': None,
-                        'min': None,
-                        'show': False,
-                    }
-                ],
-                'yaxis': {
-                    'align': False,
-                    'alignLevel': None
-                }
-            },
         ],
     }

bundles/hedgedoc/files/hedgedoc.service

@@ -33,7 +33,11 @@ ProtectSystem=strict
 ProtectHome=true
 PrivateTmp=true
 SystemCallArchitectures=native
-SystemCallFilter=@system-service
+# FIXME
+# causes problems on bookworm
+# see https://github.com/hedgedoc/hedgedoc/issues/4686
+# cmmented out for now ...
+#SystemCallFilter=@system-service
 
 # You may have to adjust these settings
 User=hedgedoc

bundles/hedgedoc/items.py

@@ -1,3 +1,5 @@
+from semver import compare
+
 repo.libs.tools.require_bundle(node, 'nodejs')
 
 git_deploy = {
@@ -47,12 +49,26 @@ directories = {
     },
 }
 
+if compare(node.metadata.get('hedgedoc/version'), '1.9.7') <= 0:
+    command = ' && '.join([
+        'cd /opt/hedgedoc',
+        'yarn workspaces focus --production',
+        'yarn install --ignore-scripts',
+        'yarn build',
+    ])
+elif compare(node.metadata.get('hedgedoc/version'), '1.9.9') >= 0:
+    command = ' && '.join([
+        'cd /opt/hedgedoc',
+        'bin/setup',
+        'yarn install --immutable',
+        'yarn build',
+    ])
+
 actions = {
     'hedgedoc_yarn': {
         'command': ' && '.join([
             'cd /opt/hedgedoc',
-            'yarn install --production=true --pure-lockfile --ignore-scripts',
-            'yarn install --ignore-scripts',
+            'yarn install --immutable',
             'yarn build',
         ]),
         'needs': {

bundles/homeassistant/files/homeassistant.service

@@ -8,6 +8,8 @@ User=homeassistant
 WorkingDirectory=/var/opt/homeassistant
 ExecStart=/opt/homeassistant/venv/bin/hass -c "/var/opt/homeassistant"
 RestartForceExitStatus=100
+Restart=on-failure
+RestartSec=2
 
 [Install]
 WantedBy=multi-user.target

bundles/homeassistant/items.py

@@ -1,3 +1,9 @@
+if node.has_bundle('pyenv'):
+    python_version = sorted(node.metadata.get('pyenv/python_versions'))[-1]
+    python_path = f'/opt/pyenv/versions/{python_version}/bin/python'
+else:
+    python_path = '/usr/bin/python3'
+
 users = {
     'homeassistant': {
         'home': '/var/opt/homeassistant',
@@ -32,11 +38,18 @@ files = {
 
 actions = {
     'homeassistant_create_virtualenv': {
-        'command': 'sudo -u homeassistant /usr/bin/python3 -m virtualenv -p python3 /opt/homeassistant/venv/',
+        'command': f'sudo -u homeassistant virtualenv -p {python_path} /opt/homeassistant/venv/',
         'unless': 'test -d /opt/homeassistant/venv/',
         'needs': {
             'directory:/opt/homeassistant',
             'user:homeassistant',
+        },
+    },
+    'homeassistant_install': {
+        'command': 'sudo -u homeassistant /opt/homeassistant/venv/bin/pip install homeassistant',
+        'unless': 'test -f /opt/homeassistant/venv/bin/hass',
+        'needs': {
+            'action:homeassistant_create_virtualenv',
             'pkg_apt:bluez',
             'pkg_apt:libffi-dev',
             'pkg_apt:libssl-dev',
@@ -45,17 +58,10 @@ actions = {
             'pkg_apt:autoconf',
             'pkg_apt:build-essential',
             'pkg_apt:libopenjp2-7',
-            'pkg_apt:libtiff5',
+            'pkg_apt:libtiff6',
             'pkg_apt:libturbojpeg0-dev',
             'pkg_apt:tzdata',
         },
-    },
-    'homeassistant_install': {
-        'command': 'sudo -u homeassistant /opt/homeassistant/venv/bin/pip install homeassistant',
-        'unless': 'test -f /opt/homeassistant/venv/bin/hass',
-        'needs': {
-            'action:homeassistant_create_virtualenv',
-        },
         'triggers': {
             'svc_systemd:homeassistant:restart',
         },

bundles/homeassistant/metadata.py

@@ -4,11 +4,12 @@ defaults = {
             'autoconf': {},
             'bluez':  {},
             'build-essential': {},
+            'ffmpeg': {},
             'libffi-dev': {},
             'libjpeg-dev': {},
             'libopenjp2-7': {},
             'libssl-dev': {},
-            'libtiff5': {},
+            'libtiff6': {},
             'libturbojpeg0-dev': {},
             'python3-packaging': {},
             'tzdata': {},
@@ -22,6 +23,8 @@ defaults = {
         },
     },
 }
+
+
 @metadata_reactor.provides(
     'icinga2_api/homeassistant/services',
 )
@@ -31,15 +34,17 @@ def icinga_check_for_new_release(metadata):
             'homeassistant': {
                 'services': {
                     'HOMEASSISTANT UPDATE': {
+                        'check_interval': '60m',
                         'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_homeassistant_update',
                         'vars.notification.mail': True,
-                        'check_interval': '60m',
+                        'vars.sshmon_timeout': 20,
                     },
                 },
             },
         },
     }
 
+
 @metadata_reactor.provides(
     'nginx/vhosts/homeassistant',
 )

bundles/icinga2-statuspage/files/icinga2-statuspage.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=icinga2-statuspage
+After=network.target
+Requires=postgresql.service
+
+[Service]
+User=www-data
+Group=www-data
+Environment=APP_CONFIG=/opt/icinga2-statuspage/config.json
+WorkingDirectory=/opt/icinga2-statuspage/src
+ExecStart=/usr/bin/gunicorn statuspage:app --workers 4 --max-requests 1200  --max-requests-jitter 50 --log-level=info --bind=127.0.0.1:22110
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target

bundles/icinga2-statuspage/items.py

@@ -0,0 +1,34 @@
+directories['/opt/icinga2-statuspage/src'] = {}
+
+git_deploy['/opt/icinga2-statuspage/src'] = {
+    'repo': 'https://git.franzi.business/kunsi/icinga-dynamic-statuspage.git',
+    'rev': 'main',
+    'triggers': {
+        'svc_systemd:icinga2-statuspage:restart',
+    },
+}
+
+files['/opt/icinga2-statuspage/config.json'] = {
+    'content': repo.libs.faults.dict_as_json(node.metadata.get('icinga2-statuspage')),
+    'triggers': {
+        'svc_systemd:icinga2-statuspage:restart',
+    },
+}
+
+files['/usr/local/lib/systemd/system/icinga2-statuspage.service'] = {
+    'triggers': {
+        'action:systemd-reload',
+        'svc_systemd:icinga2-statuspage:restart',
+    },
+}
+
+
+svc_systemd['icinga2-statuspage'] = {
+    'needs': {
+        'file:/opt/icinga2-statuspage/config.json',
+        'git_deploy:/opt/icinga2-statuspage/src',
+        'pkg_apt:gunicorn',
+        'pkg_apt:python3-flask',
+        'pkg_apt:python3-psycopg2',
+    },
+}

bundles/icinga2-statuspage/metadata.py

@@ -0,0 +1,47 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'gunicorn': {},
+            'python3-flask': {},
+            'python3-psycopg2': {},
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'icinga2-statuspage',
+)
+def import_db_settings_from_icinga(metadata):
+    return {
+        'icinga2-statuspage': {
+            'DB_USER': 'icinga2',
+            'DB_PASS': metadata.get('postgresql/roles/icinga2/password'),
+            'DB_NAME': 'icinga2',
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'nginx/vhosts/icinga2-statuspage',
+)
+def nginx(metadata):
+    if not node.has_bundle('nginx'):
+        raise DoNotRunAgain
+
+    return {
+        'nginx': {
+            'vhosts': {
+                'icinga2-statuspage': {
+                    'domain': metadata.get('icinga2-statuspage/DOMAIN'),
+                    'locations': {
+                        '/': {
+                            'target': 'http://127.0.0.1:22110',
+                        },
+                    },
+                    'website_check_path': '/',
+                    'website_check_string': 'status page',
+                },
+            },
+        },
+    }

bundles/icinga2/files/check_sipgate_account_balance

@@ -1,16 +1,17 @@
 #!/usr/bin/env python3
 
+from json import load
 from sys import exit
 
 from requests import get
 
-SIPGATE_USER = '${node.metadata['icinga2']['sipgate_user']}'
-SIPGATE_PASS = '${node.metadata['icinga2']['sipgate_pass']}'
+with open('/etc/icinga2/notification_config.json') as f:
+    CONFIG = load(f)
 
 try:
     r = get(
         'https://api.sipgate.com/v2/balance',
-        auth=(SIPGATE_USER, SIPGATE_PASS),
+        auth=(CONFIG['sipgate']['user'], CONFIG['sipgate']['password']),
         headers={'Accept': 'application/json'},
     )
 

bundles/icinga2/files/check_spam_blocklist

@@ -5,30 +5,33 @@ from ipaddress import IPv6Address, ip_address
 from subprocess import check_output
 from sys import argv, exit
 
-BLOCKLISTS = [
-    '0spam.fusionzero.com',
-    'bl.mailspike.org',
-    'bl.spamcop.net',
-    'blackholes.brainerd.net',
-    'dnsbl-1.uceprotect.net',
-    'dnsbl-2.uceprotect.net',
-    'l2.spews.dnsbl.sorbs.net',
-    'list.dsbl.org',
-    'map.spam-rbl.com',
-    'multihop.dsbl.org',
-    'ns1.unsubscore.com',
-    'opm.blitzed.org',
-    'psbl.surriel.com',
-    'rbl.efnet.org',
-    'rbl.schulte.org',
-    'spamguard.leadmon.net',
-    'ubl.unsubscore.com',
-    'unconfirmed.dsbl.org',
-    'virbl.dnsbl.bit.nl',
-    'zen.spamhaus.org',
-]
+BLOCKLISTS = {
+    '0spam.fusionzero.com': set(),
+    'bl.mailspike.org': set(),
+    'bl.spamcop.net': set(),
+    'blackholes.brainerd.net': set(),
+    'dnsbl-1.uceprotect.net': set(),
+    'l2.spews.dnsbl.sorbs.net': set(),
+    'list.dsbl.org': set(),
+    'multihop.dsbl.org': set(),
+    'ns1.unsubscore.com': set(),
+    'opm.blitzed.org': set(),
+    'psbl.surriel.com': set(),
+    'rbl.efnet.org': set(),
+    'rbl.schulte.org': set(),
+    'spamguard.leadmon.net': set(),
+    'ubl.unsubscore.com': set(),
+    'unconfirmed.dsbl.org': set(),
+    'virbl.dnsbl.bit.nl': set(),
+    'zen.spamhaus.org': {
+        # https://www.spamhaus.org/news/article/807/using-our-public-mirrors-check-your-return-codes-now.
+        '127.255.255.252', # Typing Error
+        '127.255.255.254', # public resolver / generic rdns
+        '127.255.255.255', # rate limited
+    },
+}
 
-def check_list(ip_list, blocklist):
+def check_list(ip_list, blocklist, warn_ips):
     dns_name = '{}.{}'.format(
         '.'.join(ip_list),
         blocklist,
@@ -41,17 +44,26 @@ def check_list(ip_list, blocklist):
         result = check_output([
             'dig',
             '+tries=2',
-            '+time=5',
+            '+time=10',
             '+short',
             dns_name
         ]).decode().splitlines()
         for item in result:
-            msgs.append('{} listed in {} as {}'.format(
-                ip,
-                blocklist,
-                item,
-            ))
-            returncode = 2
+            if item.startswith(';;'):
+                msgs.append('{} - {}'.format(
+                    blocklist,
+                    item,
+                ))
+            else:
+                msgs.append('{} listed in {} as {}'.format(
+                    ip,
+                    blocklist,
+                    item,
+                ))
+            if (item in warn_ips or item.startswith(';;')) and returncode < 2:
+                returncode = 1
+            else:
+                returncode = 2
     except Exception as e:
         if e.returncode == 9:
             # no reply from server
@@ -78,8 +90,8 @@ exitcode = 0
 with ThreadPoolExecutor(max_workers=len(BLOCKLISTS)) as executor:
     futures = set()
 
-    for blocklist in BLOCKLISTS:
-        futures.add(executor.submit(check_list, ip_list, blocklist))
+    for blocklist, warn_ips in BLOCKLISTS.items():
+        futures.add(executor.submit(check_list, ip_list, blocklist, warn_ips))
 
     for future in as_completed(futures):
         msgs, this_exitcode = future.result()

bundles/icinga2/files/icinga2/downtimes.conf

@@ -1,31 +1,18 @@
-% for monitored_node in sorted(repo.nodes):
-<%
-    auto_updates_enabled = (
-        monitored_node.has_any_bundle(['apt', 'c3voc-addons'])
-        or (
-            monitored_node.has_bundle('pacman')
-            and monitored_node.metadata.get('pacman/unattended-upgrades/is_enabled', False)
-        )
-    ) and not monitored_node.metadata.get('icinga_options/exclude_from_monitoring', False)
-%>\
-%  if auto_updates_enabled:
-object ScheduledDowntime "unattended_upgrades" {
-    host_name = "${monitored_node.name}"
+% for dt in downtimes:
+object ScheduledDowntime "${dt['name']}" {
+    host_name = "${dt['host']}"
 
-    author = "unattended-upgrades"
-    comment = "Downtime for upgrade-and-reboot of node ${monitored_node.name}"
+    author = "${dt['name']}"
+    comment = "${dt['comment']}"
 
     fixed = true
 
     ranges = {
-% if monitored_node.has_bundle('pacman'):
-        "${days[monitored_node.metadata.get('pacman/unattended-upgrades/day')]}" = "${monitored_node.metadata.get('pacman/unattended-upgrades/hour')}:${monitored_node.magic_number%30}-${monitored_node.metadata.get('pacman/unattended-upgrades/hour')}:${(monitored_node.magic_number%30)+30}"
-% else:
-        "${days[monitored_node.metadata.get('apt/unattended-upgrades/day')]}" = "${monitored_node.metadata.get('apt/unattended-upgrades/hour')}:${monitored_node.magic_number%30}-${monitored_node.metadata.get('apt/unattended-upgrades/hour')}:${(monitored_node.magic_number%30)+30}"
-% endif
+% for d,t in dt['times'].items():
+        "${d}" = "${t}"
+% endfor
     }
 
     child_options = "DowntimeTriggeredChildren"
 }
-%   endif
 % endfor

bundles/icinga2/files/icinga2/groups.conf

@@ -33,3 +33,11 @@ object ServiceGroup "checks_with_sms" {
     assign where service.vars.notification.sms == true
     ignore where host.vars.notification.sms == false
 }
+
+object ServiceGroup "statuspage" {
+    display_name = "Checks which are show on the public status page"
+
+    assign where service.vars.notification.sms == true
+    ignore where host.vars.notification.sms == false
+    ignore where host.vars.show_on_statuspage == false
+}

bundles/icinga2/files/icinga2/hosts_template.conf

@@ -14,7 +14,8 @@ object Host "${rnode.name}" {
     vars.os = "${rnode.os}"
 
     # used for status page
-    vars.pretty_name = "${rnode.metadata.get('icinga_options/pretty_name', rnode.name)}"
+    vars.pretty_name = "${rnode.metadata.get('icinga_options/pretty_name', rnode.metadata.get('hostname'))}"
+    vars.show_on_statuspage = ${str(rnode.metadata.get('icinga_options/show_on_statuspage', True)).lower()}
 
     vars.period = "${rnode.metadata.get('icinga_options/period', '24x7')}"
 

bundles/icinga2/files/icinga_statusmonitor.py

@@ -9,6 +9,11 @@ app = Flask(__name__)
 @app.route('/status')
 def statuspage():
     everything_fine = True
+    try:
+        check_output(['/usr/local/share/icinga/plugins/check_mounts'])
+    except:
+        everything_fine = False
+
     try:
         check_output(['/usr/lib/nagios/plugins/check_procs', '-C', 'icinga2', '-c', '1:'])
     except:

bundles/icinga2/files/icinga_statusmonitor.service

@@ -3,8 +3,6 @@ Description=Icinga2 Statusmonitor
 After=network.target
 
 [Service]
-User=nagios
-Group=nagios
 Environment="FLASK_APP=/etc/icinga2/icinga_statusmonitor.py"
 ExecStart=/usr/bin/python3 -m flask run
 WorkingDirectory=/tmp

bundles/icinga2/files/icingaweb2/monitoring_config.ini

@@ -0,0 +1,5 @@
+[settings]
+acknowledge_sticky = 1
+hostdowntime_all_services = 1
+hostdowntime_end_fixed = P1W
+servicedowntime_end_fixed = P2D

bundles/icinga2/files/scripts/icinga_notification_wrapper

@@ -3,22 +3,14 @@
 import email.mime.text
 import smtplib
 from argparse import ArgumentParser
-from json import dumps
+from json import dumps, load
 from subprocess import run
 from sys import argv
 
 from requests import post
 
-SIPGATE_USER='${node.metadata['icinga2']['sipgate_user']}'
-SIPGATE_PASS='${node.metadata['icinga2']['sipgate_pass']}'
-
-STATUS_TO_EMOJI = {
-    'critical': '🔥',
-    'down': '🚨🚨🚨',
-    'ok': '🆗',
-    'up': '👌',
-    'warning': '⚡',
-}
+with open('/etc/icinga2/notification_config.json') as f:
+    CONFIG = load(f)
 
 parser = ArgumentParser(
     prog='icinga_notification_wrapper',
@@ -73,36 +65,31 @@ def notify_per_sms():
         output_text = ''
     else:
         output_text = '\n\n{}'.format(args.output)
-    if args.state.lower() in STATUS_TO_EMOJI:
-        message_text = '{emoji} {host}{service} {emoji}{output}'.format(
-            emoji=STATUS_TO_EMOJI[args.state.lower()],
-            host=args.host_name,
-            service=('/'+args.service_name if args.service_name else ''),
-            state=args.state.upper(),
-            output=output_text,
-        )
-    else:
-        message_text = 'ICINGA: {host}{service} is {state}{output}'.format(
-            host=args.host_name,
-            service=('/'+args.service_name if args.service_name else ''),
-            state=args.state.upper(),
-            output=output_text,
-        )
+
+    message_text = 'ICINGA: {host}{service} is {state}{output}'.format(
+        host=args.host_name,
+        service=('/'+args.service_name if args.service_name else ''),
+        state=args.state.upper(),
+        output=output_text,
+    )
+
     message = {
         'message': message_text,
         'smsId': 's0', # XXX what does this mean? Documentation is unclear
         'recipient': args.sms
     }
+
     headers = {
         'Content-type': 'application/json',
         'Accept': 'application/json'
     }
+
     try:
         r = post(
             'https://api.sipgate.com/v2/sessions/sms',
             json=message,
             headers=headers,
-            auth=(SIPGATE_USER, SIPGATE_PASS),
+            auth=(CONFIG['sipgate']['user'], CONFIG['sipgate']['password']),
         )
 
         if r.status_code == 204:
@@ -113,6 +100,42 @@ def notify_per_sms():
         log_to_syslog('Sending a SMS to "{}" failed: {}'.format(args.sms, repr(e)))
 
 
+def notify_per_ntfy():
+    message_text = 'ICINGA: {host}{service} is {state}\n\n{output}'.format(
+        host=args.host_name,
+        service=('/'+args.service_name if args.service_name else ''),
+        state=args.state.upper(),
+        output=args.output,
+    )
+
+    if args.service_name:
+        subject = '[ICINGA] {}/{}'.format(args.host_name, args.service_name)
+    else:
+        subject = '[ICINGA] {}'.format(args.host_name)
+
+    if args.notification_type.lower() == 'recovery':
+        priority = 'default'
+    else:
+        priority = 'urgent'
+
+    headers = {
+        'Title': subject,
+        'Priority': priority,
+    }
+
+    try:
+        r = post(
+            CONFIG['ntfy']['url'],
+            data=message_text,
+            headers=headers,
+            auth=(CONFIG['ntfy']['user'], CONFIG['ntfy']['password']),
+        )
+
+        r.raise_for_status()
+    except Exception as e:
+        log_to_syslog('Sending a Notification failed: {}'.format(repr(e)))
+
+
 def notify_per_mail():
     if args.notification_type.lower() == 'recovery':
         # Do not send recovery emails.
@@ -176,4 +199,7 @@ if __name__ == '__main__':
         notify_per_mail()
 
     if args.sms:
-        notify_per_sms()
+        if args.service_name:
+            notify_per_sms()
+        if CONFIG['ntfy']['user']:
+            notify_per_ntfy()

bundles/icinga2/items.py

@@ -76,8 +76,6 @@ files = {
     },
     '/usr/local/share/icinga/plugins/check_sipgate_account_balance': {
         'mode': '0755',
-        'content_type': 'mako',
-        'cascade_skip': False, # contains faults
     },
     '/usr/local/share/icinga/plugins/check_freifunk_node': {
         'mode': '0755',
@@ -114,11 +112,22 @@ files = {
             'svc_systemd:icinga2:restart',
         },
     },
+    '/etc/icinga2/notification_config.json': {
+        'content': repo.libs.faults.dict_as_json({
+            'sipgate': {
+                'user': node.metadata.get('icinga2/sipgate/user'),
+                'password': node.metadata.get('icinga2/sipgate/pass'),
+            },
+            'ntfy': {
+                'url': node.metadata.get('icinga2/ntfy/url'),
+                'user': node.metadata.get('icinga2/ntfy/user'),
+                'password': node.metadata.get('icinga2/ntfy/pass'),
+            },
+        }),
+    },
     '/etc/icinga2/scripts/icinga_notification_wrapper': {
         'source': 'scripts/icinga_notification_wrapper',
-        'content_type': 'mako',
         'mode': '0755',
-        'cascade_skip': False, # contains faults
     },
     '/etc/icinga2/features-available/ido-pgsql.conf': {
         'source': 'icinga2/ido-pgsql.conf',
@@ -245,6 +254,11 @@ files = {
         'mode': '0660',
         'group': 'icingaweb2',
     },
+    '/etc/icingaweb2/modules/monitoring/config.ini': {
+        'source': 'icingaweb2/monitoring_config.ini',
+        'mode': '0660',
+        'group': 'icingaweb2',
+    },
     '/etc/icingaweb2/groups.ini': {
         'source': 'icingaweb2/groups.ini',
         'mode': '0660',
@@ -262,13 +276,13 @@ files = {
         'group': 'icingaweb2',
     },
 
-    # Statusmonitor
+    # monitoring
     '/etc/icinga2/icinga_statusmonitor.py': {
         'triggers': {
             'svc_systemd:icinga_statusmonitor:restart',
         },
     },
-    '/etc/systemd/system/icinga_statusmonitor.service': {
+    '/usr/local/lib/systemd/system/icinga_statusmonitor.service': {
         'triggers': {
             'action:systemd-reload',
             'svc_systemd:icinga_statusmonitor:restart',
@@ -276,8 +290,12 @@ files = {
     },
 }
 
-pkg_pip = {
-    'easysnmp': {}, # for check_usv_snmp
+svc_systemd['icinga_statusmonitor'] = {
+    'needs': {
+        'file:/etc/icinga2/icinga_statusmonitor.py',
+        'file:/usr/local/lib/systemd/system/icinga_statusmonitor.service',
+        'pkg_apt:python3-flask',
+    },
 }
 
 actions = {
@@ -319,44 +337,30 @@ for name in files:
 for name in symlinks:
     icinga_run_deps.add(f'symlink:{name}')
 
-svc_systemd = {
-    'icinga2': {
-        'needs': icinga_run_deps,
-    },
-    'icinga_statusmonitor': {
-        'needs': {
-            'file:/etc/icinga2/icinga_statusmonitor.py',
-            'file:/etc/systemd/system/icinga_statusmonitor.service',
-            'pkg_apt:python3-flask',
-        },
-    },
+svc_systemd['icinga2'] = {
+    'needs': icinga_run_deps,
 }
 
 
 
-
 # The actual hosts and services management starts here
 bundles = set()
-for rnode in repo.nodes:
+downtimes = []
+for rnode in sorted(repo.nodes):
     if rnode.metadata.get('icinga_options/exclude_from_monitoring', False):
         continue
 
-    host_ips = repo.libs.tools.resolve_identifier(repo, rnode.name)
+    host_ips = repo.libs.tools.resolve_identifier(repo, rnode.name, only_physical=True)
     icinga_ips = {}
 
-    # XXX for the love of god, PLEASE remove this once DNS is no longer
-    # hosted at GCE
-    if rnode.in_group('gce'):
-        icinga_ips['ipv4'] = rnode.metadata.get('external_ipv4')
-    else:
-        for ip_type in ('ipv4', 'ipv6'):
-            for ip in sorted(host_ips[ip_type]):
-                if ip.is_private and not ip.is_link_local:
-                    icinga_ips[ip_type] = str(ip)
-                    break
-            else:
-                if host_ips[ip_type]:
-                    icinga_ips[ip_type] = sorted(host_ips[ip_type])[0]
+    for ip_type in ('ipv4', 'ipv6'):
+        for ip in sorted(host_ips[ip_type]):
+            if ip.is_private and not ip.is_link_local:
+                icinga_ips[ip_type] = str(ip)
+                break
+        else:
+            if host_ips[ip_type]:
+                icinga_ips[ip_type] = sorted(host_ips[ip_type])[0]
 
     if not icinga_ips:
         raise ValueError(f'{rnode.name} requests monitoring, but has neither IPv4 nor IPv6 addresses!')
@@ -379,6 +383,41 @@ for rnode in repo.nodes:
 
     bundles |= set(rnode.metadata.get('icinga2_api', {}).keys())
 
+    if rnode.has_any_bundle(['apt', 'c3voc-addons']):
+        day = rnode.metadata.get('apt/unattended-upgrades/day')
+        hour = rnode.metadata.get('apt/unattended-upgrades/hour')
+        minute = rnode.magic_number%30
+
+        spread = rnode.metadata.get('apt/unattended-upgrades/spread_in_group', None)
+        if spread is not None:
+            spread_nodes = sorted(repo.nodes_in_group(spread))
+            day += spread_nodes.index(rnode)
+
+        downtimes.append({
+            'name': 'unattended-upgrades',
+            'host': rnode.name,
+            'comment': f'Downtime for upgrade-and-reboot of node {rnode.name}',
+            'times': {
+                DAYS_TO_STRING[day%7]: f'{hour}:{minute}-{hour}:{minute+15}',
+            },
+        })
+    elif (
+        rnode.has_bundle('pacman')
+        and rnode.metadata.get('pacman/unattended-upgrades/is_enabled', False)
+    ):
+        day = rnode.metadata.get('pacman/unattended-upgrades/day')
+        hour = rnode.metadata.get('pacman/unattended-upgrades/hour')
+        minute = rnode.magic_number%30
+
+        downtimes.append({
+            'name': 'unattended-upgrades',
+            'host': rnode.name,
+            'comment': f'Downtime for upgrade-and-reboot of node {rnode.name}',
+            'times': {
+                DAYS_TO_STRING[day%7]: f'{hour}:{minute}-{hour}:{minute+15}',
+            },
+        })
+
 files['/etc/icinga2/conf.d/groups.conf'] = {
     'source': 'icinga2/groups.conf',
     'content_type': 'mako',
@@ -399,7 +438,7 @@ files['/etc/icinga2/conf.d/downtimes.conf'] = {
     'source': 'icinga2/downtimes.conf',
     'content_type': 'mako',
     'context': {
-        'days': DAYS_TO_STRING,
+        'downtimes': downtimes,
     },
     'owner': 'nagios',
     'group': 'nagios',

bundles/icinga2/metadata.py

@@ -17,12 +17,10 @@ defaults = {
             'icinga2': {},
             'icinga2-ido-pgsql': {},
             'icingaweb2': {},
-
-            # apparently no longer needed
-            #'icingaweb2-module-monitoring': {},
-
-            # neeeded for statusmonitor
+            'icingaweb2-module-monitoring': {},
+            'python3-easysnmp': {},
             'python3-flask': {},
+            'snmp': {},
         }
     },
     'icinga2': {
@@ -43,9 +41,6 @@ defaults = {
                     'check_interval': '30m',
                     'vars.notification.mail': True,
                 },
-                'ICINGA STATUSMONITOR': {
-                    'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_systemd_unit icinga_statusmonitor',
-                },
                 'IDO-PGSQL': {
                     'check_command': 'ido',
                     'vars.ido_type': 'IdoPgsqlConnection',
@@ -59,6 +54,21 @@ defaults = {
     'icingaweb2': {
         'setup-token': repo.vault.password_for(f'{node.name} icingaweb2 setup-token'),
     },
+    'php': {
+        'version': '8.2',
+        'packages': {
+            'curl',
+            'gd',
+            'intl',
+            'imagick',
+            'ldap',
+            'mysql',
+            'opcache',
+            'pgsql',
+            'readline',
+            'xml',
+        },
+    },
     'postgresql': {
         'roles': {
             'icinga2': {
@@ -105,13 +115,29 @@ def add_users_from_json(metadata):
 
 
 @metadata_reactor.provides(
-    'firewall/port_rules/5665',
+    'nginx/vhosts/icingaweb2',
+    'nginx/vhosts/icinga_statusmonitor',
 )
-def firewall(metadata):
+def nginx(metadata):
+    if not node.has_bundle('nginx'):
+        raise DoNotRunAgain
+
     return {
-        'firewall': {
-            'port_rules': {
-                '5665': atomic(metadata.get('icinga2/restrict-to', set())),
+        'nginx': {
+            'vhosts': {
+                'icingaweb2': {
+                    'domain': metadata.get('icinga2/web_domain'),
+                    'webroot': '/usr/share/icingaweb2/public',
+                    'locations': {
+                        '/api/': {
+                            'target': 'https://127.0.0.1:5665/',
+                        },
+                        '/statusmonitor/': {
+                            'target': 'http://127.0.0.1:5000/',
+                        },
+                    },
+                    'extras': True,
+                },
             },
         },
     }

bundles/influxdb2/metadata.py

@@ -10,7 +10,7 @@ defaults = {
         'repos': {
             'influxdb': {
                 'items': {
-                    'deb https://repos.influxdata.com/{os} {os_release} stable',
+                    'deb https://repos.influxdata.com/{os} stable main',
                 },
             },
         },

bundles/infobeamer-cms/files/infobeamer-cms-runperiodic.service

@@ -4,7 +4,8 @@ After=network.target
 Requires=infobeamer-cms.service
 
 [Service]
+Environment=SETTINGS=/opt/infobeamer-cms/settings.toml
+WorkingDirectory=/opt/infobeamer-cms/src
 User=infobeamer-cms
 Group=infobeamer-cms
-WorkingDirectory=/opt/infobeamer-cms
-ExecStart=curl -s -H "Host: ${domain}" http://127.0.0.1:8000/sync
+ExecStart=/opt/infobeamer-cms/venv/bin/python syncer.py

bundles/infobeamer-cms/files/infobeamer-cms-runperiodic.timer

@@ -2,7 +2,7 @@
 Description=Run infobeamer-cms sync
 
 [Timer]
-OnCalendar=*:0/5
+OnCalendar=minutely
 Persistent=true
 
 [Install]

bundles/infobeamer-cms/files/settings.toml

@@ -1,4 +0,0 @@
-<%
-    from tomlkit import dumps as toml_dumps
-    from bundlewrap.utils.text import toml_clean
-%>${toml_clean(toml_dumps(repo.libs.faults.resolve_faults(config), sort_keys=True))}

bundles/infobeamer-cms/items.py

@@ -1,8 +1,4 @@
 actions = {
-    'infobeamer-cms_set_directory_permissions': {
-        'triggered': True,
-        'command': 'chown -R infobeamer-cms:infobeamer-cms /opt/infobeamer-cms/src/static/'
-    },
     'infobeamer-cms_create_virtualenv': {
         'command': '/usr/bin/python3 -m virtualenv -p python3 /opt/infobeamer-cms/venv/',
         'unless': 'test -d /opt/infobeamer-cms/venv/',
@@ -12,7 +8,11 @@ actions = {
         },
     },
     'infobeamer-cms_install_requirements': {
-        'command': 'cd /opt/infobeamer-cms/src && /opt/infobeamer-cms/venv/bin/pip install --upgrade pip gunicorn -r requirements.txt',
+        'command': ' && '.join([
+            'cd /opt/infobeamer-cms/src',
+            '/opt/infobeamer-cms/venv/bin/pip install --upgrade pip gunicorn -r requirements.txt',
+            'rsync /opt/infobeamer-cms/src/static/* /opt/infobeamer-cms/static/',
+        ]),
         'needs': {
             'action:infobeamer-cms_create_virtualenv',
         },
@@ -29,7 +29,6 @@ git_deploy = {
         },
         'triggers': {
             'svc_systemd:infobeamer-cms:restart',
-            'action:infobeamer-cms_set_directory_permissions',
             'action:infobeamer-cms_install_requirements',
         },
     },
@@ -37,6 +36,9 @@ git_deploy = {
 
 directories = {
     '/opt/infobeamer-cms/src': {},
+    '/opt/infobeamer-cms/static': {
+        'owner': 'infobeamer-cms',
+    },
 }
 
 config = node.metadata.get('infobeamer-cms/config', {})
@@ -66,10 +68,7 @@ for room, device_id in sorted(node.metadata.get('infobeamer-cms/rooms', {}).item
 
 files = {
     '/opt/infobeamer-cms/settings.toml': {
-        'content_type': 'mako',
-        'context': {
-            'config': config,
-        },
+        'content': repo.libs.faults.dict_as_toml(config),
         'triggers': {
             'svc_systemd:infobeamer-cms:restart',
         },
@@ -109,7 +108,7 @@ svc_systemd = {
     'infobeamer-cms': {
         'needs': {
             'action:infobeamer-cms_install_requirements',
-            'action:infobeamer-cms_set_directory_permissions',
+            'directory:/opt/infobeamer-cms/static',
             'file:/etc/systemd/system/infobeamer-cms.service',
             'file:/opt/infobeamer-cms/settings.toml',
             'git_deploy:/opt/infobeamer-cms/src',
@@ -117,8 +116,12 @@ svc_systemd = {
     },
     'infobeamer-cms-runperiodic.timer': {
         'needs': {
-            'file:/etc/systemd/system/infobeamer-cms-runperiodic.timer',
+            'action:infobeamer-cms_install_requirements',
+            'directory:/opt/infobeamer-cms/static',
             'file:/etc/systemd/system/infobeamer-cms-runperiodic.service',
+            'file:/etc/systemd/system/infobeamer-cms-runperiodic.timer',
+            'file:/opt/infobeamer-cms/settings.toml',
+            'git_deploy:/opt/infobeamer-cms/src',
         },
     },
 }

bundles/infobeamer-cms/metadata.py

@@ -6,6 +6,7 @@ defaults = {
             'MAX_UPLOADS': 5,
             'PREFERRED_URL_SCHEME': 'https',
             'SESSION_COOKIE_NAME': '__Host-sess',
+            'STATIC_PATH': '/opt/infobeamer-cms/static',
             'URL_KEY': repo.vault.password_for(f'{node.name} infobeamer-cms url key'),
             'VERSION': 1,
         },
@@ -29,15 +30,13 @@ def nginx(metadata):
                         '/': {
                             'target': 'http://127.0.0.1:8000',
                         },
-                        '/sync': {
-                            'return': 403,
-                        },
                         '/static': {
-                            'alias': '/opt/infobeamer-cms/src/static',
+                            'alias': '/opt/infobeamer-cms/static',
                         },
                     },
                     'website_check_path': '/',
                     'website_check_string': 'Share your projects',
+                    'do_not_set_content_security_headers': True,
                 },
             },
         },
@@ -45,6 +44,7 @@ def nginx(metadata):
 
 
 @metadata_reactor.provides(
+    'infobeamer-cms/config/DOMAIN',
     'infobeamer-cms/config/TIME_MAX',
     'infobeamer-cms/config/TIME_MIN',
 )
@@ -57,6 +57,7 @@ def event_times(metadata):
     return {
         'infobeamer-cms': {
             'config': {
+                'DOMAIN': metadata.get('infobeamer-cms/domain'),
                 'TIME_MAX': int(event_end.timestamp()),
                 'TIME_MIN': int(event_start.timestamp()),
             },

bundles/infobeamer-monitor/files/infobeamer-monitor.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=infobeamer-monitor
+After=network.target
+
+[Service]
+Type=exec
+Restart=always
+RestartSec=5s
+ExecStart=/opt/infobeamer-cms/venv/bin/python monitor.py
+User=infobeamer-cms
+Group=infobeamer-cms
+WorkingDirectory=/opt/infobeamer-monitor/
+
+[Install]
+WantedBy=multi-user.target

bundles/infobeamer-monitor/files/monitor.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+
+import logging
+from datetime import datetime, timezone
+from json import dumps
+from time import sleep
+
+import paho.mqtt.client as mqtt
+from requests import RequestException, get
+
+try:
+    # python 3.11
+    from tomllib import loads as toml_load
+except ImportError:
+    from rtoml import load as toml_load
+
+with open("config.toml") as f:
+    CONFIG = toml_load(f.read())
+
+
+logging.basicConfig(
+    format="[%(levelname)s %(name)s] %(message)s",
+    level=logging.INFO,
+)
+
+LOG = logging.getLogger("main")
+MLOG = logging.getLogger("mqtt")
+
+state = None
+
+client = mqtt.Client()
+client.username_pw_set(CONFIG["mqtt"]["user"], CONFIG["mqtt"]["password"])
+client.connect(CONFIG["mqtt"]["host"], 1883, 60)
+client.loop_start()
+
+
+def mqtt_out(message, level="INFO", device=None):
+    key = "infobeamer"
+    if device:
+        key += f"/{device['id']}"
+        message = f"[{device['description']}] {message}"
+
+    client.publish(
+        CONFIG["mqtt"]["topic"],
+        dumps(
+            {
+                "level": level,
+                "component": key,
+                "msg": message,
+            }
+        ),
+    )
+
+
+def mqtt_dump_state(device):
+    if not device["is_online"]:
+        return
+
+    out = []
+    if device["location"]:
+        out.append("Location: {}".format(device["location"]))
+    out.append("Setup: {} ({})".format(device["setup"]["name"], device["setup"]["id"]))
+    out.append("Resolution: {}".format(device["run"].get("resolution", "unknown")))
+    if not device["is_synced"]:
+        out.append("syncing ...")
+
+    mqtt_out(
+        " - ".join(out),
+        device=device,
+    )
+
+
+mqtt_out("Monitor starting up")
+while True:
+    try:
+        try:
+            r = get(
+                "https://info-beamer.com/api/v1/device/list",
+                auth=("", CONFIG["api_key"]),
+            )
+            r.raise_for_status()
+            ib_state = r.json()["devices"]
+        except RequestException as e:
+            LOG.exception("Could not get data from info-beamer")
+            mqtt_out(
+                f"Could not get data from info-beamer: {e!r}",
+                level="WARN",
+            )
+        else:
+            new_state = {}
+            online_devices = set()
+            for device in ib_state:
+                did = str(device["id"])
+
+                if did in new_state:
+                    mqtt_out("DUPLICATE DETECTED!", level="ERROR", device=device)
+                    continue
+
+                new_state[did] = device
+                must_dump_state = False
+
+                if state is not None:
+                    if did not in state:
+                        LOG.info(
+                            "new device found: {} [{}]".format(
+                                did,
+                                device["description"],
+                            )
+                        )
+                        mqtt_out(
+                            "new device found!",
+                            device=device,
+                        )
+                        must_dump_state = True
+
+                    else:
+                        if device["is_online"] != state[did]["is_online"]:
+                            online_status = (
+                                "online from {}".format(device["run"]["public_addr"])
+                                if device["is_online"]
+                                else "offline"
+                            )
+
+                            LOG.info("device {} is now {}".format(did, online_status))
+                            mqtt_out(
+                                f"status changed to {online_status}",
+                                level="INFO" if device["is_online"] else "WARN",
+                                device=device,
+                            )
+                            must_dump_state = True
+
+                        if device["description"] != state[did]["description"]:
+                            LOG.info(
+                                "device {} changed name to {}".format(
+                                    did, device["description"]
+                                )
+                            )
+                            must_dump_state = True
+
+                        if device["is_online"]:
+                            if device["maintenance"]:
+                                mqtt_out(
+                                    "maintenance required: {}".join(
+                                        sorted(device["maintenance"])
+                                    ),
+                                    level="WARN",
+                                    device=device,
+                                )
+                                must_dump_state = True
+
+                        if (
+                            device["is_synced"] != state[did]["is_synced"]
+                            or device["location"] != state[did]["location"]
+                            or device["setup"]["id"] != state[did]["setup"]["id"]
+                            or device["run"].get("resolution")
+                            != state[did]["run"].get("resolution")
+                        ):
+                            must_dump_state = True
+
+                    if must_dump_state:
+                        mqtt_dump_state(device)
+                else:
+                    LOG.info("adding device {} to empty state".format(device["id"]))
+
+                if device["is_online"]:
+                    online_devices.add(
+                        "{} ({})".format(
+                            device["id"],
+                            device["description"],
+                        )
+                    )
+
+            state = new_state
+
+            if (
+                datetime.now(timezone.utc).strftime("%H%M") == "1312"
+                and online_devices
+                and int(datetime.now(timezone.utc).strftime("%S")) < 30
+            ):
+                mqtt_out("Online Devices: {}".format(", ".join(sorted(online_devices))))
+        sleep(30)
+    except KeyboardInterrupt:
+        break
+
+mqtt_out("Monitor exiting")

bundles/infobeamer-monitor/items.py

@@ -0,0 +1,30 @@
+assert node.has_bundle('infobeamer-cms') # uses same venv
+
+files['/opt/infobeamer-monitor/config.toml'] = {
+    'content': repo.libs.faults.dict_as_toml(node.metadata.get('infobeamer-monitor')),
+    'triggers': {
+        'svc_systemd:infobeamer-monitor:restart',
+    },
+}
+
+files['/opt/infobeamer-monitor/monitor.py'] = {
+    'mode': '0755',
+    'triggers': {
+        'svc_systemd:infobeamer-monitor:restart',
+    },
+}
+
+files['/usr/local/lib/systemd/system/infobeamer-monitor.service'] = {
+    'triggers': {
+        'action:systemd-reload',
+        'svc_systemd:infobeamer-monitor:restart',
+    },
+}
+
+svc_systemd['infobeamer-monitor'] = {
+    'needs': {
+        'file:/opt/infobeamer-monitor/config.toml',
+        'file:/opt/infobeamer-monitor/monitor.py',
+        'file:/usr/local/lib/systemd/system/infobeamer-monitor.service',
+    },
+}

bundles/jellyfin/files/jellyfin-sudoers

@@ -0,0 +1,7 @@
+Cmnd_Alias RESTARTSERVER_SYSTEMD = /usr/bin/systemd-run systemctl restart jellyfin
+Cmnd_Alias STARTSERVER_SYSTEMD = /usr/bin/systemd-run systemctl start jellyfin
+Cmnd_Alias STOPSERVER_SYSTEMD = /usr/bin/systemd-run systemctl stop jellyfin
+
+jellyfin ALL=(ALL) NOPASSWD: RESTARTSERVER_SYSTEMD
+jellyfin ALL=(ALL) NOPASSWD: STARTSERVER_SYSTEMD
+jellyfin ALL=(ALL) NOPASSWD: STOPSERVER_SYSTEMD

bundles/jellyfin/items.py

@@ -0,0 +1,5 @@
+files['/etc/sudoers.d/jellyfin-sudoers'] = {
+    'after': {
+        'pkg_apt:jellyfin',
+    },
+}

bundles/jellyfin/metadata.py

@@ -0,0 +1,69 @@
+from bundlewrap.metadata import atomic
+
+defaults = {
+    'apt': {
+        'packages': {
+            'jellyfin': {},
+        },
+        'repos': {
+            'jellyfin': {
+                'uris': {
+                    'https://repo.jellyfin.org/{os}'
+                },
+            },
+        },
+    },
+    'backups': {
+        'paths': {
+            f'/var/lib/jellyfin/{x}' for x in ('data', 'metadata', 'plugins', 'root')
+        },
+    },
+    'icinga2_api': {
+        'transmission': {
+            'services': {
+                'JELLYFIN PROCESS': {
+                    'command_on_monitored_host': '/usr/lib/nagios/plugins/check_procs -C jellyfin -c 1:',
+                },
+            },
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'nginx/vhosts/jellyfin',
+)
+def nginx(metadata):
+    if not node.has_bundle('nginx'):
+        raise DoNotRunAgain
+
+    if 'jellyfin' not in metadata.get('nginx/vhosts', {}):
+        return {}
+
+    return {
+        'nginx': {
+            'vhosts': {
+                'jellyfin': {
+                    'do_not_add_content_security_headers': True,
+                    'locations': {
+                        '/': {
+                            'target': 'http://127.0.0.1:8096',
+                            'websockets': True,
+                        },
+                    },
+                },
+            },
+        },
+    }
+
+@metadata_reactor.provides(
+    'firewall/port_rules',
+)
+def firewall(metadata):
+    return {
+        'firewall': {
+            'port_rules': {
+                '8096/tcp': atomic(metadata.get('jellyfin/restrict-to', {'*'})),
+            },
+        },
+    }

bundles/jool/items.py

@@ -0,0 +1,15 @@
+actions['modprobe_jool'] = {
+    'command': 'modprobe jool',
+    'unless': 'lsmod | grep -F jool',
+}
+
+actions['jool_add_nat64_instance'] = {
+    'command': 'jool instance add "nat64" --netfilter --pool6 64:ff9b::/96',
+    'unless': 'jool instance display --no-headers --csv | grep -E ",nat64,netfilter$"',
+    'needs': {
+        'action:modprobe_jool',
+        'pkg_apt:jool-dkms',
+        'pkg_apt:jool-tools',
+        'pkg_apt:linux-headers-amd64',
+    },
+}

bundles/jool/metadata.py

@@ -0,0 +1,14 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'jool-dkms': {},
+            'jool-tools': {},
+            'linux-headers-amd64': {},
+        },
+    },
+    'modules': {
+        'jool': [
+            'jool',
+        ],
+    },
+}

bundles/jugendhackt_tools/files/jugendhackt_tools.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=jugendhackt_tools web service
+After=network.target
+Requires=postgresql.service
+
+[Service]
+User=jugendhackt_tools
+Group=jugendhackt_tools
+Environment=CONFIG_PATH=/opt/jugendhackt_tools/config.toml
+WorkingDirectory=/opt/jugendhackt_tools/src
+ExecStart=/opt/jugendhackt_tools/venv/bin/gunicorn jugendhackt_tools.wsgi --name jugendhackt_tools --workers 4 --max-requests 1200  --max-requests-jitter 50 --log-level=info --bind=127.0.0.1:22090
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

bundles/jugendhackt_tools/items.py

@@ -0,0 +1,75 @@
+directories['/opt/jugendhackt_tools/src'] = {}
+
+git_deploy['/opt/jugendhackt_tools/src'] = {
+    'repo': 'https://github.com/kunsi/jugendhackt_schedule.git',
+    'rev': 'main',
+    'triggers': {
+        'action:jugendhackt_tools_install',
+        'action:jugendhackt_tools_migrate',
+        'svc_systemd:jugendhackt_tools:restart',
+    },
+}
+
+actions['jugendhackt_tools_create_virtualenv'] = {
+    'command': '/usr/bin/python3 -m virtualenv -p python3 /opt/jugendhackt_tools/venv/',
+    'unless': 'test -d /opt/jugendhackt_tools/venv/',
+    'needs': {
+        # actually /opt/jugendhackt_tools, but we don't create that
+        'directory:/opt/jugendhackt_tools/src',
+    },
+}
+
+actions['jugendhackt_tools_install'] = {
+    'command': ' && '.join([
+        'cd /opt/jugendhackt_tools/src',
+        '/opt/jugendhackt_tools/venv/bin/pip install --upgrade pip wheel gunicorn psycopg2-binary',
+        '/opt/jugendhackt_tools/venv/bin/pip install --upgrade -r requirements.txt',
+    ]),
+    'needs': {
+        'action:jugendhackt_tools_create_virtualenv',
+    },
+    'triggered': True,
+}
+
+actions['jugendhackt_tools_migrate'] = {
+    'command': ' && '.join([
+        'cd /opt/jugendhackt_tools/src',
+        'CONFIG_PATH=/opt/jugendhackt_tools/config.toml /opt/jugendhackt_tools/venv/bin/python manage.py migrate',
+        'CONFIG_PATH=/opt/jugendhackt_tools/config.toml /opt/jugendhackt_tools/venv/bin/python manage.py collectstatic --noinput',
+    ]),
+    'needs': {
+        'action:jugendhackt_tools_install',
+        'file:/opt/jugendhackt_tools/config.toml',
+        'postgres_db:jugendhackt_tools',
+        'postgres_role:jugendhackt_tools',
+    },
+    'triggered': True,
+}
+
+files['/opt/jugendhackt_tools/config.toml'] = {
+    'content': repo.libs.faults.dict_as_toml(node.metadata.get('jugendhackt_tools')),
+    'triggers': {
+        'svc_systemd:jugendhackt_tools:restart',
+    },
+}
+
+files['/usr/local/lib/systemd/system/jugendhackt_tools.service'] = {
+    'triggers': {
+        'action:systemd-reload',
+        'svc_systemd:jugendhackt_tools:restart',
+    },
+}
+
+svc_systemd['jugendhackt_tools'] = {
+    'needs': {
+        'action:jugendhackt_tools_migrate',
+        'file:/opt/jugendhackt_tools/config.toml',
+        'file:/usr/local/lib/systemd/system/jugendhackt_tools.service',
+        'git_deploy:/opt/jugendhackt_tools/src',
+        'user:jugendhackt_tools',
+    },
+}
+
+users['jugendhackt_tools'] = {
+    'home': '/opt/jugendhackt_tools/src',
+}

bundles/jugendhackt_tools/metadata.py

@@ -0,0 +1,28 @@
+defaults = {
+    'jugendhackt_tools': {
+        'django_secret': repo.vault.random_bytes_as_base64_for(f'{node.name} jugendhackt_tools django_secret'),
+        'django_debug': False,
+        'static_root': '/opt/jugendhackt_tools/src/static/',
+        'database': {
+            'ENGINE': 'django.db.backends.postgresql',
+            'NAME': 'jugendhackt_tools',
+            'USER': 'jugendhackt_tools',
+            'PASSWORD': repo.vault.password_for(f'{node.name} postgresql jugendhackt_tools'),
+            'HOST': 'localhost',
+            'PORT': '5432'
+        },
+    },
+    'postgresql': {
+        'roles': {
+            'jugendhackt_tools': {
+                'password': repo.vault.password_for(f'{node.name} postgresql jugendhackt_tools'),
+            },
+        },
+        'databases': {
+            'jugendhackt_tools': {
+                'owner': 'jugendhackt_tools',
+            },
+        },
+    },
+}
+

bundles/kea-dhcp-server/files/kea-lease-list

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+from csv import DictReader
+from datetime import datetime, timezone
+from os import scandir
+from os.path import join
+
+
+def parse():
+    NOW = datetime.now()
+    active_leases = {}
+    for file in scandir("/var/lib/kea/"):
+        with open(file.path) as f:
+            for row in DictReader(f):
+                expires = datetime.fromtimestamp(int(row["expire"]))
+
+                if expires >= NOW:
+                    if (
+                        row["address"] not in active_leases
+                        or active_leases[row["address"]]["expires_dt"] < expires
+                    ):
+                        row["expires_dt"] = expires
+                        active_leases[row["address"]] = row
+    return active_leases.values()
+
+
+def print_table(leases):
+    print(""" address         | MAC               | expires | hostname
+-----------------+-------------------+---------+----------""")
+    for lease in sorted(leases, key=lambda r: r["address"]):
+        print(
+            f' {lease["address"]:<15} | {lease["hwaddr"].lower()} |  {lease["expires_dt"]:%H:%M}  | {lease["hostname"]}'
+        )
+
+
+if __name__ == "__main__":
+    print_table(parse())

bundles/kea-dhcp-server/items.py

@@ -0,0 +1,56 @@
+kea_config = {
+    'Dhcp4': {
+        **node.metadata.get('kea-dhcp-server/config'),
+        'interfaces-config': {
+            'interfaces': sorted(node.metadata.get('kea-dhcp-server/subnets', {}).keys()),
+        },
+        'subnet4': [],
+        'loggers': [{
+            'name': 'kea-dhcp4',
+            'output_options': [{
+                # -> journal
+                'output': 'stdout',
+            }],
+            'severity': 'WARN',
+        }],
+    },
+}
+
+for iface, config in sorted(node.metadata.get('kea-dhcp-server/subnets', {}).items()):
+    kea_config['Dhcp4']['subnet4'].append({
+        'subnet': config['subnet'],
+        'pools': [{
+            'pool': f'{config["lower"]} - {config["higher"]}',
+        }],
+        'option-data': [
+            {
+                'name': k,
+                'data': v,
+            } for k, v in sorted(config.get('options', {}).items())
+        ],
+        'reservations': [
+            {
+                'ip-address': v['ip'],
+                'hw-address': v['mac'],
+                'hostname': k,
+            } for k, v in sorted(node.metadata.get(f'kea-dhcp-server/fixed_allocations/{iface}', {}).items())
+        ]
+    })
+
+files['/etc/kea/kea-dhcp4.conf'] = {
+    'content': repo.libs.faults.dict_as_json(kea_config),
+    'triggers': {
+        'svc_systemd:kea-dhcp4-server:restart',
+    },
+}
+
+files['/usr/local/bin/kea-lease-list'] = {
+    'mode': '0500',
+}
+
+svc_systemd['kea-dhcp4-server'] = {
+    'needs': {
+        'file:/etc/kea/kea-dhcp4.conf',
+        'pkg_apt:kea-dhcp4-server',
+    },
+}

bundles/kea-dhcp-server/metadata.py

@@ -0,0 +1,83 @@
+from ipaddress import ip_address, ip_network
+
+defaults = {
+    'apt': {
+        'packages': {
+            'kea-dhcp4-server': {},
+        },
+    },
+    'kea-dhcp-server': {
+        'config': {
+            'authoritative': True,
+            'rebind-timer': 450,
+            'renew-timer': 300,
+            'valid-lifetime': 600,
+            'expired-leases-processing': {
+                'max-reclaim-leases': 0,
+                'max-reclaim-time': 0,
+            },
+            'lease-database': {
+                'lfc-interval': 3600,
+                'name': '/var/lib/kea/kea-leases4.csv',
+                'persist': True,
+                'type': 'memfile',
+            },
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'kea-dhcp-server/fixed_allocations',
+)
+def get_static_allocations(metadata):
+    result = {}
+    mapping = {}
+
+    for iface, config in metadata.get('kea-dhcp-server/subnets', {}).items():
+        result[iface] = {}
+        mapping[iface] = ip_network(config['subnet'])
+
+    for rnode in repo.nodes:
+        if (
+            rnode.metadata.get('location', '') != metadata.get('location', '')
+            or rnode == node
+        ):
+            continue
+
+        for iface_name, iface_config in rnode.metadata.get('interfaces', {}).items():
+            if iface_config.get('dhcp', False) and iface_config.get('mac'):
+                for ip in iface_config.get('ips', set()):
+                    ipaddr = ip_address(ip)
+
+                    for kea_iface, kea_subnet in mapping.items():
+                        if ipaddr in kea_subnet:
+                            result[kea_iface][f'{rnode.name}_{iface_name}'] = {
+                                'ip': ip,
+                                'mac': iface_config['mac'],
+                            }
+                            break
+
+    return {
+        'kea-dhcp-server': {
+            'fixed_allocations': result,
+        }
+    }
+
+
+@metadata_reactor.provides(
+    'nftables/input/10-kea-dhcp-server',
+)
+def nftables(metadata):
+    rules = set()
+    for iface in node.metadata.get('kea-dhcp-server/subnets', {}):
+        rules.add(f'udp dport {{ 67, 68 }} iifname {iface} accept')
+
+    return {
+        'nftables': {
+            'input': {
+                # can't use port_rules here, because we're generating interface based rules.
+                '10-kea-dhcp-server': sorted(rules),
+            },
+        }
+    }

bundles/kernel-modules/files/modules

@@ -0,0 +1,8 @@
+# This file is managed using bundlewrap
+% for identifier, modules in sorted(node.metadata.get('modules', {}).items()):
+
+# ${identifier}
+%  for module in modules:
+${module}
+%  endfor
+% endfor

bundles/kernel-modules/items.py

@@ -0,0 +1,3 @@
+files['/etc/modules'] = {
+    'content_type': 'mako',
+}

bundles/kodi/metadata.py

@@ -43,15 +43,15 @@ defaults = {
 
 
 @metadata_reactor.provides(
-    'firewall/port_rules/8080',
-    'firewall/port_rules/9090',
+    'firewall/port_rules',
+    'firewall/port_rules',
 )
 def firewall(metadata):
     return {
         'firewall': {
             'port_rules': {
-                '8080': atomic(metadata.get('kodi/restrict-to', {'*'})),
-                '9090': atomic(metadata.get('kodi/restrict-to', {'*'})),
+                '8080/tcp': atomic(metadata.get('kodi/restrict-to', {'*'})),
+                '9090/tcp': atomic(metadata.get('kodi/restrict-to', {'*'})),
             },
         },
     }

bundles/letsencrypt/metadata.py

@@ -39,6 +39,7 @@ def cron(metadata):
                             '/usr/bin/dehydrated --cleanup',
                         ],
                         'when': '04:{}:00'.format(node.magic_number % 60),
+                        'exclude_from_monitoring': True,
                     },
                 },
             },

bundles/matrix-dimension/files/matrix-dimension.service

@@ -1,14 +0,0 @@
-[Unit]
-Description=Matrix Dimension
-After=network.target
-
-[Service]
-User=matrix-dimension
-Group=matrix-dimension
-Environment="NODE_ENV=production"
-ExecStart=/usr/bin/node ${config['install_dir']}/build/app/index.js
-WorkingDirectory=${config['install_dir']}
-Restart=on-failure
-
-[Install]
-WantedBy=multi-user.target

bundles/matrix-dimension/files/production.yaml

@@ -1,93 +0,0 @@
-# The web settings for the service (API and UI).
-# It is best to have this run on localhost and use a reverse proxy to access Dimension.
-web:
-  port: 20030
-  address: '127.0.0.1'
-
-# Homeserver configuration
-homeserver:
-  # The domain name of the homeserver. This is used in many places, such as with go-neb
-  # setups, to identify the homeserver.
-  name: "${config['homeserver']['name']}"
-
-  # The URL that Dimension, go-neb, and other services provisioned by Dimension should
-  # use to access the homeserver with.
-  clientServerUrl: "${config['homeserver']['clientServerUrl']}"
-
-  # The URL that Dimension should use when trying to communicate with federated APIs on
-  # the homeserver. If not supplied or left empty Dimension will try to resolve the address
-  # through the normal federation process.
-  #federationUrl: "https://t2bot.io:8448"
-
-  # The URL that Dimension will redirect media requests to for downloading media such as
-  # stickers. If not supplied or left empty Dimension will use the clientServerUrl.
-  #mediaUrl: "https://t2bot.io"
-
-  # The access token Dimension should use for miscellaneous access to the homeserver, and
-  # for tracking custom sticker pack updates. This should be a user configured on the homeserver
-  # and be dedicated to Dimension (create a user named "dimension" on your homeserver). For
-  # information on how to acquire an access token, visit https://t2bot.io/docs/access_tokens
-  accessToken: "${config['homeserver']['accessToken']}"
-
-# These users can modify the integrations this Dimension supports.
-# To access the admin interface, open Dimension in Riot and click the settings icon.
-admins:
-% for i in config['admins']:
-  - "${i}"
-% endfor
-# IPs and CIDR ranges listed here will be blocked from being widgets.
-# Note: Widgets may still be embedded with restricted content, although not through Dimension directly.
-widgetBlacklist:
-  - 10.0.0.0/8
-  - 172.16.0.0/12
-  - 192.168.0.0/16
-  - 127.0.0.0/8
-
-database:
-  # Where the database for Dimension is
-  uri: "postgres://${node.metadata['matrix-dimension']['database']['user']}:${node.metadata['matrix-dimension']['database']['password']}@${node.metadata['matrix-dimension']['database'].get('host', 'localhost')}/${node.metadata['matrix-dimension']['database']['database']}"
-
-  # Where to store misc information for the utility bot account.
-  botData: "${config['data_dir']}/dimension.bot.json"
-
-# Display settings that apply to self-hosted go-neb instances
-goneb:
-  # The avatars to set for each bot. Usually these don't need to be changed, however if your homeserver
-  # is not able to reach t2bot.io then you should specify your own here. To not use an avatar for a bot,
-  # make the bot's avatar an empty string.
-  avatars:
-    giphy: "mxc://t2bot.io/c5eaab3ef0133c1a61d3c849026deb27"
-    imgur: "mxc://t2bot.io/6749eaf2b302bb2188ae931b2eeb1513"
-    github: "mxc://t2bot.io/905b64b3cd8e2347f91a60c5eb0832e1"
-    wikipedia: "mxc://t2bot.io/7edfb54e9ad9e13fec0df22636feedf1"
-    travisci: "mxc://t2bot.io/7f4703126906fab8bb27df34a17707a8"
-    rss: "mxc://t2bot.io/aace4fcbd045f30afc1b4e5f0928f2f3"
-    google: "mxc://t2bot.io/636ad10742b66c4729bf89881a505142"
-    guggy: "mxc://t2bot.io/e7ef0ed0ba651aaf907655704f9a7526"
-    echo: "mxc://t2bot.io/3407ff2db96b4e954fcbf2c6c0415a13"
-    circleci: "mxc://t2bot.io/cf7d875845a82a6b21f5f66de78f6bee"
-    jira: "mxc://t2bot.io/f4a38ebcc4280ba5b950163ca3e7c329"
-
-# Settings for interacting with Telegram. Currently only applies for importing
-# sticker packs from Telegram.
-telegram:
-  # Talk to @BotFather on Telegram to get a token
-  botToken: "${config['telegram']['botToken']}"
-
-# Custom sticker pack options.
-# Largely based on https://github.com/turt2live/matrix-sticker-manager
-stickers:
-  # Whether or not to allow people to add custom sticker packs
-  enabled: true
-
-  # The sticker manager bot to promote
-  stickerBot: "@stickers:t2bot.io"
-
-  # The sticker manager URL to promote
-  managerUrl: "https://stickers.t2bot.io"
-
-
-# Settings for controlling how logging works
-logging:
-  console: true
-  consoleLevel: info

bundles/matrix-dimension/items.py

@@ -1,78 +0,0 @@
-repo.libs.tools.require_bundle(node, 'nodejs')
-
-
-directories = {
-    node.metadata['matrix-dimension']['install_dir']: {
-        'owner': 'matrix-dimension',
-        'group': 'matrix-dimension',
-    },
-}
-
-git_deploy = {
-    node.metadata['matrix-dimension']['install_dir']: {
-        'rev': node.metadata.get('matrix-dimension/version', 'master'), # doesn't have releases yet
-        'repo': 'https://github.com/turt2live/matrix-dimension.git',
-        'triggers': {
-            'action:matrix_dimension_build',
-        },
-        'needs': {
-            'directory:{}'.format(node.metadata.get('matrix-dimension/install_dir')),
-            'directory:{}'.format(node.metadata.get('matrix-dimension/data_dir')),
-        },
-    },
-}
-
-files = {
-    '{}/config/production.yaml'.format(node.metadata.get('matrix-dimension/install_dir')): {
-        'owner': 'matrix-dimension',
-        'group': 'matrix-dimension',
-        'content_type': 'mako',
-        'context': {
-            'config': node.metadata.get('matrix-dimension', {}),
-        },
-        'needs': {
-            'git_deploy:{}'.format(node.metadata.get('matrix-dimension/install_dir')),
-        },
-        'triggers': {
-            'svc_systemd:matrix-dimension:restart',
-        },
-    },
-    '/etc/systemd/system/matrix-dimension.service': {
-        'content_type': 'mako',
-        'context': {
-            'config': node.metadata.get('matrix-dimension', {}),
-        },
-        'triggers': {
-            'action:systemd-reload',
-            'svc_systemd:matrix-dimension:restart',
-        },
-    },
-}
-
-actions = {
-    'matrix_dimension_build': {
-        'command': ' && '.join([
-            'cd ' + node.metadata.get('matrix-dimension/install_dir'),
-            'sudo -u matrix-dimension npm install --legacy-peer-deps',
-            'sudo -u matrix-dimension NODE_OPTIONS=--openssl-legacy-provider npm run build',
-        ]),
-        'needs': {
-            'pkg_apt:nodejs',
-        },
-        'triggered': True,
-        'triggers': {
-            'svc_systemd:matrix-dimension:restart',
-        },
-    },
-}
-
-svc_systemd = {
-    'matrix-dimension': {
-        'needs': {
-            'action:matrix_dimension_build',
-            'file:{}/config/production.yaml'.format(node.metadata.get('matrix-dimension/install_dir')),
-            'postgres_db:matrix-dimension',
-            'postgres_role:matrix-dimension',
-        },
-    },
-}

bundles/matrix-dimension/metadata.py

@@ -1,110 +0,0 @@
-defaults = {
-    'backups': {
-        'paths': {
-            '/opt/matrix-dimension',
-            '/var/opt/matrix-dimension',
-        },
-    },
-    'icinga2_api': {
-        'matrix-dimension': {
-            'services': {
-                'MATRIX-DIMENSION PROCESS': {
-                    'command_on_monitored_host': '/usr/lib/nagios/plugins/check_procs -a matrix-dimension -c 1:',
-                },
-            },
-        },
-    },
-    'matrix-dimension': {
-        'install_dir': '/opt/matrix-dimension',
-        'data_dir': '/var/opt/matrix-dimension',
-        'database': {
-            'user': 'matrix-dimension',
-            'password': repo.vault.password_for('{} postgresql matrix-dimension'.format(node.name)),
-            'database': 'matrix-dimension',
-        },
-    },
-    'postgresql': {
-        'roles': {
-            'matrix-dimension': {
-                'password': repo.vault.password_for('{} postgresql matrix-dimension'.format(node.name)),
-            },
-        },
-        'databases': {
-            'matrix-dimension': {
-                'owner': 'matrix-dimension',
-            },
-        },
-    },
-    'users': {
-        'matrix-dimension': {
-            'home': '/var/opt/matrix-dimension',
-        },
-    },
-}
-
-
-@metadata_reactor.provides(
-    'nginx/vhosts/matrix-dimension',
-)
-def nginx_config(metadata):
-    return {
-        'nginx': {
-            'vhosts': {
-                'matrix-dimension': {
-                    'domain': metadata.get('matrix-dimension/url'),
-                    'do_not_set_content_security_headers': True,
-                    'max_body_size': '50M',
-                    'locations': {
-                        '/': {
-                            'target': 'http://127.0.0.1:20030',
-                        },
-                    },
-                },
-            },
-        },
-    }
-
-
-@metadata_reactor.provides(
-    'zfs/datasets',
-)
-def zfs(metadata):
-    return {
-        'zfs': {
-            'datasets': {
-                'tank/matrix-dimension': {},
-                'tank/matrix-dimension/install': {
-                    'mountpoint': metadata.get('matrix-dimension/install_dir'),
-                    'needed_by': {
-                        'directory:{}'.format(metadata.get('matrix-dimension/install_dir')),
-                    },
-                },
-                'tank/matrix-dimension/var': {
-                    'mountpoint': metadata.get('matrix-dimension/data_dir'),
-                    'needed_by': {
-                        'directory:{}'.format(metadata.get('matrix-dimension/data_dir')),
-                    },
-                },
-            },
-        },
-    }
-
-
-# XXX enable this once there are releases for matrix-dimension
-#@metadata_reactor.provides(
-#    'icinga2_api/matrix-dimension/services',
-#)
-#def icinga_check_for_new_release(metadata):
-#    return {
-#        'icinga2_api': {
-#            'matrix-dimension': {
-#                'services': {
-#                    'MATRIX-DIMENSION UPDATE': {
-#                        'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_github_for_new_release turt2live/matrix-dimension {}'.format(metadata.get('matrix-dimension/version')),
-#                        'vars.notification.mail': True,
-#                        'check_interval': '60m',
-#                    },
-#                },
-#            },
-#        },
-#    }

bundles/matrix-media-repo/files/config.yaml

@@ -1,7 +1,7 @@
 # General repo configuration
 repo:
-  bindAddress: '${node.metadata['matrix-media-repo'].get('listen-addr', '127.0.0.1')}'
-  port: ${node.metadata['matrix-media-repo'].get('port', 20090)}
+  bindAddress: '${node.metadata.get('matrix-media-repo/listen-addr', '127.0.0.1')}'
+  port: ${node.metadata.get('matrix-media-repo/port', 20090)}
   logDirectory: '-'
   trustAnyForwardedAddress: false
   useForwardedHost: true
@@ -10,14 +10,14 @@ federation:
   backoffAt: 20
 
 database:
-  postgres: "postgres://${node.metadata['matrix-media-repo']['database']['user']}:${node.metadata['matrix-media-repo']['database']['password']}@${node.metadata['matrix-media-repo']['database'].get('host', 'localhost')}/${node.metadata['matrix-media-repo']['database']['database']}?sslmode=disable"
+  postgres: "postgres://${node.metadata.get('matrix-media-repo/database/user')}:${node.metadata.get('matrix-media-repo/database/password')}@${node.metadata.get('matrix-media-repo/database/host', 'localhost')}/${node.metadata.get('matrix-media-repo/database/database')}?sslmode=disable"
 
   pool:
     maxConnections: 25
     maxIdleConnections: 5
 
 homeservers:
-% for homeserver, config in node.metadata['matrix-media-repo'].get('homeservers', {}).items():
+% for homeserver, config in node.metadata.get('matrix-media-repo/homeservers').items():
   - name: ${homeserver}
     csApi: "${config['domain']}"
     backoffAt: ${config.get('backoff_at', 10)}
@@ -29,45 +29,42 @@ accessTokens:
   useLocalAppserviceConfig: false
 
 admins:
-% for user in sorted(node.metadata['matrix-media-repo']['admins']):
+% for user in sorted(node.metadata.get('matrix-media-repo/admins')):
   - "${user}"
 % endfor
 
 sharedSecretAuth:
   enabled: false
-  token: "${node.metadata['matrix-media-repo']['shared-secret-token']}"
+  token: "${node.metadata.get('matrix-media-repo/shared-secret-token')}"
 
 datastores:
   - type: file
+    id: "${node.metadata.get('matrix-media-repo/datastore_id')}"
     enabled: true
-    forKinds:
-      - 'thumbnails'
-      - 'remote_media'
-      - 'local_media'
-      - 'archives'
+    forKinds: ['all']
     opts:
       path: /var/matrix/media
 
 archiving:
   enabled: true
-  selfService: ${str(node.metadata['matrix-media-repo']['archive']['self-service']).lower()}
-  targetBytesPerPart: ${node.metadata['matrix-media-repo']['archive'].get('mb_per_part', node.metadata['matrix-media-repo']['upload_max_mb']*2)*1024*1024}
+  selfService: ${str(node.metadata.get('matrix-media-repo/archive/self-service')).lower()}
+  targetBytesPerPart: ${node.metadata.get('matrix-media-repo/archive/mb_per_part', node.metadata.get('matrix-media-repo/upload_max_mb')*2)*1024*1024}
 
 uploads:
-  maxBytes: ${node.metadata['matrix-media-repo']['upload_max_mb']*1024*1024}
+  maxBytes: ${node.metadata.get('matrix-media-repo/upload_max_mb')*1024*1024}
   minBytes: 100
   reportedMaxBytes: 0
   quotas:
     enabled: false
 
 downloads:
-  maxBytes: ${node.metadata['matrix-media-repo']['download_max_mb']*1024*1024}
-  numWorkers: ${node.metadata['matrix-media-repo']['workers']}
+  maxBytes: ${node.metadata.get('matrix-media-repo/download_max_mb')*1024*1024}
+  numWorkers: ${node.metadata.get('matrix-media-repo/workers')}
   failureCacheMinutes: 5
   cache:
     enabled: true
-    maxSizeBytes: ${node.metadata['matrix-media-repo']['download_max_mb']*10*1024*1024}
-    maxFileSizeBytes: ${node.metadata['matrix-media-repo']['upload_max_mb']*1024*1024}
+    maxSizeBytes: ${node.metadata.get('matrix-media-repo/download_max_mb')*10*1024*1024}
+    maxFileSizeBytes: ${node.metadata.get('matrix-media-repo/download_max_mb')*1024*1024}
     trackedMinutes: 30
     minDownloads: 5
     minCacheTimeSeconds: 300
@@ -76,7 +73,7 @@ downloads:
 
 urlPreviews:
   enabled: true
-  maxPageSizeBytes: ${node.metadata['matrix-media-repo']['preview_max_mb']*1024*1024}
+  maxPageSizeBytes: ${node.metadata.get('matrix-media-repo/preview_max_mb')*1024*1024}
   previewUnsafeCertificates: false
   numWords: 50
   maxLength: 200
@@ -84,7 +81,7 @@ urlPreviews:
   maxTitleLength: 150
   filePreviewTypes:
     - "image/*"
-  numWorkers: ${node.metadata['matrix-media-repo']['workers']}
+  numWorkers: ${node.metadata.get('matrix-media-repo/workers')}
   disallowedNetworks:
     - "127.0.0.1/8"
     - "10.0.0.0/8"
@@ -103,8 +100,8 @@ urlPreviews:
   oEmbed: false
 
 thumbnails:
-  maxSourceBytes: ${node.metadata['matrix-media-repo']['preview_max_mb']*1024*1024}
-  numWorkers: ${node.metadata['matrix-media-repo']['workers']}
+  maxSourceBytes: ${node.metadata.get('matrix-media-repo/preview_max_mb')*1024*1024}
+  numWorkers: ${node.metadata.get('matrix-media-repo/workers')}
   sizes:
     - width: 32
       height: 32
@@ -134,7 +131,7 @@ thumbnails:
     - "video/mp4"
   allowAnimated: true
   defaultAnimated: false
-  maxAnimateSizeBytes: ${node.metadata['matrix-media-repo']['preview_max_mb']*1024*1024}
+  maxAnimateSizeBytes: ${node.metadata.get('matrix-media-repo/preview_max_mb')*1024*1024}
   stillFrame: 0.5
   expireAfterDays: 0
 

bundles/matrix-registration/files/config.yaml

@@ -0,0 +1,40 @@
+server_location: 'http://[::1]:20080'
+server_name: '${server_name}'
+registration_shared_secret: '${reg_secret}'
+admin_api_shared_secret: '${admin_secret}'
+base_url: '${base_url}'
+client_redirect: '${client_redirect}'
+client_logo: 'static/images/element-logo.png' # use '{cwd}' for current working directory
+#db: 'sqlite:///opt/matrix-registration/data/db.sqlite3'
+db: 'postgresql://${database['user']}:${database['password']}@localhost/${database['database']}'
+host: 'localhost'
+port: 20100
+rate_limit: ["100 per day", "10 per minute"]
+allow_cors: false
+ip_logging: false
+logging:
+  disable_existing_loggers: false
+  version: 1
+  root:
+    level: DEBUG
+    handlers: [console]
+  formatters:
+    brief:
+      format: '%(name)s - %(levelname)s - %(message)s'
+  handlers:
+    console:
+      class: logging.StreamHandler
+      level: INFO
+      formatter: brief
+      stream: ext://sys.stdout
+# password requirements
+password:
+  min_length: 8
+# username requirements
+username:
+  validation_regex: [] #list of regexes that the selected username must match.        Example: '[a-zA-Z]\.[a-zA-Z]'
+  invalidation_regex:  #list of regexes that the selected username must NOT match.  Example: '(admin|support)'
+    - '^abuse'
+    - 'admin'
+    - 'support'
+    - 'help'

bundles/matrix-registration/files/matrix-registration.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=matrix-registration
+After=network.target
+
+[Service]
+User=matrix-registration
+Group=matrix-registration
+WorkingDirectory=/opt/matrix-registration/src
+ExecStart=/opt/matrix-registration/venv/bin/matrix-registration --config-path /opt/matrix-registration/config.yaml serve
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

bundles/matrix-registration/items.py

@@ -0,0 +1,65 @@
+actions['matrix-registration_create_virtualenv'] = {
+    'command': '/usr/bin/python3 -m virtualenv -p python3 /opt/matrix-registration/venv/',
+    'unless': 'test -d /opt/matrix-registration/venv/',
+    'needs': {
+        # actually /opt/matrix-registration, but we don't create that
+        'directory:/opt/matrix-registration/src',
+    },
+}
+
+actions['matrix-registration_install'] = {
+    'command': ' && '.join([
+        'cd /opt/matrix-registration/src',
+        '/opt/matrix-registration/venv/bin/pip install psycopg2-binary',
+        '/opt/matrix-registration/venv/bin/pip install -e .',
+    ]),
+    'needs': {
+        'action:matrix-registration_create_virtualenv',
+    },
+    'triggered': True,
+}
+
+users['matrix-registration'] = {
+    'home': '/opt/matrix-registration',
+}
+
+directories['/opt/matrix-registration/src'] = {}
+
+git_deploy['/opt/matrix-registration/src'] = {
+    'repo': 'https://github.com/zeratax/matrix-registration.git',
+    'rev': 'master',
+    'triggers': {
+        'action:matrix-registration_install',
+        'svc_systemd:matrix-registration:restart',
+    },
+}
+
+files['/opt/matrix-registration/config.yaml'] = {
+    'content_type': 'mako',
+    'context': {
+        'admin_secret': node.metadata.get('matrix-registration/admin_secret'),
+        'base_url': node.metadata.get('matrix-registration/base_path', ''),
+        'client_redirect': node.metadata.get('matrix-registration/client_redirect'),
+        'database': node.metadata.get('matrix-registration/database'),
+        'reg_secret': node.metadata.get('matrix-synapse/registration_shared_secret'),
+        'server_name': node.metadata.get('matrix-synapse/server_name'),
+    },
+    'triggers': {
+        'svc_systemd:matrix-registration:restart',
+    },
+}
+
+files['/usr/local/lib/systemd/system/matrix-registration.service'] = {
+    'triggers': {
+        'action:systemd-reload',
+        'svc_systemd:matrix-registration:restart',
+    },
+}
+
+svc_systemd['matrix-registration'] = {
+    'needs': {
+        'action:matrix-registration_install',
+        'file:/opt/matrix-registration/config.yaml',
+        'file:/usr/local/lib/systemd/system/matrix-registration.service',
+    },
+}

bundles/matrix-registration/metadata.py

@@ -0,0 +1,25 @@
+defaults = {
+    'bash_aliases': {
+        'matrix-registration': '/opt/matrix-registration/venv/bin/matrix-registration --config-path /opt/matrix-registration/config.yaml',
+    },
+    'matrix-registration': {
+        'admin_secret': repo.vault.password_for(f'{node.name} matrix-registration admin secret'),
+        'database': {
+            'user': 'matrix-registration',
+            'password': repo.vault.password_for(f'{node.name} postgresql matrix-registration'),
+            'database': 'matrix-registration',
+        },
+    },
+    'postgresql': {
+        'roles': {
+            'matrix-registration': {
+                'password': repo.vault.password_for(f'{node.name} postgresql matrix-registration'),
+            },
+        },
+        'databases': {
+            'matrix-registration': {
+                'owner': 'matrix-registration',
+            },
+        },
+    },
+}

bundles/matrix-stickerpicker/files/sticker-import

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+/opt/matrix-stickerpicker/venv/bin/sticker-import \
+    --config /opt/matrix-stickerpicker/config.json \
+    --session /opt/matrix-stickerpicker/sticker-import.session \
+    --output-dir /var/opt/matrix-stickerpicker/ \
+    "$@"

bundles/matrix-stickerpicker/items.py

@@ -0,0 +1,47 @@
+actions['matrix-stickerpicker_create_virtualenv'] = {
+    'command': '/usr/bin/python3 -m virtualenv -p python3 /opt/matrix-stickerpicker/venv/',
+    'unless': 'test -d /opt/matrix-stickerpicker/venv/',
+    'needs': {
+        # actually /opt/matrix-stickerpicker, but we don't create that
+        'directory:/opt/matrix-stickerpicker/src',
+    },
+}
+
+actions['matrix-stickerpicker_install'] = {
+    'command': 'cd /opt/matrix-stickerpicker/src && /opt/matrix-stickerpicker/venv/bin/pip install --upgrade pip .',
+    'needs': {
+        'action:matrix-stickerpicker_create_virtualenv',
+    },
+    'triggered': True,
+}
+
+users['matrix-stickerpicker'] = {
+    'home': '/opt/matrix-stickerpicker',
+}
+
+files['/usr/local/bin/sticker-import'] = {
+    'mode': '0700',
+}
+
+files['/opt/matrix-stickerpicker/config.json'] = {
+    'content': repo.libs.faults.dict_as_json(node.metadata.get('matrix-stickerpicker/config')),
+}
+
+directories['/opt/matrix-stickerpicker/src'] = {}
+
+directories['/var/opt/matrix-stickerpicker'] = {}
+
+git_deploy['/opt/matrix-stickerpicker/src'] = {
+    'repo': 'https://github.com/maunium/stickerpicker.git',
+    'rev': node.metadata.get('matrix-stickerpicker/version', 'master'),
+    'triggers': {
+        'action:matrix-stickerpicker_install',
+    },
+}
+
+symlinks['/opt/matrix-stickerpicker/src/web/packs'] = {
+    'target': '/var/opt/matrix-stickerpicker',
+    'after': {
+        'git_deploy:/opt/matrix-stickerpicker/src',
+    },
+}

bundles/matrix-stickerpicker/metadata.py

@@ -0,0 +1,37 @@
+defaults = {
+    'backups': {
+        'paths': {
+            '/var/opt/matrix-stickerpicker',
+        },
+    },
+    'zfs': {
+        'datasets': {
+            'tank/matrix-stickerpicker': {
+                'mountpoint': '/var/opt/matrix-stickerpicker',
+                'needed_by': {
+                    'directory:/var/opt/matrix-stickerpicker',
+                },
+            },
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'nginx/vhosts/matrix-stickerpicker',
+)
+def nginx(metadata):
+    if not node.has_bundle('nginx'):
+        raise DoNotRunAgain
+
+    return {
+        'nginx': {
+            'vhosts': {
+                'matrix-stickerpicker': {
+                    'domain': metadata.get('matrix-stickerpicker/domain'),
+                    'do_not_set_content_security_headers': True,
+                    'webroot': '/opt/matrix-stickerpicker/src/web/',
+                },
+            },
+        },
+    }

bundles/matrix-synapse/files/homeserver.yaml

@@ -62,10 +62,14 @@ allow_guest_access: false
 
 enable_metrics: True
 
+% if appservice_configs:
 app_service_config_files:
-% for config in sorted(appservice_configs):
+%  for config in sorted(appservice_configs):
   - "${config}"
-% endfor
+%  endfor
+% else:
+app_service_config_files: []
+% endif
 
 signing_key_path: "/etc/matrix-synapse/homeserver.signing.key"
 trusted_key_servers:
@@ -81,7 +85,7 @@ password_config:
 
 email:
   enable_notifs: false
-  notif_from: "Matrix <noreply@${server_name}"
+  notif_from: "Matrix <noreply@${server_name}>"
 
 enable_group_creation: true
 

bundles/matrix-synapse/files/log.yaml

@@ -0,0 +1,35 @@
+version: 1
+
+formatters:
+    precise:
+        format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+    journal:
+        format: '%(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+
+handlers:
+    file:
+        class: logging.handlers.TimedRotatingFileHandler
+        formatter: precise
+        filename: /var/log/matrix-synapse/homeserver.log
+        when: midnight
+        backupCount: 1  # Does not include the current log file.
+        encoding: utf8
+    buffer:
+        class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler
+        target: file
+        capacity: 10
+        flushLevel: 30  # Flush immediately for WARNING logs and higher
+        period: 5
+    console:
+        class: logging.StreamHandler
+        formatter: journal
+
+loggers:
+    synapse.storage.SQL:
+        level: WARNING
+
+root:
+    level: WARNING
+    handlers: [buffer]
+
+disable_existing_loggers: false

bundles/matrix-synapse/files/matrix-sliding-sync.service

@@ -0,0 +1,27 @@
+<%
+    database = node.metadata.get('matrix-synapse/database')
+    db_string = 'postgresql://{}:{}@{}/{}?sslmode=disable'.format(
+        database['user'],
+        database['password'],
+        database.get('host', 'localhost'),
+        database['database'],
+    )
+%>\
+[Unit]
+Description=matrix-org sliding-sync proxy
+After=network.target
+Requires=postgresql.service
+
+[Service]
+User=matrix-synapse
+Group=matrix-synapse
+Environment=SYNCV3_SERVER=https://${node.metadata.get('matrix-synapse/baseurl')}
+Environment=SYNCV3_DB=${db_string}
+Environment=SYNCV3_SECRET=${node.metadata.get('matrix-synapse/sliding_sync/secret')}
+Environment=SYNCV3_BINDADDR=127.0.0.1:20070
+ExecStart=/usr/local/bin/matrix-sliding-sync
+Restart=always
+RestartSec=10s
+
+[Install]
+WantedBy=multi-user.target

bundles/matrix-synapse/items.py

@@ -1,7 +1,15 @@
 files = {
     '/etc/matrix-synapse/homeserver.yaml': {
         'content_type': 'mako',
-        'context': node.metadata['matrix-synapse'],
+        'context': node.metadata.get('matrix-synapse'),
+        'needs': {
+            'pkg_apt:matrix-synapse-py3',
+        },
+        'triggers': {
+            'svc_systemd:matrix-synapse:restart',
+        },
+    },
+    '/etc/matrix-synapse/log.yaml': {
         'needs': {
             'pkg_apt:matrix-synapse-py3',
         },
@@ -24,9 +32,6 @@ files = {
             'svc_systemd:matrix-synapse:restart',
         },
     },
-    '/etc/matrix-synapse/homeserver.signing.key': {
-        'content': repo.vault.decrypt_file('matrix-synapse/{}/homeserver_signing.key.vault'.format(node.name)),
-    },
     '/etc/matrix-synapse/conf.d/server_name.yaml': {
         # We don't actually need this file. However, if we don't put the
         # server name in there, synapse will somehow remove it from
@@ -34,7 +39,7 @@ files = {
         # Our override.conf ensures this file is never read, so we don't
         # need to restart synapse after changing stuff in here.
         'content_type': 'mako',
-        'context': node.metadata['matrix-synapse'],
+        'context': node.metadata.get('matrix-synapse'),
     },
     '/etc/matrix-synapse/conf.d/report_stats.yaml': {
         # see comment above
@@ -45,9 +50,39 @@ svc_systemd = {
     'matrix-synapse': {
         'needs': {
             'file:/etc/matrix-synapse/homeserver.yaml',
+            'file:/etc/matrix-synapse/log.yaml',
             'file:/etc/systemd/system/matrix-synapse.service.d/override.conf',
             'pkg_apt:matrix-synapse-py3',
             'postgres_db:synapse',
         },
     },
 }
+
+if node.metadata.get('matrix-synapse/sliding_sync/version', None):
+    files['/usr/local/bin/matrix-sliding-sync'] = {
+        'content_type': 'download',
+        'source': 'https://github.com/matrix-org/sliding-sync/releases/download/{}/syncv3_linux_amd64'.format(
+            node.metadata.get('matrix-synapse/sliding_sync/version'),
+        ),
+        'content_hash': node.metadata.get('matrix-synapse/sliding_sync/sha1', None),
+        'mode': '0755',
+        'triggers': {
+            'svc_systemd:matrix-sliding-sync:restart',
+        },
+    }
+
+    files['/usr/local/lib/systemd/system/matrix-sliding-sync.service'] = {
+        'content_type': 'mako',
+        'triggers': {
+            'action:systemd-reload',
+            'svc_systemd:matrix-sliding-sync:restart',
+        },
+    }
+
+    svc_systemd['matrix-sliding-sync'] = {
+        'needs': {
+            'file:/usr/local/bin/matrix-sliding-sync',
+            'file:/usr/local/lib/systemd/system/matrix-sliding-sync.service',
+            'postgres_db:synapse',
+        },
+    }

bundles/matrix-synapse/metadata.py

@@ -15,6 +15,7 @@ defaults = {
     },
     'backups': {
         'paths': {
+            '/etc/matrix-synapse', # to backup the signing key
             '/var/lib/matrix-synapse',
         },
     },
@@ -87,6 +88,14 @@ def nginx(metadata):
     if not node.has_bundle('nginx'):
         raise DoNotRunAgain
 
+    wellknown_client_sliding_sync = {}
+    if metadata.get('matrix-synapse/sliding_sync/version', None):
+        wellknown_client_sliding_sync = {
+            'org.matrix.msc3575.proxy': {
+                'url': 'https://{}'.format(metadata.get('matrix-synapse/baseurl')),
+            },
+        }
+
     wellknown = {
         '/.well-known/matrix/client': {
             'content': dumps({
@@ -96,6 +105,7 @@ def nginx(metadata):
                 'm.identity_server': {
                     'base_url': metadata.get('matrix-synapse/identity_server', 'https://matrix.org'),
                 },
+                **wellknown_client_sliding_sync,
                 **metadata.get('matrix-synapse/additional_client_config', {}),
             }, sort_keys=True),
             'return': 200,
@@ -117,10 +127,16 @@ def nginx(metadata):
     }
 
     locations = {
+        '/_client/': {
+            'target': 'http://127.0.0.1:20070',
+        },
         '/_matrix': {
             'target': 'http://[::1]:20080',
             'max_body_size': '50M',
         },
+        '/_matrix/client/unstable/org.matrix.msc3575/sync': {
+            'target': 'http://127.0.0.1:20070',
+        },
         '/_synapse': {
             'target': 'http://[::1]:20080',
         },
@@ -155,3 +171,20 @@ def nginx(metadata):
             'vhosts': vhosts
         },
     }
+
+@metadata_reactor.provides(
+    'matrix-synapse/trusted_key_servers',
+)
+def autotrust_our_own_servers(metadata):
+    domains = set()
+    for rnode in repo.nodes:
+        if not rnode.has_bundle('matrix-synapse'):
+            continue
+
+        domains.add(rnode.metadata.get('matrix-synapse/server_name'))
+
+    return {
+        'matrix-synapse': {
+            'trusted_key_servers': domains,
+        },
+    }

bundles/mautrix-telegram/files/config.yaml

@@ -180,7 +180,7 @@ logging:
         telethon:
             level: INFO
         aiohttp:
-            level: INFO
+            level: WARNING
     root:
         level: INFO
         handlers: [console]

bundles/minecraft/metadata.py

@@ -150,13 +150,13 @@ def heap_to_java_opts(metadata):
 
 
 @metadata_reactor.provides(
-    'firewall/port_rules/25565',
+    'firewall/port_rules',
 )
 def firewall(metadata):
     return {
         'firewall': {
             'port_rules': {
-                '25565': atomic(metadata.get('minecraft/restrict-to', set())),
+                '25565/tcp': atomic(metadata.get('minecraft/restrict-to', set())),
             },
         },
     }

bundles/miniflux/items.py

@@ -2,8 +2,8 @@ files = {
     '/etc/miniflux.conf': {
         'content_type': 'mako',
         'context': {
-            'dbpassword': node.metadata['postgresql']['roles']['miniflux']['password'],
-            'base_url': node.metadata['miniflux']['domain'],
+            'dbpassword': node.metadata.get('postgresql/roles/miniflux/password'),
+            'base_url': node.metadata.get('miniflux/domain'),
         },
         'triggers': {
             'svc_systemd:miniflux:restart',