bundles/wireguard: add wg_health_check

bundles/wireguard/files/wg_health_check

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+now="$(date +%s)"
+
+everything_up=1
+
+% for peer, ip in sorted(peers.items()):
+# ${peer}
+if ! /usr/bin/ping -c 4 ${ip}
+then
+    everything_up=0
+fi
+
+% endfor
+if [[ "$everything_up" -eq 1 ]]
+then
+    echo "Everything is up as expected"
+    echo "$now" > /var/tmp/wg_all_reached
+    exit 0
+fi
+
+five_min_ago="$(expr $now - 300)"
+last_reached="$(cat /var/tmp/wg_all_reached)"
+
+if [[ "$last_reached" -lt "$five_min_ago" ]]
+then
+    echo "RESTART"
+
+    systemctl restart systemd-networkd
+
+    # only restart once an hour
+    echo "$(expr $now + 3300)" > /var/tmp/wg_all_reached
+elif [[ "$last_reached" -gt "$now" ]]
+then
+    echo "Something's broken, but we have recently restarted"
+else
+    echo "Something's broken, but still in grace time"
+fi

bundles/wireguard/items.py

@@ -8,6 +8,7 @@ files = {
     },
 }
 
+health_checks = {}
 for number, (peer, config) in enumerate(sorted(node.metadata.get('wireguard/peers', {}).items())):
     files[f'/etc/systemd/network/wg{number}.netdev'] = {
         'content_type': 'mako',
@@ -29,6 +30,21 @@ for number, (peer, config) in enumerate(sorted(node.metadata.get('wireguard/peer
         },
     }
 
+    if config.get('health_check', False):
+        health_checks[peer] = config['their_ip']
+
+if health_checks:
+    files['/usr/local/bin/wg_health_check'] = {
+        'content_type': 'mako',
+        'context': {
+            'peers': health_checks,
+        },
+        'mode': '0755',
+    }
+    files['/etc/cron.d/wg_health_check'] = {
+        'content': '* * * * *    root    /usr/local/bin/wg_health_check | logger -t wg_health_check\n',
+    }
+
 if node.has_bundle('pppd'):
     files['/etc/ppp/ip-up.d/reconnect-wireguard'] = {
         'source': 'pppd-ip-up',

nodes/ovh/wireguard.py

@@ -32,7 +32,9 @@ nodes['ovh.wireguard'] = {
         'wireguard': {
             'peers': {
                 'ovh.icinga2': {},
-                'home.router': {},
+                'home.router': {
+                    'health_check': True,
+                },
                 'htz-cloud.wireguard': {},
                 'kunsi-oneplus3': {
                     'their_ip': '172.19.136.65',