bundles/pacman: add opt-in unattended-upgrades

bundles/icinga2/files/icinga2/downtimes.conf

@@ -1,5 +1,14 @@
 % for monitored_node in sorted(monitored_nodes):
-%   if monitored_node.has_any_bundle(['apt', 'c3voc-addons']):
+<%
+    auto_updates_enabled = (
+        monitored_node.has_any_bundle(['apt', 'c3voc-addons'])
+        or (
+            monitored_node.has_bundle('pacman')
+            and monitored_node.metadata.get('pacman/unattended-upgrades/is_enabled', False)
+        )
+    )
+%>\
+%  if auto_updates_enabled:
 object ScheduledDowntime "unattended_upgrades" {
     host_name = "${monitored_node.name}"
 
@@ -9,7 +18,11 @@ object ScheduledDowntime "unattended_upgrades" {
     fixed = true
 
     ranges = {
+% if monitored_node.has_bundle('pacman'):
+        "${days[monitored_node.metadata.get('pacman/unattended-upgrades/day')]}" = "${monitored_node.metadata.get('pacman/unattended-upgrades/hour')}:${monitored_node.magic_number%30}-${monitored_node.metadata.get('pacman/unattended-upgrades/hour')}:${(monitored_node.magic_number%30)+30}"
+% else:
         "${days[monitored_node.metadata.get('apt/unattended-upgrades/day')]}" = "${monitored_node.metadata.get('apt/unattended-upgrades/hour')}:${monitored_node.magic_number%30}-${monitored_node.metadata.get('apt/unattended-upgrades/hour')}:${(monitored_node.magic_number%30)+30}"
+% endif
     }
 
     child_options = "DowntimeTriggeredChildren"

bundles/pacman/files/check_unattended_upgrades

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+statusfile="/var/tmp/unattended_upgrades.status"
+if ! [[ -f "$statusfile" ]]
+then
+    echo "Status file not found"
+    exit 3
+fi
+
+mtime=$(stat -c %Y $statusfile)
+now=$(date +%s)
+if (( $now - $mtime > 60*60*24*8 ))
+then
+    echo "Status file is older than 8 days!"
+    exit 3
+fi
+
+exitcode=$(cat $statusfile)
+case "$exitcode" in
+    abort_ssh)
+        echo "Upgrades skipped due to active SSH login"
+        exit 1
+        ;;
+    0)
+        if [[ -f /var/run/reboot-required ]]
+        then
+            echo "OK, but updates require a reboot"
+            exit 1
+        else
+            echo "OK"
+            exit 0
+        fi
+        ;;
+    *)
+        echo "Last exitcode was $exitcode"
+        exit 2
+        ;;
+esac

bundles/pacman/files/do-unattended-upgrades

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -xeuo pipefail
+
+pacman -Syu
+
+% for affected, restarts in sorted(restart_triggers.items()):
+    up_since=$(systemctl show "${affected}" | sed -n 's/^ActiveEnterTimestamp=//p' || echo 0)
+    up_since_ts=$(date -d "$up_since" +%s || echo 0)
+    now=$(date +%s)
+
+    if [ $((now - up_since_ts)) -lt 3600 ]
+    then
+% for restart in sorted(restarts):
+        systemctl restart "${restart}" || true
+% endfor
+    fi
+% endfor

bundles/pacman/files/upgrade-and-reboot

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# With systemd, we can force logging to the journal. This is better than
+# spamming the world with cron mails. You can then view these logs using
+# "journalctl -rat upgrade-and-reboot".
+if which logger >/dev/null 2>&1
+then
+    # Dump stdout and stderr to logger, which will then put everything
+    # into the journal.
+    exec 1> >(logger -t upgrade-and-reboot -p user.info)
+    exec 2> >(logger -t upgrade-and-reboot -p user.error)
+fi
+
+. /etc/upgrade-and-reboot.conf
+
+echo "Starting upgrade-and-reboot for node $nodename ..."
+
+statusfile="/var/tmp/unattended_upgrades.status"
+# Workaround, because /var/tmp is usually 1777
+[[ "$UID" == 0 ]] && chown root:root "$statusfile"
+
+logins=$(ps h -C sshd -o euser | awk '$1 != "root" && $1 != "sshd" && $1 != "sshmon"')
+if [[ -n "$logins" ]]
+then
+    echo "Will abort now, there are active SSH logins: $logins"
+    echo "abort_ssh" > "$statusfile"
+    exit 1
+fi
+
+softlockdir=/var/lib/bundlewrap/soft-$nodename
+mkdir -p "$softlockdir"
+printf '{"comment": "UPDATE", "date": %s, "expiry": %s, "id": "UNATTENDED", "items": ["*"], "user": "root@localhost"}\n' \
+    $(date +%s) \
+    $(date -d 'now + 30 mins' +%s) \
+    >"$softlockdir"/UNATTENDED
+trap 'rm -f "$softlockdir"/UNATTENDED' EXIT
+
+do-unattended-upgrades
+ret=$?
+
+echo "$ret" > "$statusfile"
+if (( $ret != 0 ))
+then
+    exit 1
+fi
+
+if [[ -n "$reboot_mail_to" ]]
+then
+    date | mail -s "SYSREBOOTNOW $nodename" "$reboot_mail_to"
+fi
+systemctl reboot
+
+echo "upgrade-and-reboot for node $nodename is DONE"

bundles/pacman/files/upgrade-and-reboot.conf

@@ -0,0 +1,3 @@
+nodename="${node.name}"
+reboot_mail_to="${node.metadata.get('apt/unattended-upgrades/reboot_mail_to', '')}"
+auto_reboot_enabled="${node.metadata.get('apt/unattended-upgrades/reboot_enabled', True)}"

bundles/pacman/items.py

@@ -7,6 +7,22 @@ files = {
     '/etc/pacman.conf': {
         'content_type': 'mako',
     },
+    '/etc/upgrade-and-reboot.conf': {
+        'content_type': 'mako',
+    },
+    '/usr/local/sbin/upgrade-and-reboot': {
+        'mode': '0700',
+    },
+    '/usr/local/sbin/do-unattended-upgrades': {
+        'content_type': 'mako',
+        'mode': '0700',
+        'context': {
+            'restart_triggers': node.metadata.get('pacman/restart_triggers', {}),
+        }
+    },
+    '/usr/local/share/icinga/plugins/check_unattended_upgrades': {
+        'mode': '0755',
+    },
 }
 
 svc_systemd['paccache.timer'] = {

bundles/pacman/metadata.py

@@ -6,5 +6,42 @@ defaults = {
         },
         'parallel_downloads': 4,
         'repository': 'http://ftp.uni-kl.de/pub/linux/archlinux/$repo/os/$arch',
+        'unattended-upgrades': {
+            'day': 5,
+            'hour': 21,
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'cron/jobs/upgrade-and-reboot',
+    'icinga2_api/pacman/services',
+)
+def patchday(metadata):
+    if not metadata.get('pacman/unattended-upgrades/is_enabled', False):
+        return {}
+
+    day = metadata.get('pacman/unattended-upgrades/day')
+    hour = metadata.get('pacman/unattended-upgrades/hour')
+
+    return {
+        'cron': {
+            'jobs': {
+                'upgrade-and-reboot': '{minute} {hour} * * {day}    root    /usr/local/sbin/upgrade-and-reboot'.format(
+                    minute=node.magic_number % 30,
+                    hour=hour,
+                    day=day,
+                ),
+            },
+        },
+        'icinga2_api': {
+            'pacman': {
+                'services': {
+                    'UNATTENDED UPGRADES': {
+                        'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_unattended_upgrades',
+                    },
+                },
+            },
         },
     }

nodes/aurto.py

@@ -54,6 +54,9 @@ nodes['aurto'] = {
             'additional_config': {
                 'Include = /etc/pacman.d/aurto',
             },
+            'unattended-upgrades': {
+                'is_enabled': True,
+            },
         },
         'sudo': {
             'extra_configs': {