bundles/apt: rework upgrade-and-reboot mechanics to be more robust

bundles/apt/files/do-unattended-upgrades

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -xeuo pipefail
+
+apt-get update
+
+DEBIAN_FRONTEND=noninteractive apt-get -y -q -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-confdef dist-upgrade
+
+DEBIAN_FRONTEND=noninteractive apt-get -y -q autoclean
+
+DEBIAN_FRONTEND=noninteractive apt-get -y -q autoremove
+
+% if clean_old_kernels:
+existing=$(dpkg --get-selections | grep -E '^linux-(image|headers)-[0-9]' || true)
+
+if [[ -z "$existing" ]]
+then
+    echo "ERROR: No installed kernels found! Aborting!" >&2
+    exit 1
+fi
+
+current=$(uname -r | sed -r 's/-[a-zA-Z]+$//')
+latest=$(echo "$existing" | sort --version-sort -t- -k 3,4 | tail -n 1 | sed -r 's/[^0-9]+([0-9]\.[^-]+-[0-9]+).*/\1/')
+todelete=$(echo "$existing" | grep -v -E "($current|$latest)" | awk '{ print $1 }' || true)
+
+if [[ -n "$todelete" ]]
+then
+    DEBIAN_FRONTEND=noninteractive apt-get -qy purge $todelete
+fi
+% endif
+
+% for affected, restarts in sorted(restart_triggers.items()):
+    up_since=$(systemctl show "${affected}" | sed -n 's/^ActiveEnterTimestamp=//p' || echo 0)
+    up_since_ts=$(date -d "$up_since" +%s || echo 0)
+    now=$(date +%s)
+
+    if [ $((now - up_since_ts)) -lt 3600 ]
+    then
+% for restart in sorted(restarts):
+        systemctl restart "${restart}" || true
+% endfor
+    fi
+% endfor

bundles/apt/files/upgrade-and-reboot

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-set -euo pipefail
-
 # With systemd, we can force logging to the journal. This is better than
 # spamming the world with cron mails. You can then view these logs using
 # "journalctl -rat upgrade-and-reboot".
@@ -13,7 +11,9 @@ then
     exec 2> >(logger -t upgrade-and-reboot -p user.error)
 fi
 
-set -x
+. /etc/upgrade-and-reboot.conf
+
+echo "Starting upgrade-and-reboot for node $nodename ..."
 
 statusfile="/var/tmp/unattended_upgrades.status"
 # Workaround, because /var/tmp is usually 1777
@@ -27,7 +27,7 @@ then
     exit 1
 fi
 
-softlockdir=/var/lib/bundlewrap/soft-${node.name}
+softlockdir=/var/lib/bundlewrap/soft-$nodename
 mkdir -p "$softlockdir"
 printf '{"comment": "UPDATE", "date": %s, "expiry": %s, "id": "UNATTENDED", "items": ["*"], "user": "root@localhost"}\n' \
     $(date +%s) \
@@ -35,62 +35,22 @@ printf '{"comment": "UPDATE", "date": %s, "expiry": %s, "id": "UNATTENDED", "ite
     >"$softlockdir"/UNATTENDED
 trap 'rm -f "$softlockdir"/UNATTENDED' EXIT
 
-(
-    apt-get update
-    DEBIAN_FRONTEND=noninteractive apt-get -y -q -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-confdef dist-upgrade
-)
+do-unattended-upgrades
 ret=$?
 
-echo "apt-get dist-upgrade exited $ret"
 echo "$ret" > "$statusfile"
-
 if (( $ret != 0 ))
 then
     exit 1
 fi
 
-DEBIAN_FRONTEND=noninteractive apt-get -y -q autoclean
-DEBIAN_FRONTEND=noninteractive apt-get -y -q autoremove
-
-% if clean_old_kernels:
-existing=$(dpkg --get-selections | grep -E '^linux-(image|headers)-[0-9]' || true)
-
-if [[ -z "$existing" ]]
-then
-    echo "ERROR: No installed kernels found! Aborting!" >&2
-    exit 1
-fi
-
-current=$(uname -r | sed -r 's/-[a-zA-Z]+$//')
-latest=$(echo "$existing" | sort --version-sort -t- -k 3,4 | tail -n 1 | sed -r 's/[^0-9]+([0-9]\.[^-]+-[0-9]+).*/\1/')
-todelete=$(echo "$existing" | grep -v -E "($current|$latest)" | awk '{ print $1 }' || true)
-
-if [[ -n "$todelete" ]]
-then
-    DEBIAN_FRONTEND=noninteractive apt-get -qy purge $todelete
-fi
-% endif
-
-
 if [[ -f /var/run/reboot-required ]]
 then
-% if 'mail' in data:
-    date | mail -s "SYSREBOOTNOW ${node.name}" ${data['mail']}
-% endif
-    systemctl reboot
-% if restart_triggers:
-else
-% for affected, restarts in sorted(restart_triggers.items()):
-    up_since=$(systemctl show "${affected}" | sed -n 's/^ActiveEnterTimestamp=//p' || echo 0)
-    up_since_ts=$(date -d "$up_since" +%s || echo 0)
-    now=$(date +%s)
-
-    if [ $((now - up_since_ts)) -lt 3600 ]
+    if [[ -n "$reboot_mail_to" ]]
     then
-% for restart in sorted(restarts):
-        systemctl restart "${restart}" || true
-% endfor
+        date | mail -s "SYSREBOOTNOW $nodename" "$reboot_mail_to"
     fi
-% endfor
-% endif
+    systemctl reboot
+else
+    echo "upgrade-and-reboot for node $nodename is DONE"
 fi

bundles/apt/files/upgrade-and-reboot.conf

@@ -0,0 +1,2 @@
+nodename="${node.name}"
+reboot_mail_to="${node.metadata.get('apt/unattended-upgrades/reboot_mail_to', '')}"

bundles/apt/items.py

@@ -29,37 +29,42 @@ actions = {
 }
 
 files = {
-    '/usr/local/share/icinga/plugins/check_unattended_upgrades': {
-        'mode': '0755',
-    },
-    '/usr/local/sbin/upgrade-and-reboot': {
-        'content_type': 'mako',
-        'mode': '0700',
-        'context': {
-            'clean_old_kernels': node.metadata.get('apt/clean_old_kernels', True),
-            'data': node.metadata.get('apt/unattended-upgrades', {}),
-            'restart_triggers': node.metadata.get('apt/restart_triggers', {}),
-        }
-    },
-    '/etc/cloud': {
-        'delete': True,
-    },
-    '/etc/netplan': {
-        'delete': True,
-    },
-    '/var/lib/cloud': {
-        'delete': True,
-    },
     '/etc/apt/sources.list': {
         'source': 'sources.list-{}-{}'.format(node.os, supported_os[node.os][node.os_version[0]]),
         'triggers': {
             'action:apt_update',
         },
     },
+    '/etc/cloud': {
+        'delete': True,
+    },
     '/etc/kernel/postinst.d/unattended-upgrades': {
         'source': 'kernel-postinst.d',
         'mode': '0755',
     },
+    '/etc/netplan': {
+        'delete': True,
+    },
+    '/etc/upgrade-and-reboot.conf': {
+        'content_type': 'mako',
+    },
+    '/usr/local/sbin/upgrade-and-reboot': {
+        'mode': '0700',
+    },
+    '/usr/local/sbin/do-unattended-upgrades': {
+        'content_type': 'mako',
+        'mode': '0700',
+        'context': {
+            'clean_old_kernels': node.metadata.get('apt/clean_old_kernels', True),
+            'restart_triggers': node.metadata.get('apt/restart_triggers', {}),
+        }
+    },
+    '/usr/local/share/icinga/plugins/check_unattended_upgrades': {
+        'mode': '0755',
+    },
+    '/var/lib/cloud': {
+        'delete': True,
+    },
 }
 
 directories = {