bundles/apt: add monitoring for unattended upgrades

bundles/apt/files/check_unattended_upgrades

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+statusfile=/var/tmp/unattended_upgrades.status
+if ! [[ -f "$statusfile" ]]
+then
+    echo "Status file not found"
+    exit 3
+fi
+
+mtime=$(stat -c %Y $statusfile)
+now=$(date +$s)
+if (( $now - $mtime > 60*60*8 ))
+then
+    echo "Status file is older than 8 days!"
+    exit 3
+fi
+
+exitcode=$(cat $statusfile)
+case "$exitcode" in
+    abort_ssh)
+        echo "Upgrades skipped due to active SSH login"
+        exit 1
+        ;;
+    0)
+        echo "OK"
+        exit 0
+        ;;
+    *)
+        echo "Last exitcode was $exitcode"
+        exit 2
+        ;;
+esac

bundles/apt/files/upgrade-and-reboot

@@ -16,11 +16,15 @@ then
     fi
 fi
 
+statusfile=/var/tmp/unattended_upgrades.status
+# Workaround, because /var/tmp is usually 1777
+[[ "$UID" == 0 ]] && chown root:root "$statusfile"
 
-logins=$(ps h -C sshd -o euser | awk '$1 != "root" && $1 != "sshd"')
+logins=$(ps h -C sshd -o euser | awk '$1 != "root" && $1 != "sshd" && $1 != "sshmon"')
 if [[ -n "$logins" ]]
 then
     echo "Will abort now, there are active SSH logins: $logins"
+    echo "abort_ssh" > "$statusfile"
     exit 1
 fi
 
@@ -32,21 +36,15 @@ printf '{"comment": "UPDATE", "date": %s, "expiry": %s, "id": "UNATTENDED", "ite
     >"$softlockdir"/UNATTENDED
 trap 'rm -f "$softlockdir"/UNATTENDED' EXIT
 
-apt-get update
+(
-
+    apt-get update
-ret=$?
+    DEBIAN_FRONTEND=noninteractive apt-get -y -q -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-confdef dist-upgrade
-if (( $ret != 0 ))
+)
-then
-    echo "apt-get update exited $ret"
-    exit 1
-fi
-
-DEBIAN_FRONTEND=noninteractive apt-get -y -q -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-confdef dist-upgrade
-
 ret=$?
 if (( $ret != 0 ))
 then
     echo "apt-get dist-upgrade exited $ret"
+    echo "$ret" > "$statusfile"
     exit 1
 fi
 

bundles/apt/items.py

@@ -23,6 +23,9 @@ actions = {
 }
 
 files = {
+    '/usr/local/share/icinga/plugins/check_unattended_upgrades': {
+        'mode': '0755',
+    },
     '/usr/local/sbin/upgrade-and-reboot': {
         'content_type': 'mako',
         'mode': '0700',

bundles/apt/metadata.py

@@ -4,4 +4,13 @@ defaults = {
             minute=node.magic_number % 30,
         ),
     },
+    'icinga2_api': {
+        'apt': {
+            'services': {
+                'UNATTENDED UPGRADES': {
+                    'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_unattended_upgrades',
+                },
+            },
+        },
+    },
 }