diff --git a/bundles/icinga2/files/icinga2/downtimes.conf b/bundles/icinga2/files/icinga2/downtimes.conf index a4dd0b0..96ba242 100644 --- a/bundles/icinga2/files/icinga2/downtimes.conf +++ b/bundles/icinga2/files/icinga2/downtimes.conf @@ -1,5 +1,14 @@ % for monitored_node in sorted(monitored_nodes): -% if monitored_node.has_any_bundle(['apt', 'c3voc-addons']): +<% + auto_updates_enabled = ( + monitored_node.has_any_bundle(['apt', 'c3voc-addons']) + or ( + monitored_node.has_bundle('pacman') + and monitored_node.metadata.get('pacman/unattended-upgrades/is_enabled', False) + ) + ) +%>\ +% if auto_updates_enabled: object ScheduledDowntime "unattended_upgrades" { host_name = "${monitored_node.name}" @@ -9,7 +18,11 @@ object ScheduledDowntime "unattended_upgrades" { fixed = true ranges = { +% if monitored_node.has_bundle('pacman'): + "${days[monitored_node.metadata.get('pacman/unattended-upgrades/day')]}" = "${monitored_node.metadata.get('pacman/unattended-upgrades/hour')}:${monitored_node.magic_number%30}-${monitored_node.metadata.get('pacman/unattended-upgrades/hour')}:${(monitored_node.magic_number%30)+30}" +% else: "${days[monitored_node.metadata.get('apt/unattended-upgrades/day')]}" = "${monitored_node.metadata.get('apt/unattended-upgrades/hour')}:${monitored_node.magic_number%30}-${monitored_node.metadata.get('apt/unattended-upgrades/hour')}:${(monitored_node.magic_number%30)+30}" +% endif } child_options = "DowntimeTriggeredChildren" diff --git a/bundles/pacman/files/check_unattended_upgrades b/bundles/pacman/files/check_unattended_upgrades new file mode 100644 index 0000000..1cafab5 --- /dev/null +++ b/bundles/pacman/files/check_unattended_upgrades @@ -0,0 +1,38 @@ +#!/bin/bash + +statusfile="/var/tmp/unattended_upgrades.status" +if ! [[ -f "$statusfile" ]] +then + echo "Status file not found" + exit 3 +fi + +mtime=$(stat -c %Y $statusfile) +now=$(date +%s) +if (( $now - $mtime > 60*60*24*8 )) +then + echo "Status file is older than 8 days!" + exit 3 +fi + +exitcode=$(cat $statusfile) +case "$exitcode" in + abort_ssh) + echo "Upgrades skipped due to active SSH login" + exit 1 + ;; + 0) + if [[ -f /var/run/reboot-required ]] + then + echo "OK, but updates require a reboot" + exit 1 + else + echo "OK" + exit 0 + fi + ;; + *) + echo "Last exitcode was $exitcode" + exit 2 + ;; +esac diff --git a/bundles/pacman/files/do-unattended-upgrades b/bundles/pacman/files/do-unattended-upgrades new file mode 100644 index 0000000..79d38aa --- /dev/null +++ b/bundles/pacman/files/do-unattended-upgrades @@ -0,0 +1,18 @@ +#!/bin/bash + +set -xeuo pipefail + +pacman -Syu + +% for affected, restarts in sorted(restart_triggers.items()): + up_since=$(systemctl show "${affected}" | sed -n 's/^ActiveEnterTimestamp=//p' || echo 0) + up_since_ts=$(date -d "$up_since" +%s || echo 0) + now=$(date +%s) + + if [ $((now - up_since_ts)) -lt 3600 ] + then +% for restart in sorted(restarts): + systemctl restart "${restart}" || true +% endfor + fi +% endfor diff --git a/bundles/pacman/files/upgrade-and-reboot b/bundles/pacman/files/upgrade-and-reboot new file mode 100644 index 0000000..b8339ce --- /dev/null +++ b/bundles/pacman/files/upgrade-and-reboot @@ -0,0 +1,53 @@ +#!/bin/bash + +# With systemd, we can force logging to the journal. This is better than +# spamming the world with cron mails. You can then view these logs using +# "journalctl -rat upgrade-and-reboot". +if which logger >/dev/null 2>&1 +then + # Dump stdout and stderr to logger, which will then put everything + # into the journal. + exec 1> >(logger -t upgrade-and-reboot -p user.info) + exec 2> >(logger -t upgrade-and-reboot -p user.error) +fi + +. /etc/upgrade-and-reboot.conf + +echo "Starting upgrade-and-reboot for node $nodename ..." + +statusfile="/var/tmp/unattended_upgrades.status" +# Workaround, because /var/tmp is usually 1777 +[[ "$UID" == 0 ]] && chown root:root "$statusfile" + +logins=$(ps h -C sshd -o euser | awk '$1 != "root" && $1 != "sshd" && $1 != "sshmon"') +if [[ -n "$logins" ]] +then + echo "Will abort now, there are active SSH logins: $logins" + echo "abort_ssh" > "$statusfile" + exit 1 +fi + +softlockdir=/var/lib/bundlewrap/soft-$nodename +mkdir -p "$softlockdir" +printf '{"comment": "UPDATE", "date": %s, "expiry": %s, "id": "UNATTENDED", "items": ["*"], "user": "root@localhost"}\n' \ + $(date +%s) \ + $(date -d 'now + 30 mins' +%s) \ + >"$softlockdir"/UNATTENDED +trap 'rm -f "$softlockdir"/UNATTENDED' EXIT + +do-unattended-upgrades +ret=$? + +echo "$ret" > "$statusfile" +if (( $ret != 0 )) +then + exit 1 +fi + +if [[ -n "$reboot_mail_to" ]] +then + date | mail -s "SYSREBOOTNOW $nodename" "$reboot_mail_to" +fi +systemctl reboot + +echo "upgrade-and-reboot for node $nodename is DONE" diff --git a/bundles/pacman/files/upgrade-and-reboot.conf b/bundles/pacman/files/upgrade-and-reboot.conf new file mode 100644 index 0000000..ca71dce --- /dev/null +++ b/bundles/pacman/files/upgrade-and-reboot.conf @@ -0,0 +1,3 @@ +nodename="${node.name}" +reboot_mail_to="${node.metadata.get('apt/unattended-upgrades/reboot_mail_to', '')}" +auto_reboot_enabled="${node.metadata.get('apt/unattended-upgrades/reboot_enabled', True)}" diff --git a/bundles/pacman/items.py b/bundles/pacman/items.py index 9bc9833..6ef5c89 100644 --- a/bundles/pacman/items.py +++ b/bundles/pacman/items.py @@ -7,6 +7,22 @@ files = { '/etc/pacman.conf': { 'content_type': 'mako', }, + '/etc/upgrade-and-reboot.conf': { + 'content_type': 'mako', + }, + '/usr/local/sbin/upgrade-and-reboot': { + 'mode': '0700', + }, + '/usr/local/sbin/do-unattended-upgrades': { + 'content_type': 'mako', + 'mode': '0700', + 'context': { + 'restart_triggers': node.metadata.get('pacman/restart_triggers', {}), + } + }, + '/usr/local/share/icinga/plugins/check_unattended_upgrades': { + 'mode': '0755', + }, } svc_systemd['paccache.timer'] = { diff --git a/bundles/pacman/metadata.py b/bundles/pacman/metadata.py index d4e077f..401ffd1 100644 --- a/bundles/pacman/metadata.py +++ b/bundles/pacman/metadata.py @@ -6,5 +6,42 @@ defaults = { }, 'parallel_downloads': 4, 'repository': 'http://ftp.uni-kl.de/pub/linux/archlinux/$repo/os/$arch', + 'unattended-upgrades': { + 'day': 5, + 'hour': 21, + }, }, } + + +@metadata_reactor.provides( + 'cron/jobs/upgrade-and-reboot', + 'icinga2_api/pacman/services', +) +def patchday(metadata): + if not metadata.get('pacman/unattended-upgrades/is_enabled', False): + return {} + + day = metadata.get('pacman/unattended-upgrades/day') + hour = metadata.get('pacman/unattended-upgrades/hour') + + return { + 'cron': { + 'jobs': { + 'upgrade-and-reboot': '{minute} {hour} * * {day} root /usr/local/sbin/upgrade-and-reboot'.format( + minute=node.magic_number % 30, + hour=hour, + day=day, + ), + }, + }, + 'icinga2_api': { + 'pacman': { + 'services': { + 'UNATTENDED UPGRADES': { + 'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_unattended_upgrades', + }, + }, + }, + }, + } diff --git a/nodes/aurto.py b/nodes/aurto.py index 9d73b19..54a18f6 100644 --- a/nodes/aurto.py +++ b/nodes/aurto.py @@ -54,6 +54,9 @@ nodes['aurto'] = { 'additional_config': { 'Include = /etc/pacman.d/aurto', }, + 'unattended-upgrades': { + 'is_enabled': True, + }, }, 'sudo': { 'extra_configs': {