diff --git a/bundles/ldap-frontend/metadata.py b/bundles/ldap-frontend/metadata.py index 86b9212..819c9fa 100644 --- a/bundles/ldap-frontend/metadata.py +++ b/bundles/ldap-frontend/metadata.py @@ -17,5 +17,17 @@ defaults = { }, 'title': 'Usermanagement QZWI', }, + 'monit': { + 'services': { + 'ldap-frontend': { + 'bin': '/opt/ldap-frontend/venv/bin/python /opt/ldap-frontend/venv/bin/gunicorn', + 'ports': { + '23000': { + 'protocol': 'http', + }, + }, + }, + }, + }, } diff --git a/bundles/monit/files/monitrc b/bundles/monit/files/monitrc new file mode 100644 index 0000000..9f5d35a --- /dev/null +++ b/bundles/monit/files/monitrc @@ -0,0 +1,60 @@ +set daemon 30 + with start delay 30 +set log syslog + +set mailserver localhost + +set mail-format { from: ${monit['from_address']} } +% for alert_address in monit['alert_addresses']: +set alert ${alert_address} +% endfor + +set httpd unixsocket /var/run/monit.sock + use address 127.0.0.1 + allow 127.0.0.1 + +check system $HOST + if cpu usage > 95% for 10 cycles then alert + if memory usage > 80% then alert + if swap usage > 25% then alert + +check filesystem rootfs with path / + if space usage > 80% for 5 times within 15 cycles then alert + if space usage > 90% then alert + if inode usage > 90% then alert + +check process cron matching "/usr/sbin/cron" + start program = "/usr/bin/systemctl start cron.service" + stop program = "/usr/bin/systemctl stop cron.service" + +% for systemd_service in ('systemd-timesyncd', 'systemd-networkd', 'systemd-journald'): +check process ${systemd_service} matching "/lib/systemd/${systemd_service}" + start program = "/usr/bin/systemctl start ${systemd_service}.service" + stop program = "/usr/bin/systemctl stop ${systemd_service}.service" +% endfor + +% for service,options in sorted(monit.get('services', {}).items()): +check process ${service} matching "${options['bin']}" + start program = "/bin/systemctl start ${options.get('systemd_unit', service)}.service" + stop program = "/bin/systemctl stop ${options.get('systemd_unit', service)}.service" +% for port,port_options in sorted(options.get('ports', {}).items()): + if failed port ${port} +% if port_options.get('protocol', {}): + protocol ${port_options['protocol']} +% endif + for ${port_options.get('cycles', '5')} cycles + then restart +% endfor +% for domain,http_options in sorted(options.get('http', {}).items()): + if failed host ${domain} +% if http_options['scheme'] == 'https': + port 443 + protocol https +% else: + port 80 + protocol http +% endif + then restart +% endfor + +% endfor diff --git a/bundles/monit/items.py b/bundles/monit/items.py new file mode 100644 index 0000000..e5f8a2b --- /dev/null +++ b/bundles/monit/items.py @@ -0,0 +1,38 @@ +svc_systemd = { + 'monit': { + 'needs': [ + 'pkg_apt:monit', + ], + }, +} + +files = { + '/etc/monit/monitrc': { + 'mode': '0400', + 'content_type': 'mako', + 'needs': [ + 'pkg_apt:monit', + ], + 'triggers': [ + 'svc_systemd:monit:restart', + ], + 'context': { + 'monit': node.metadata['monit'], + }, + }, +} + +directories = { + '/etc/monit/conf-enabled': { + 'purge': True, + }, + '/etc/monit/conf-available': { + 'purge': True, + }, + '/etc/monit/conf.d': { + 'purge': True, + }, + '/etc/monit/templates': { + 'purge': True, + }, +} diff --git a/bundles/monit/metadata.py b/bundles/monit/metadata.py new file mode 100644 index 0000000..499f943 --- /dev/null +++ b/bundles/monit/metadata.py @@ -0,0 +1,7 @@ +defaults = { + 'apt': { + 'packages': { + 'monit': {}, + }, + }, +} diff --git a/bundles/nginx/metadata.py b/bundles/nginx/metadata.py index f802ea7..f2b2855 100644 --- a/bundles/nginx/metadata.py +++ b/bundles/nginx/metadata.py @@ -18,6 +18,13 @@ defaults = { 'nginx': { 'worker_connections': 768, }, + 'monit': { + 'services': { + 'nginx': { + 'bin': '/usr/sbin/nginx', + }, + }, + }, } @@ -92,10 +99,10 @@ def index_files(metadata): @metadata_reactor.provides( - 'icinga2_api/nginx/services', + 'monit/services/nginx/http', ) -def monitoring(metadata): - services = {} +def monithttp(metadata): + http = {} for vname, vconfig in metadata.get('nginx/vhosts', {}).items(): domain = vconfig.get('domain', vname) @@ -105,33 +112,16 @@ def monitoring(metadata): else: scheme = 'http' - if 'website_check_path' in vconfig and 'website_check_string' in vconfig: - services['NGINX VHOST {} CONTENT'.format(vname)] = { - 'check_command': 'check_http_wget', - 'vars.http_wget_contains': vconfig['website_check_string'], - 'vars.http_wget_url': '{}://{}{}'.format(scheme, domain, vconfig['website_check_path']), - 'vars.notification.sms': True, - } - - if vconfig.get('check_ssl', vconfig['ssl']): - services['NGINX VHOST {} CERTIFICATE'.format(vname)] = { - 'check_command': 'check_https_cert_at_url', - 'vars.domain': domain, - 'vars.notification.mail': True, - } - - max_connections = metadata.get('nginx/worker_connections') * metadata.get('nginx/worker_processes') - connections_warn = int(max_connections * 0.8) - connections_crit = int(max_connections * 0.9) - - services['NGINX STATUS'] = { - 'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_nginx_status --warn={},-1,-1 --critical={},-1,-1 -H 127.0.0.1:22999'.format(connections_warn, connections_crit), - } + http[domain] = { + 'scheme': scheme, + } return { - 'icinga2_api': { - 'nginx': { - 'services': services, + 'monit': { + 'services': { + 'nginx': { + 'http': http, + }, }, }, } @@ -150,29 +140,3 @@ def firewall(metadata): }, }, } - - -@metadata_reactor.provides( - 'telegraf/input_plugins/tail', -) -def telegraf_anon_timing(metadata): - result = {} - - for vhost in metadata.get('nginx/vhosts', {}): - result[f'nginx-{vhost}'] = { - 'files': [f'/var/log/nginx-timing/{vhost}.log'], - 'from_beginning': False, - 'grok_patterns': ['%{LOGPATTERN}'], - 'grok_custom_patterns': 'LOGPATTERN \[%{HTTPDATE:ts:ts-httpd}\] %{NUMBER:request_time:float} (?:%{NUMBER:upstream_response_time:float}|-) "%{WORD:verb:tag} %{NOTSPACE:request} HTTP/%{NUMBER:http_version:float}" %{NUMBER:resp_code:tag}', - 'data_format': 'grok', - 'name_override': 'nginx_timing', - } - - return { - 'telegraf': { - 'input_plugins': { - 'tail': result, - }, - }, - } - diff --git a/bundles/openldap/metadata.py b/bundles/openldap/metadata.py index f1e5d6b..fdd07a1 100644 --- a/bundles/openldap/metadata.py +++ b/bundles/openldap/metadata.py @@ -29,6 +29,18 @@ defaults = { }, }, }, + 'monit': { + 'services': { + 'openldap': { + 'bin': '/usr/sbin/slapd', + 'systemd_unit': 'slapd', + 'ports': { + '389': {}, + '636': {}, + }, + }, + }, + }, 'openldap': { 'rootpw': repo.vault.password_for(f'{node.name} openldap rootpw'), }, diff --git a/bundles/openssh/metadata.py b/bundles/openssh/metadata.py index c533fcb..7e5ef8b 100644 --- a/bundles/openssh/metadata.py +++ b/bundles/openssh/metadata.py @@ -1,5 +1,21 @@ from bundlewrap.metadata import atomic +defaults = { + 'monit': { + 'services': { + 'openssh': { + 'bin': '/usr/sbin/sshd', + 'systemd_unit': 'sshd', + 'ports': { + '22': { + 'protocol': 'ssh', + }, + }, + }, + }, + }, +} + @metadata_reactor.provides( 'firewall/port_rules/22', ) diff --git a/bundles/postfix/metadata.py b/bundles/postfix/metadata.py index 266fcb1..0e443ee 100644 --- a/bundles/postfix/metadata.py +++ b/bundles/postfix/metadata.py @@ -4,4 +4,16 @@ defaults = { 'postfix': {}, }, }, + 'monit': { + 'services': { + 'postfix': { + 'bin': '/usr/lib/postfix/sbin/master', + 'ports': { + '25': { + 'protocol': 'smtp', + }, + }, + }, + }, + }, } \ No newline at end of file diff --git a/bundles/redis/metadata.py b/bundles/redis/metadata.py index ff33513..8298bd9 100644 --- a/bundles/redis/metadata.py +++ b/bundles/redis/metadata.py @@ -9,4 +9,14 @@ defaults = { '/var/lib/redis', }, }, + 'monit': { + 'services': { + 'redis': { + 'bin': '/usr/bin/redis-server', + 'ports': { + '6379': {}, + }, + }, + }, + }, } diff --git a/nodes/qzwi.toml b/nodes/qzwi.toml index a580fc1..c069cd9 100644 --- a/nodes/qzwi.toml +++ b/nodes/qzwi.toml @@ -3,6 +3,7 @@ hostname = "31.47.232.108" bundles = [ "ldap-frontend", "letsencrypt", + "monit", "nginx", "nextcloud", "openldap", @@ -75,3 +76,9 @@ manage = [ [metadata.vm] cpu = 4 ram = 4 + +[metadata.monit] +from_address = "monit@qzwi.de" +alert_addresses = [ + "rico@qzwi.de", +] \ No newline at end of file