monit #4

Merged
stillbeben merged 19 commits from monit into main 2022-01-04 18:25:08 +00:00
10 changed files with 192 additions and 54 deletions

View file

@ -17,5 +17,17 @@ defaults = {
}, },
'title': 'Usermanagement QZWI', 'title': 'Usermanagement QZWI',
}, },
'monit': {
'services': {
'ldap-frontend': {
'bin': '/opt/ldap-frontend/venv/bin/python /opt/ldap-frontend/venv/bin/gunicorn',
'ports': {
'23000': {
'protocol': 'http',
},
},
},
},
},
} }

View file

@ -0,0 +1,60 @@
set daemon 30
with start delay 30
set log syslog
set mailserver localhost
set mail-format { from: ${monit['from_address']} }
% for alert_address in monit['alert_addresses']:
set alert ${alert_address}
% endfor
set httpd unixsocket /var/run/monit.sock
use address 127.0.0.1
allow 127.0.0.1
check system $HOST
if cpu usage > 95% for 10 cycles then alert
if memory usage > 80% then alert
if swap usage > 25% then alert
check filesystem rootfs with path /
if space usage > 80% for 5 times within 15 cycles then alert
if space usage > 90% then alert
if inode usage > 90% then alert
check process cron matching "/usr/sbin/cron"
start program = "/usr/bin/systemctl start cron.service"
stop program = "/usr/bin/systemctl stop cron.service"
% for systemd_service in ('systemd-timesyncd', 'systemd-networkd', 'systemd-journald'):
check process ${systemd_service} matching "/lib/systemd/${systemd_service}"
start program = "/usr/bin/systemctl start ${systemd_service}.service"
stop program = "/usr/bin/systemctl stop ${systemd_service}.service"
% endfor
% for service,options in sorted(monit.get('services', {}).items()):
check process ${service} matching "${options['bin']}"
start program = "/bin/systemctl start ${options.get('systemd_unit', service)}.service"
stop program = "/bin/systemctl stop ${options.get('systemd_unit', service)}.service"
% for port,port_options in sorted(options.get('ports', {}).items()):
if failed port ${port}
% if port_options.get('protocol', {}):
protocol ${port_options['protocol']}
% endif
for ${port_options.get('cycles', '5')} cycles
then restart
% endfor
% for domain,http_options in sorted(options.get('http', {}).items()):
if failed host ${domain}
% if http_options['scheme'] == 'https':
port 443
protocol https
% else:
port 80
protocol http
% endif
then restart
% endfor
% endfor

38
bundles/monit/items.py Normal file
View file

@ -0,0 +1,38 @@
svc_systemd = {
'monit': {
'needs': [
'pkg_apt:monit',
],
},
}
files = {
'/etc/monit/monitrc': {
'mode': '0400',
'content_type': 'mako',
'needs': [
'pkg_apt:monit',
],
'triggers': [
'svc_systemd:monit:restart',
],
'context': {
'monit': node.metadata['monit'],
},
},
}
directories = {
'/etc/monit/conf-enabled': {
'purge': True,
},
'/etc/monit/conf-available': {
'purge': True,
},
'/etc/monit/conf.d': {
'purge': True,
},
'/etc/monit/templates': {
'purge': True,
},
}

View file

@ -0,0 +1,7 @@
defaults = {
'apt': {
'packages': {
'monit': {},
},
},
}

View file

@ -18,6 +18,13 @@ defaults = {
'nginx': { 'nginx': {
'worker_connections': 768, 'worker_connections': 768,
}, },
'monit': {
'services': {
'nginx': {
'bin': '/usr/sbin/nginx',
},
},
},
} }
@ -92,10 +99,10 @@ def index_files(metadata):
@metadata_reactor.provides( @metadata_reactor.provides(
'icinga2_api/nginx/services', 'monit/services/nginx/http',
) )
def monitoring(metadata): def monithttp(metadata):
services = {} http = {}
for vname, vconfig in metadata.get('nginx/vhosts', {}).items(): for vname, vconfig in metadata.get('nginx/vhosts', {}).items():
domain = vconfig.get('domain', vname) domain = vconfig.get('domain', vname)
@ -105,33 +112,16 @@ def monitoring(metadata):
else: else:
scheme = 'http' scheme = 'http'
if 'website_check_path' in vconfig and 'website_check_string' in vconfig: http[domain] = {
services['NGINX VHOST {} CONTENT'.format(vname)] = { 'scheme': scheme,
'check_command': 'check_http_wget', }
'vars.http_wget_contains': vconfig['website_check_string'],
'vars.http_wget_url': '{}://{}{}'.format(scheme, domain, vconfig['website_check_path']),
'vars.notification.sms': True,
}
if vconfig.get('check_ssl', vconfig['ssl']):
services['NGINX VHOST {} CERTIFICATE'.format(vname)] = {
'check_command': 'check_https_cert_at_url',
'vars.domain': domain,
'vars.notification.mail': True,
}
max_connections = metadata.get('nginx/worker_connections') * metadata.get('nginx/worker_processes')
connections_warn = int(max_connections * 0.8)
connections_crit = int(max_connections * 0.9)
services['NGINX STATUS'] = {
'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_nginx_status --warn={},-1,-1 --critical={},-1,-1 -H 127.0.0.1:22999'.format(connections_warn, connections_crit),
}
return { return {
'icinga2_api': { 'monit': {
'nginx': { 'services': {
'services': services, 'nginx': {
'http': http,
},
}, },
}, },
} }
@ -150,29 +140,3 @@ def firewall(metadata):
}, },
}, },
} }
@metadata_reactor.provides(
'telegraf/input_plugins/tail',
)
def telegraf_anon_timing(metadata):
result = {}
for vhost in metadata.get('nginx/vhosts', {}):
result[f'nginx-{vhost}'] = {
'files': [f'/var/log/nginx-timing/{vhost}.log'],
'from_beginning': False,
'grok_patterns': ['%{LOGPATTERN}'],
'grok_custom_patterns': 'LOGPATTERN \[%{HTTPDATE:ts:ts-httpd}\] %{NUMBER:request_time:float} (?:%{NUMBER:upstream_response_time:float}|-) "%{WORD:verb:tag} %{NOTSPACE:request} HTTP/%{NUMBER:http_version:float}" %{NUMBER:resp_code:tag}',
'data_format': 'grok',
'name_override': 'nginx_timing',
}
return {
'telegraf': {
'input_plugins': {
'tail': result,
},
},
}

View file

@ -29,6 +29,18 @@ defaults = {
}, },
}, },
}, },
'monit': {
'services': {
'openldap': {
'bin': '/usr/sbin/slapd',
'systemd_unit': 'slapd',
'ports': {
'389': {},
'636': {},
},
},
},
},
'openldap': { 'openldap': {
'rootpw': repo.vault.password_for(f'{node.name} openldap rootpw'), 'rootpw': repo.vault.password_for(f'{node.name} openldap rootpw'),
}, },

View file

@ -1,5 +1,21 @@
from bundlewrap.metadata import atomic from bundlewrap.metadata import atomic
defaults = {
'monit': {
'services': {
'openssh': {
'bin': '/usr/sbin/sshd',
'systemd_unit': 'sshd',
'ports': {
'22': {
'protocol': 'ssh',
},
},
},
},
},
}
@metadata_reactor.provides( @metadata_reactor.provides(
'firewall/port_rules/22', 'firewall/port_rules/22',
) )

View file

@ -4,4 +4,16 @@ defaults = {
'postfix': {}, 'postfix': {},
}, },
}, },
'monit': {
'services': {
'postfix': {
'bin': '/usr/lib/postfix/sbin/master',
'ports': {
'25': {
'protocol': 'smtp',
},
},
},
},
},
} }

View file

@ -9,4 +9,14 @@ defaults = {
'/var/lib/redis', '/var/lib/redis',
}, },
}, },
'monit': {
'services': {
'redis': {
'bin': '/usr/bin/redis-server',
'ports': {
'6379': {},
},
},
},
},
} }

View file

@ -3,6 +3,7 @@ hostname = "31.47.232.108"
bundles = [ bundles = [
"ldap-frontend", "ldap-frontend",
"letsencrypt", "letsencrypt",
"monit",
"nginx", "nginx",
"nextcloud", "nextcloud",
"openldap", "openldap",
@ -75,3 +76,9 @@ manage = [
[metadata.vm] [metadata.vm]
cpu = 4 cpu = 4
ram = 4 ram = 4
[metadata.monit]
from_address = "monit@qzwi.de"
alert_addresses = [
"rico@qzwi.de",
]