bundles/ldap-frontend/metadata.py

@@ -17,5 +17,17 @@ defaults = {
         },
         'title': 'Usermanagement QZWI',
     },
+    'monit': {
+        'services': {
+            'ldap-frontend': {
+                'bin': '/opt/ldap-frontend/venv/bin/python /opt/ldap-frontend/venv/bin/gunicorn',
+                'ports': {
+                    '23000': {
+                        'protocol': 'http',
+                    },
+                },
+            },
+        },
+    },
 }
 

bundles/monit/files/monitrc

@@ -0,0 +1,60 @@
+set daemon  30
+   with start delay 30
+set log syslog
+
+set mailserver localhost
+
+set mail-format { from: ${monit['from_address']} }
+% for alert_address in monit['alert_addresses']:
+set alert ${alert_address}
+% endfor
+
+set httpd unixsocket /var/run/monit.sock
+    use address 127.0.0.1
+    allow 127.0.0.1
+
+check system $HOST
+    if cpu usage > 95% for 10 cycles then alert
+    if memory usage > 80% then alert
+    if swap usage > 25% then alert
+
+check filesystem rootfs with path /
+    if space usage > 80% for 5 times within 15 cycles then alert
+    if space usage > 90% then alert
+    if inode usage > 90% then alert
+
+check process cron matching "/usr/sbin/cron"
+    start program = "/usr/bin/systemctl start cron.service"
+    stop program = "/usr/bin/systemctl stop cron.service"
+
+% for systemd_service in ('systemd-timesyncd', 'systemd-networkd', 'systemd-journald'):
+check process ${systemd_service} matching "/lib/systemd/${systemd_service}"
+    start program = "/usr/bin/systemctl start ${systemd_service}.service"
+    stop program = "/usr/bin/systemctl stop ${systemd_service}.service"
+% endfor
+
+% for service,options in sorted(monit.get('services', {}).items()):
+check process ${service} matching "${options['bin']}"
+    start program = "/bin/systemctl start ${options.get('systemd_unit', service)}.service"
+    stop program = "/bin/systemctl stop ${options.get('systemd_unit', service)}.service"
+% for port,port_options in sorted(options.get('ports', {}).items()):
+    if failed port ${port}
+% if port_options.get('protocol', {}):
+        protocol ${port_options['protocol']}
+% endif
+        for ${port_options.get('cycles', '5')} cycles
+        then restart
+% endfor
+% for domain,http_options in sorted(options.get('http', {}).items()):
+    if failed host ${domain}
+% if http_options['scheme'] == 'https':
+        port 443
+        protocol https
+% else:
+        port 80
+        protocol http
+% endif
+        then restart
+% endfor
+
+% endfor

bundles/monit/items.py

@@ -0,0 +1,38 @@
+svc_systemd = {
+    'monit': {
+        'needs': [
+            'pkg_apt:monit',
+        ],
+    },
+}
+
+files = {
+    '/etc/monit/monitrc': {
+        'mode': '0400',
+        'content_type': 'mako',
+        'needs': [
+            'pkg_apt:monit',
+        ],
+        'triggers': [
+            'svc_systemd:monit:restart',
+        ],
+        'context': {
+            'monit': node.metadata['monit'],
+        },
+    },
+}
+
+directories = {
+    '/etc/monit/conf-enabled': {
+        'purge': True,
+    },
+    '/etc/monit/conf-available': {
+        'purge': True,
+    },
+    '/etc/monit/conf.d': {
+        'purge': True,
+    },
+    '/etc/monit/templates': {
+        'purge': True,
+    },
+}

bundles/monit/metadata.py

@@ -0,0 +1,7 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'monit': {},
+        },
+    },
+}

bundles/nginx/metadata.py

@@ -18,6 +18,13 @@ defaults = {
     'nginx': {
         'worker_connections': 768,
     },
+    'monit': {
+        'services': {
+            'nginx': {
+                'bin': '/usr/sbin/nginx',
+            },
+        },
+    },
 }
 
 
@@ -92,10 +99,10 @@ def index_files(metadata):
 
 
 @metadata_reactor.provides(
-    'icinga2_api/nginx/services',
+    'monit/services/nginx/http',
 )
-def monitoring(metadata):
+def monithttp(metadata):
-    services = {}
+    http = {}
 
     for vname, vconfig in metadata.get('nginx/vhosts', {}).items():
         domain = vconfig.get('domain', vname)
@@ -105,33 +112,16 @@ def monitoring(metadata):
         else:
             scheme = 'http'
 
-        if 'website_check_path' in vconfig and 'website_check_string' in vconfig:
+        http[domain] = {
-            services['NGINX VHOST {} CONTENT'.format(vname)] = {
+            'scheme': scheme,
-                'check_command': 'check_http_wget',
+        }
-                'vars.http_wget_contains': vconfig['website_check_string'],
-                'vars.http_wget_url': '{}://{}{}'.format(scheme, domain, vconfig['website_check_path']),
-                'vars.notification.sms': True,
-            }
-
-        if vconfig.get('check_ssl', vconfig['ssl']):
-            services['NGINX VHOST {} CERTIFICATE'.format(vname)] = {
-                'check_command': 'check_https_cert_at_url',
-                'vars.domain': domain,
-                'vars.notification.mail': True,
-            }
-
-    max_connections = metadata.get('nginx/worker_connections') * metadata.get('nginx/worker_processes')
-    connections_warn = int(max_connections * 0.8)
-    connections_crit = int(max_connections * 0.9)
-
-    services['NGINX STATUS'] = {
-        'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_nginx_status --warn={},-1,-1 --critical={},-1,-1 -H 127.0.0.1:22999'.format(connections_warn, connections_crit),
-    }
 
     return {
-        'icinga2_api': {
+        'monit': {
-            'nginx': {
+            'services': {
-                'services': services,
+                'nginx': {
+                    'http': http,
+                },
             },
         },
     }
@@ -150,29 +140,3 @@ def firewall(metadata):
             },
         },
     }
-
-
-@metadata_reactor.provides(
-    'telegraf/input_plugins/tail',
-)
-def telegraf_anon_timing(metadata):
-    result = {}
-
-    for vhost in metadata.get('nginx/vhosts', {}):
-        result[f'nginx-{vhost}'] = {
-            'files': [f'/var/log/nginx-timing/{vhost}.log'],
-            'from_beginning': False,
-            'grok_patterns': ['%{LOGPATTERN}'],
-            'grok_custom_patterns': 'LOGPATTERN \[%{HTTPDATE:ts:ts-httpd}\] %{NUMBER:request_time:float} (?:%{NUMBER:upstream_response_time:float}|-) "%{WORD:verb:tag} %{NOTSPACE:request} HTTP/%{NUMBER:http_version:float}" %{NUMBER:resp_code:tag}',
-            'data_format': 'grok',
-            'name_override': 'nginx_timing',
-        }
-
-    return {
-        'telegraf': {
-            'input_plugins': {
-                'tail': result,
-            },
-        },
-    }
-

bundles/openldap/metadata.py

@@ -29,6 +29,18 @@ defaults = {
             },
         },
     },
+    'monit': {
+        'services': {
+            'openldap': {
+                'bin': '/usr/sbin/slapd',
+                'systemd_unit': 'slapd',
+                'ports': {
+                    '389': {},
+                    '636': {},
+                },
+            },
+        },
+    },
     'openldap': {
         'rootpw': repo.vault.password_for(f'{node.name} openldap rootpw'),
     },

bundles/openssh/metadata.py

@@ -1,5 +1,21 @@
 from bundlewrap.metadata import atomic
 
+defaults = {
+    'monit': {
+        'services': {
+            'openssh': {
+                'bin': '/usr/sbin/sshd',
+                'systemd_unit': 'sshd',
+                'ports': {
+                    '22': {
+                        'protocol': 'ssh',
+                    },
+                },
+            },
+        },
+    },
+}
+
 @metadata_reactor.provides(
     'firewall/port_rules/22',
 )

bundles/postfix/metadata.py

@@ -4,4 +4,16 @@ defaults = {
             'postfix': {},
         },
     },
+    'monit': {
+        'services': {
+            'postfix': {
+                'bin': '/usr/lib/postfix/sbin/master',
+                'ports': {
+                    '25': {
+                        'protocol': 'smtp',
+                    },
+                },
+            },
+        },
+    },
 }

bundles/redis/metadata.py

@@ -9,4 +9,14 @@ defaults = {
             '/var/lib/redis',
         },
     },
+    'monit': {
+        'services': {
+            'redis': {
+                'bin': '/usr/bin/redis-server',
+                'ports': {
+                    '6379': {},
+                },
+            },
+        },
+    },
 }

nodes/qzwi.toml

@@ -3,6 +3,7 @@ hostname = "31.47.232.108"
 bundles = [
   "ldap-frontend",
   "letsencrypt",
+  "monit",
   "nginx",
   "nextcloud",
   "openldap",
@@ -75,3 +76,9 @@ manage = [
 [metadata.vm]
 cpu = 4
 ram = 4
+
+[metadata.monit]
+from_address = "monit@qzwi.de"
+alert_addresses = [
+  "rico@qzwi.de",
+]