bundles/icinga2: initial working draft

bundles/icinga2/files/icinga2/api-users.conf

@@ -0,0 +1,7 @@
+% for user, password in sorted(node.metadata.get('icinga2', {}).get('api_users', {}).items()):
+object ApiUser "${user}" {
+  password = "${password}"
+  permissions = [ "*" ]
+}
+
+% endfor

bundles/icinga2/files/icinga2/app.conf

@@ -0,0 +1 @@
+object IcingaApplication "app" { }

bundles/icinga2/files/icinga2/check_commands.conf

@@ -0,0 +1,93 @@
+object CheckCommand "sshmon" {
+    import "plugin-check-command"
+    import "ipv4-or-ipv6"
+
+    command = [ "/usr/local/share/icinga/plugins/check_by_sshmon" ]
+
+    arguments = {
+        "-c" = {
+            value = "$sshmon_command$"
+        }
+        "-h" = {
+            value = "$address$"
+        }
+        "-t" = {
+            set_if = bool("$sshmon_timeout$")
+            value = "$sshmon_timeout$"
+        }
+    }
+}
+
+object CheckCommand "dummy_hostalive" {
+    import "plugin-check-command"
+    import "ipv4-or-ipv6"
+
+    command = [ "true" ]
+}
+
+object CheckCommand "check_http_wget" {
+    import "plugin-check-command"
+    import "ipv4-or-ipv6"
+
+    command = [ "/usr/local/share/icinga/plugins/check_http_wget" ]
+
+    arguments = {
+        "--check-string" = {
+            set_if = bool("$http_wget_contains$")
+            value = "$http_wget_contains$"
+        }
+        "--no-follow-redirects" = {
+            set_if = "$http_wget_nofollow$"
+        }
+        "--no-verify-ssl" = {
+            set_if = "$http_wget_noverify$"
+        }
+        "--useragent" = {
+            set_if = bool("$http_wget_useragent$")
+            value = "$http_wget_useragent$"
+        }
+        "--url" = {
+            value = "$http_wget_url$"
+        }
+    }
+}
+
+object CheckCommand "check_https_cert_at_url" {
+    import "plugin-check-command"
+    import "ipv4-or-ipv6"
+
+    command = [ "/usr/local/share/icinga/plugins/check_https_certificate_at_url", "$domain$", "$port$" ]
+    vars.port = "443"
+}
+
+object CheckCommand "check_imap" {
+    import "plugin-check-command"
+    import "ipv4-or-ipv6"
+
+    command = [ "/usr/lib/nagios/plugins/check_imap" ]
+
+    arguments = {
+        "-S" = {
+            set_if = "$imap_ssl$"
+        }
+        "-p" = {
+            value = "$imap_port$"
+        }
+        "-H" = {
+            value = "$address$"
+        }
+    }
+}
+
+object CheckCommand "check_smtp" {
+    import "plugin-check-command"
+    import "ipv4-or-ipv6"
+
+    command = [ "/usr/lib/nagios/plugins/check_smtp" ]
+
+    arguments = {
+        "-H" = {
+            value = "$address$"
+        }
+    }
+}

bundles/icinga2/files/icinga2/hosts.conf

@@ -0,0 +1,16 @@
+% for monitored_node in sorted(monitored_nodes):
+object Host "${monitored_node.name}" {
+  import "generic-host"
+
+  vars.os = "Linux"
+  vars.sla = "${monitored_node.metadata.get('sla', '24x7')}"
+  vars.period = "${sla_info[monitored_node.metadata.get('sla', '24x7')]}"
+  address = "${monitored_node.hostname}"
+}
+% endfor
+
+apply Dependency "disable-service-checks-on-host-down" to Service {
+  disable_checks = true
+  ignore_soft_states = true
+  assign where true
+}

bundles/icinga2/files/icinga2/icinga2.conf

@@ -0,0 +1,44 @@
+/**
+ * Icinga 2 configuration file
+ * - this is where you define settings for the Icinga application including
+ * which hosts/services to check.
+ *
+ * For an overview of all available configuration options please refer
+ * to the documentation that is distributed as part of Icinga 2.
+ */
+
+/**
+ * The constants.conf defines global constants.
+ */
+include "constants.conf"
+
+/**
+ * The zones.conf defines zones for a cluster setup.
+ * Not required for single instance setups.
+ */
+include "zones.conf"
+
+/**
+ * The Icinga Template Library (ITL) provides a number of useful templates
+ * and command definitions.
+ * Common monitoring plugin command definitions are included separately.
+ */
+include <itl>
+include <plugins>
+include <plugins-contrib>
+
+/**
+ * The features-available directory contains a number of configuration
+ * files for features which can be enabled and disabled using the
+ * icinga2 feature enable / icinga2 feature disable CLI commands.
+ * These commands work by creating and removing symbolic links in
+ * the features-enabled directory.
+ */
+include "features-enabled/*.conf"
+
+/**
+ * Although in theory you could define all your objects in this file
+ * the preferred way is to create separate directories and files in the conf.d
+ * directory. Each of these files must have the file extension ".conf".
+ */
+include_recursive "conf.d"

bundles/icinga2/files/icinga2/ido-pgsql.conf

@@ -0,0 +1,8 @@
+library "db_ido_pgsql"
+
+object IdoPgsqlConnection "ido-pgsql" {
+  user = "icinga2",
+  password = "${node.metadata['postgresql']['roles']['icinga2']['password']}",
+  host = "localhost",
+  database = "icinga2"
+}

bundles/icinga2/files/icinga2/services_template.conf

@@ -0,0 +1,20 @@
+% for node_data, icingaconf in sorted(bundle_metadata.items()):
+%     for service, config in sorted(icingaconf.items()):
+object Service "${service}" {
+  import "generic-service"
+  host_name = "${node_data.name}"
+%         for k, v in sorted(config.items()):
+%             if k != 'import' and k != 'command_on_monitored_host':
+%                 if isinstance(v, bool) or isinstance(v, int) or k in ["check_interval", "retry_interval"]:
+  ${k} = ${str(v).lower()}
+%                 elif isinstance(v, list):
+  ${k} = [${", ".join(['"' + i + '"' for i in v])}]
+%                 else:
+  ${k} = "${v}"
+%                 endif
+%             endif
+%         endfor
+  vars.notification.sms = ${str(config.get('vars.notification.sms', False)).lower()}
+}
+%     endfor
+% endfor

bundles/icinga2/files/icinga2/templates.conf

@@ -0,0 +1,83 @@
+template Host "generic-host" {
+    max_check_attempts = 3
+    check_interval = 5m
+    retry_interval = 1m
+    check_command = "hostalive"
+    enable_notifications = true
+    enable_event_handler = true
+    enable_flapping = false
+    enable_perfdata = false
+    vars.notification.sms = true
+    vars.notification_type = "sms"
+
+}
+
+template Host "host-active" {
+    import "generic-host"
+    enable_active_checks = true
+    enable_passive_checks = false
+}
+
+template Host "host-active-nohostcheck" {
+    import "generic-host"
+    enable_active_checks = true
+    enable_passive_checks = false
+    check_command = "dummy_ok"
+}
+
+template Host "host-passive" {
+    import "generic-host"
+    enable_active_checks = false
+    enable_passive_checks = true
+}
+
+
+template Service "generic-service" {
+    max_check_attempts = 4
+    check_interval = 5m
+    retry_interval = 2m
+    vars.notification.sms = true
+    enable_notifications = true
+    enable_event_handler = true
+    enable_flapping = false
+    enable_perfdata = false
+    volatile = false
+}
+
+template Service "service-active" {
+    import "generic-service"
+    enable_active_checks = true
+    enable_passive_checks = false
+}
+
+template Service "service-passive" {
+    import "generic-service"
+    enable_active_checks = false
+    enable_passive_checks = true
+}
+
+
+template Notification "host-notification" {
+    command = "send-host-notification"
+    states = [ Up, Down ]
+    types = [ Problem, Recovery, Custom ]
+    user_groups = [ "on-call_sms" ]
+    period = host.vars.period
+
+    vars.notification_type = "sms"
+}
+
+template Notification "service-notification" {
+    command = "send-service-notification"
+    states = [ OK, Critical ]
+    types = [ Problem, Recovery, Custom ]
+    user_groups = [ "on-call_sms" ]
+
+    vars.notification_type = "sms"
+
+    if(service.vars.period) {
+      period = service.vars.period
+    } else {
+      period = host.vars.period
+    }
+}

bundles/icinga2/files/icinga2/timeperiods.conf

@@ -0,0 +1,35 @@
+object TimePeriod "24x7" {
+  import "legacy-timeperiod"
+
+  display_name = "24x7"
+  ranges = {
+    "monday"    = "00:00-24:00"
+    "tuesday"   = "00:00-24:00"
+    "wednesday" = "00:00-24:00"
+    "thursday"  = "00:00-24:00"
+    "friday"    = "00:00-24:00"
+    "saturday"  = "00:00-24:00"
+    "sunday"    = "00:00-24:00"
+  }
+}
+
+object TimePeriod "weekdays_9to6" {
+  import "legacy-timeperiod"
+
+  display_name = "9to6 (UTC)"
+  ranges = {
+    "monday"    = "09:00-18:00"
+    "tuesday"   = "09:00-18:00"
+    "wednesday" = "09:00-18:00"
+    "thursday"  = "09:00-18:00"
+    "friday"    = "09:00-18:00"
+  }
+}
+
+object TimePeriod "never" {
+  import "legacy-timeperiod"
+
+  display_name = "never"
+  ranges = {
+  }
+}

bundles/icinga2/files/icinga2/users.conf

@@ -0,0 +1,14 @@
+object UserGroup "on-call_sms" {
+    display_name = "On-Call Support"
+}
+
+% for username in sorted(node.metadata.get('icinga2', {}).get('icinga_users', {})):
+object User "${username}" {
+  display_name = "${username}"
+  enable_notifications = true
+  period = "24x7"  # Is overwritten in notifications.conf
+  states = [ OK, Critical, Up, Down ]
+  types = [ Problem, Recovery ]
+}
+
+% endfor

bundles/icinga2/files/icingaweb2/authentication.ini

@@ -0,0 +1,3 @@
+[icingaweb2]
+backend = "db"
+resource = "icingaweb_db"

bundles/icinga2/files/icingaweb2/config.ini

@@ -0,0 +1,11 @@
+[global]
+show_stacktraces = "1"
+show_application_state_messages = "1"
+config_backend = "db"
+config_resource = "icingaweb_db"
+
+[logging]
+log = "syslog"
+level = "ERROR"
+application = "icingaweb2"
+facility = "user"

bundles/icinga2/files/icingaweb2/groups.ini

@@ -0,0 +1,3 @@
+[icingaweb2]
+backend = "db"
+resource = "icingaweb_db"

bundles/icinga2/files/icingaweb2/resources.ini

@@ -0,0 +1,21 @@
+[icingaweb_db]
+type = "db"
+db = "pgsql"
+host = "localhost"
+port = "5432"
+dbname = "icingaweb2"
+username = "icinga2"
+password = "${node.metadata['postgresql']['roles']['icinga2']['password']}"
+charset = ""
+use_ssl = "0"
+
+[icinga_ido]
+type = "db"
+db = "pgsql"
+host = "localhost"
+port = "5432"
+dbname = "icinga2"
+username = "icinga2"
+password = "${node.metadata['postgresql']['roles']['icinga2']['password']}"
+charset = ""
+use_ssl = "0"

bundles/icinga2/items.py

@@ -3,6 +3,20 @@ assert node.has_bundle('sshmon')
 
 from os.path import join
 
+ENABLED_FEATURES = {
+    'api',
+    'checker',
+    'command',
+    'ido-pgsql',
+    'mainlog',
+    'notification',
+}
+
+SLA_INFO = {
+    '24x7': '24x7',
+    'never': 'never',
+}
+
 directories = {
     '/etc/icingaweb2': {
         'group': 'icingaweb2',
@@ -11,9 +25,46 @@ directories = {
             'pkg_apt:icingaweb2',
         },
     },
+    '/etc/icinga2/features-enabled': {
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0555',
+        'purge': True,
+        'needs': {
+            'pkg_apt:icinga2-ido-pgsql',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    },
+    '/etc/icinga2/conf.d': {
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0555',
+        'purge': True,
+        'needs': {
+            'pkg_apt:icinga2',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    },
+    '/etc/icinga2/conf.d/services': {
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0555',
+        'purge': True,
+        'needs': {
+            'pkg_apt:icinga2',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    },
 }
 
 files = {
+    ### Checks
     '/usr/local/share/icinga/plugins/check_rbl': {
         'mode': '0755',
     },
@@ -29,9 +80,198 @@ files = {
             'pkg_apt:icinga2-ido-pgsql',
         },
     },
-    '/etc/icingaweb2/setup.token': {
-        'content': node.metadata['icingaweb2']['setup-token'],
+
+    # Icinga2
+    '/etc/icinga2/icinga2.conf': {
+        'source': 'icinga2/icinga2.conf',
+        'needs': {
+            'pkg_apt:icinga2',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    },
+    '/etc/icinga2/features-available/ido-pgsql.conf': {
+        'source': 'icinga2/ido-pgsql.conf',
+        'content_type': 'mako',
+        'needs': {
+            'pkg_apt:icinga2',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    },
+    '/etc/icinga2/conf.d/api-users.conf': {
+        'source': 'icinga2/api-users.conf',
+        'content_type': 'mako',
+        'needs': {
+            'pkg_apt:icinga2',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    },
+    '/etc/icinga2/conf.d/app.conf': {
+        'source': 'icinga2/app.conf',
+        'needs': {
+            'pkg_apt:icinga2',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    },
+    '/etc/icinga2/conf.d/check_commands.conf': {
+        'source': 'icinga2/check_commands.conf',
+        'needs': {
+            'pkg_apt:icinga2',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    },
+    '/etc/icinga2/conf.d/templates.conf': {
+        'source': 'icinga2/templates.conf',
+        'needs': {
+            'pkg_apt:icinga2',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    },
+    '/etc/icinga2/conf.d/timeperiods.conf': {
+        'source': 'icinga2/timeperiods.conf',
+        'needs': {
+            'pkg_apt:icinga2',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    },
+    '/etc/icinga2/conf.d/users.conf': {
+        'source': 'icinga2/users.conf',
+        'content_type': 'mako',
+        'needs': {
+            'pkg_apt:icinga2',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    },
+
+    # IcingaWeb2
+    '/etc/icingaweb2/authentication.ini': {
+        'source': 'icingaweb2/authentication.ini',
+        'mode': '0660',
+        'group': 'icingaweb2',
+    },
+    '/etc/icingaweb2/config.ini': {
+        'source': 'icingaweb2/config.ini',
+        'mode': '0660',
+        'group': 'icingaweb2',
+    },
+    '/etc/icingaweb2/groups.ini': {
+        'source': 'icingaweb2/groups.ini',
+        'mode': '0660',
+        'group': 'icingaweb2',
+    },
+    '/etc/icingaweb2/resources.ini': {
+        'source': 'icingaweb2/resources.ini',
+        'content_type': 'mako',
         'mode': '0660',
         'group': 'icingaweb2',
     },
 }
+
+actions = {
+    'icinga2_api_setup': {
+        'command': 'icinga2 api setup',
+        'unless': 'test -e /var/lib/icinga2/certs/{}.crt'.format(node.metadata['hostname']),
+        'needs': {
+            'pkg_apt:icinga2',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    },
+}
+
+for feature in ENABLED_FEATURES:
+    symlinks[f'/etc/icinga2/features-enabled/{feature}.conf'] = {
+        'target': f'/etc/icinga2/features-available/{feature}.conf',
+        'needs': {
+            'pkg_apt:icinga2',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    }
+
+svc_systemd = {
+    'icinga2': {
+        'needs': {
+            'action:icinga2_api_setup',
+            'file:',
+            'pkg_apt:',
+            'symlink:',
+        },
+    },
+}
+
+
+
+
+# The actual hosts and services management starts here
+monitored_nodes = repo.nodes
+
+for n in monitored_nodes[:]:
+    if n.metadata.get('icinga_options', {}).get('exclude_from_monitoring', False):
+        monitored_nodes.remove(n)
+
+bundle_metadata = {}
+for monitored_node in monitored_nodes:
+    node_metadata = monitored_node.metadata.copy()
+
+    for bundle, config in sorted(node_metadata.get('icinga2_api', {}).items()):
+        if bundle not in bundle_metadata:
+            bundle_metadata[bundle] = {
+                'services': {}
+            }
+
+        bundle_metadata[bundle]['services'].update({
+            monitored_node: config['services']
+        })
+
+        for serv, conf in bundle_metadata[bundle]['services'][monitored_node].items():
+            if 'check_command' not in conf:
+                # This default is also set in sshmon bundle
+                conf['check_command'] = 'sshmon'
+
+for bundle, metadata in bundle_metadata.items():
+    files[f'/etc/icinga2/conf.d/services/{bundle}.conf'] = {
+        'source': 'icinga2/services_template.conf',
+        'content_type': 'mako',
+        'context': {
+            'bundle_metadata': metadata['services'],
+        },
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0440',
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    }
+
+files['/etc/icinga2/conf.d/hosts.conf'] = {
+    'source': 'icinga2/hosts.conf',
+    'content_type': 'mako',
+    'context': {
+        'monitored_nodes': monitored_nodes,
+        'sla_info': SLA_INFO,
+    },
+    'owner': 'nagios',
+    'group': 'nagios',
+    'mode': '0440',
+    'triggers': {
+        'svc_systemd:icinga2:restart',
+    },
+}

bundles/icinga2/metadata.py

@@ -22,6 +22,11 @@ defaults = {
             'libreadonly-perl': {},
         }
     },
+    'icinga2': {
+        'api_users': {
+            'root': repo.vault.password_for(f'{node.name} icinga2 api root'),
+        },
+    },
     'icingaweb2': {
         'setup-token': repo.vault.password_for(f'{node.name} icingaweb2 setup-token'),
     },

nodes/ovh/icinga2.py

@@ -24,6 +24,11 @@ nodes['ovh.icinga2'] = {
                 'php-imagick': {},
             },
         },
+        'icinga2': {
+            'icinga_users': {
+                'kunsi',
+            },
+        },
         'nginx': {
             'vhosts': {
                 'icingaweb': {