From 11071914e086703c71e99eb886fad773510daabc Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Sat, 21 Nov 2020 10:29:36 +0100 Subject: [PATCH] bundles/icinga2: initial working draft --- bundles/icinga2/files/icinga2/api-users.conf | 7 + bundles/icinga2/files/icinga2/app.conf | 1 + .../icinga2/files/icinga2/check_commands.conf | 93 +++++++ bundles/icinga2/files/icinga2/hosts.conf | 16 ++ bundles/icinga2/files/icinga2/icinga2.conf | 44 ++++ bundles/icinga2/files/icinga2/ido-pgsql.conf | 8 + .../files/icinga2/services_template.conf | 20 ++ bundles/icinga2/files/icinga2/templates.conf | 83 ++++++ .../icinga2/files/icinga2/timeperiods.conf | 35 +++ bundles/icinga2/files/icinga2/users.conf | 14 + .../files/icingaweb2/authentication.ini | 3 + bundles/icinga2/files/icingaweb2/config.ini | 11 + bundles/icinga2/files/icingaweb2/groups.ini | 3 + .../icinga2/files/icingaweb2/resources.ini | 21 ++ bundles/icinga2/items.py | 244 +++++++++++++++++- bundles/icinga2/metadata.py | 5 + nodes/ovh/icinga2.py | 5 + 17 files changed, 611 insertions(+), 2 deletions(-) create mode 100644 bundles/icinga2/files/icinga2/api-users.conf create mode 100644 bundles/icinga2/files/icinga2/app.conf create mode 100644 bundles/icinga2/files/icinga2/check_commands.conf create mode 100644 bundles/icinga2/files/icinga2/hosts.conf create mode 100644 bundles/icinga2/files/icinga2/icinga2.conf create mode 100644 bundles/icinga2/files/icinga2/ido-pgsql.conf create mode 100644 bundles/icinga2/files/icinga2/services_template.conf create mode 100644 bundles/icinga2/files/icinga2/templates.conf create mode 100644 bundles/icinga2/files/icinga2/timeperiods.conf create mode 100644 bundles/icinga2/files/icinga2/users.conf create mode 100644 bundles/icinga2/files/icingaweb2/authentication.ini create mode 100644 bundles/icinga2/files/icingaweb2/config.ini create mode 100644 bundles/icinga2/files/icingaweb2/groups.ini create mode 100644 bundles/icinga2/files/icingaweb2/resources.ini diff --git a/bundles/icinga2/files/icinga2/api-users.conf b/bundles/icinga2/files/icinga2/api-users.conf new file mode 100644 index 0000000..59be6f0 --- /dev/null +++ b/bundles/icinga2/files/icinga2/api-users.conf @@ -0,0 +1,7 @@ +% for user, password in sorted(node.metadata.get('icinga2', {}).get('api_users', {}).items()): +object ApiUser "${user}" { + password = "${password}" + permissions = [ "*" ] +} + +% endfor diff --git a/bundles/icinga2/files/icinga2/app.conf b/bundles/icinga2/files/icinga2/app.conf new file mode 100644 index 0000000..3e4be0d --- /dev/null +++ b/bundles/icinga2/files/icinga2/app.conf @@ -0,0 +1 @@ +object IcingaApplication "app" { } diff --git a/bundles/icinga2/files/icinga2/check_commands.conf b/bundles/icinga2/files/icinga2/check_commands.conf new file mode 100644 index 0000000..9f59c72 --- /dev/null +++ b/bundles/icinga2/files/icinga2/check_commands.conf @@ -0,0 +1,93 @@ +object CheckCommand "sshmon" { + import "plugin-check-command" + import "ipv4-or-ipv6" + + command = [ "/usr/local/share/icinga/plugins/check_by_sshmon" ] + + arguments = { + "-c" = { + value = "$sshmon_command$" + } + "-h" = { + value = "$address$" + } + "-t" = { + set_if = bool("$sshmon_timeout$") + value = "$sshmon_timeout$" + } + } +} + +object CheckCommand "dummy_hostalive" { + import "plugin-check-command" + import "ipv4-or-ipv6" + + command = [ "true" ] +} + +object CheckCommand "check_http_wget" { + import "plugin-check-command" + import "ipv4-or-ipv6" + + command = [ "/usr/local/share/icinga/plugins/check_http_wget" ] + + arguments = { + "--check-string" = { + set_if = bool("$http_wget_contains$") + value = "$http_wget_contains$" + } + "--no-follow-redirects" = { + set_if = "$http_wget_nofollow$" + } + "--no-verify-ssl" = { + set_if = "$http_wget_noverify$" + } + "--useragent" = { + set_if = bool("$http_wget_useragent$") + value = "$http_wget_useragent$" + } + "--url" = { + value = "$http_wget_url$" + } + } +} + +object CheckCommand "check_https_cert_at_url" { + import "plugin-check-command" + import "ipv4-or-ipv6" + + command = [ "/usr/local/share/icinga/plugins/check_https_certificate_at_url", "$domain$", "$port$" ] + vars.port = "443" +} + +object CheckCommand "check_imap" { + import "plugin-check-command" + import "ipv4-or-ipv6" + + command = [ "/usr/lib/nagios/plugins/check_imap" ] + + arguments = { + "-S" = { + set_if = "$imap_ssl$" + } + "-p" = { + value = "$imap_port$" + } + "-H" = { + value = "$address$" + } + } +} + +object CheckCommand "check_smtp" { + import "plugin-check-command" + import "ipv4-or-ipv6" + + command = [ "/usr/lib/nagios/plugins/check_smtp" ] + + arguments = { + "-H" = { + value = "$address$" + } + } +} diff --git a/bundles/icinga2/files/icinga2/hosts.conf b/bundles/icinga2/files/icinga2/hosts.conf new file mode 100644 index 0000000..9e63392 --- /dev/null +++ b/bundles/icinga2/files/icinga2/hosts.conf @@ -0,0 +1,16 @@ +% for monitored_node in sorted(monitored_nodes): +object Host "${monitored_node.name}" { + import "generic-host" + + vars.os = "Linux" + vars.sla = "${monitored_node.metadata.get('sla', '24x7')}" + vars.period = "${sla_info[monitored_node.metadata.get('sla', '24x7')]}" + address = "${monitored_node.hostname}" +} +% endfor + +apply Dependency "disable-service-checks-on-host-down" to Service { + disable_checks = true + ignore_soft_states = true + assign where true +} diff --git a/bundles/icinga2/files/icinga2/icinga2.conf b/bundles/icinga2/files/icinga2/icinga2.conf new file mode 100644 index 0000000..522c505 --- /dev/null +++ b/bundles/icinga2/files/icinga2/icinga2.conf @@ -0,0 +1,44 @@ +/** + * Icinga 2 configuration file + * - this is where you define settings for the Icinga application including + * which hosts/services to check. + * + * For an overview of all available configuration options please refer + * to the documentation that is distributed as part of Icinga 2. + */ + +/** + * The constants.conf defines global constants. + */ +include "constants.conf" + +/** + * The zones.conf defines zones for a cluster setup. + * Not required for single instance setups. + */ +include "zones.conf" + +/** + * The Icinga Template Library (ITL) provides a number of useful templates + * and command definitions. + * Common monitoring plugin command definitions are included separately. + */ +include +include +include + +/** + * The features-available directory contains a number of configuration + * files for features which can be enabled and disabled using the + * icinga2 feature enable / icinga2 feature disable CLI commands. + * These commands work by creating and removing symbolic links in + * the features-enabled directory. + */ +include "features-enabled/*.conf" + +/** + * Although in theory you could define all your objects in this file + * the preferred way is to create separate directories and files in the conf.d + * directory. Each of these files must have the file extension ".conf". + */ +include_recursive "conf.d" diff --git a/bundles/icinga2/files/icinga2/ido-pgsql.conf b/bundles/icinga2/files/icinga2/ido-pgsql.conf new file mode 100644 index 0000000..a252e01 --- /dev/null +++ b/bundles/icinga2/files/icinga2/ido-pgsql.conf @@ -0,0 +1,8 @@ +library "db_ido_pgsql" + +object IdoPgsqlConnection "ido-pgsql" { + user = "icinga2", + password = "${node.metadata['postgresql']['roles']['icinga2']['password']}", + host = "localhost", + database = "icinga2" +} diff --git a/bundles/icinga2/files/icinga2/services_template.conf b/bundles/icinga2/files/icinga2/services_template.conf new file mode 100644 index 0000000..7ee5b30 --- /dev/null +++ b/bundles/icinga2/files/icinga2/services_template.conf @@ -0,0 +1,20 @@ +% for node_data, icingaconf in sorted(bundle_metadata.items()): +% for service, config in sorted(icingaconf.items()): +object Service "${service}" { + import "generic-service" + host_name = "${node_data.name}" +% for k, v in sorted(config.items()): +% if k != 'import' and k != 'command_on_monitored_host': +% if isinstance(v, bool) or isinstance(v, int) or k in ["check_interval", "retry_interval"]: + ${k} = ${str(v).lower()} +% elif isinstance(v, list): + ${k} = [${", ".join(['"' + i + '"' for i in v])}] +% else: + ${k} = "${v}" +% endif +% endif +% endfor + vars.notification.sms = ${str(config.get('vars.notification.sms', False)).lower()} +} +% endfor +% endfor diff --git a/bundles/icinga2/files/icinga2/templates.conf b/bundles/icinga2/files/icinga2/templates.conf new file mode 100644 index 0000000..b1865e0 --- /dev/null +++ b/bundles/icinga2/files/icinga2/templates.conf @@ -0,0 +1,83 @@ +template Host "generic-host" { + max_check_attempts = 3 + check_interval = 5m + retry_interval = 1m + check_command = "hostalive" + enable_notifications = true + enable_event_handler = true + enable_flapping = false + enable_perfdata = false + vars.notification.sms = true + vars.notification_type = "sms" + +} + +template Host "host-active" { + import "generic-host" + enable_active_checks = true + enable_passive_checks = false +} + +template Host "host-active-nohostcheck" { + import "generic-host" + enable_active_checks = true + enable_passive_checks = false + check_command = "dummy_ok" +} + +template Host "host-passive" { + import "generic-host" + enable_active_checks = false + enable_passive_checks = true +} + + +template Service "generic-service" { + max_check_attempts = 4 + check_interval = 5m + retry_interval = 2m + vars.notification.sms = true + enable_notifications = true + enable_event_handler = true + enable_flapping = false + enable_perfdata = false + volatile = false +} + +template Service "service-active" { + import "generic-service" + enable_active_checks = true + enable_passive_checks = false +} + +template Service "service-passive" { + import "generic-service" + enable_active_checks = false + enable_passive_checks = true +} + + +template Notification "host-notification" { + command = "send-host-notification" + states = [ Up, Down ] + types = [ Problem, Recovery, Custom ] + user_groups = [ "on-call_sms" ] + period = host.vars.period + + vars.notification_type = "sms" +} + +template Notification "service-notification" { + command = "send-service-notification" + states = [ OK, Critical ] + types = [ Problem, Recovery, Custom ] + user_groups = [ "on-call_sms" ] + + vars.notification_type = "sms" + + if(service.vars.period) { + period = service.vars.period + } else { + period = host.vars.period + } +} diff --git a/bundles/icinga2/files/icinga2/timeperiods.conf b/bundles/icinga2/files/icinga2/timeperiods.conf new file mode 100644 index 0000000..d45d31b --- /dev/null +++ b/bundles/icinga2/files/icinga2/timeperiods.conf @@ -0,0 +1,35 @@ +object TimePeriod "24x7" { + import "legacy-timeperiod" + + display_name = "24x7" + ranges = { + "monday" = "00:00-24:00" + "tuesday" = "00:00-24:00" + "wednesday" = "00:00-24:00" + "thursday" = "00:00-24:00" + "friday" = "00:00-24:00" + "saturday" = "00:00-24:00" + "sunday" = "00:00-24:00" + } +} + +object TimePeriod "weekdays_9to6" { + import "legacy-timeperiod" + + display_name = "9to6 (UTC)" + ranges = { + "monday" = "09:00-18:00" + "tuesday" = "09:00-18:00" + "wednesday" = "09:00-18:00" + "thursday" = "09:00-18:00" + "friday" = "09:00-18:00" + } +} + +object TimePeriod "never" { + import "legacy-timeperiod" + + display_name = "never" + ranges = { + } +} diff --git a/bundles/icinga2/files/icinga2/users.conf b/bundles/icinga2/files/icinga2/users.conf new file mode 100644 index 0000000..0a05dc3 --- /dev/null +++ b/bundles/icinga2/files/icinga2/users.conf @@ -0,0 +1,14 @@ +object UserGroup "on-call_sms" { + display_name = "On-Call Support" +} + +% for username in sorted(node.metadata.get('icinga2', {}).get('icinga_users', {})): +object User "${username}" { + display_name = "${username}" + enable_notifications = true + period = "24x7" # Is overwritten in notifications.conf + states = [ OK, Critical, Up, Down ] + types = [ Problem, Recovery ] +} + +% endfor diff --git a/bundles/icinga2/files/icingaweb2/authentication.ini b/bundles/icinga2/files/icingaweb2/authentication.ini new file mode 100644 index 0000000..6852ece --- /dev/null +++ b/bundles/icinga2/files/icingaweb2/authentication.ini @@ -0,0 +1,3 @@ +[icingaweb2] +backend = "db" +resource = "icingaweb_db" diff --git a/bundles/icinga2/files/icingaweb2/config.ini b/bundles/icinga2/files/icingaweb2/config.ini new file mode 100644 index 0000000..890a475 --- /dev/null +++ b/bundles/icinga2/files/icingaweb2/config.ini @@ -0,0 +1,11 @@ +[global] +show_stacktraces = "1" +show_application_state_messages = "1" +config_backend = "db" +config_resource = "icingaweb_db" + +[logging] +log = "syslog" +level = "ERROR" +application = "icingaweb2" +facility = "user" diff --git a/bundles/icinga2/files/icingaweb2/groups.ini b/bundles/icinga2/files/icingaweb2/groups.ini new file mode 100644 index 0000000..6852ece --- /dev/null +++ b/bundles/icinga2/files/icingaweb2/groups.ini @@ -0,0 +1,3 @@ +[icingaweb2] +backend = "db" +resource = "icingaweb_db" diff --git a/bundles/icinga2/files/icingaweb2/resources.ini b/bundles/icinga2/files/icingaweb2/resources.ini new file mode 100644 index 0000000..69a0f44 --- /dev/null +++ b/bundles/icinga2/files/icingaweb2/resources.ini @@ -0,0 +1,21 @@ +[icingaweb_db] +type = "db" +db = "pgsql" +host = "localhost" +port = "5432" +dbname = "icingaweb2" +username = "icinga2" +password = "${node.metadata['postgresql']['roles']['icinga2']['password']}" +charset = "" +use_ssl = "0" + +[icinga_ido] +type = "db" +db = "pgsql" +host = "localhost" +port = "5432" +dbname = "icinga2" +username = "icinga2" +password = "${node.metadata['postgresql']['roles']['icinga2']['password']}" +charset = "" +use_ssl = "0" diff --git a/bundles/icinga2/items.py b/bundles/icinga2/items.py index a085c0b..7063f2a 100644 --- a/bundles/icinga2/items.py +++ b/bundles/icinga2/items.py @@ -3,6 +3,20 @@ assert node.has_bundle('sshmon') from os.path import join +ENABLED_FEATURES = { + 'api', + 'checker', + 'command', + 'ido-pgsql', + 'mainlog', + 'notification', +} + +SLA_INFO = { + '24x7': '24x7', + 'never': 'never', +} + directories = { '/etc/icingaweb2': { 'group': 'icingaweb2', @@ -11,9 +25,46 @@ directories = { 'pkg_apt:icingaweb2', }, }, + '/etc/icinga2/features-enabled': { + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0555', + 'purge': True, + 'needs': { + 'pkg_apt:icinga2-ido-pgsql', + }, + 'triggers': { + 'svc_systemd:icinga2:restart', + }, + }, + '/etc/icinga2/conf.d': { + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0555', + 'purge': True, + 'needs': { + 'pkg_apt:icinga2', + }, + 'triggers': { + 'svc_systemd:icinga2:restart', + }, + }, + '/etc/icinga2/conf.d/services': { + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0555', + 'purge': True, + 'needs': { + 'pkg_apt:icinga2', + }, + 'triggers': { + 'svc_systemd:icinga2:restart', + }, + }, } files = { + ### Checks '/usr/local/share/icinga/plugins/check_rbl': { 'mode': '0755', }, @@ -29,9 +80,198 @@ files = { 'pkg_apt:icinga2-ido-pgsql', }, }, - '/etc/icingaweb2/setup.token': { - 'content': node.metadata['icingaweb2']['setup-token'], + + # Icinga2 + '/etc/icinga2/icinga2.conf': { + 'source': 'icinga2/icinga2.conf', + 'needs': { + 'pkg_apt:icinga2', + }, + 'triggers': { + 'svc_systemd:icinga2:restart', + }, + }, + '/etc/icinga2/features-available/ido-pgsql.conf': { + 'source': 'icinga2/ido-pgsql.conf', + 'content_type': 'mako', + 'needs': { + 'pkg_apt:icinga2', + }, + 'triggers': { + 'svc_systemd:icinga2:restart', + }, + }, + '/etc/icinga2/conf.d/api-users.conf': { + 'source': 'icinga2/api-users.conf', + 'content_type': 'mako', + 'needs': { + 'pkg_apt:icinga2', + }, + 'triggers': { + 'svc_systemd:icinga2:restart', + }, + }, + '/etc/icinga2/conf.d/app.conf': { + 'source': 'icinga2/app.conf', + 'needs': { + 'pkg_apt:icinga2', + }, + 'triggers': { + 'svc_systemd:icinga2:restart', + }, + }, + '/etc/icinga2/conf.d/check_commands.conf': { + 'source': 'icinga2/check_commands.conf', + 'needs': { + 'pkg_apt:icinga2', + }, + 'triggers': { + 'svc_systemd:icinga2:restart', + }, + }, + '/etc/icinga2/conf.d/templates.conf': { + 'source': 'icinga2/templates.conf', + 'needs': { + 'pkg_apt:icinga2', + }, + 'triggers': { + 'svc_systemd:icinga2:restart', + }, + }, + '/etc/icinga2/conf.d/timeperiods.conf': { + 'source': 'icinga2/timeperiods.conf', + 'needs': { + 'pkg_apt:icinga2', + }, + 'triggers': { + 'svc_systemd:icinga2:restart', + }, + }, + '/etc/icinga2/conf.d/users.conf': { + 'source': 'icinga2/users.conf', + 'content_type': 'mako', + 'needs': { + 'pkg_apt:icinga2', + }, + 'triggers': { + 'svc_systemd:icinga2:restart', + }, + }, + + # IcingaWeb2 + '/etc/icingaweb2/authentication.ini': { + 'source': 'icingaweb2/authentication.ini', + 'mode': '0660', + 'group': 'icingaweb2', + }, + '/etc/icingaweb2/config.ini': { + 'source': 'icingaweb2/config.ini', + 'mode': '0660', + 'group': 'icingaweb2', + }, + '/etc/icingaweb2/groups.ini': { + 'source': 'icingaweb2/groups.ini', + 'mode': '0660', + 'group': 'icingaweb2', + }, + '/etc/icingaweb2/resources.ini': { + 'source': 'icingaweb2/resources.ini', + 'content_type': 'mako', 'mode': '0660', 'group': 'icingaweb2', }, } + +actions = { + 'icinga2_api_setup': { + 'command': 'icinga2 api setup', + 'unless': 'test -e /var/lib/icinga2/certs/{}.crt'.format(node.metadata['hostname']), + 'needs': { + 'pkg_apt:icinga2', + }, + 'triggers': { + 'svc_systemd:icinga2:restart', + }, + }, +} + +for feature in ENABLED_FEATURES: + symlinks[f'/etc/icinga2/features-enabled/{feature}.conf'] = { + 'target': f'/etc/icinga2/features-available/{feature}.conf', + 'needs': { + 'pkg_apt:icinga2', + }, + 'triggers': { + 'svc_systemd:icinga2:restart', + }, + } + +svc_systemd = { + 'icinga2': { + 'needs': { + 'action:icinga2_api_setup', + 'file:', + 'pkg_apt:', + 'symlink:', + }, + }, +} + + + + +# The actual hosts and services management starts here +monitored_nodes = repo.nodes + +for n in monitored_nodes[:]: + if n.metadata.get('icinga_options', {}).get('exclude_from_monitoring', False): + monitored_nodes.remove(n) + +bundle_metadata = {} +for monitored_node in monitored_nodes: + node_metadata = monitored_node.metadata.copy() + + for bundle, config in sorted(node_metadata.get('icinga2_api', {}).items()): + if bundle not in bundle_metadata: + bundle_metadata[bundle] = { + 'services': {} + } + + bundle_metadata[bundle]['services'].update({ + monitored_node: config['services'] + }) + + for serv, conf in bundle_metadata[bundle]['services'][monitored_node].items(): + if 'check_command' not in conf: + # This default is also set in sshmon bundle + conf['check_command'] = 'sshmon' + +for bundle, metadata in bundle_metadata.items(): + files[f'/etc/icinga2/conf.d/services/{bundle}.conf'] = { + 'source': 'icinga2/services_template.conf', + 'content_type': 'mako', + 'context': { + 'bundle_metadata': metadata['services'], + }, + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0440', + 'triggers': { + 'svc_systemd:icinga2:restart', + }, + } + +files['/etc/icinga2/conf.d/hosts.conf'] = { + 'source': 'icinga2/hosts.conf', + 'content_type': 'mako', + 'context': { + 'monitored_nodes': monitored_nodes, + 'sla_info': SLA_INFO, + }, + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0440', + 'triggers': { + 'svc_systemd:icinga2:restart', + }, +} diff --git a/bundles/icinga2/metadata.py b/bundles/icinga2/metadata.py index aee752c..06ea36c 100644 --- a/bundles/icinga2/metadata.py +++ b/bundles/icinga2/metadata.py @@ -22,6 +22,11 @@ defaults = { 'libreadonly-perl': {}, } }, + 'icinga2': { + 'api_users': { + 'root': repo.vault.password_for(f'{node.name} icinga2 api root'), + }, + }, 'icingaweb2': { 'setup-token': repo.vault.password_for(f'{node.name} icingaweb2 setup-token'), }, diff --git a/nodes/ovh/icinga2.py b/nodes/ovh/icinga2.py index 5bfba21..50eaadb 100644 --- a/nodes/ovh/icinga2.py +++ b/nodes/ovh/icinga2.py @@ -24,6 +24,11 @@ nodes['ovh.icinga2'] = { 'php-imagick': {}, }, }, + 'icinga2': { + 'icinga_users': { + 'kunsi', + }, + }, 'nginx': { 'vhosts': { 'icingaweb': {