diff --git a/bundles/grafana/dashboard-rows/rspamd.py b/bundles/grafana/dashboard-rows/rspamd.py new file mode 100644 index 0000000..b60f8b5 --- /dev/null +++ b/bundles/grafana/dashboard-rows/rspamd.py @@ -0,0 +1,296 @@ +def dashboard_row_rspamd(panel_id, node): + return { + 'title': 'rspamd', + 'collapse': False, + 'editable': False, + 'height': '250px', + 'panels': [ + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.name}' + }, + 'overrides': [] + }, + 'fill': 1, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 1, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 6, + 'stack': False, + 'steppedLine': False, + 'targets': [{ + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "rspamd_actions" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> derivative(unit: 1s, nonNegative: true) + |> yield(name: "value")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }], + 'thresholds': [], + 'timeRegions': [], + 'title': 'rspamd actions', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': 0, + 'show': True, + 'decimals': 0, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.name}' + }, + 'overrides': [] + }, + 'fill': 1, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 1, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 6, + 'stack': False, + 'steppedLine': False, + 'targets': [ + { + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "rspamd_stats" and + r["_field"] == "scanned" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> derivative(unit: 1s, nonNegative: true) + |> yield(name: "avg")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }, + { + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "rspamd_stats" and + r["_field"] == "learned" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> derivative(unit: 1s, nonNegative: true) + |> yield(name: "mean")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }, + { + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "rspamd_stats" and + r["_field"] == "spam_count" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> derivative(unit: 1s, nonNegative: true) + |> yield(name: "mean")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }, + { + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "rspamd_stats" and + r["_field"] == "ham_count" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> derivative(unit: 1s, nonNegative: true) + |> yield(name: "mean")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }, + ], + 'thresholds': [], + 'timeRegions': [], + 'title': 'rspamd stats', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': 0, + 'show': True, + 'decimals': 0, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + ], + } diff --git a/bundles/grafana/items.py b/bundles/grafana/items.py index a9cac31..5ddda20 100644 --- a/bundles/grafana/items.py +++ b/bundles/grafana/items.py @@ -116,6 +116,10 @@ for rnode in repo.nodes: dashboard['rows'].append(dashboard_row_dovecot(panel_id, rnode)) dashboard['tags'].add('dovecot') + if rnode.has_bundle('rspamd'): + dashboard['rows'].append(dashboard_row_rspamd(panel_id, rnode)) + dashboard['tags'].add('rspamd') + if rnode.has_bundle('postgresql'): dashboard['rows'].append(dashboard_row_postgresql(panel_id, rnode)) dashboard['tags'].add('postgresql') diff --git a/bundles/rspamd/files/telegraf-rspamd-plugin b/bundles/rspamd/files/telegraf-rspamd-plugin new file mode 100644 index 0000000..9cb2c3d --- /dev/null +++ b/bundles/rspamd/files/telegraf-rspamd-plugin @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +from requests import get +from sys import argv, stderr + +try: + r = get('http://127.0.0.1:11334/stat') + r.raise_for_status() + + json = r.json() + + actions = set() + for k, v in json['actions'].items(): + actions.add('{}={}i'.format(k.replace(' ', '_'), v)) + + print('rspamd_actions {}'.format(','.join(sorted(actions)))) + + stats = set() + for i in { + 'scanned', + 'learned', + 'spam_count', + 'ham_count', + 'connections', + 'control_connections', + 'pools_allocated', + 'pools_freed', + 'bytes_allocated', + 'chunks_allocated', + 'shared_chunks_allocated', + 'chunks_freed', + 'chunks_oversized', + 'fragmented', + 'total_learns', + }: + stats.add('{}={}i'.format(i, json[i])) + + print('rspamd_stats {}'.format(','.join(sorted(stats)))) + + for domain, value in json['fuzzy_hashes'].items(): + print('rspamd_fuzzy,domain={} value={}i'.format(domain, value)) +except Exception as e: + print(repr(e), file=stderr) diff --git a/bundles/rspamd/items.py b/bundles/rspamd/items.py index 2411ea5..c94d0e4 100644 --- a/bundles/rspamd/items.py +++ b/bundles/rspamd/items.py @@ -55,6 +55,9 @@ files = { '/etc/rspamd/local.d/ip_whitelist.map': { 'content_type': 'mako', }, + '/usr/local/bin/telegraf-rspamd-plugin': { + 'mode': '0755', + }, } @@ -92,7 +95,7 @@ if 'dkim' in node.metadata.get('rspamd', {}): }, } -if 'password' in node.metadata.get('rspamd', {}): +if node.metadata.get('rspamd/password', None): files['/etc/rspamd/local.d/worker-controller.inc'] = { 'content_type': 'mako', 'triggers': { diff --git a/bundles/rspamd/metadata.py b/bundles/rspamd/metadata.py index 1ae0b6e..334f2ae 100644 --- a/bundles/rspamd/metadata.py +++ b/bundles/rspamd/metadata.py @@ -55,7 +55,20 @@ defaults = { }, }, 'rspamd': { - 'dkim': repo.vault.password_for(node.name + ' rspamd dkim key'), + 'dkim': repo.vault.password_for(f'{node.name} rspamd dkim key'), + }, + 'telegraf': { + 'input_plugins': { + 'exec': { + 'rspamd': { + 'commands': [ + '/usr/local/bin/telegraf-rspamd-plugin', + ], + 'data_format': 'influx', + 'timeout': '5s', + }, + }, + }, }, 'zfs': { 'datasets': {