bundles/rspamd: add telegraf metrics
Some checks failed
kunsi/bundlewrap/pipeline/head There was a failure building this commit

This commit is contained in:
Franzi 2021-08-23 22:19:31 +02:00
parent ad159cc29a
commit 2940bfd3d8
Signed by: kunsi
GPG key ID: 12E3D2136B818350
5 changed files with 361 additions and 2 deletions

View file

@ -0,0 +1,296 @@
def dashboard_row_rspamd(panel_id, node):
return {
'title': 'rspamd',
'collapse': False,
'editable': False,
'height': '250px',
'panels': [
{
'aliasColors': {},
'bars': False,
'dashLength': 10,
'dashes': False,
'datasource': None,
'fieldConfig': {
'defaults': {
'displayName': '${__field.name}'
},
'overrides': []
},
'fill': 1,
'fillGradient': 0,
'hiddenSeries': False,
'id': next(panel_id),
'legend': {
'alignAsTable': False,
'avg': False,
'current': False,
'max': False,
'min': False,
'rightSide': False,
'show': True,
'total': False,
'values': False
},
'lines': True,
'linewidth': 1,
'NonePointMode': 'None',
'options': {
'alertThreshold': True
},
'percentage': False,
'pluginVersion': '7.5.5',
'pointradius': 2,
'points': False,
'renderer': 'flot',
'seriesOverrides': [],
'spaceLength': 10,
'span': 6,
'stack': False,
'steppedLine': False,
'targets': [{
'groupBy': [
{'type': 'time', 'params': ['$__interval']},
{'type': 'fill', 'params': ['linear']},
],
'orderByTime': "ASC",
'policy': "default",
'query': f"""from(bucket: "telegraf")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) =>
r["_measurement"] == "rspamd_actions" and
r["host"] == "{node.name}"
)
|> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)
|> derivative(unit: 1s, nonNegative: true)
|> yield(name: "value")""",
'resultFormat': 'time_series',
'select': [[
{'type': 'field', 'params': ['value']},
{'type': 'mean', 'params': []},
]],
"tags": []
}],
'thresholds': [],
'timeRegions': [],
'title': 'rspamd actions',
'tooltip': {
'shared': True,
'sort': 0,
'value_type': 'individual'
},
'type': 'graph',
'xaxis': {
'buckets': None,
'mode': 'time',
'name': None,
'show': True,
'values': []
},
'yaxes': [
{
'format': 'short',
'label': None,
'logBase': 1,
'max': None,
'min': 0,
'show': True,
'decimals': 0,
},
{
'format': 'short',
'label': None,
'logBase': 1,
'max': None,
'min': None,
'show': False,
}
],
'yaxis': {
'align': False,
'alignLevel': None
}
},
{
'aliasColors': {},
'bars': False,
'dashLength': 10,
'dashes': False,
'datasource': None,
'fieldConfig': {
'defaults': {
'displayName': '${__field.name}'
},
'overrides': []
},
'fill': 1,
'fillGradient': 0,
'hiddenSeries': False,
'id': next(panel_id),
'legend': {
'alignAsTable': False,
'avg': False,
'current': False,
'max': False,
'min': False,
'rightSide': False,
'show': True,
'total': False,
'values': False
},
'lines': True,
'linewidth': 1,
'NonePointMode': 'None',
'options': {
'alertThreshold': True
},
'percentage': False,
'pluginVersion': '7.5.5',
'pointradius': 2,
'points': False,
'renderer': 'flot',
'seriesOverrides': [],
'spaceLength': 10,
'span': 6,
'stack': False,
'steppedLine': False,
'targets': [
{
'groupBy': [
{'type': 'time', 'params': ['$__interval']},
{'type': 'fill', 'params': ['linear']},
],
'orderByTime': "ASC",
'policy': "default",
'query': f"""from(bucket: "telegraf")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) =>
r["_measurement"] == "rspamd_stats" and
r["_field"] == "scanned" and
r["host"] == "{node.name}"
)
|> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)
|> derivative(unit: 1s, nonNegative: true)
|> yield(name: "avg")""",
'resultFormat': 'time_series',
'select': [[
{'type': 'field', 'params': ['value']},
{'type': 'mean', 'params': []},
]],
"tags": []
},
{
'groupBy': [
{'type': 'time', 'params': ['$__interval']},
{'type': 'fill', 'params': ['linear']},
],
'orderByTime': "ASC",
'policy': "default",
'query': f"""from(bucket: "telegraf")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) =>
r["_measurement"] == "rspamd_stats" and
r["_field"] == "learned" and
r["host"] == "{node.name}"
)
|> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)
|> derivative(unit: 1s, nonNegative: true)
|> yield(name: "mean")""",
'resultFormat': 'time_series',
'select': [[
{'type': 'field', 'params': ['value']},
{'type': 'mean', 'params': []},
]],
"tags": []
},
{
'groupBy': [
{'type': 'time', 'params': ['$__interval']},
{'type': 'fill', 'params': ['linear']},
],
'orderByTime': "ASC",
'policy': "default",
'query': f"""from(bucket: "telegraf")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) =>
r["_measurement"] == "rspamd_stats" and
r["_field"] == "spam_count" and
r["host"] == "{node.name}"
)
|> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)
|> derivative(unit: 1s, nonNegative: true)
|> yield(name: "mean")""",
'resultFormat': 'time_series',
'select': [[
{'type': 'field', 'params': ['value']},
{'type': 'mean', 'params': []},
]],
"tags": []
},
{
'groupBy': [
{'type': 'time', 'params': ['$__interval']},
{'type': 'fill', 'params': ['linear']},
],
'orderByTime': "ASC",
'policy': "default",
'query': f"""from(bucket: "telegraf")
|> range(start: v.timeRangeStart, stop: v.timeRangeStop)
|> filter(fn: (r) =>
r["_measurement"] == "rspamd_stats" and
r["_field"] == "ham_count" and
r["host"] == "{node.name}"
)
|> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)
|> derivative(unit: 1s, nonNegative: true)
|> yield(name: "mean")""",
'resultFormat': 'time_series',
'select': [[
{'type': 'field', 'params': ['value']},
{'type': 'mean', 'params': []},
]],
"tags": []
},
],
'thresholds': [],
'timeRegions': [],
'title': 'rspamd stats',
'tooltip': {
'shared': True,
'sort': 0,
'value_type': 'individual'
},
'type': 'graph',
'xaxis': {
'buckets': None,
'mode': 'time',
'name': None,
'show': True,
'values': []
},
'yaxes': [
{
'format': 'short',
'label': None,
'logBase': 1,
'max': None,
'min': 0,
'show': True,
'decimals': 0,
},
{
'format': 'short',
'label': None,
'logBase': 1,
'max': None,
'min': None,
'show': False,
}
],
'yaxis': {
'align': False,
'alignLevel': None
}
},
],
}

View file

@ -116,6 +116,10 @@ for rnode in repo.nodes:
dashboard['rows'].append(dashboard_row_dovecot(panel_id, rnode)) dashboard['rows'].append(dashboard_row_dovecot(panel_id, rnode))
dashboard['tags'].add('dovecot') dashboard['tags'].add('dovecot')
if rnode.has_bundle('rspamd'):
dashboard['rows'].append(dashboard_row_rspamd(panel_id, rnode))
dashboard['tags'].add('rspamd')
if rnode.has_bundle('postgresql'): if rnode.has_bundle('postgresql'):
dashboard['rows'].append(dashboard_row_postgresql(panel_id, rnode)) dashboard['rows'].append(dashboard_row_postgresql(panel_id, rnode))
dashboard['tags'].add('postgresql') dashboard['tags'].add('postgresql')

View file

@ -0,0 +1,43 @@
#!/usr/bin/env python3
from requests import get
from sys import argv, stderr
try:
r = get('http://127.0.0.1:11334/stat')
r.raise_for_status()
json = r.json()
actions = set()
for k, v in json['actions'].items():
actions.add('{}={}i'.format(k.replace(' ', '_'), v))
print('rspamd_actions {}'.format(','.join(sorted(actions))))
stats = set()
for i in {
'scanned',
'learned',
'spam_count',
'ham_count',
'connections',
'control_connections',
'pools_allocated',
'pools_freed',
'bytes_allocated',
'chunks_allocated',
'shared_chunks_allocated',
'chunks_freed',
'chunks_oversized',
'fragmented',
'total_learns',
}:
stats.add('{}={}i'.format(i, json[i]))
print('rspamd_stats {}'.format(','.join(sorted(stats))))
for domain, value in json['fuzzy_hashes'].items():
print('rspamd_fuzzy,domain={} value={}i'.format(domain, value))
except Exception as e:
print(repr(e), file=stderr)

View file

@ -55,6 +55,9 @@ files = {
'/etc/rspamd/local.d/ip_whitelist.map': { '/etc/rspamd/local.d/ip_whitelist.map': {
'content_type': 'mako', 'content_type': 'mako',
}, },
'/usr/local/bin/telegraf-rspamd-plugin': {
'mode': '0755',
},
} }
@ -92,7 +95,7 @@ if 'dkim' in node.metadata.get('rspamd', {}):
}, },
} }
if 'password' in node.metadata.get('rspamd', {}): if node.metadata.get('rspamd/password', None):
files['/etc/rspamd/local.d/worker-controller.inc'] = { files['/etc/rspamd/local.d/worker-controller.inc'] = {
'content_type': 'mako', 'content_type': 'mako',
'triggers': { 'triggers': {

View file

@ -55,7 +55,20 @@ defaults = {
}, },
}, },
'rspamd': { 'rspamd': {
'dkim': repo.vault.password_for(node.name + ' rspamd dkim key'), 'dkim': repo.vault.password_for(f'{node.name} rspamd dkim key'),
},
'telegraf': {
'input_plugins': {
'exec': {
'rspamd': {
'commands': [
'/usr/local/bin/telegraf-rspamd-plugin',
],
'data_format': 'influx',
'timeout': '5s',
},
},
},
}, },
'zfs': { 'zfs': {
'datasets': { 'datasets': {