From 6b641890c3d2518419276f857182e08de9304188 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Sun, 7 Aug 2022 10:16:07 +0200 Subject: [PATCH] bundles/grafana: replace the useless builtin of telegraf with something more useful --- bundles/grafana/dashboard-rows/smartd.py | 228 +++++++++++++++++++++++ bundles/grafana/items.py | 4 + bundles/smartd/files/telegraf_plugin | 47 +++++ bundles/smartd/items.py | 4 + bundles/smartd/metadata.py | 39 ++-- 5 files changed, 299 insertions(+), 23 deletions(-) create mode 100644 bundles/grafana/dashboard-rows/smartd.py create mode 100644 bundles/smartd/files/telegraf_plugin diff --git a/bundles/grafana/dashboard-rows/smartd.py b/bundles/grafana/dashboard-rows/smartd.py new file mode 100644 index 0000000..f2fb257 --- /dev/null +++ b/bundles/grafana/dashboard-rows/smartd.py @@ -0,0 +1,228 @@ +def dashboard_row_smartd(panel_id, node): + return { + 'title': 'smartd', + 'collapse': False, + 'editable': False, + 'height': '250px', + 'panels': [ + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.labels.device}' + }, + 'overrides': [] + }, + 'fill': 0, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'hideEmpty': True, + 'hideZero': True, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 1, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 8, + 'stack': False, + 'steppedLine': False, + 'targets': [ + { + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "smartd_stats" and + r["_field"] == "temperature" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> yield(name: "cpu")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }, + ], + 'thresholds': [], + 'timeRegions': [], + 'title': 'temperatures', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'celsius', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': 0, + 'show': True, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.labels.device}' + }, + 'overrides': [] + }, + 'fill': 0, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'hideEmpty': True, + 'hideZero': True, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 1, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 4, + 'stack': False, + 'steppedLine': False, + 'targets': [ + { + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "smartd_stats" and + r["_field"] == "power_on_hours" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> yield(name: "fan")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }, + ], + 'thresholds': [], + 'timeRegions': [], + 'title': 'fans', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'hours', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': True, + 'decimals': 0, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + ], + } diff --git a/bundles/grafana/items.py b/bundles/grafana/items.py index 6abf19e..4e61d39 100644 --- a/bundles/grafana/items.py +++ b/bundles/grafana/items.py @@ -103,6 +103,10 @@ for rnode in repo.nodes: dashboard['rows'].append(dashboard_row_sensors(panel_id, rnode)) dashboard['tags'].add('lm-sensors') + if rnode.has_bundle('smartd'): + dashboard['rows'].append(dashboard_row_smartd(panel_id, rnode)) + dashboard['tags'].add('smartd') + if rnode.has_bundle('telegraf-battery-usage'): dashboard['rows'].append(dashboard_row_battery(panel_id, rnode)) diff --git a/bundles/smartd/files/telegraf_plugin b/bundles/smartd/files/telegraf_plugin new file mode 100644 index 0000000..5a7a1a5 --- /dev/null +++ b/bundles/smartd/files/telegraf_plugin @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +from subprocess import check_output +from json import loads +from sys import stderr + +devices = check_output(['smartctl', '--scan']).decode().splitlines() + +for device in devices: + device = device.split(' ')[0] + + try: + json = loads(check_output(['smartctl', '-n', 'standby', '-A', '--json=c', device])) + + telegraf_output = set() + + if 'power_on_time' in json: + telegraf_output.add('power_on_hours={}'.format(json['power_on_time']['hours'])) + + if 'temperature' in json: + telegraf_output.add('temperature={}'.format(json['temperature']['current'])) + + print('smartd_stats,device={device} {values}'.format( + device=device, + values=','.join(sorted(telegraf_output)), + )) + + telegraf_output = set() + + if 'nvme_smart_health_information_log' in json: + for k, v in json['nvme_smart_health_information_log'].items(): + telegraf_output.add(f'{k}={v}') + + if 'ata_smart_attributes' in json: + for entry in json['ata_smart_attributes']['table']: + telegraf_output.add('{}={}'.format( + entry['name'], + entry['raw']['value'], + )) + + print('smartd_health,device={device},type={type} {values}'.format( + device=device, + type=json['device']['type'], + values=','.join(sorted(telegraf_output)), + )) + except Exception as e: + print(f'{device} {repr(e)}', file=stderr) diff --git a/bundles/smartd/items.py b/bundles/smartd/items.py index 62bdbe7..540a6a9 100644 --- a/bundles/smartd/items.py +++ b/bundles/smartd/items.py @@ -8,6 +8,10 @@ files = { '/usr/local/share/icinga/plugins/check_smart': { 'mode': '0755', }, + '/usr/local/sbin/telegraf-smartd': { + 'source': 'telegraf_plugin', + 'mode': '0755', + }, } svc_systemd = { diff --git a/bundles/smartd/metadata.py b/bundles/smartd/metadata.py index bef9390..63af59a 100644 --- a/bundles/smartd/metadata.py +++ b/bundles/smartd/metadata.py @@ -24,6 +24,22 @@ defaults = { }, } +if node.has_bundle('telegraf'): + defaults['telegraf'] = { + 'input_plugins': { + 'exec': { + 'smartd': { + 'commands': ['sudo /usr/local/sbin/telegraf-smartd'], + 'data_format': 'influx', + 'timeout': '5s', + }, + }, + }, + 'sudo_commands': { + '/usr/local/sbin/telegraf-smartd', + }, + } + @metadata_reactor.provides( 'smartd/disks', @@ -67,29 +83,6 @@ def icinga(metadata): } -@metadata_reactor.provides( - 'telegraf/input_plugins/builtin/smart', -) -def telegraf(metadata): - if not node.has_bundle('telegraf'): - raise DoNotRunAgain - - if metadata.get('smartd/disks', set()): - return { - 'telegraf': { - 'input_plugins': { - 'builtin': { - 'smart': [{ - 'devices': list(sorted(metadata.get('smartd/disks'))), - }], - }, - }, - }, - } - - return {} - - @metadata_reactor.provides( 'cron/jobs/smartd', )