From c548a88ee72e019406fa04c56b3bee7c6b371ac2 Mon Sep 17 00:00:00 2001 From: Franziska Kunsmann Date: Sun, 2 May 2021 10:44:50 +0200 Subject: [PATCH] bundles/grafana: introduce, add to htz-cloud.influxdb --- PORT_MAP.md | 2 + bundles/grafana/dashboard-rows/cpu.py | 240 ++++++++++ bundles/grafana/dashboard-rows/disk_iops.py | 441 ++++++++++++++++++ bundles/grafana/dashboard-rows/disk_space.py | 236 ++++++++++ bundles/grafana/dashboard-rows/memory.py | 125 +++++ .../dashboard-rows/traffic_by_ip_version.py | 284 +++++++++++ bundles/grafana/files/dashboards.yaml | 15 + bundles/grafana/files/grafana.ini | 104 +++++ bundles/grafana/items.py | 99 ++++ bundles/grafana/metadata.py | 48 ++ data/apt/files/gpg-keys/grafana.asc | 30 ++ data/powerdns/files/bind-zones/kunsmann.eu | 3 + nodes.py | 1 - nodes/htz-cloud/influxdb.py | 4 + 14 files changed, 1631 insertions(+), 1 deletion(-) create mode 100644 bundles/grafana/dashboard-rows/cpu.py create mode 100644 bundles/grafana/dashboard-rows/disk_iops.py create mode 100644 bundles/grafana/dashboard-rows/disk_space.py create mode 100644 bundles/grafana/dashboard-rows/memory.py create mode 100644 bundles/grafana/dashboard-rows/traffic_by_ip_version.py create mode 100644 bundles/grafana/files/dashboards.yaml create mode 100644 bundles/grafana/files/grafana.ini create mode 100644 bundles/grafana/items.py create mode 100644 bundles/grafana/metadata.py create mode 100644 data/apt/files/gpg-keys/grafana.asc diff --git a/PORT_MAP.md b/PORT_MAP.md index fe5117b..5a85294 100644 --- a/PORT_MAP.md +++ b/PORT_MAP.md @@ -25,6 +25,7 @@ Rule of thumb: keep ports below 10000 free for stuff that reserves ports. | 6379 | | redis | | 6667 | | bitlbee | | 8010 | | matrix-media-repo | +| 8086 | influxdb2 | influx | | 8184 | | matrix-dimension | | 11332-11334 | rspamd | rspamd | | 20000 | mx-puppet-discord | Bridge | @@ -34,6 +35,7 @@ Rule of thumb: keep ports below 10000 free for stuff that reserves ports. | 20081 | matrix-synapse | prometheus metrics | | 20090 | matrix-media-repo | media_repo | | 21000 | pleroma | pleroma | +| 21010 | grafana | grafana | | 22000 | gitea | gitea | | 22010 | jenkins-ci | Jenkins CI | | 22020 | travelynx | Travelynx Web | diff --git a/bundles/grafana/dashboard-rows/cpu.py b/bundles/grafana/dashboard-rows/cpu.py new file mode 100644 index 0000000..c0609c7 --- /dev/null +++ b/bundles/grafana/dashboard-rows/cpu.py @@ -0,0 +1,240 @@ +def dashboard_row_cpu(panel_id, node): + queries_cpu = [] + queries_load = [] + + for measurement in [ + 'user', + 'system', + 'steal', + 'iowait', + 'nice', + 'softirq', + ]: + queries_cpu.append({ + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "cpu" and + r["_field"] == "usage_{measurement}" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> yield(name: "{measurement}")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }) + + for measurement in [ + 'load1', + 'load5', + 'load15', + ]: + queries_load.append({ + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "system" and + r["_field"] == "{measurement}" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> yield(name: "{measurement}")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }) + + return { + 'title': 'cpu/load', + 'collapse': False, + 'editable': False, + 'height': '250px', + 'panels': [ + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.name}' + }, + 'overrides': [] + }, + 'fill': 10, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 0, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 6, + 'stack': True, + 'steppedLine': False, + 'targets': queries_cpu, + 'thresholds': [], + 'timeRegions': [], + 'title': 'cpu', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'percent', + 'label': None, + 'logBase': 1, + 'max': 100, + 'min': 0, + 'show': True, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.name}' + }, + 'overrides': [] + }, + 'fill': 1, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 1, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 6, + 'stack': False, + 'steppedLine': False, + 'targets': queries_load, + 'thresholds': [], + 'timeRegions': [], + 'title': 'load', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': True, + 'decimals': 0, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + ], + } diff --git a/bundles/grafana/dashboard-rows/disk_iops.py b/bundles/grafana/dashboard-rows/disk_iops.py new file mode 100644 index 0000000..40f267e --- /dev/null +++ b/bundles/grafana/dashboard-rows/disk_iops.py @@ -0,0 +1,441 @@ +def dashboard_row_disk_iops(panel_id, node): + return { + 'title': 'disk iops', + 'collapse': False, + 'editable': False, + 'height': '200px', + 'panels': [ + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.labels.name}' + }, + 'overrides': [] + }, + 'fill': 1, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 1, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 6, + 'stack': True, + 'steppedLine': False, + 'targets': [ + { + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "diskio" and + r["_field"] == "reads" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> derivative(unit: 1s) + |> yield(name: "read")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }, + ], + 'thresholds': [], + 'timeRegions': [], + 'title': 'read IOPS', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': 0, + 'show': True, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.labels.name}' + }, + 'overrides': [] + }, + 'fill': 1, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 1, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 6, + 'stack': True, + 'steppedLine': False, + 'targets': [ + { + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "diskio" and + r["_field"] == "writes" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> derivative(unit: 1s) + |> yield(name: "write")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }, + ], + 'thresholds': [], + 'timeRegions': [], + 'title': 'write IOPS', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'binBps', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': 0, + 'show': True, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.labels.name}' + }, + 'overrides': [] + }, + 'fill': 1, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 1, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 6, + 'stack': True, + 'steppedLine': False, + 'targets': [ + { + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "diskio" and + r["_field"] == "read_bytes" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> derivative(unit: 1s) + |> yield(name: "read")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }, + ], + 'thresholds': [], + 'timeRegions': [], + 'title': 'read bytes', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'binBps', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': 0, + 'show': True, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.labels.name}' + }, + 'overrides': [] + }, + 'fill': 1, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 1, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 6, + 'stack': True, + 'steppedLine': False, + 'targets': [ + { + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "diskio" and + r["_field"] == "write_bytes" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> derivative(unit: 1s) + |> yield(name: "write")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }, + ], + 'thresholds': [], + 'timeRegions': [], + 'title': 'write bytes', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'binBps', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': True, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + ], + } diff --git a/bundles/grafana/dashboard-rows/disk_space.py b/bundles/grafana/dashboard-rows/disk_space.py new file mode 100644 index 0000000..c788956 --- /dev/null +++ b/bundles/grafana/dashboard-rows/disk_space.py @@ -0,0 +1,236 @@ +def dashboard_row_disk_space(panel_id, node): + queries_bytes = [] + queries_inodes = [] + + for measurement in [ + 'used', + 'free', + ]: + queries_bytes.append({ + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "disk" and + r["_field"] == "{measurement}" and + r["fstype"] == "ext4" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> yield(name: "{measurement}")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }) + + for measurement in [ + 'inodes_used', + 'inodes_free', + ]: + queries_inodes.append({ + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "disk" and + r["_field"] == "{measurement}" and + r["fstype"] == "ext4" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> yield(name: "{measurement}")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }) + + return { + 'title': 'disk space', + 'collapse': False, + 'editable': False, + 'height': '200px', + 'panels': [ + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.name} ${__field.labels.path}' + }, + 'overrides': [] + }, + 'fill': 1, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 1, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 6, + 'stack': True, + 'steppedLine': False, + 'targets': queries_bytes, + 'thresholds': [], + 'timeRegions': [], + 'title': 'disk space', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'bytes', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': 0, + 'show': True, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.name} ${__field.labels.path}' + }, + 'overrides': [] + }, + 'fill': 1, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 1, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 6, + 'stack': True, + 'steppedLine': False, + 'targets': queries_inodes, + 'thresholds': [], + 'timeRegions': [], + 'title': 'disk inodes', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'bytes', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': 0, + 'show': True, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + ], + } diff --git a/bundles/grafana/dashboard-rows/memory.py b/bundles/grafana/dashboard-rows/memory.py new file mode 100644 index 0000000..9402890 --- /dev/null +++ b/bundles/grafana/dashboard-rows/memory.py @@ -0,0 +1,125 @@ +def dashboard_row_memory(panel_id, node): + queries_mem = [] + + for measurement in [ + 'used', + 'buffered', + 'cached', + 'sreclaimable', + 'sunreclaim', + ]: + queries_mem.append({ + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "mem" and + r["_field"] == "{measurement}" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> yield(name: "{measurement}")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }) + + return { + 'title': 'memory', + 'collapse': False, + 'editable': False, + 'height': '200px', + 'panels': [ + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.name}' + }, + 'overrides': [] + }, + 'fill': 1, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 1, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 12, + 'stack': True, + 'steppedLine': False, + 'targets': queries_mem, + 'thresholds': [], + 'timeRegions': [], + 'title': 'memory usage', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'bytes', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': 0, + 'show': True, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + ], + } diff --git a/bundles/grafana/dashboard-rows/traffic_by_ip_version.py b/bundles/grafana/dashboard-rows/traffic_by_ip_version.py new file mode 100644 index 0000000..fa8bdbc --- /dev/null +++ b/bundles/grafana/dashboard-rows/traffic_by_ip_version.py @@ -0,0 +1,284 @@ +def dashboard_row_ip_traffic(panel_id, node): + return { + 'title': 'ip traffic', + 'collapse': False, + 'editable': False, + 'height': '250px', + 'panels': [ + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.name}' + }, + 'overrides': [] + }, + 'fill': 1, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 1, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 6, + 'stack': True, + 'steppedLine': False, + 'targets': [ + { + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "nstat" and + r["_field"] == "IpExtInOctets" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> derivative(unit: 1s) + |> yield(name: "in")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }, + { + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "nstat" and + r["_field"] == "IpExtOutOctets" and + r["host"] == "{node.name}" + ) + |> map(fn: (r) => ({{ + r with + _value: r._value * -1 + }}) + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> derivative(unit: 1s) + |> yield(name: "out")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }, + ], + 'thresholds': [], + 'timeRegions': [], + 'title': 'IPv4', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'binBps', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': True, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + { + 'aliasColors': {}, + 'bars': False, + 'dashLength': 10, + 'dashes': False, + 'datasource': None, + 'fieldConfig': { + 'defaults': { + 'displayName': '${__field.name}' + }, + 'overrides': [] + }, + 'fill': 1, + 'fillGradient': 0, + 'hiddenSeries': False, + 'id': next(panel_id), + 'legend': { + 'alignAsTable': False, + 'avg': False, + 'current': False, + 'max': False, + 'min': False, + 'rightSide': False, + 'show': True, + 'total': False, + 'values': False + }, + 'lines': True, + 'linewidth': 1, + 'NonePointMode': 'None', + 'options': { + 'alertThreshold': True + }, + 'percentage': False, + 'pluginVersion': '7.5.5', + 'pointradius': 2, + 'points': False, + 'renderer': 'flot', + 'seriesOverrides': [], + 'spaceLength': 10, + 'span': 6, + 'stack': False, + 'steppedLine': False, + 'targets': [ + { + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "nstat" and + r["_field"] == "Ip6InOctets" and + r["host"] == "{node.name}" + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> derivative(unit: 1s) + |> yield(name: "in")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }, + { + 'groupBy': [ + {'type': 'time', 'params': ['$__interval']}, + {'type': 'fill', 'params': ['linear']}, + ], + 'orderByTime': "ASC", + 'policy': "default", + 'query': f"""from(bucket: "telegraf") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => + r["_measurement"] == "nstat" and + r["_field"] == "Ip6OutOctets" and + r["host"] == "{node.name}" + ) + |> map(fn: (r) => ({{ + r with + _value: r._value * -1 + }}) + ) + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> derivative(unit: 1s) + |> yield(name: "out")""", + 'resultFormat': 'time_series', + 'select': [[ + {'type': 'field', 'params': ['value']}, + {'type': 'mean', 'params': []}, + ]], + "tags": [] + }, + ], + 'thresholds': [], + 'timeRegions': [], + 'title': 'IPv6', + 'tooltip': { + 'shared': True, + 'sort': 0, + 'value_type': 'individual' + }, + 'type': 'graph', + 'xaxis': { + 'buckets': None, + 'mode': 'time', + 'name': None, + 'show': True, + 'values': [] + }, + 'yaxes': [ + { + 'format': 'binBps', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': True, + 'decimals': 0, + }, + { + 'format': 'short', + 'label': None, + 'logBase': 1, + 'max': None, + 'min': None, + 'show': False, + } + ], + 'yaxis': { + 'align': False, + 'alignLevel': None + } + }, + ], + } diff --git a/bundles/grafana/files/dashboards.yaml b/bundles/grafana/files/dashboards.yaml new file mode 100644 index 0000000..1f176af --- /dev/null +++ b/bundles/grafana/files/dashboards.yaml @@ -0,0 +1,15 @@ +apiVersion: 1 + +providers: + - name: 'managed by bundlewrap' + orgId: 1 + folder: 'Managed by BundleWrap' + folderUid: '222af3a08b' + type: file + disableDeletion: true + updateIntervalSeconds: 10 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false + diff --git a/bundles/grafana/files/grafana.ini b/bundles/grafana/files/grafana.ini new file mode 100644 index 0000000..0a3fdd2 --- /dev/null +++ b/bundles/grafana/files/grafana.ini @@ -0,0 +1,104 @@ +app_mode = production +instance_name = ${node.name} + +[paths] +data = /var/lib/grafana +;temp_data_lifetime = 24h +logs = /var/log/grafana +plugins = /var/lib/grafana/plugins +provisioning = /etc/grafana/provisioning + +[server] +protocol = http +http_port = 21010 +domain = ${domain} +root_url = https://${domain}/ + +[database] +type = sqlite3 + +# for postgres +;host = 127.0.0.1:3306 +;name = grafana +;user = root +;password = +;ssl_mode = disable + +# for sqlite +;path = grafana.db +;cache_mode = private + +[remote_cache] +% if node.has_bundle('redis'): +type = redis +connstr = addr=127.0.0.1:6379 +% else: +type = database +% endif + +[analytics] +reporting_enabled = false +check_for_updates = false + +[security] +disable_initial_admin_creation = false +secret_key = ${secret_key} +disable_gravatar = true +cookie_secure = true +allow_embedding = ${str(allow_embedding).lower()} + +[dashboards] +min_refresh_interval = 10s + +[users] +allow_sign_up = ${str(allow_sign_up).lower()} +allow_org_create = false +auto_assign_org = false +verify_email_enabled = true +default_theme = dark +viewers_can_edit = false +editors_can_admin = false + +[auth] +login_maximum_inactive_lifetime_duration = ${login_max_duration} +login_maximum_lifetime_duration = ${login_max_duration} + +[auth.anonymous] +enabled = ${str(allow_anonymous).lower()} +org_name = ${anonymous_org} +org_role = Viewer + +[smtp] +enabled = ${str(enable_smtp).lower()} +host = localhost:25 +from_address = noreply@${domain} +from_name = Grafana + +[emails] +welcome_email_on_sign_up = false +templates_pattern = emails/*.html + +[log] +mode = syslog + +[alerting] +enabled = false + +[explore] +enabled = true + +[plugins] +enable_alpha = true + +[date_formats] +full_date = YYYY-MM-DD HH:mm:ss +interval_second = HH:mm:ss +interval_minute = HH:mm +interval_hour = YYYY-MM-DD HH:mm +interval_day = YYYY-MM-DD +interval_month = YYYY-MM +interval_year = YYYY +default_timezone = browser + +[expressions] +enabled = true diff --git a/bundles/grafana/items.py b/bundles/grafana/items.py new file mode 100644 index 0000000..fe2d462 --- /dev/null +++ b/bundles/grafana/items.py @@ -0,0 +1,99 @@ +from itertools import count +from os.path import join +from pathlib import Path +from uuid import UUID + +from bundlewrap.metadata import metadata_to_json + +for row in Path(join(repo.path, 'bundles', 'grafana', 'dashboard-rows')).rglob("*.py"): + with open(row, 'r') as f: + exec(f.read()) + +directories = { + '/etc/grafana/provisioning/dashboards': { + 'purge': True, + }, + '/etc/grafana/provisioning/datasources': { + 'purge': True, + }, + '/etc/grafana/provisioning/notifiers': { + 'purge': True, + }, + '/etc/grafana/provisioning/plugins': { + 'purge': True, + }, + '/var/lib/grafana/dashboards': { + 'purge': True, + 'triggers': { + 'svc_systemd:grafana-server:restart', + }, + }, +} + +files = { + '/etc/grafana/grafana.ini': { + 'content_type': 'mako', + 'context': node.metadata['grafana'], + 'owner': 'grafana', + 'mode': '0640', + 'triggers': { + 'svc_systemd:grafana-server:restart', + }, + }, + '/etc/grafana/provisioning/dashboards/bundlewrap.yaml': { + 'source': 'dashboards.yaml', + 'owner': 'grafana', + 'mode': '0640', + 'triggers': { + 'svc_systemd:grafana-server:restart', + }, + }, +} + +svc_systemd = { + 'grafana-server': { + 'needs': { + 'file:/etc/grafana/grafana.ini', + 'pkg_apt:grafana', + }, + }, +} + +### dashboard management starts here +for rnode in repo.nodes: + if not rnode.has_bundle('telegraf'): + continue + + panel_id = count(start=1) + dashboard = { + 'title': rnode.name, + 'uid': UUID(int=rnode.magic_number).hex[:10], + + 'editable': False, + 'graphTooltip': 1, + 'refresh': '30s', + 'schemaVersion': 12, + 'style': 'dark', + 'tags': {'bw'}, + 'time': { + 'from': 'now-1d', + 'to': 'now' + }, + 'version': 1, + + 'rows': [ + dashboard_row_cpu(panel_id, rnode), + dashboard_row_ip_traffic(panel_id, rnode), + dashboard_row_memory(panel_id, rnode), + dashboard_row_disk_space(panel_id, rnode), + dashboard_row_disk_iops(panel_id, rnode), + ], + } + + files[f'/var/lib/grafana/dashboards/{rnode.name}.json'] = { + # use metadata_to_json, because this supports sets + 'content': metadata_to_json(dashboard), + 'triggers': { + 'svc_systemd:grafana-server:restart', + }, + } diff --git a/bundles/grafana/metadata.py b/bundles/grafana/metadata.py new file mode 100644 index 0000000..b4d8577 --- /dev/null +++ b/bundles/grafana/metadata.py @@ -0,0 +1,48 @@ +defaults = { + 'apt': { + 'packages': { + 'grafana': {}, + }, + 'repos': { + 'grafana': { + 'items': { + 'deb https://packages.grafana.com/oss/deb stable main', + }, + }, + }, + }, + 'grafana': { + 'allow_anonymous': False, + 'allow_embedding': False, + 'allow_sign_up': False, + 'anonymous_org': 'public', + 'enable_smtp': True, + 'login_max_duration': '24h', + 'secret_key': repo.vault.random_bytes_as_base64_for(f'{node.name} grafana secret_key'), + }, +} + + +@metadata_reactor.provides( + 'nginx/vhosts/grafana', +) +def nginx(metadata): + if not node.has_bundle('nginx'): + raise DoNotRunAgain + + return { + 'nginx': { + 'vhosts': { + 'grafana': { + 'domain': metadata.get('grafana/domain'), + 'proxy': { + '/': { + 'target': 'http://127.0.0.1:21010', + }, + }, + 'website_check_path': '/login', + 'website_check_string': 'Grafana', + }, + }, + }, + } diff --git a/data/apt/files/gpg-keys/grafana.asc b/data/apt/files/gpg-keys/grafana.asc new file mode 100644 index 0000000..c74f292 --- /dev/null +++ b/data/apt/files/gpg-keys/grafana.asc @@ -0,0 +1,30 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mQENBFiHXVIBCADr3VDEAGpq9Sg/xrPVu1GGqWGXdbnTbbNKeveCtFHZz7/GSATW +iwiY1skvlAOBiIKCqJEji0rZZgd8WxuhdfugiCBk1hDTMWCpjI0P+YymV77jHjYB +jHrKNlhb+aLjEd9Gf2EtbKUT1fvGUkzlVrcRGSX/XR9MBZlgja7NIyuVbn3uwZQ4 +jflWSNSlvMpohNxTFkrBFTRrCJXhbDLfCS46+so22CP3+1VQyqJ7/6RWK9v9KYdS +AVNgILXMggSrMqha4WA1a/ktczVQXNtP8IuPxTdp9pNYsklOTmrFVeq3mXsvWh9Q +lIhpYHIZlTZ5wVBq4wTRchsXC5MubIhz+ASDABEBAAG0GkdyYWZhbmEgPGluZm9A +Z3JhZmFuYS5jb20+iQE4BBMBAgAiBQJYh11SAhsDBgsJCAcDAgYVCAIJCgsEFgID +AQIeAQIXgAAKCRCMjDTFJAmMthxJB/9Id6JrwqRkJW+eSBb71FGQmRsJvNFR8J+3 +NPVhJNkTFFOM7TnjAMUIv+LYEURqGcceTNAN1aHq/7n/8ybXucCS0CnDYyNYpyVs +tWJ3FOQK3jPrmziDCWPQATqMM/Z2auXVFWrDFqfh2xKZNjuix0w2nyuWB8U0CG2U +89w+ksPJblGGU5xLPPzDQoAqyZXY3gpGGTkCuohMq2RWYbp/QJSQagYhQkKZoJhr +XJlnw4At6R1A5UUPzDw6WJqMRkGrkieE6ApIgf1vZSmnLRpXkqquRTAEyGT8Pugg +ee6YkD19/LK6ED6gn32StY770U9ti560U7oRjrOPK/Kjp4+qBtkQuQENBFiHXVIB +CACz4hO1g/4fKO9QWLcbSWpB75lbNgt1kHXP0UcW8TE0DIgqrifod09lC85adIz0 +zdhs+00lLqckM5wNbp2r+pd5rRaxOsMw2V+c/y1Pt3qZxupmPc5l5lL6jzbEVR9g +ygPaE+iabTk9Np2OZQ7Qv5gIDzivqK2mRHXaHTzoQn2dA/3xpFcxnen9dvu7LCpA +CdScSj9/UIRKk9PHIgr2RJhcjzLx0u1PxN9MEqfIsIJUUgZOoDsr8oCs44PGGIMm +cK1CKALLLiC4ZM58B56jRyXo18MqB6VYsC1X9wkcIs72thL3tThXO70oDGcoXzoo +ywAHBH63EzEyduInOhecDIKlABEBAAGJAR8EGAECAAkFAliHXVICGwwACgkQjIw0 +xSQJjLbWSwf/VIM5wEFBY4QLGUAfqfjDyfGXpcha58Y24Vv3n6MwJqnCIbTAaeWf +30CZ/wHg3NNIMB7I31vgmMOEbHQdv0LPTi9TG205VQeehcpNtZRZQ0D8TIetbxyi +Emmn9osig9U3/7jaAWBabE/9bGx4TF3eLlEH9wmFrNYeXvgRqmyqVoqhIMCNAAOY +REYyHyy9mzr9ywkwl0aroBqhzKIPyFlatZy9oRKllY/CCKO9RJy4DZidLphuwzqU +ymdQ1sqe5nKvwG5GvcncPc3O7LMevDBWnpNNkgERnVxCqpm90TuE3ONbirnU4+/S +tUsVU1DERc1fjOCnAm4pKIlNYphISIE7OQ== +=0pMC +-----END PGP PUBLIC KEY BLOCK----- diff --git a/data/powerdns/files/bind-zones/kunsmann.eu b/data/powerdns/files/bind-zones/kunsmann.eu index 70107bc..e5fde09 100644 --- a/data/powerdns/files/bind-zones/kunsmann.eu +++ b/data/powerdns/files/bind-zones/kunsmann.eu @@ -13,6 +13,9 @@ dav IN AAAA 2a01:4f8:10b:2a5f::2 git IN A 94.130.52.224 git IN AAAA 2a01:4f8:10b:2a5f::2 +grafana IN A 116.203.84.44 +grafana IN AAAA 2a01:4f8:c0c:8d56::1 + icinga IN A 51.195.44.8 icinga IN AAAA 2001:41d0:701:1100::2618 diff --git a/nodes.py b/nodes.py index 810ec8e..439c306 100644 --- a/nodes.py +++ b/nodes.py @@ -1,7 +1,6 @@ from os.path import join from pathlib import Path -nodes = {} for node in Path(join(repo_path, "nodes")).rglob("*.py"): with open(node, 'r') as f: exec(f.read()) diff --git a/nodes/htz-cloud/influxdb.py b/nodes/htz-cloud/influxdb.py index 98b8caa..ab0c4b8 100644 --- a/nodes/htz-cloud/influxdb.py +++ b/nodes/htz-cloud/influxdb.py @@ -1,6 +1,7 @@ nodes['htz-cloud.influxdb'] = { 'hostname': '116.203.84.44', 'bundles': { + 'grafana', 'influxdb2', 'zfs', }, @@ -33,6 +34,9 @@ nodes['htz-cloud.influxdb'] = { 'backups': { 'exclude_from_backups': True, }, + 'grafana': { + 'domain': 'grafana.kunsmann.eu', + }, 'nginx': { 'vhosts': { 'influxdb': {