From 0a09344a5fb8782cfd5af7aa8267208ee2d9cbbd Mon Sep 17 00:00:00 2001 From: Fotis Nikolaidis Date: Mon, 5 Dec 2022 20:43:44 +0200 Subject: [PATCH] improve grafana plots Signed-off-by: Fotis Nikolaidis --- .../fedbed/templates/framework.yaml | 3 + charts/system/dashboards/analytics.json | 113 +- charts/system/dashboards/summaries.json | 3262 ++++++++--------- .../templates/telemetry/agent/agent.yml | 2 +- 4 files changed, 1561 insertions(+), 1819 deletions(-) diff --git a/charts/federated-learning/fedbed/templates/framework.yaml b/charts/federated-learning/fedbed/templates/framework.yaml index e1e86d91..b5c3cfd7 100644 --- a/charts/federated-learning/fedbed/templates/framework.yaml +++ b/charts/federated-learning/fedbed/templates/framework.yaml @@ -48,6 +48,8 @@ spec: - -c # Read from string - | # Multi-line str set -eux + cut -d ' ' -f 4 /proc/self/stat > /dev/shm/app # Sidecar: use it for entering the cgroup + python server.py @@ -109,6 +111,7 @@ spec: - -c # Read from string - | # Multi-line str set -eux + cut -d ' ' -f 4 /proc/self/stat > /dev/shm/app # Sidecar: use it for entering the cgroup # do it like that until I find a better way to go from fl.dataset.cifar10 to cifar10. FL_DATASET=$(basename {{"{{.inputs.parameters.dataset}}"}}) diff --git a/charts/system/dashboards/analytics.json b/charts/system/dashboards/analytics.json index 66ee40f0..f44341a8 100644 --- a/charts/system/dashboards/analytics.json +++ b/charts/system/dashboards/analytics.json @@ -87,7 +87,7 @@ "gnetId": 315, "graphTooltip": 0, "id": 1, - "iteration": 1656342183612, + "iteration": 1670246624998, "links": [], "liveNow": false, "panels": [ @@ -163,6 +163,7 @@ "exemplar": true, "expr": "sum(container_memory_working_set_bytes{node=~\"$node\"}) / max (machine_memory_bytes{node=~\"$node\"}) * 100", "hide": false, + "instant": false, "interval": "", "legendFormat": "", "refId": "usage" @@ -794,7 +795,7 @@ "exemplar": true, "expr": "sum(max(machine_cpu_cores) by (node))", "format": "table", - "instant": true, + "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "", @@ -953,8 +954,8 @@ { "datasource": null, "gridPos": { - "h": 6, - "w": 24, + "h": 8, + "w": 6, "x": 0, "y": 10 }, @@ -1051,7 +1052,8 @@ }, "custom": { "align": "auto", - "displayMode": "auto" + "displayMode": "auto", + "filterable": false }, "mappings": [], "thresholds": { @@ -1080,10 +1082,10 @@ "overrides": [] }, "gridPos": { - "h": 10, + "h": 8, "w": 12, - "x": 0, - "y": 16 + "x": 6, + "y": 10 }, "id": 74, "options": { @@ -1097,7 +1099,7 @@ "expr": "max(up) by (node, instance)", "format": "table", "hide": false, - "instant": true, + "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "", @@ -1108,15 +1110,44 @@ "timeShift": null, "title": "Placement Table", "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Time": { + "aggregations": [ + "first", + "last" + ], + "operation": "aggregate" + }, + "Value": { + "aggregations": [], + "operation": "groupby" + }, + "instance": { + "aggregations": [], + "operation": "groupby" + }, + "node": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, { "id": "organize", "options": { "excludeByName": { - "Time": true, + "Time": false, "Value": true }, "indexByName": {}, "renameByName": { + "Time": "Time", + "Time (first)": "Started", + "Time (last)": "To", "Value": "", "instance": "Instance", "node": "Node" @@ -1158,10 +1189,10 @@ "overrides": [] }, "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 16 + "h": 8, + "w": 6, + "x": 18, + "y": 10 }, "id": 137, "links": [], @@ -1201,7 +1232,7 @@ "h": 1, "w": 24, "x": 0, - "y": 26 + "y": 18 }, "id": 129, "panels": [], @@ -1214,7 +1245,7 @@ "h": 2, "w": 24, "x": 0, - "y": 27 + "y": 19 }, "id": 119, "options": { @@ -1302,7 +1333,7 @@ "h": 9, "w": 24, "x": 0, - "y": 29 + "y": 21 }, "id": 78, "options": { @@ -1397,7 +1428,7 @@ "h": 9, "w": 24, "x": 0, - "y": 38 + "y": 30 }, "id": 110, "options": { @@ -1446,7 +1477,7 @@ "h": 1, "w": 24, "x": 0, - "y": 47 + "y": 39 }, "id": 131, "panels": [], @@ -1459,7 +1490,7 @@ "h": 2, "w": 24, "x": 0, - "y": 48 + "y": 40 }, "id": 134, "options": { @@ -1586,7 +1617,7 @@ "h": 8, "w": 24, "x": 0, - "y": 50 + "y": 42 }, "id": 109, "links": [], @@ -1686,7 +1717,7 @@ "h": 8, "w": 24, "x": 0, - "y": 58 + "y": 50 }, "id": 112, "links": [], @@ -1778,7 +1809,7 @@ "h": 8, "w": 24, "x": 0, - "y": 66 + "y": 58 }, "id": 139, "links": [], @@ -1869,7 +1900,7 @@ "h": 8, "w": 24, "x": 0, - "y": 74 + "y": 66 }, "id": 140, "links": [], @@ -1960,7 +1991,7 @@ "h": 8, "w": 24, "x": 0, - "y": 82 + "y": 74 }, "id": 141, "links": [], @@ -2012,7 +2043,7 @@ "h": 8, "w": 24, "x": 0, - "y": 90 + "y": 82 }, "heatmap": {}, "hideZeroBuckets": false, @@ -2118,7 +2149,7 @@ "h": 8, "w": 24, "x": 0, - "y": 98 + "y": 90 }, "id": 126, "links": [], @@ -2159,7 +2190,7 @@ "h": 1, "w": 24, "x": 0, - "y": 106 + "y": 98 }, "id": 133, "panels": [], @@ -2172,7 +2203,7 @@ "h": 2, "w": 24, "x": 0, - "y": 107 + "y": 99 }, "id": 122, "options": { @@ -2255,7 +2286,7 @@ "h": 11, "w": 24, "x": 0, - "y": 109 + "y": 101 }, "id": 123, "links": [], @@ -2360,7 +2391,7 @@ "h": 11, "w": 24, "x": 0, - "y": 120 + "y": 112 }, "id": 113, "links": [], @@ -2456,7 +2487,7 @@ "h": 11, "w": 24, "x": 0, - "y": 131 + "y": 123 }, "id": 115, "links": [], @@ -2560,7 +2591,7 @@ "h": 11, "w": 24, "x": 0, - "y": 142 + "y": 134 }, "id": 117, "links": [], @@ -2667,7 +2698,7 @@ "h": 11, "w": 24, "x": 0, - "y": 153 + "y": 145 }, "id": 114, "links": [], @@ -2773,7 +2804,7 @@ "h": 11, "w": 24, "x": 0, - "y": 164 + "y": 156 }, "id": 118, "links": [], @@ -2831,7 +2862,7 @@ "h": 11, "w": 24, "x": 0, - "y": 175 + "y": 167 }, "heatmap": {}, "hideZeroBuckets": true, @@ -2949,7 +2980,7 @@ "h": 11, "w": 24, "x": 0, - "y": 186 + "y": 178 }, "id": 121, "links": [], @@ -3080,7 +3111,7 @@ "h": 11, "w": 24, "x": 0, - "y": 197 + "y": 189 }, "id": 125, "links": [], @@ -3134,7 +3165,7 @@ "h": 1, "w": 24, "x": 0, - "y": 208 + "y": 200 }, "id": 136, "panels": [], @@ -3224,7 +3255,7 @@ "h": 11, "w": 24, "x": 0, - "y": 209 + "y": 201 }, "id": 97, "links": [], @@ -3355,7 +3386,7 @@ "h": 11, "w": 24, "x": 0, - "y": 220 + "y": 212 }, "id": 100, "links": [], diff --git a/charts/system/dashboards/summaries.json b/charts/system/dashboards/summaries.json index c3e12092..7f6b91b8 100644 --- a/charts/system/dashboards/summaries.json +++ b/charts/system/dashboards/summaries.json @@ -16,56 +16,6 @@ }, "type": "dashboard" }, - { - "datasource": "-- Grafana --", - "enable": true, - "expr": "", - "iconColor": "rgba(0, 211, 255, 1)", - "iconSize": 0, - "lineColor": "", - "name": "Annotations & Alerts", - "query": "", - "showLine": false, - "step": "", - "tagKeys": "", - "tags": null, - "tagsField": "", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "textField": "", - "textFormat": "", - "titleFormat": "", - "type": "dashboard" - }, - { - "datasource": "-- Grafana --", - "enable": true, - "expr": "", - "iconColor": "rgba(0, 211, 255, 1)", - "iconSize": 0, - "lineColor": "", - "name": "Annotations & Alerts", - "query": "", - "showLine": false, - "step": "", - "tagKeys": "", - "tags": null, - "tagsField": "", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "textField": "", - "textFormat": "", - "titleFormat": "", - "type": "dashboard" - }, { "datasource": "-- Grafana --", "enable": true, @@ -158,7 +108,7 @@ "editable": true, "fiscalYearStartMonth": 0, "gnetId": null, - "graphTooltip": 0, + "graphTooltip": 1, "id": 2, "links": [], "liveNow": false, @@ -1107,15 +1057,111 @@ "valueName": "" }, { + "columns": null, "datasource": null, "editable": false, "error": false, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "displayMode": "auto" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, "gridPos": { - "h": 6, - "w": 24, + "h": 7, + "w": 9, "x": 0, "y": 10 }, + "id": 74, + "isNew": false, + "options": { + "showHeader": true + }, + "pluginVersion": "8.2.3", + "scroll": false, + "span": 0, + "styles": null, + "targets": [ + { + "exemplar": true, + "expr": "max(up) by (node, instance)", + "format": "table", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "Used" + } + ], + "title": "Placement Table", + "transform": "", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Time": { + "aggregations": [ + "first", + "last" + ], + "operation": "aggregate" + }, + "instance": { + "aggregations": [], + "operation": "groupby" + }, + "node": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Time (first)": "From", + "Time (last)": "To", + "instance": "Instance", + "node": "Node" + } + } + } + ], + "type": "table" + }, + { + "datasource": null, + "editable": false, + "error": false, + "gridPos": { + "h": 7, + "w": 9, + "x": 9, + "y": 10 + }, "id": 70, "isNew": false, "options": { @@ -1202,62 +1248,6 @@ ], "type": "alertlist" }, - { - "columns": null, - "datasource": null, - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "displayMode": "auto" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 74, - "isNew": false, - "options": { - "showHeader": true - }, - "pluginVersion": "8.2.3", - "scroll": false, - "span": 0, - "styles": null, - "targets": [ - { - "expr": "max(up) by (node, instance)", - "format": "table", - "instant": true, - "intervalFactor": 1, - "refId": "Used" - } - ], - "title": "Placement Table", - "transform": "", - "type": "table" - }, { "datasource": "Prometheus", "editable": false, @@ -1283,10 +1273,10 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 16 + "h": 7, + "w": 6, + "x": 18, + "y": 10 }, "id": 137, "isNew": false, @@ -1322,149 +1312,93 @@ "type": "bargauge" }, { - "collapsed": false, "datasource": null, - "editable": false, - "error": false, "gridPos": { - "h": 1, + "h": 7, "w": 24, "x": 0, - "y": 25 + "y": 17 }, - "id": 129, - "isNew": false, - "panels": [], - "span": 0, - "title": "CPU Metrics", - "type": "row" - }, - { - "datasource": null, - "description": "container_cpu_usage_seconds_total which gives you how long the CPU has been occupied. 1s = a single Core for a whole second.\n\n", - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "vCore(s)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 2, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] + "id": 160, + "options": { + "limit": 10, + "navigateAfter": "10m", + "navigateBefore": "10m", + "navigateToPanel": true, + "onlyFromThisDashboard": false, + "onlyInTimeRange": true, + "showTags": true, + "showTime": true, + "showUser": true }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 78, - "isNew": false, - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "last" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.2.1", - "span": 0, + "pluginVersion": "8.2.3", "targets": [ { - "exemplar": true, - "expr": "max(rate(container_cpu_usage_seconds_total{id=\"/\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "legendFormat": "max", - "refId": "max" - }, - { - "exemplar": true, - "expr": "avg(rate(container_cpu_usage_seconds_total{id=\"/\"}[$__rate_interval])) != 0", + "exemplar": false, + "expr": "container_memory_usage_bytes{}", + "format": "table", "hide": false, + "instant": true, "interval": "", - "legendFormat": "avg", - "refId": "avg" + "intervalFactor": 2, + "legendFormat": "Used", + "refId": "Used" }, { "exemplar": true, - "expr": "min(rate(container_cpu_usage_seconds_total{id=\"/\"}[$__rate_interval])) != 0", + "expr": "sum(container_spec_memory_limit_bytes{} - container_memory_usage_bytes{}) by (instance) ", + "format": "table", "hide": false, + "instant": true, "interval": "", - "legendFormat": "min", - "refId": "min" + "intervalFactor": 2, + "legendFormat": "Remaining", + "refId": "Remaining" } ], - "thresholds": [ + "timeFrom": null, + "timeShift": null, + "title": "Scenario Events", + "transformations": [ { - "colorMode": "critical", - "op": "lt", - "value": 0.20000000298023224, - "visible": true + "id": "merge", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "node", + "instance", + "Value #Used", + "Value #Remaining" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #Remaining": 3, + "Value #Used": 2, + "instance": 0, + "node": 1 + }, + "renameByName": { + "Value #Remaining": "Remaining Memory", + "Value #Used": "Used Memory" + } + } } ], - "timeFrom": null, - "timeShift": null, - "title": "Total CPU Usage", - "type": "timeseries" + "type": "annolist" }, { "datasource": null, - "description": "container_cpu_usage_seconds_total which gives you how long the CPU has been occupied. 1s = a single Core for a whole second.\n\n", + "description": "\n\n", "editable": false, "error": false, "fieldConfig": { @@ -1473,12 +1407,12 @@ "mode": "palette-classic" }, "custom": { - "axisLabel": "vCore(s)", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 2, - "gradientMode": "opacity", + "fillOpacity": 54, + "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, @@ -1488,13 +1422,13 @@ "lineStyle": { "fill": "solid" }, - "lineWidth": 2, + "lineWidth": 0, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, + "showPoints": "never", + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -1518,26 +1452,77 @@ } ] }, - "unit": "short" + "unit": "decbytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "cpu" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "min", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "memory" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "custom.pointSize", + "value": 6 + }, + { + "id": "custom.showPoints", + "value": "always" + } + ] + } + ] }, "gridPos": { "h": 9, "w": 24, "x": 0, - "y": 35 + "y": 24 }, - "id": 149, + "id": 78, "isNew": false, "options": { "legend": { - "calcs": [ - "mean", - "max", - "last" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom" }, "tooltip": { @@ -1549,39 +1534,54 @@ "targets": [ { "exemplar": true, - "expr": "max(rate(container_cpu_system_seconds_total{id=\"/\"}[$__rate_interval])) != 0 ", + "expr": "avg(rate(container_cpu_usage_seconds_total{id=\"/\"}[$__rate_interval])) != 0", "hide": false, - "instant": false, "interval": "", - "legendFormat": "max", - "refId": "max" + "legendFormat": "cpu", + "refId": "cpu" }, { "exemplar": true, - "expr": "avg(rate(container_cpu_system_seconds_total{id=\"/\"}[$__rate_interval])) != 0 ", + "expr": "avg(rate(container_memory_working_set_bytes{id=\"/\"}[$__rate_interval])) != 0", "hide": false, - "instant": false, "interval": "", - "legendFormat": "avg", - "refId": "avg" - }, + "legendFormat": "memory", + "refId": "memory" + } + ], + "thresholds": [ { - "exemplar": true, - "expr": "min(rate(container_cpu_system_seconds_total{id=\"/\"}[$__rate_interval])) != 0 ", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "min", - "refId": "min" + "colorMode": "critical", + "op": "lt", + "value": 0.20000000298023224, + "visible": true } ], "timeFrom": null, "timeShift": null, - "title": "Kernel CPU Usage", + "title": "CPU / RAM", "type": "timeseries" }, { + "collapsed": false, "datasource": null, + "editable": false, + "error": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 129, + "isNew": false, + "panels": [], + "span": 0, + "title": "CPU Metrics", + "type": "row" + }, + { + "datasource": "Prometheus", "description": "container_cpu_usage_seconds_total which gives you how long the CPU has been occupied. 1s = a single Core for a whole second.\n\n", "editable": false, "error": false, @@ -1595,33 +1595,29 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 2, - "gradientMode": "opacity", + "fillOpacity": 35, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, + "lineInterpolation": "linear", "lineWidth": 2, - "pointSize": 5, + "pointSize": 6, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { - "mode": "line" + "mode": "off" } }, - "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -1629,25 +1625,73 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "avg" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ed0e0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "custom.fillBelowTo", + "value": "min" + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "color", + "value": { + "fixedColor": "#447ebc", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "min" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 0 + } + ] + } + ] }, "gridPos": { - "h": 9, + "h": 8, "w": 24, "x": 0, - "y": 44 + "y": 34 }, - "id": 139, + "id": 109, "isNew": false, + "links": [], "options": { "legend": { "calcs": [ @@ -1655,11 +1699,11 @@ "max", "last" ], - "displayMode": "table", + "displayMode": "list", "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "multi" } }, "pluginVersion": "8.2.1", @@ -1667,27 +1711,24 @@ "targets": [ { "exemplar": true, - "expr": "max(rate(container_cpu_user_seconds_total{id=\"/\"}[$__rate_interval])) != 0 ", + "expr": "max(rate(container_cpu_usage_seconds_total{id=\"/\"}[$__rate_interval])) != 0", "hide": false, - "instant": false, "interval": "", "legendFormat": "max", "refId": "max" }, { "exemplar": true, - "expr": "avg(rate(container_cpu_user_seconds_total{id=\"/\"}[$__rate_interval])) != 0 ", + "expr": "avg(rate(container_cpu_usage_seconds_total{id=\"/\"}[$__rate_interval])) != 0", "hide": false, - "instant": false, "interval": "", "legendFormat": "avg", "refId": "avg" }, { "exemplar": true, - "expr": "min(rate(container_cpu_user_seconds_total{id=\"/\"}[$__rate_interval])) != 0 ", + "expr": "min(rate(container_cpu_usage_seconds_total{id=\"/\"}[$__rate_interval])) != 0", "hide": false, - "instant": false, "interval": "", "legendFormat": "min", "refId": "min" @@ -1695,30 +1736,12 @@ ], "timeFrom": null, "timeShift": null, - "title": "Userspace CPU Usage ", + "title": "CPU Utilization [ Min / Avg / Max ]", "type": "timeseries" }, { - "collapsed": false, "datasource": null, - "editable": false, - "error": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 53 - }, - "id": 131, - "isNew": false, - "panels": [], - "span": 0, - "title": "In-RAM Memory Metrics", - "type": "row" - }, - { - "datasource": "Prometheus", - "description": "Current memory usage, this includes all memory regardless of when it was accessed.", + "description": "container_cpu_usage_seconds_total which gives you how long the CPU has been occupied. 1s = a single Core for a whole second.\n\n", "editable": false, "error": false, "fieldConfig": { @@ -1727,33 +1750,37 @@ "mode": "palette-classic" }, "custom": { - "axisLabel": "", + "axisLabel": "vCore(s)", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 2, - "gradientMode": "none", + "fillOpacity": 41, + "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 6, + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { - "mode": "off" + "mode": "line" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -1768,62 +1795,49 @@ } ] }, - "unit": "decbytes" + "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", - "options": "server (Limit)" + "options": "user" }, "properties": [ { - "id": "thresholds", + "id": "color", "value": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 80 - }, - { - "color": "red", - "value": 90 - } - ] + "fixedColor": "#5195ce", + "mode": "fixed" } - }, + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "kernel" + }, + "properties": [ { - "id": "custom.lineStyle", + "id": "color", "value": { - "dash": [ - 10, - 10 - ], - "fill": "dash" + "fixedColor": "#e24d42", + "mode": "fixed" } - }, - { - "id": "custom.fillOpacity", - "value": 5 } ] } ] }, "gridPos": { - "h": 8, + "h": 9, "w": 24, "x": 0, - "y": 54 + "y": 42 }, - "id": 109, + "id": 154, "isNew": false, - "links": [], "options": { "legend": { "calcs": [ @@ -1831,7 +1845,7 @@ "max", "last" ], - "displayMode": "table", + "displayMode": "list", "placement": "bottom" }, "tooltip": { @@ -1843,34 +1857,52 @@ "targets": [ { "exemplar": true, - "expr": "max(container_memory_usage_bytes{id=\"/\"}) != 0", + "expr": "avg(rate(container_cpu_system_seconds_total{id=\"/\"}[$__rate_interval])) != 0", "hide": false, "interval": "", - "legendFormat": "max", - "refId": "max" + "legendFormat": "kernel", + "refId": "kernel" }, { "exemplar": true, - "expr": "avg(container_memory_usage_bytes{id=\"/\"}) != 0", + "expr": "avg(rate(container_cpu_user_seconds_total{id=\"/\"}[$__rate_interval])) != 0", "hide": false, "interval": "", - "legendFormat": "avg", - "refId": "avg" - }, + "legendFormat": "user", + "refId": "user" + } + ], + "thresholds": [ { - "exemplar": true, - "expr": "min(container_memory_usage_bytes{id=\"/\"}) != 0", - "hide": false, - "interval": "", - "legendFormat": "min", - "refId": "min" + "colorMode": "critical", + "op": "lt", + "value": 0.20000000298023224, + "visible": true } ], "timeFrom": null, "timeShift": null, - "title": "Virtual Memory Usage ", + "title": "CPU Utilization [ User / Kernel ]", "type": "timeseries" }, + { + "collapsed": false, + "datasource": null, + "editable": false, + "error": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 131, + "isNew": false, + "panels": [], + "span": 0, + "title": "RAM Memory Metrics", + "type": "row" + }, { "datasource": "Prometheus", "description": "hot: The amount of working set memory, this includes recently accessed memory, dirty memory, and kernel memory. ", @@ -1886,20 +1918,20 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 2, + "fillOpacity": 41, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 6, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -1925,13 +1957,29 @@ }, "unit": "decbytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "WorkingSet" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#b877d9", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 8, "w": 24, "x": 0, - "y": 62 + "y": 52 }, "id": 112, "isNew": false, @@ -1956,38 +2004,30 @@ "targets": [ { "exemplar": true, - "expr": "max(container_memory_working_set_bytes{id=\"/\"} ) != 0", + "expr": "avg(container_memory_usage_bytes{id=\"/\"}) != 0", "hide": false, "interval": "", - "legendFormat": "max", - "refId": "max" + "legendFormat": "MemoryUsage", + "refId": "MemoryUsage" }, { "exemplar": true, "expr": "avg(container_memory_working_set_bytes{id=\"/\"}) != 0", "hide": false, "interval": "", - "legendFormat": "avg", - "refId": "avg" - }, - { - "exemplar": true, - "expr": "min(container_memory_working_set_bytes{id=\"/\"}) != 0", - "hide": false, - "interval": "", - "legendFormat": "min", - "refId": "min" + "legendFormat": "WorkingSet", + "refId": "WorkingSet" } ], "timeFrom": null, "timeShift": null, - "title": "Hot Memory Usage", + "title": "Memory Usage", "transformations": [], "type": "timeseries" }, { "datasource": "Prometheus", - "description": "cache: Number of bytes of page cache memory.", + "description": "hot: The amount of working set memory, this includes recently accessed memory, dirty memory, and kernel memory. ", "editable": false, "error": false, "fieldConfig": { @@ -2000,20 +2040,20 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 2, + "fillOpacity": 41, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 6, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -2039,15 +2079,31 @@ }, "unit": "decbytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "WorkingSet" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#b877d9", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 8, "w": 24, "x": 0, - "y": 70 + "y": 60 }, - "id": 141, + "id": 171, "isNew": false, "links": [], "options": { @@ -2069,132 +2125,46 @@ "targets": [ { "exemplar": true, - "expr": "max(container_memory_cache{id=\"/\"}) != 0", + "expr": "avg(container_memory_rss{id=\"/\"}) != 0", "hide": false, "interval": "", - "legendFormat": "max", - "refId": "max" + "legendFormat": "RSS", + "refId": "RSS" }, { "exemplar": true, - "expr": "avg(container_memory_cache{id=\"/\"}) != 0", + "expr": "avg(container_memory_working_set_bytes{id=\"/\"}) != 0", "hide": false, "interval": "", - "legendFormat": "avg", - "refId": "avg" + "legendFormat": "WorkingSet", + "refId": "WorkingSet" }, { "exemplar": true, - "expr": "min(container_memory_cache{id=\"/\"}) != 0", + "expr": "avg(container_memory_cache{id=\"/\"}) != 0", "hide": false, "interval": "", - "legendFormat": "min", - "refId": "min" + "legendFormat": "Cache", + "refId": "Cache" + }, + { + "exemplar": true, + "expr": "avg(container_memory_mapped_file{id=\"/\"}) != 0", + "hide": false, + "interval": "", + "legendFormat": "MMAP", + "refId": "MMAP" } ], "timeFrom": null, "timeShift": null, - "title": "Cache Memory Usage", + "title": "Memory Usage", "transformations": [], "type": "timeseries" }, { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "timeseries", - "datasource": "Prometheus", - "description": "", - "editable": false, - "error": false, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 78 - }, - "hideZeroBuckets": false, - "highlightCards": true, - "id": 106, - "isNew": false, - "legend": { - "show": true - }, - "reverseYBuckets": false, - "span": 0, - "targets": [ - { - "expr": "max(rate(container_memory_failures_total{id=\"/\", failure_type=\"pgfault\"}[$__rate_interval])) != 0", - "hide": true, - "legendFormat": "max", - "refId": "max" - }, - { - "expr": "avg(rate(container_memory_failures_total{id=\"/\", failure_type=\"pgfault\"}[$__rate_interval])) != 0", - "legendFormat": "avg", - "refId": "avg" - }, - { - "expr": "min(rate(container_memory_failures_total{id=\"/\", failure_type=\"pgfault\"}[$__rate_interval])) != 0", - "hide": true, - "legendFormat": "min", - "refId": "min" - } - ], - "title": "Page Faults", - "tooltip": { - "show": true, - "showHistogram": false - }, - "tooltipDecimals": 0, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "short", - "logBase": 10, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "collapsed": false, "datasource": null, - "editable": false, - "error": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 86 - }, - "id": 151, - "isNew": false, - "panels": [], - "span": 0, - "title": "Out-of-RAM Memory Metrics", - "type": "row" - }, - { - "datasource": "Prometheus", - "description": "", + "description": "\n\n", "editable": false, "error": false, "fieldConfig": { @@ -2207,29 +2177,33 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 2, - "gradientMode": "opacity", + "fillOpacity": 54, + "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", - "lineWidth": 2, + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 0, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, + "showPoints": "never", + "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { - "mode": "off" + "mode": "line" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -2237,898 +2211,222 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "decbytes" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 87 - }, - "id": 126, - "isNew": false, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "last" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.2.1", - "span": 0, - "targets": [ - { - "exemplar": true, - "expr": "max(rate(container_oom_events_total{id=\"/\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "max" - }, - { - "exemplar": true, - "expr": "avg(rate(container_oom_events_total{id=\"/\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "avg", - "refId": "avg" - }, - { - "exemplar": true, - "expr": "min(rate(container_oom_events_total{id=\"/\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "min", - "refId": "min" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "OOM-Killer events", - "type": "timeseries" - }, - { - "datasource": "Prometheus", - "description": "mmap: \tThe amount of memory used for mapped files (includes tmpfs/shmem)\n", - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 2, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 6, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 95 - }, - "id": 140, - "isNew": false, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "last" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.2.1", - "span": 0, - "targets": [ - { - "exemplar": true, - "expr": "max(container_memory_mapped_file{id=\"/\"}) != 0", - "hide": false, - "interval": "", - "legendFormat": "max", - "refId": "max" - }, - { - "exemplar": true, - "expr": "avg(container_memory_mapped_file{id=\"/\"}) != 0", - "hide": false, - "interval": "", - "legendFormat": "avg", - "refId": "avg" - }, - { - "exemplar": true, - "expr": "min(container_memory_mapped_file{id=\"/\"}) != 0", - "hide": false, - "interval": "", - "legendFormat": "min", - "refId": "min" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Mapped File Usage", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": "Prometheus", - "description": "hot: The amount of working set memory, this includes recently accessed memory, dirty memory, and kernel memory. \n\nswap: The amount of swap currently used by the processes in this cgroup\nmmap: \tThe amount of memory used for mapped files (includes tmpfs/shmem)\n\ncache: Number of bytes of page cache memory.", - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 2, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 6, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 103 - }, - "id": 142, - "isNew": false, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "last" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.2.1", - "span": 0, - "targets": [ - { - "exemplar": true, - "expr": "max(container_memory_swap{id=\"/\"}) != 0", - "hide": false, - "interval": "", - "legendFormat": "max", - "refId": "max" - }, - { - "exemplar": true, - "expr": "avg(container_memory_swap{id=\"/\"}) != 0", - "hide": false, - "interval": "", - "legendFormat": "avg", - "refId": "avg" - }, - { - "exemplar": true, - "expr": "min(container_memory_swap{id=\"/\"}) != 0", - "hide": false, - "interval": "", - "legendFormat": "min", - "refId": "min" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Swap Memory Usage", - "transformations": [], - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": null, - "editable": false, - "error": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 111 - }, - "id": 133, - "isNew": false, - "panels": [], - "span": 0, - "title": "I/O Metrics", - "type": "row" - }, - { - "datasource": "Prometheus", - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 112 - }, - "id": 113, - "isNew": false, - "options": { - "colorMode": "", - "content": "", - "displayMode": "basic", - "graphMode": "", - "justifyMode": "", - "mode": "", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "text": {}, - "textMode": "" - }, - "pluginVersion": "8.2.3", - "span": 0, - "targets": [ - { - "expr": "max(container_blkio_device_usage_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}) by (operation) != 0", - "legendFormat": "{{operation}}", - "refId": "ops" - } - ], - "title": "Operation Distribution", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": "Prometheus", - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 2, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 123 - }, - "id": 123, - "isNew": false, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "last" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.2.1", - "span": 0, - "targets": [ - { - "exemplar": true, - "expr": "max(rate(container_fs_writes_bytes_total{ id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "max" - }, - { - "exemplar": true, - "expr": "avg(rate(container_fs_writes_bytes_total{ id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "avg", - "refId": "avg" - }, - { - "exemplar": true, - "expr": "min(rate(container_fs_writes_bytes_total{ id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "min", - "refId": "min" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Write Throughput", - "transformations": [], - "transparent": true, - "type": "timeseries" - }, - { - "datasource": "Prometheus", - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 2, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 134 - }, - "id": 143, - "isNew": false, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "last" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.2.1", - "span": 0, - "targets": [ - { - "exemplar": true, - "expr": "max(rate(container_fs_reads_bytes_total{ id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "max" - }, - { - "exemplar": true, - "expr": "avg(rate(container_fs_reads_bytes_total{ id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "avg", - "refId": "avg" - }, - { - "exemplar": true, - "expr": "min(rate(container_fs_reads_bytes_total{ id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "min", - "refId": "min" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Read Throughput ", - "transformations": [], - "transparent": true, - "type": "timeseries" - }, - { - "datasource": "Prometheus", - "description": "This value counts the number of I/O requests that have been issued to the device driver but have not yet completed. It does not include I/O requests that are in the queue but not yet issued to the device driver", - "editable": false, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 2, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PageFaults" }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ + "properties": [ { - "color": "green", - "value": null + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } }, { - "color": "red", - "value": 80 + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "min", + "value": 0 } ] }, - "unit": "short" - }, - "overrides": [] + { + "matcher": { + "id": "byName", + "options": "RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B877D9", + "mode": "fixed" + } + }, + { + "id": "custom.pointSize", + "value": 6 + }, + { + "id": "custom.showPoints", + "value": "always" + } + ] + } + ] }, "gridPos": { - "h": 11, + "h": 9, "w": 24, "x": 0, - "y": 145 + "y": 68 }, - "id": 118, + "id": 170, "isNew": false, - "links": [], "options": { "legend": { - "calcs": [ - "mean", - "max", - "last" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, - "pluginVersion": "8.2.3", + "pluginVersion": "8.2.1", "span": 0, "targets": [ { "exemplar": true, - "expr": "max(container_fs_io_current{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\"}) != 0", - "format": "time_series", + "expr": "avg(rate(container_memory_failures_total{id=\"/\"}[$__rate_interval])) != 0", "hide": false, - "instant": false, "interval": "", - "legendFormat": "max", - "refId": "max" + "legendFormat": "PageFaults", + "refId": "PageFaults" }, { "exemplar": true, - "expr": "avg(container_fs_io_current{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\"}) != 0", - "format": "time_series", + "expr": "avg(rate(container_memory_working_set_bytes{id=\"/\"}[$__rate_interval])) != 0", "hide": false, - "instant": false, "interval": "", - "legendFormat": "avg", - "refId": "avg" - }, + "legendFormat": "RAM", + "refId": "RAM" + } + ], + "thresholds": [ { - "exemplar": true, - "expr": "min(container_fs_io_current{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\"}) != 0", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "min", - "refId": "min" + "colorMode": "critical", + "op": "lt", + "value": 0.20000000298023224, + "visible": true } ], - "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "IOPS In Progress ", - "transformations": [], - "transparent": true, + "title": "Memory Allocation (Page Faults / RAM )", "type": "timeseries" }, { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", "datasource": "Prometheus", "description": "", "editable": false, "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 2, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, "gridPos": { - "h": 11, + "h": 8, "w": 24, "x": 0, - "y": 156 + "y": 77 }, - "id": 115, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 106, "isNew": false, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "last" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } + "legend": { + "show": true }, - "pluginVersion": "8.2.3", + "reverseYBuckets": false, "span": 0, "targets": [ { - "exemplar": true, - "expr": "max(rate(container_fs_writes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, + "expr": "max(rate(container_memory_failures_total{id=\"/\", failure_type=\"pgfault\"}[$__rate_interval])) != 0", + "hide": true, "legendFormat": "max", "refId": "max" }, { "exemplar": true, - "expr": "avg(rate(container_fs_writes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", - "format": "time_series", - "hide": false, + "expr": "avg(rate(container_memory_failures_total{id=\"/\", failure_type=\"pgfault\"}[$__rate_interval])) != 0", "interval": "", - "intervalFactor": 1, "legendFormat": "avg", "refId": "avg" }, { - "exemplar": true, - "expr": "min(rate(container_fs_writes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, + "expr": "min(rate(container_memory_failures_total{id=\"/\", failure_type=\"pgfault\"}[$__rate_interval])) != 0", + "hide": true, "legendFormat": "min", "refId": "min" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Completed Write IOPS", - "transformations": [], - "transparent": true, - "type": "timeseries" + "title": "Page Faults", + "tooltip": { + "show": true, + "showHistogram": false + }, + "tooltipDecimals": 0, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 10, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null }, { - "datasource": "Prometheus", - "description": "", + "collapsed": false, + "datasource": null, "editable": false, "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 2, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 85 + }, + "id": 133, + "isNew": false, + "panels": [], + "span": 0, + "title": "I/O Metrics", + "type": "row" + }, + { + "datasource": "Prometheus", + "editable": false, + "error": false, + "fieldConfig": { + "defaults": { "mappings": [], "thresholds": { "mode": "absolute", @@ -3145,100 +2443,53 @@ }, "unit": "short" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "clients-0:/dev/sda (Write)" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 11, "w": 24, "x": 0, - "y": 167 + "y": 86 }, - "id": 144, + "id": 113, "isNew": false, - "links": [], "options": { - "legend": { + "colorMode": "", + "content": "", + "displayMode": "basic", + "graphMode": "", + "justifyMode": "", + "mode": "", + "orientation": "auto", + "reduceOptions": { "calcs": [ - "mean", - "max", "last" ], - "displayMode": "table", - "placement": "bottom" + "fields": "", + "values": false }, - "tooltip": { - "mode": "single" - } + "showUnfilled": true, + "text": {}, + "textMode": "" }, "pluginVersion": "8.2.3", "span": 0, "targets": [ { "exemplar": true, - "expr": "max(rate(container_fs_reads_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "max" - }, - { - "exemplar": true, - "expr": "avg(rate(container_fs_reads_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "avg", - "refId": "avg" - }, - { - "exemplar": true, - "expr": "min(rate(container_fs_reads_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", - "format": "time_series", - "hide": false, + "expr": "max(container_blkio_device_usage_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}) by (operation) != 0", "interval": "", - "intervalFactor": 1, - "legendFormat": "min", - "refId": "min" + "legendFormat": "{{operation}}", + "refId": "ops" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Completed Read IOPS", - "transformations": [], + "title": "Operation Distribution", "transparent": true, - "type": "timeseries" + "type": "bargauge" }, { "datasource": "Prometheus", + "description": "This value counts the number of I/O requests that have been issued to the device driver but have not yet completed. It does not include I/O requests that are in the queue but not yet issued to the device driver", "editable": false, "error": false, "fieldConfig": { @@ -3250,22 +2501,21 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, + "drawStyle": "line", + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "stepBefore", - "lineWidth": 2, + "lineInterpolation": "smooth", + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "log": 10, - "type": "log" + "type": "linear" }, - "showPoints": "always", + "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", @@ -3285,7 +2535,7 @@ } ] }, - "unit": "decbytes" + "unit": "Bps" }, "overrides": [] }, @@ -3293,16 +2543,17 @@ "h": 11, "w": 24, "x": 0, - "y": 178 + "y": 97 }, - "id": 117, + "id": 118, "isNew": false, "links": [], "options": { "legend": { "calcs": [ + "max", "mean", - "lastNotNull" + "last" ], "displayMode": "table", "placement": "bottom" @@ -3311,141 +2562,79 @@ "mode": "single" } }, - "pluginVersion": "8.2.1", + "pluginVersion": "8.2.3", "span": 0, "targets": [ { "exemplar": true, - "expr": "sum(rate(container_fs_writes_bytes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) / sum(rate(container_fs_writes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", + "expr": "avg(rate(container_fs_writes_bytes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\"}[$__rate_interval])) != 0", "format": "time_series", "hide": false, + "instant": false, "interval": "", - "intervalFactor": 1, "legendFormat": "Write", - "refId": "write" + "refId": "Write" }, { "exemplar": true, - "expr": "sum(rate(container_fs_reads_bytes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) / sum(rate(container_fs_reads_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", - "format": "time_series", + "expr": "avg(rate(container_fs_reads_bytes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\"}[$__rate_interval])) != 0", "hide": false, - "instant": false, "interval": "", - "intervalFactor": 1, "legendFormat": "Read", - "refId": "read" + "refId": "Read" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Avg I/O Req Size", + "title": "I/O Throughput (Filesystem)", "transformations": [], "transparent": true, "type": "timeseries" }, { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "timeseries", - "datasource": "Prometheus", - "description": "io_time:\nThis value counts the number of milliseconds during which the device has had I/O requests queued.\n\nweighted_io_time:\nThis value counts the number of milliseconds that I/O requests have waited on this block device. If there are multiple I/O requests waiting, this value will increase as the product of the number of milliseconds times the number of requests waiting ", - "editable": false, - "error": false, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 189 - }, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 124, - "isNew": false, - "legend": { - "show": true - }, - "reverseYBuckets": false, - "span": 0, - "targets": [ - { - "expr": "max(rate(container_fs_io_time_weighted_seconds_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\"}[$__rate_interval])) != 0", - "intervalFactor": 1, - "refId": "avg" - } - ], - "title": "Max I/O Queue wait time", - "tooltip": { - "show": true, - "showHistogram": false - }, - "tooltipDecimals": 0, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "datasource": "Prometheus", - "description": "read_time & write_time:\nThese values count the number of milliseconds that I/O requests have waited on this block device. If there are multiple I/O requests waiting, these values will increase at a rate greater than 1000/second; for example, if 60 read requests wait for an average of 30 ms, the read_time field will increase by 60*30 = 1800.", + "datasource": null, + "description": "\n\n", "editable": false, "error": false, "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "fixedColor": "dark-green", + "mode": "fixed" }, "custom": { - "axisLabel": "", + "axisLabel": "ops/second", "axisPlacement": "auto", "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 2, - "gradientMode": "none", + "drawStyle": "line", + "fillOpacity": 41, + "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", - "lineWidth": 2, + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, + "showPoints": "never", + "spanNulls": true, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { - "mode": "off" + "mode": "line" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -3460,24 +2649,39 @@ } ] }, - "unit": "s" + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Read" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-yellow", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { - "h": 11, + "h": 9, "w": 24, "x": 0, - "y": 200 + "y": 108 }, - "id": 121, + "id": 166, "isNew": false, - "links": [], "options": { "legend": { "calcs": [ - "mean", "max", + "mean", "last" ], "displayMode": "table", @@ -3487,48 +2691,42 @@ "mode": "single" } }, - "pluginVersion": "8.2.3", + "pluginVersion": "8.2.1", "span": 0, "targets": [ { "exemplar": true, - "expr": "max(rate(container_fs_write_seconds_total{id=\"/\", device=~\"^/dev/mapper/.*$\", }[$__rate_interval])) != 0", + "expr": "avg(rate(container_fs_writes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "max" + "legendFormat": "Write", + "refId": "Write" }, { "exemplar": true, - "expr": "avg(rate(container_fs_write_seconds_total{id=\"/\", device=~\"^/dev/mapper/.*$\", }[$__rate_interval])) != 0", + "expr": "avg(rate(container_fs_reads_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "avg", - "refId": "avg" - }, + "legendFormat": "Read", + "refId": "Read" + } + ], + "thresholds": [ { - "exemplar": true, - "expr": "min(rate(container_fs_write_seconds_total{id=\"/\", device=~\"^/dev/mapper/.*$\", }[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "min", - "refId": "min" + "colorMode": "critical", + "op": "lt", + "value": 0.20000000298023224, + "visible": true } ], - "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Write Queue wait time", - "transformations": [], - "transparent": true, + "title": "IOPS (Filesystem)", "type": "timeseries" }, { "datasource": "Prometheus", - "description": "read_time & write_time:\nThese values count the number of milliseconds that I/O requests have waited on this block device. If there are multiple I/O requests waiting, these values will increase at a rate greater than 1000/second; for example, if 60 read requests wait for an average of 30 ms, the read_time field will increase by 60*30 = 1800.", + "description": "Contains the number of 512-bytes sectors read and written by the processes member of the cgroup, device by device. \n\n", "editable": false, "error": false, "fieldConfig": { @@ -3537,24 +2735,24 @@ "mode": "palette-classic" }, "custom": { - "axisLabel": "", + "axisLabel": "ops/s", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", - "fillOpacity": 2, + "fillOpacity": 35, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "smooth", + "lineInterpolation": "linear", "lineWidth": 2, - "pointSize": 5, + "pointSize": 6, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -3571,24 +2769,71 @@ { "color": "green", "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "avg" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "custom.fillBelowTo", + "value": "min" }, { - "color": "red", - "value": 80 + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "color", + "value": { + "fixedColor": "dark-green", + "mode": "fixed" + } } ] }, - "unit": "s" - }, - "overrides": [] + { + "matcher": { + "id": "byName", + "options": "min" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 0 + } + ] + } + ] }, "gridPos": { - "h": 11, + "h": 8, "w": 24, "x": 0, - "y": 211 + "y": 117 }, - "id": 146, + "id": 168, "isNew": false, "links": [], "options": { @@ -3598,55 +2843,49 @@ "max", "last" ], - "displayMode": "table", + "displayMode": "list", "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "multi" } }, - "pluginVersion": "8.2.3", + "pluginVersion": "8.2.1", "span": 0, "targets": [ { "exemplar": true, - "expr": "max(rate(container_fs_read_seconds_total{id=\"/\", device=~\"^/dev/mapper/.*$\", }[$__rate_interval])) != 0", + "expr": "max(rate(container_fs_sector_writes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\", }[$__rate_interval])) != 0", "hide": false, "interval": "", - "intervalFactor": 1, "legendFormat": "max", "refId": "max" }, { "exemplar": true, - "expr": "avg(rate(container_fs_read_seconds_total{id=\"/\", device=~\"^/dev/mapper/.*$\", }[$__rate_interval])) != 0", + "expr": "avg(rate(container_fs_sector_writes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\", }[$__rate_interval])) != 0", "hide": false, "interval": "", - "intervalFactor": 1, "legendFormat": "avg", "refId": "avg" }, { "exemplar": true, - "expr": "min(rate(container_fs_read_seconds_total{id=\"/\", device=~\"^/dev/mapper/.*$\", }[$__rate_interval])) != 0", + "expr": "min(rate(container_fs_sector_writes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\", }[$__rate_interval])) != 0", "hide": false, "interval": "", - "intervalFactor": 1, "legendFormat": "min", "refId": "min" } ], - "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Read Queue wait time", - "transformations": [], - "transparent": true, + "title": "Sectors Written", "type": "timeseries" }, { "datasource": "Prometheus", - "description": "Contains the number of 512-bytes sectors read and written by the processes member of the cgroup, device by device. ", + "description": "Contains the number of 512-bytes sectors read and written by the processes member of the cgroup, device by device. \n\n", "editable": false, "error": false, "fieldConfig": { @@ -3655,24 +2894,24 @@ "mode": "palette-classic" }, "custom": { - "axisLabel": "", + "axisLabel": "ops/s", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", - "fillOpacity": 2, + "fillOpacity": 35, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "smooth", + "lineInterpolation": "linear", "lineWidth": 2, - "pointSize": 5, + "pointSize": 6, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -3689,24 +2928,71 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "avg" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "custom.fillBelowTo", + "value": "min" + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "min" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 0 + } + ] + } + ] }, "gridPos": { - "h": 11, + "h": 8, "w": 24, "x": 0, - "y": 222 + "y": 125 }, - "id": 114, + "id": 169, "isNew": false, "links": [], "options": { @@ -3716,11 +3002,11 @@ "max", "last" ], - "displayMode": "table", + "displayMode": "list", "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "multi" } }, "pluginVersion": "8.2.1", @@ -3728,43 +3014,36 @@ "targets": [ { "exemplar": true, - "expr": "max(rate(container_fs_sector_writes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\", }[$__rate_interval])) != 0", + "expr": "max(rate(container_fs_sector_reads_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\", }[$__rate_interval])) != 0", "hide": false, "interval": "", - "intervalFactor": 1, "legendFormat": "max", "refId": "max" }, { "exemplar": true, - "expr": "avg(rate(container_fs_sector_writes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\", }[$__rate_interval])) != 0", + "expr": "avg(rate(container_fs_sector_reads_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\", }[$__rate_interval])) != 0", "hide": false, "interval": "", - "intervalFactor": 1, "legendFormat": "avg", "refId": "avg" }, { "exemplar": true, - "expr": "min(rate(container_fs_sector_writes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\", }[$__rate_interval])) != 0", + "expr": "min(rate(container_fs_sector_reads_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\", }[$__rate_interval])) != 0", "hide": false, "interval": "", - "intervalFactor": 1, "legendFormat": "min", "refId": "min" } ], - "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Written Sectors", - "transformations": [], - "transparent": true, + "title": "Sectors Read", "type": "timeseries" }, { "datasource": "Prometheus", - "description": "Contains the number of 512-bytes sectors read and written by the processes member of the cgroup, device by device. ", "editable": false, "error": false, "fieldConfig": { @@ -3776,21 +3055,22 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 2, + "drawStyle": "points", + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "smooth", + "lineInterpolation": "stepBefore", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { - "type": "linear" + "log": 10, + "type": "log" }, - "showPoints": "auto", + "showPoints": "always", "spanNulls": false, "stacking": { "group": "A", @@ -3813,8 +3093,7 @@ "value": 80 } ] - }, - "unit": "short" + } }, "overrides": [] }, @@ -3822,67 +3101,61 @@ "h": 11, "w": 24, "x": 0, - "y": 233 + "y": 133 }, - "id": 145, + "id": 117, "isNew": false, "links": [], "options": { "legend": { "calcs": [ - "mean", "max", + "mean", "last" ], - "displayMode": "table", + "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, - "pluginVersion": "8.2.1", + "pluginVersion": "8.2.3", "span": 0, "targets": [ { "exemplar": true, - "expr": "max(rate(container_fs_sector_reads_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\", }[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "max", - "refId": "max" - }, - { - "exemplar": true, - "expr": "avg(rate(container_fs_sector_reads_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\", }[$__rate_interval])) != 0", + "expr": "sum(rate(container_fs_writes_bytes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) / sum(rate(container_fs_writes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", + "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "avg", - "refId": "avg" + "legendFormat": "Write", + "refId": "write" }, { "exemplar": true, - "expr": "min(rate(container_fs_sector_reads_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\", }[$__rate_interval])) != 0", + "expr": "sum(rate(container_fs_reads_bytes_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) / sum(rate(container_fs_reads_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/[sv]d[a-z][1-9]*)$\"}[$__rate_interval])) != 0", + "format": "heatmap", "hide": false, + "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "min", - "refId": "min" + "legendFormat": "Read", + "refId": "read" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Read Sectors", + "title": "Avg I/O Req Size", "transformations": [], "transparent": true, "type": "timeseries" }, { "datasource": "Prometheus", - "description": "Reads and writes which are adjacent to each other may be merged for efficiency. Thus, two 4K reads may become one 8K read before it is ultimately handed to the disk, and so it will be counted (and queued) as only one I/O. These fields lets you know how often this was done.", + "description": "read_time & write_time:\nThese values count the number of milliseconds that I/O requests have waited on this block device. If there are multiple I/O requests waiting, these values will increase at a rate greater than 1000/second; for example, if 60 read requests wait for an average of 30 ms, the read_time field will increase by 60*30 = 1800.", "editable": false, "error": false, "fieldConfig": { @@ -3894,7 +3167,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "drawStyle": "line", + "drawStyle": "points", "fillOpacity": 2, "gradientMode": "none", "hideFrom": { @@ -3934,47 +3207,23 @@ }, "unit": "s" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "clients-1 (Write)" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 11, "w": 24, "x": 0, - "y": 244 + "y": 144 }, - "id": 125, + "id": 121, "isNew": false, "links": [], "options": { "legend": { "calcs": [ + "max", "mean", - "lastNotNull" + "last" ], "displayMode": "table", "placement": "bottom" @@ -3983,37 +3232,104 @@ "mode": "single" } }, - "pluginVersion": "8.2.1", + "pluginVersion": "8.2.3", "span": 0, "targets": [ { "exemplar": true, - "expr": "sum(rate(container_fs_writes_merged_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)|(/dev/[sv]d[a-z][1-9]*)$\", }[$__rate_interval])) by (instance, device) != 0", + "expr": "max(rate(container_fs_write_seconds_total{id=\"/\", device=~\"^/dev/mapper/.*$\", }[$__rate_interval])) != 0", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}:{{device}} (Write)", - "refId": "write" + "legendFormat": "writes", + "refId": "writes" }, { "exemplar": true, - "expr": "- sum(rate(container_fs_reads_merged_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)|(/dev/[sv]d[a-z][1-9]*)$\",}[$__rate_interval])) by (instance,device) != 0", + "expr": "max(rate(container_fs_read_seconds_total{id=\"/\", device=~\"^/dev/mapper/.*$\", }[$__rate_interval])) != 0", "hide": false, - "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}:{{device}} (Read)", - "refId": "read" + "legendFormat": "reads", + "refId": "reads" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Merged I/O Operations", + "title": "Tiime spent in I/O", "transformations": [], "transparent": true, "type": "timeseries" }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "Prometheus", + "description": "io_time:\nThis value counts the number of milliseconds during which the device has had I/O requests queued.\n\nweighted_io_time:\nThis value counts the number of milliseconds that I/O requests have waited on this block device. If there are multiple I/O requests waiting, this value will increase as the product of the number of milliseconds times the number of requests waiting ", + "editable": false, + "error": false, + "gridPos": { + "h": 9, + "w": 23, + "x": 0, + "y": 155 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 155, + "isNew": false, + "legend": { + "show": true + }, + "reverseYBuckets": false, + "span": 0, + "targets": [ + { + "exemplar": true, + "expr": "max(rate(container_fs_io_time_weighted_seconds_total{id=\"/\", device=~\"^(^$)|(overlay_.*)|(/dev/mapper/.*)$\"}[$__rate_interval])) != 0", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "avg" + } + ], + "title": "Max I/O Queue wait time", + "tooltip": { + "show": true, + "showHistogram": false + }, + "tooltipDecimals": 0, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, { "collapsed": false, "datasource": null, @@ -4023,7 +3339,7 @@ "h": 1, "w": 24, "x": 0, - "y": 255 + "y": 164 }, "id": 136, "isNew": false, @@ -4033,7 +3349,8 @@ "type": "row" }, { - "datasource": "Prometheus", + "datasource": null, + "description": "\n\n", "editable": false, "error": false, "fieldConfig": { @@ -4046,29 +3363,33 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 2, - "gradientMode": "none", + "fillOpacity": 54, + "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", - "lineWidth": 2, + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 0, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, + "showPoints": "never", + "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { - "mode": "off" + "mode": "line" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -4076,33 +3397,80 @@ { "color": "green", "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Throughput" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } }, { - "color": "red", - "value": 80 + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "unit", + "value": "Bps" + }, + { + "id": "min", + "value": 0 } ] }, - "unit": "Bps" - }, - "overrides": [] + { + "matcher": { + "id": "byName", + "options": "Packet Size" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "custom.pointSize", + "value": 6 + }, + { + "id": "custom.showPoints", + "value": "always" + } + ] + } + ] }, "gridPos": { "h": 11, "w": 24, "x": 0, - "y": 256 + "y": 165 }, - "id": 152, + "id": 167, "isNew": false, - "links": [], "options": { "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom" }, "tooltip": { @@ -4114,75 +3482,37 @@ "targets": [ { "exemplar": true, - "expr": " max(rate(container_network_transmit_bytes_total{id=\"/\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "TX (max)", - "refId": "tx-max" - }, - { - "exemplar": true, - "expr": " avg(rate(container_network_transmit_bytes_total{id=\"/\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "TX (avg)", - "refId": "tx-avg" - }, - { - "exemplar": true, - "expr": " min(rate(container_network_transmit_bytes_total{id=\"/\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "TX (min)", - "refId": "tx-min" - }, - { - "exemplar": true, - "expr": "- max(rate(container_network_receive_bytes_total{id=\"/\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "RX (max)", - "refId": "rx-max" - }, - { - "exemplar": true, - "expr": "- avg(rate(container_network_receive_bytes_total{id=\"/\"}[$__rate_interval])) != 0", + "expr": "avg(rate(container_network_receive_packets_total{id=\"/\"}[$__rate_interval])) != 0", "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "RX (avg)", - "refId": "rx-avg" + "legendFormat": "Throughput", + "refId": "cpu" }, { "exemplar": true, - "expr": "- min(rate(container_network_receive_bytes_total{id=\"/\"}[$__rate_interval])) != 0", + "expr": "avg(rate(container_network_receive_bytes_total{id=\"/\"}[$__rate_interval])) != 0", "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "RX (min)", - "refId": "rx-min" + "legendFormat": "Packet Size", + "refId": "memory" } ], "thresholds": [ { "colorMode": "critical", "op": "lt", - "value": -3000000000, + "value": 0.20000000298023224, "visible": true } ], "timeFrom": null, "timeShift": null, - "title": "Network Throughput ", - "transparent": true, + "title": "Inbound Network Traffic", "type": "timeseries" }, { - "datasource": "Prometheus", + "datasource": null, + "description": "\n\n", "editable": false, "error": false, "fieldConfig": { @@ -4195,29 +3525,33 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", + "fillOpacity": 54, + "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", - "lineWidth": 2, + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 0, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, + "showPoints": "never", + "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { - "mode": "off" + "mode": "line" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -4225,34 +3559,80 @@ { "color": "green", "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Throughput" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } }, { - "color": "red", - "value": 80 + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "unit", + "value": "Bps" + }, + { + "id": "min", + "value": 0 } ] }, - "unit": "short" - }, - "overrides": [] + { + "matcher": { + "id": "byName", + "options": "Packet Size" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "custom.pointSize", + "value": 6 + }, + { + "id": "custom.showPoints", + "value": "always" + } + ] + } + ] }, "gridPos": { - "h": 11, + "h": 9, "w": 24, "x": 0, - "y": 267 + "y": 176 }, - "id": 100, + "id": 158, "isNew": false, - "links": [], "options": { "legend": { - "calcs": [ - "mean", - "max", - "last" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom" }, "tooltip": { @@ -4262,67 +3642,395 @@ "pluginVersion": "8.2.1", "span": 0, "targets": [ - { - "exemplar": true, - "expr": "max(rate(container_network_transmit_packets_total{id=\"/\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "TX (max)", - "refId": "tx-max" - }, { "exemplar": true, "expr": "avg(rate(container_network_transmit_packets_total{id=\"/\"}[$__rate_interval])) != 0", "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "TX (avg)", - "refId": "tx-avg" - }, - { - "exemplar": true, - "expr": "min(rate(container_network_transmit_packets_total{id=\"/\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "TX (min)", - "refId": "tx-min" - }, - { - "exemplar": true, - "expr": "- max(rate(container_network_transmit_packets_total{id=\"/\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "RX (max)", - "refId": "rx-mac" + "legendFormat": "Throughput", + "refId": "cpu" }, { "exemplar": true, - "expr": "- avg(rate(container_network_transmit_packets_total{id=\"/\"}[$__rate_interval])) != 0", + "expr": "avg(rate(container_network_transmit_bytes_total{id=\"/\"}[$__rate_interval])) != 0", "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "RX (avg)", - "refId": "rx-avg" - }, + "legendFormat": "Packet Size", + "refId": "memory" + } + ], + "thresholds": [ { - "exemplar": true, - "expr": "- min(rate(container_network_transmit_packets_total{id=\"/\"}[$__rate_interval])) != 0", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "RX (min)", - "refId": "rx-min" + "colorMode": "critical", + "op": "lt", + "value": 0.20000000298023224, + "visible": true } ], - "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Network Packet", - "transparent": true, + "title": "Outbound Network Traffic", "type": "timeseries" + }, + { + "collapsed": true, + "datasource": null, + "editable": false, + "error": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 185 + }, + "id": 151, + "isNew": false, + "panels": [ + { + "datasource": "Prometheus", + "description": "", + "editable": false, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 2, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 77 + }, + "id": 126, + "isNew": false, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.2.1", + "span": 0, + "targets": [ + { + "exemplar": true, + "expr": "max(rate(container_oom_events_total{id=\"/\"}[$__rate_interval])) != 0", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "max", + "refId": "max" + }, + { + "exemplar": true, + "expr": "avg(rate(container_oom_events_total{id=\"/\"}[$__rate_interval])) != 0", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg", + "refId": "avg" + }, + { + "exemplar": true, + "expr": "min(rate(container_oom_events_total{id=\"/\"}[$__rate_interval])) != 0", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "min", + "refId": "min" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "OOM-Killer events", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "description": "mmap: \tThe amount of memory used for mapped files (includes tmpfs/shmem)\n", + "editable": false, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 2, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 6, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 85 + }, + "id": 140, + "isNew": false, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.2.1", + "span": 0, + "targets": [ + { + "exemplar": true, + "expr": "max(container_memory_mapped_file{id=\"/\"}) != 0", + "hide": false, + "interval": "", + "legendFormat": "max", + "refId": "max" + }, + { + "exemplar": true, + "expr": "avg(container_memory_mapped_file{id=\"/\"}) != 0", + "hide": false, + "interval": "", + "legendFormat": "avg", + "refId": "avg" + }, + { + "exemplar": true, + "expr": "min(container_memory_mapped_file{id=\"/\"}) != 0", + "hide": false, + "interval": "", + "legendFormat": "min", + "refId": "min" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Mapped File Usage", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "description": "hot: The amount of working set memory, this includes recently accessed memory, dirty memory, and kernel memory. \n\nswap: The amount of swap currently used by the processes in this cgroup\nmmap: \tThe amount of memory used for mapped files (includes tmpfs/shmem)\n\ncache: Number of bytes of page cache memory.", + "editable": false, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 2, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 6, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 142, + "isNew": false, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.2.1", + "span": 0, + "targets": [ + { + "exemplar": true, + "expr": "max(container_memory_swap{id=\"/\"}) != 0", + "hide": false, + "interval": "", + "legendFormat": "max", + "refId": "max" + }, + { + "exemplar": true, + "expr": "avg(container_memory_swap{id=\"/\"}) != 0", + "hide": false, + "interval": "", + "legendFormat": "avg", + "refId": "avg" + }, + { + "exemplar": true, + "expr": "min(container_memory_swap{id=\"/\"}) != 0", + "hide": false, + "interval": "", + "legendFormat": "min", + "refId": "min" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Swap Memory Usage", + "transformations": [], + "type": "timeseries" + } + ], + "span": 0, + "title": "Out-of-RAM Memory Metrics", + "type": "row" } ], "refresh": "5s", diff --git a/charts/system/templates/telemetry/agent/agent.yml b/charts/system/templates/telemetry/agent/agent.yml index 896fd031..0ab47c37 100644 --- a/charts/system/templates/telemetry/agent/agent.yml +++ b/charts/system/templates/telemetry/agent/agent.yml @@ -18,7 +18,7 @@ spec: "sidecar.frisbee.dev/telemetry": cadvisor containers: - name: cadvisor - image: gcr.io/cadvisor/cadvisor:v0.44.0 + image: gcr.io/cadvisor/cadvisor:v0.46.0 ports: - name: tel-sys containerPort: 9442