From 2da45a8368364c184831e6ef40db74cb239d72f8 Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk Date: Mon, 10 Jun 2024 12:49:00 +0300 Subject: [PATCH] vmagent: updated dashboard and alert for stream aggregation (#6427) ### Describe Your Changes Added streaming aggregation section to vmagent dashboards Added alert for streaming aggregation and deduplication flush timeouts Removed deprecated compose versions from compose files Signed-off-by: hagen1778 Co-authored-by: hagen1778 --- dashboards/vm/vmagent.json | 816 +++++++++++++++--- dashboards/vmagent.json | 816 +++++++++++++++--- deployment/docker/alerts-vmagent.yml | 20 + deployment/docker/docker-compose-cluster.yml | 1 - .../docker/docker-compose-victorialogs.yml | 1 - deployment/docker/docker-compose.yml | 1 - .../vm-datasource/docker-compose-cluster.yml | 1 - .../docker/vm-datasource/docker-compose.yml | 1 - docs/CHANGELOG.md | 4 + 9 files changed, 1456 insertions(+), 205 deletions(-) diff --git a/dashboards/vm/vmagent.json b/dashboards/vm/vmagent.json index d9e5fdc98..c78ef1cab 100644 --- a/dashboards/vm/vmagent.json +++ b/dashboards/vm/vmagent.json @@ -86,7 +86,7 @@ } ] }, - "description": "Overview for VictoriaMetrics vmagent v1.80.0 or higher", + "description": "Overview for VictoriaMetrics vmagent v1.102.0 or higher", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -1336,7 +1336,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1344,8 +1345,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1440,7 +1440,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1448,8 +1449,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1627,8 +1627,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -1636,10 +1635,9 @@ "h": 8, "w": 12, "x": 0, - "y": 3 + "y": 11 }, "id": 109, - "links": [], "options": { "legend": { "calcs": [ @@ -1743,8 +1741,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -1752,10 +1749,9 @@ "h": 8, "w": 12, "x": 12, - "y": 3 + "y": 11 }, "id": 111, - "links": [], "options": { "legend": { "calcs": [ @@ -1856,8 +1852,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [ { @@ -1878,10 +1873,9 @@ "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 19 }, "id": 81, - "links": [], "options": { "legend": { "calcs": [ @@ -1993,8 +1987,7 @@ } ] }, - "unit": "bps", - "unitScale": true + "unit": "bps" }, "overrides": [ { @@ -2015,7 +2008,7 @@ "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 19 }, "id": 7, "options": { @@ -2125,8 +2118,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -2134,10 +2126,9 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 27 }, "id": 83, - "links": [], "options": { "legend": { "calcs": [ @@ -2232,8 +2223,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2241,10 +2231,9 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 27 }, "id": 39, - "links": [], "options": { "legend": { "calcs": [ @@ -2340,8 +2329,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -2349,10 +2337,9 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 35 }, "id": 135, - "links": [], "options": { "legend": { "calcs": [ @@ -2447,8 +2434,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2456,10 +2442,9 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 35 }, "id": 41, - "links": [], "options": { "legend": { "calcs": [ @@ -2534,6 +2519,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2547,6 +2533,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2584,7 +2571,7 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 12 }, "id": 92, "options": { @@ -2636,6 +2623,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2649,6 +2637,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2686,7 +2675,7 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 12 }, "id": 95, "options": { @@ -2738,6 +2727,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2752,6 +2742,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2791,7 +2782,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 20 }, "id": 98, "options": { @@ -2843,6 +2834,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2857,6 +2849,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2896,7 +2889,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 20 }, "id": 99, "options": { @@ -3000,10 +2993,9 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 28 }, "id": 79, - "links": [], "options": { "legend": { "calcs": [ @@ -3105,7 +3097,7 @@ "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 28 }, "id": 18, "links": [ @@ -3215,10 +3207,9 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 36 }, "id": 127, - "links": [], "options": { "legend": { "calcs": [ @@ -3318,7 +3309,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 36 }, "id": 50, "options": { @@ -3418,7 +3409,7 @@ "h": 7, "w": 24, "x": 0, - "y": 36 + "y": 44 }, "id": 129, "options": { @@ -3565,8 +3556,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3574,7 +3564,7 @@ "h": 7, "w": 12, "x": 0, - "y": 37 + "y": 13 }, "id": 48, "options": { @@ -3672,8 +3662,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3681,7 +3670,7 @@ "h": 7, "w": 12, "x": 12, - "y": 37 + "y": 13 }, "id": 76, "options": { @@ -3777,8 +3766,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3786,7 +3774,7 @@ "h": 7, "w": 12, "x": 0, - "y": 44 + "y": 20 }, "id": 132, "options": { @@ -3884,8 +3872,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3893,7 +3880,7 @@ "h": 7, "w": 12, "x": 12, - "y": 44 + "y": 20 }, "id": 133, "options": { @@ -3990,8 +3977,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3999,7 +3985,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 27 }, "id": 20, "options": { @@ -4095,8 +4081,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -4104,7 +4089,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 27 }, "id": 126, "options": { @@ -4199,8 +4184,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [] }, @@ -4208,7 +4192,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 35 }, "id": 46, "options": { @@ -4303,8 +4287,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -4312,7 +4295,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 35 }, "id": 31, "options": { @@ -4437,6 +4420,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4450,6 +4434,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4489,10 +4474,9 @@ "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 14 }, "id": 73, - "links": [], "options": { "legend": { "calcs": [ @@ -4553,6 +4537,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4566,6 +4551,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4605,10 +4591,9 @@ "h": 8, "w": 12, "x": 12, - "y": 38 + "y": 14 }, "id": 131, - "links": [], "options": { "legend": { "calcs": [ @@ -4656,6 +4641,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4669,6 +4655,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4708,10 +4695,9 @@ "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 22 }, "id": 130, - "links": [], "options": { "legend": { "calcs": [ @@ -4772,6 +4758,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4785,6 +4772,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4824,10 +4812,9 @@ "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 22 }, "id": 77, - "links": [], "options": { "legend": { "calcs": [ @@ -4885,6 +4872,620 @@ "title": "Ingestion", "type": "row" }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 136, + "panels": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "The 99th percentile of avg flush duration for the aggregated data. \n\nSmaller is better.\n\nAggregation can produce incorrect results ff flush duration exceeds configured deduplication interval. See \"Flush Timeouts\" panel.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 137, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, vm_streamaggr_dedup_flush_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Dedup flush duration (0.99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Amount of ignored samples during aggregation. \nStream aggregation will drop samples with NaN values, or samples with too old timestamps. See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 143, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_ignored_samples_total[5m]) > 0", + "instant": false, + "legendFormat": "{{reason}}: {{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Ignored samples", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows events when deduplication or aggregation couldn't be finished in the configured interval. Such events may result into bad accuracy of produced data.\n\nPossible solutions:\n* increase interval; \n* use match filter matching smaller number of series;\n* reduce samples ingestion rate to stream aggregation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": -5, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 139, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "instant": false, + "legendFormat": "aggregate: {{instance}} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "hide": false, + "instant": false, + "legendFormat": "dedup: {{instance}} ({{job}})", + "range": true, + "refId": "B" + } + ], + "title": "Flush timeouts", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows the lag between average samples timestamp and aggregation interval.\n\nLower is better.\n\nToo high lag or lag exceeding the interval might be a sign of data delay before aggregation or resource insufficiency on aggregator. Samples with high lag may affect accuracy of aggregation.\n\nSee https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 142, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Aggregated samples lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows the size of Label Compressor in bytes.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 140, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "vm_streamaggr_labels_compressor_size_bytes{job=~\"$job\",instance=~\"$instance\"} > 0", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Labels compressor bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows the size of Label Compressor in number of entries.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 141, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "vm_streamaggr_labels_compressor_items_count{job=~\"$job\",instance=~\"$instance\"} > 0", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Labels compressor items count", + "type": "timeseries" + } + ], + "title": "Streaming aggregation", + "type": "row" + }, { "collapsed": true, "datasource": { @@ -4895,7 +5496,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 39 }, "id": 58, "panels": [ @@ -4911,6 +5512,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4924,6 +5526,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4964,7 +5567,7 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 16 }, "id": 60, "options": { @@ -5014,6 +5617,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5027,6 +5631,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5066,7 +5671,7 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 16 }, "id": 66, "options": { @@ -5116,6 +5721,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5129,6 +5735,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5168,7 +5775,7 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 24 }, "id": 61, "options": { @@ -5218,6 +5825,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5231,6 +5839,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5270,7 +5879,7 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 24 }, "id": 65, "options": { @@ -5371,7 +5980,7 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 32 }, "id": 88, "options": { @@ -5469,7 +6078,7 @@ "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 32 }, "id": 84, "options": { @@ -5570,7 +6179,7 @@ "h": 8, "w": 12, "x": 0, - "y": 79 + "y": 40 }, "id": 90, "options": { @@ -5623,7 +6232,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 40 }, "id": 113, "panels": [ @@ -5636,7 +6245,7 @@ "h": 2, "w": 24, "x": 0, - "y": 87 + "y": 17 }, "id": 115, "options": { @@ -5648,7 +6257,7 @@ "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.", "mode": "markdown" }, - "pluginVersion": "9.2.6", + "pluginVersion": "10.4.2", "transparent": true, "type": "text" }, @@ -5664,6 +6273,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5677,6 +6287,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5712,10 +6323,9 @@ "h": 8, "w": 12, "x": 0, - "y": 89 + "y": 19 }, "id": 119, - "links": [], "options": { "legend": { "calcs": [ @@ -5767,6 +6377,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5780,6 +6391,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5815,10 +6427,9 @@ "h": 8, "w": 12, "x": 12, - "y": 89 + "y": 19 }, "id": 117, - "links": [], "options": { "legend": { "calcs": [ @@ -5868,6 +6479,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5881,6 +6493,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5920,7 +6533,7 @@ "h": 8, "w": 12, "x": 0, - "y": 97 + "y": 27 }, "id": 125, "links": [ @@ -5976,6 +6589,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5989,6 +6603,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -6040,7 +6655,7 @@ "h": 8, "w": 12, "x": 12, - "y": 97 + "y": 27 }, "id": 123, "options": { @@ -6104,6 +6719,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -6117,6 +6733,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -6168,10 +6785,9 @@ "h": 8, "w": 12, "x": 0, - "y": 105 + "y": 35 }, "id": 121, - "links": [], "options": { "legend": { "calcs": [ diff --git a/dashboards/vmagent.json b/dashboards/vmagent.json index 80c6bddb8..06e0b0422 100644 --- a/dashboards/vmagent.json +++ b/dashboards/vmagent.json @@ -85,7 +85,7 @@ } ] }, - "description": "Overview for VictoriaMetrics vmagent v1.80.0 or higher", + "description": "Overview for VictoriaMetrics vmagent v1.102.0 or higher", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -1335,7 +1335,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1343,8 +1344,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1439,7 +1439,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1447,8 +1448,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1626,8 +1626,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -1635,10 +1634,9 @@ "h": 8, "w": 12, "x": 0, - "y": 3 + "y": 11 }, "id": 109, - "links": [], "options": { "legend": { "calcs": [ @@ -1742,8 +1740,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -1751,10 +1748,9 @@ "h": 8, "w": 12, "x": 12, - "y": 3 + "y": 11 }, "id": 111, - "links": [], "options": { "legend": { "calcs": [ @@ -1855,8 +1851,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [ { @@ -1877,10 +1872,9 @@ "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 19 }, "id": 81, - "links": [], "options": { "legend": { "calcs": [ @@ -1992,8 +1986,7 @@ } ] }, - "unit": "bps", - "unitScale": true + "unit": "bps" }, "overrides": [ { @@ -2014,7 +2007,7 @@ "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 19 }, "id": 7, "options": { @@ -2124,8 +2117,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -2133,10 +2125,9 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 27 }, "id": 83, - "links": [], "options": { "legend": { "calcs": [ @@ -2231,8 +2222,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2240,10 +2230,9 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 27 }, "id": 39, - "links": [], "options": { "legend": { "calcs": [ @@ -2339,8 +2328,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -2348,10 +2336,9 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 35 }, "id": 135, - "links": [], "options": { "legend": { "calcs": [ @@ -2446,8 +2433,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2455,10 +2441,9 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 35 }, "id": 41, - "links": [], "options": { "legend": { "calcs": [ @@ -2533,6 +2518,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2546,6 +2532,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2583,7 +2570,7 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 12 }, "id": 92, "options": { @@ -2635,6 +2622,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2648,6 +2636,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2685,7 +2674,7 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 12 }, "id": 95, "options": { @@ -2737,6 +2726,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2751,6 +2741,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2790,7 +2781,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 20 }, "id": 98, "options": { @@ -2842,6 +2833,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2856,6 +2848,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2895,7 +2888,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 20 }, "id": 99, "options": { @@ -2999,10 +2992,9 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 28 }, "id": 79, - "links": [], "options": { "legend": { "calcs": [ @@ -3104,7 +3096,7 @@ "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 28 }, "id": 18, "links": [ @@ -3214,10 +3206,9 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 36 }, "id": 127, - "links": [], "options": { "legend": { "calcs": [ @@ -3317,7 +3308,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 36 }, "id": 50, "options": { @@ -3417,7 +3408,7 @@ "h": 7, "w": 24, "x": 0, - "y": 36 + "y": 44 }, "id": 129, "options": { @@ -3564,8 +3555,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3573,7 +3563,7 @@ "h": 7, "w": 12, "x": 0, - "y": 37 + "y": 13 }, "id": 48, "options": { @@ -3671,8 +3661,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3680,7 +3669,7 @@ "h": 7, "w": 12, "x": 12, - "y": 37 + "y": 13 }, "id": 76, "options": { @@ -3776,8 +3765,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3785,7 +3773,7 @@ "h": 7, "w": 12, "x": 0, - "y": 44 + "y": 20 }, "id": 132, "options": { @@ -3883,8 +3871,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3892,7 +3879,7 @@ "h": 7, "w": 12, "x": 12, - "y": 44 + "y": 20 }, "id": 133, "options": { @@ -3989,8 +3976,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3998,7 +3984,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 27 }, "id": 20, "options": { @@ -4094,8 +4080,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -4103,7 +4088,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 27 }, "id": 126, "options": { @@ -4198,8 +4183,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [] }, @@ -4207,7 +4191,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 35 }, "id": 46, "options": { @@ -4302,8 +4286,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -4311,7 +4294,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 35 }, "id": 31, "options": { @@ -4436,6 +4419,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4449,6 +4433,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4488,10 +4473,9 @@ "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 14 }, "id": 73, - "links": [], "options": { "legend": { "calcs": [ @@ -4552,6 +4536,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4565,6 +4550,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4604,10 +4590,9 @@ "h": 8, "w": 12, "x": 12, - "y": 38 + "y": 14 }, "id": 131, - "links": [], "options": { "legend": { "calcs": [ @@ -4655,6 +4640,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4668,6 +4654,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4707,10 +4694,9 @@ "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 22 }, "id": 130, - "links": [], "options": { "legend": { "calcs": [ @@ -4771,6 +4757,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4784,6 +4771,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4823,10 +4811,9 @@ "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 22 }, "id": 77, - "links": [], "options": { "legend": { "calcs": [ @@ -4884,6 +4871,620 @@ "title": "Ingestion", "type": "row" }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 136, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The 99th percentile of avg flush duration for the aggregated data. \n\nSmaller is better.\n\nAggregation can produce incorrect results ff flush duration exceeds configured deduplication interval. See \"Flush Timeouts\" panel.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 137, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, vm_streamaggr_dedup_flush_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Dedup flush duration (0.99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Amount of ignored samples during aggregation. \nStream aggregation will drop samples with NaN values, or samples with too old timestamps. See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 143, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_ignored_samples_total[5m]) > 0", + "instant": false, + "legendFormat": "{{reason}}: {{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Ignored samples", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows events when deduplication or aggregation couldn't be finished in the configured interval. Such events may result into bad accuracy of produced data.\n\nPossible solutions:\n* increase interval; \n* use match filter matching smaller number of series;\n* reduce samples ingestion rate to stream aggregation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": -5, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 139, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "instant": false, + "legendFormat": "aggregate: {{instance}} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "hide": false, + "instant": false, + "legendFormat": "dedup: {{instance}} ({{job}})", + "range": true, + "refId": "B" + } + ], + "title": "Flush timeouts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the lag between average samples timestamp and aggregation interval.\n\nLower is better.\n\nToo high lag or lag exceeding the interval might be a sign of data delay before aggregation or resource insufficiency on aggregator. Samples with high lag may affect accuracy of aggregation.\n\nSee https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 142, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Aggregated samples lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the size of Label Compressor in bytes.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 140, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "vm_streamaggr_labels_compressor_size_bytes{job=~\"$job\",instance=~\"$instance\"} > 0", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Labels compressor bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the size of Label Compressor in number of entries.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 141, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "vm_streamaggr_labels_compressor_items_count{job=~\"$job\",instance=~\"$instance\"} > 0", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Labels compressor items count", + "type": "timeseries" + } + ], + "title": "Streaming aggregation", + "type": "row" + }, { "collapsed": true, "datasource": { @@ -4894,7 +5495,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 39 }, "id": 58, "panels": [ @@ -4910,6 +5511,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4923,6 +5525,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4963,7 +5566,7 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 16 }, "id": 60, "options": { @@ -5013,6 +5616,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5026,6 +5630,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5065,7 +5670,7 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 16 }, "id": 66, "options": { @@ -5115,6 +5720,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5128,6 +5734,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5167,7 +5774,7 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 24 }, "id": 61, "options": { @@ -5217,6 +5824,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5230,6 +5838,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5269,7 +5878,7 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 24 }, "id": 65, "options": { @@ -5370,7 +5979,7 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 32 }, "id": 88, "options": { @@ -5468,7 +6077,7 @@ "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 32 }, "id": 84, "options": { @@ -5569,7 +6178,7 @@ "h": 8, "w": 12, "x": 0, - "y": 79 + "y": 40 }, "id": 90, "options": { @@ -5622,7 +6231,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 40 }, "id": 113, "panels": [ @@ -5635,7 +6244,7 @@ "h": 2, "w": 24, "x": 0, - "y": 87 + "y": 17 }, "id": 115, "options": { @@ -5647,7 +6256,7 @@ "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.", "mode": "markdown" }, - "pluginVersion": "9.2.6", + "pluginVersion": "10.4.2", "transparent": true, "type": "text" }, @@ -5663,6 +6272,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5676,6 +6286,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5711,10 +6322,9 @@ "h": 8, "w": 12, "x": 0, - "y": 89 + "y": 19 }, "id": 119, - "links": [], "options": { "legend": { "calcs": [ @@ -5766,6 +6376,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5779,6 +6390,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5814,10 +6426,9 @@ "h": 8, "w": 12, "x": 12, - "y": 89 + "y": 19 }, "id": 117, - "links": [], "options": { "legend": { "calcs": [ @@ -5867,6 +6478,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5880,6 +6492,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5919,7 +6532,7 @@ "h": 8, "w": 12, "x": 0, - "y": 97 + "y": 27 }, "id": 125, "links": [ @@ -5975,6 +6588,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5988,6 +6602,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -6039,7 +6654,7 @@ "h": 8, "w": 12, "x": 12, - "y": 97 + "y": 27 }, "id": 123, "options": { @@ -6103,6 +6718,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -6116,6 +6732,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -6167,10 +6784,9 @@ "h": 8, "w": 12, "x": 0, - "y": 105 + "y": 35 }, "id": 121, - "links": [], "options": { "legend": { "calcs": [ diff --git a/deployment/docker/alerts-vmagent.yml b/deployment/docker/alerts-vmagent.yml index fc11b87d0..15d522a7f 100644 --- a/deployment/docker/alerts-vmagent.yml +++ b/deployment/docker/alerts-vmagent.yml @@ -135,3 +135,23 @@ groups: summary: "Configuration reload failed for vmagent instance {{ $labels.instance }}" description: "Configuration hot-reload failed for vmagent on instance {{ $labels.instance }}. Check vmagent's logs for detailed error message." + + - alert: StreamAggrFlushTimeout + expr: | + increase(vm_streamaggr_flush_timeouts_total[5m]) > 0 + labels: + severity: warning + annotations: + summary: "Streaming aggregation at \"{{ $labels.job }}\" (instance {{ $labels.instance }}) can't be finished within the configured aggregation interval." + description: "Stream aggregation process can't keep up with the load and might produce incorrect aggregation results. Check logs for more details. + Possible solutions: increase aggregation interval; aggregate smaller number of series; reduce samples' ingestion rate to stream aggregation." + + - alert: StreamAggrDedupFlushTimeout + expr: | + increase(vm_streamaggr_dedup_flush_timeouts_total[5m]) > 0 + labels: + severity: warning + annotations: + summary: "Deduplication \"{{ $labels.job }}\" (instance {{ $labels.instance }}) can't be finished within configured deduplication interval." + description: "Deduplication process can't keep up with the load and might produce incorrect results. Check docs https://docs.victoriametrics.com/stream-aggregation/#deduplication and logs for more details. + Possible solutions: increase deduplication interval; deduplicate smaller number of series; reduce samples' ingestion rate." diff --git a/deployment/docker/docker-compose-cluster.yml b/deployment/docker/docker-compose-cluster.yml index 6194de252..1672f516b 100644 --- a/deployment/docker/docker-compose-cluster.yml +++ b/deployment/docker/docker-compose-cluster.yml @@ -1,4 +1,3 @@ -version: '3.5' services: # Metrics collector. # It scrapes targets defined in --promscrape.config diff --git a/deployment/docker/docker-compose-victorialogs.yml b/deployment/docker/docker-compose-victorialogs.yml index 484a80265..e2e676481 100644 --- a/deployment/docker/docker-compose-victorialogs.yml +++ b/deployment/docker/docker-compose-victorialogs.yml @@ -1,4 +1,3 @@ -version: "3.5" services: # Grafana instance configured with VictoriaLogs as datasource grafana: diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index e444bd654..8be7e6501 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.5" services: # Metrics collector. # It scrapes targets defined in --promscrape.config diff --git a/deployment/docker/vm-datasource/docker-compose-cluster.yml b/deployment/docker/vm-datasource/docker-compose-cluster.yml index 948c6d12c..5c8d806df 100644 --- a/deployment/docker/vm-datasource/docker-compose-cluster.yml +++ b/deployment/docker/vm-datasource/docker-compose-cluster.yml @@ -1,4 +1,3 @@ -version: "3.5" services: grafana: container_name: grafana diff --git a/deployment/docker/vm-datasource/docker-compose.yml b/deployment/docker/vm-datasource/docker-compose.yml index bfbf25bc6..994d09630 100644 --- a/deployment/docker/vm-datasource/docker-compose.yml +++ b/deployment/docker/vm-datasource/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.5" services: grafana: container_name: grafana diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f9093c22e..de8573c75 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -30,6 +30,10 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). ## tip +* FEATURE: [alerts-vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmagent.yml): add new alerting rules `StreamAggrFlushTimeout` and `StreamAggrDedupFlushTimeout` to notify about issues during stream aggregation. +* FEATURE: [dashboards/vmagent](https://grafana.com/grafana/dashboards/12683): add row `Streaming aggregation` with panels related to [streaming aggregation](https://docs.victoriametrics.com/stream-aggregation/) process. + + ## [v1.102.0-rc1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.0-rc1) Released at 2024-06-07