diff --git a/dashboards/vm/vmagent.json b/dashboards/vm/vmagent.json index d9e5fdc98..c78ef1cab 100644 --- a/dashboards/vm/vmagent.json +++ b/dashboards/vm/vmagent.json @@ -86,7 +86,7 @@ } ] }, - "description": "Overview for VictoriaMetrics vmagent v1.80.0 or higher", + "description": "Overview for VictoriaMetrics vmagent v1.102.0 or higher", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -1336,7 +1336,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1344,8 +1345,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1440,7 +1440,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1448,8 +1449,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1627,8 +1627,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -1636,10 +1635,9 @@ "h": 8, "w": 12, "x": 0, - "y": 3 + "y": 11 }, "id": 109, - "links": [], "options": { "legend": { "calcs": [ @@ -1743,8 +1741,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -1752,10 +1749,9 @@ "h": 8, "w": 12, "x": 12, - "y": 3 + "y": 11 }, "id": 111, - "links": [], "options": { "legend": { "calcs": [ @@ -1856,8 +1852,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [ { @@ -1878,10 +1873,9 @@ "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 19 }, "id": 81, - "links": [], "options": { "legend": { "calcs": [ @@ -1993,8 +1987,7 @@ } ] }, - "unit": "bps", - "unitScale": true + "unit": "bps" }, "overrides": [ { @@ -2015,7 +2008,7 @@ "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 19 }, "id": 7, "options": { @@ -2125,8 +2118,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -2134,10 +2126,9 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 27 }, "id": 83, - "links": [], "options": { "legend": { "calcs": [ @@ -2232,8 +2223,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2241,10 +2231,9 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 27 }, "id": 39, - "links": [], "options": { "legend": { "calcs": [ @@ -2340,8 +2329,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -2349,10 +2337,9 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 35 }, "id": 135, - "links": [], "options": { "legend": { "calcs": [ @@ -2447,8 +2434,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2456,10 +2442,9 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 35 }, "id": 41, - "links": [], "options": { "legend": { "calcs": [ @@ -2534,6 +2519,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2547,6 +2533,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2584,7 +2571,7 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 12 }, "id": 92, "options": { @@ -2636,6 +2623,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2649,6 +2637,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2686,7 +2675,7 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 12 }, "id": 95, "options": { @@ -2738,6 +2727,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2752,6 +2742,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2791,7 +2782,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 20 }, "id": 98, "options": { @@ -2843,6 +2834,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2857,6 +2849,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2896,7 +2889,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 20 }, "id": 99, "options": { @@ -3000,10 +2993,9 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 28 }, "id": 79, - "links": [], "options": { "legend": { "calcs": [ @@ -3105,7 +3097,7 @@ "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 28 }, "id": 18, "links": [ @@ -3215,10 +3207,9 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 36 }, "id": 127, - "links": [], "options": { "legend": { "calcs": [ @@ -3318,7 +3309,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 36 }, "id": 50, "options": { @@ -3418,7 +3409,7 @@ "h": 7, "w": 24, "x": 0, - "y": 36 + "y": 44 }, "id": 129, "options": { @@ -3565,8 +3556,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3574,7 +3564,7 @@ "h": 7, "w": 12, "x": 0, - "y": 37 + "y": 13 }, "id": 48, "options": { @@ -3672,8 +3662,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3681,7 +3670,7 @@ "h": 7, "w": 12, "x": 12, - "y": 37 + "y": 13 }, "id": 76, "options": { @@ -3777,8 +3766,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3786,7 +3774,7 @@ "h": 7, "w": 12, "x": 0, - "y": 44 + "y": 20 }, "id": 132, "options": { @@ -3884,8 +3872,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3893,7 +3880,7 @@ "h": 7, "w": 12, "x": 12, - "y": 44 + "y": 20 }, "id": 133, "options": { @@ -3990,8 +3977,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3999,7 +3985,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 27 }, "id": 20, "options": { @@ -4095,8 +4081,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -4104,7 +4089,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 27 }, "id": 126, "options": { @@ -4199,8 +4184,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [] }, @@ -4208,7 +4192,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 35 }, "id": 46, "options": { @@ -4303,8 +4287,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -4312,7 +4295,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 35 }, "id": 31, "options": { @@ -4437,6 +4420,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4450,6 +4434,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4489,10 +4474,9 @@ "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 14 }, "id": 73, - "links": [], "options": { "legend": { "calcs": [ @@ -4553,6 +4537,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4566,6 +4551,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4605,10 +4591,9 @@ "h": 8, "w": 12, "x": 12, - "y": 38 + "y": 14 }, "id": 131, - "links": [], "options": { "legend": { "calcs": [ @@ -4656,6 +4641,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4669,6 +4655,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4708,10 +4695,9 @@ "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 22 }, "id": 130, - "links": [], "options": { "legend": { "calcs": [ @@ -4772,6 +4758,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4785,6 +4772,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4824,10 +4812,9 @@ "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 22 }, "id": 77, - "links": [], "options": { "legend": { "calcs": [ @@ -4885,6 +4872,620 @@ "title": "Ingestion", "type": "row" }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 136, + "panels": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "The 99th percentile of avg flush duration for the aggregated data. \n\nSmaller is better.\n\nAggregation can produce incorrect results ff flush duration exceeds configured deduplication interval. See \"Flush Timeouts\" panel.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 137, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, vm_streamaggr_dedup_flush_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Dedup flush duration (0.99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Amount of ignored samples during aggregation. \nStream aggregation will drop samples with NaN values, or samples with too old timestamps. See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 143, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_ignored_samples_total[5m]) > 0", + "instant": false, + "legendFormat": "{{reason}}: {{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Ignored samples", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows events when deduplication or aggregation couldn't be finished in the configured interval. Such events may result into bad accuracy of produced data.\n\nPossible solutions:\n* increase interval; \n* use match filter matching smaller number of series;\n* reduce samples ingestion rate to stream aggregation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": -5, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 139, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "instant": false, + "legendFormat": "aggregate: {{instance}} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "hide": false, + "instant": false, + "legendFormat": "dedup: {{instance}} ({{job}})", + "range": true, + "refId": "B" + } + ], + "title": "Flush timeouts", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows the lag between average samples timestamp and aggregation interval.\n\nLower is better.\n\nToo high lag or lag exceeding the interval might be a sign of data delay before aggregation or resource insufficiency on aggregator. Samples with high lag may affect accuracy of aggregation.\n\nSee https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 142, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Aggregated samples lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows the size of Label Compressor in bytes.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 140, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "vm_streamaggr_labels_compressor_size_bytes{job=~\"$job\",instance=~\"$instance\"} > 0", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Labels compressor bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows the size of Label Compressor in number of entries.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 141, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "vm_streamaggr_labels_compressor_items_count{job=~\"$job\",instance=~\"$instance\"} > 0", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Labels compressor items count", + "type": "timeseries" + } + ], + "title": "Streaming aggregation", + "type": "row" + }, { "collapsed": true, "datasource": { @@ -4895,7 +5496,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 39 }, "id": 58, "panels": [ @@ -4911,6 +5512,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4924,6 +5526,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4964,7 +5567,7 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 16 }, "id": 60, "options": { @@ -5014,6 +5617,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5027,6 +5631,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5066,7 +5671,7 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 16 }, "id": 66, "options": { @@ -5116,6 +5721,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5129,6 +5735,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5168,7 +5775,7 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 24 }, "id": 61, "options": { @@ -5218,6 +5825,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5231,6 +5839,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5270,7 +5879,7 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 24 }, "id": 65, "options": { @@ -5371,7 +5980,7 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 32 }, "id": 88, "options": { @@ -5469,7 +6078,7 @@ "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 32 }, "id": 84, "options": { @@ -5570,7 +6179,7 @@ "h": 8, "w": 12, "x": 0, - "y": 79 + "y": 40 }, "id": 90, "options": { @@ -5623,7 +6232,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 40 }, "id": 113, "panels": [ @@ -5636,7 +6245,7 @@ "h": 2, "w": 24, "x": 0, - "y": 87 + "y": 17 }, "id": 115, "options": { @@ -5648,7 +6257,7 @@ "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.", "mode": "markdown" }, - "pluginVersion": "9.2.6", + "pluginVersion": "10.4.2", "transparent": true, "type": "text" }, @@ -5664,6 +6273,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5677,6 +6287,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5712,10 +6323,9 @@ "h": 8, "w": 12, "x": 0, - "y": 89 + "y": 19 }, "id": 119, - "links": [], "options": { "legend": { "calcs": [ @@ -5767,6 +6377,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5780,6 +6391,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5815,10 +6427,9 @@ "h": 8, "w": 12, "x": 12, - "y": 89 + "y": 19 }, "id": 117, - "links": [], "options": { "legend": { "calcs": [ @@ -5868,6 +6479,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5881,6 +6493,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5920,7 +6533,7 @@ "h": 8, "w": 12, "x": 0, - "y": 97 + "y": 27 }, "id": 125, "links": [ @@ -5976,6 +6589,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5989,6 +6603,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -6040,7 +6655,7 @@ "h": 8, "w": 12, "x": 12, - "y": 97 + "y": 27 }, "id": 123, "options": { @@ -6104,6 +6719,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -6117,6 +6733,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -6168,10 +6785,9 @@ "h": 8, "w": 12, "x": 0, - "y": 105 + "y": 35 }, "id": 121, - "links": [], "options": { "legend": { "calcs": [ diff --git a/dashboards/vmagent.json b/dashboards/vmagent.json index 80c6bddb8..06e0b0422 100644 --- a/dashboards/vmagent.json +++ b/dashboards/vmagent.json @@ -85,7 +85,7 @@ } ] }, - "description": "Overview for VictoriaMetrics vmagent v1.80.0 or higher", + "description": "Overview for VictoriaMetrics vmagent v1.102.0 or higher", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -1335,7 +1335,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1343,8 +1344,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1439,7 +1439,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1447,8 +1448,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -1626,8 +1626,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -1635,10 +1634,9 @@ "h": 8, "w": 12, "x": 0, - "y": 3 + "y": 11 }, "id": 109, - "links": [], "options": { "legend": { "calcs": [ @@ -1742,8 +1740,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -1751,10 +1748,9 @@ "h": 8, "w": 12, "x": 12, - "y": 3 + "y": 11 }, "id": 111, - "links": [], "options": { "legend": { "calcs": [ @@ -1855,8 +1851,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [ { @@ -1877,10 +1872,9 @@ "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 19 }, "id": 81, - "links": [], "options": { "legend": { "calcs": [ @@ -1992,8 +1986,7 @@ } ] }, - "unit": "bps", - "unitScale": true + "unit": "bps" }, "overrides": [ { @@ -2014,7 +2007,7 @@ "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 19 }, "id": 7, "options": { @@ -2124,8 +2117,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -2133,10 +2125,9 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 27 }, "id": 83, - "links": [], "options": { "legend": { "calcs": [ @@ -2231,8 +2222,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2240,10 +2230,9 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 27 }, "id": 39, - "links": [], "options": { "legend": { "calcs": [ @@ -2339,8 +2328,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -2348,10 +2336,9 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 35 }, "id": 135, - "links": [], "options": { "legend": { "calcs": [ @@ -2446,8 +2433,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2455,10 +2441,9 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 35 }, "id": 41, - "links": [], "options": { "legend": { "calcs": [ @@ -2533,6 +2518,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2546,6 +2532,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2583,7 +2570,7 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 12 }, "id": 92, "options": { @@ -2635,6 +2622,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2648,6 +2636,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2685,7 +2674,7 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 12 }, "id": 95, "options": { @@ -2737,6 +2726,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2751,6 +2741,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2790,7 +2781,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 20 }, "id": 98, "options": { @@ -2842,6 +2833,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2856,6 +2848,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2895,7 +2888,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 20 }, "id": 99, "options": { @@ -2999,10 +2992,9 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 28 }, "id": 79, - "links": [], "options": { "legend": { "calcs": [ @@ -3104,7 +3096,7 @@ "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 28 }, "id": 18, "links": [ @@ -3214,10 +3206,9 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 36 }, "id": 127, - "links": [], "options": { "legend": { "calcs": [ @@ -3317,7 +3308,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 36 }, "id": 50, "options": { @@ -3417,7 +3408,7 @@ "h": 7, "w": 24, "x": 0, - "y": 36 + "y": 44 }, "id": 129, "options": { @@ -3564,8 +3555,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3573,7 +3563,7 @@ "h": 7, "w": 12, "x": 0, - "y": 37 + "y": 13 }, "id": 48, "options": { @@ -3671,8 +3661,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3680,7 +3669,7 @@ "h": 7, "w": 12, "x": 12, - "y": 37 + "y": 13 }, "id": 76, "options": { @@ -3776,8 +3765,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3785,7 +3773,7 @@ "h": 7, "w": 12, "x": 0, - "y": 44 + "y": 20 }, "id": 132, "options": { @@ -3883,8 +3871,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3892,7 +3879,7 @@ "h": 7, "w": 12, "x": 12, - "y": 44 + "y": 20 }, "id": 133, "options": { @@ -3989,8 +3976,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -3998,7 +3984,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 27 }, "id": 20, "options": { @@ -4094,8 +4080,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -4103,7 +4088,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 27 }, "id": 126, "options": { @@ -4198,8 +4183,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [] }, @@ -4207,7 +4191,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 35 }, "id": 46, "options": { @@ -4302,8 +4286,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -4311,7 +4294,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 35 }, "id": 31, "options": { @@ -4436,6 +4419,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4449,6 +4433,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4488,10 +4473,9 @@ "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 14 }, "id": 73, - "links": [], "options": { "legend": { "calcs": [ @@ -4552,6 +4536,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4565,6 +4550,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4604,10 +4590,9 @@ "h": 8, "w": 12, "x": 12, - "y": 38 + "y": 14 }, "id": 131, - "links": [], "options": { "legend": { "calcs": [ @@ -4655,6 +4640,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4668,6 +4654,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4707,10 +4694,9 @@ "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 22 }, "id": 130, - "links": [], "options": { "legend": { "calcs": [ @@ -4771,6 +4757,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4784,6 +4771,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4823,10 +4811,9 @@ "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 22 }, "id": 77, - "links": [], "options": { "legend": { "calcs": [ @@ -4884,6 +4871,620 @@ "title": "Ingestion", "type": "row" }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 136, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The 99th percentile of avg flush duration for the aggregated data. \n\nSmaller is better.\n\nAggregation can produce incorrect results ff flush duration exceeds configured deduplication interval. See \"Flush Timeouts\" panel.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 137, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, vm_streamaggr_dedup_flush_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Dedup flush duration (0.99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Amount of ignored samples during aggregation. \nStream aggregation will drop samples with NaN values, or samples with too old timestamps. See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 143, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_ignored_samples_total[5m]) > 0", + "instant": false, + "legendFormat": "{{reason}}: {{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Ignored samples", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows events when deduplication or aggregation couldn't be finished in the configured interval. Such events may result into bad accuracy of produced data.\n\nPossible solutions:\n* increase interval; \n* use match filter matching smaller number of series;\n* reduce samples ingestion rate to stream aggregation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": -5, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 139, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "instant": false, + "legendFormat": "aggregate: {{instance}} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) > 0", + "hide": false, + "instant": false, + "legendFormat": "dedup: {{instance}} ({{job}})", + "range": true, + "refId": "B" + } + ], + "title": "Flush timeouts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the lag between average samples timestamp and aggregation interval.\n\nLower is better.\n\nToo high lag or lag exceeding the interval might be a sign of data delay before aggregation or resource insufficiency on aggregator. Samples with high lag may affect accuracy of aggregation.\n\nSee https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 142, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Aggregated samples lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the size of Label Compressor in bytes.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 140, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "vm_streamaggr_labels_compressor_size_bytes{job=~\"$job\",instance=~\"$instance\"} > 0", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Labels compressor bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the size of Label Compressor in number of entries.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 141, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "vm_streamaggr_labels_compressor_items_count{job=~\"$job\",instance=~\"$instance\"} > 0", + "instant": false, + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + } + ], + "title": "Labels compressor items count", + "type": "timeseries" + } + ], + "title": "Streaming aggregation", + "type": "row" + }, { "collapsed": true, "datasource": { @@ -4894,7 +5495,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 39 }, "id": 58, "panels": [ @@ -4910,6 +5511,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4923,6 +5525,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4963,7 +5566,7 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 16 }, "id": 60, "options": { @@ -5013,6 +5616,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5026,6 +5630,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5065,7 +5670,7 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 16 }, "id": 66, "options": { @@ -5115,6 +5720,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5128,6 +5734,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5167,7 +5774,7 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 24 }, "id": 61, "options": { @@ -5217,6 +5824,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5230,6 +5838,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5269,7 +5878,7 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 24 }, "id": 65, "options": { @@ -5370,7 +5979,7 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 32 }, "id": 88, "options": { @@ -5468,7 +6077,7 @@ "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 32 }, "id": 84, "options": { @@ -5569,7 +6178,7 @@ "h": 8, "w": 12, "x": 0, - "y": 79 + "y": 40 }, "id": 90, "options": { @@ -5622,7 +6231,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 40 }, "id": 113, "panels": [ @@ -5635,7 +6244,7 @@ "h": 2, "w": 24, "x": 0, - "y": 87 + "y": 17 }, "id": 115, "options": { @@ -5647,7 +6256,7 @@ "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.", "mode": "markdown" }, - "pluginVersion": "9.2.6", + "pluginVersion": "10.4.2", "transparent": true, "type": "text" }, @@ -5663,6 +6272,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5676,6 +6286,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5711,10 +6322,9 @@ "h": 8, "w": 12, "x": 0, - "y": 89 + "y": 19 }, "id": 119, - "links": [], "options": { "legend": { "calcs": [ @@ -5766,6 +6376,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5779,6 +6390,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5814,10 +6426,9 @@ "h": 8, "w": 12, "x": 12, - "y": 89 + "y": 19 }, "id": 117, - "links": [], "options": { "legend": { "calcs": [ @@ -5867,6 +6478,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5880,6 +6492,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5919,7 +6532,7 @@ "h": 8, "w": 12, "x": 0, - "y": 97 + "y": 27 }, "id": 125, "links": [ @@ -5975,6 +6588,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -5988,6 +6602,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -6039,7 +6654,7 @@ "h": 8, "w": 12, "x": 12, - "y": 97 + "y": 27 }, "id": 123, "options": { @@ -6103,6 +6718,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -6116,6 +6732,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -6167,10 +6784,9 @@ "h": 8, "w": 12, "x": 0, - "y": 105 + "y": 35 }, "id": 121, - "links": [], "options": { "legend": { "calcs": [ diff --git a/deployment/docker/alerts-vmagent.yml b/deployment/docker/alerts-vmagent.yml index fc11b87d0..15d522a7f 100644 --- a/deployment/docker/alerts-vmagent.yml +++ b/deployment/docker/alerts-vmagent.yml @@ -135,3 +135,23 @@ groups: summary: "Configuration reload failed for vmagent instance {{ $labels.instance }}" description: "Configuration hot-reload failed for vmagent on instance {{ $labels.instance }}. Check vmagent's logs for detailed error message." + + - alert: StreamAggrFlushTimeout + expr: | + increase(vm_streamaggr_flush_timeouts_total[5m]) > 0 + labels: + severity: warning + annotations: + summary: "Streaming aggregation at \"{{ $labels.job }}\" (instance {{ $labels.instance }}) can't be finished within the configured aggregation interval." + description: "Stream aggregation process can't keep up with the load and might produce incorrect aggregation results. Check logs for more details. + Possible solutions: increase aggregation interval; aggregate smaller number of series; reduce samples' ingestion rate to stream aggregation." + + - alert: StreamAggrDedupFlushTimeout + expr: | + increase(vm_streamaggr_dedup_flush_timeouts_total[5m]) > 0 + labels: + severity: warning + annotations: + summary: "Deduplication \"{{ $labels.job }}\" (instance {{ $labels.instance }}) can't be finished within configured deduplication interval." + description: "Deduplication process can't keep up with the load and might produce incorrect results. Check docs https://docs.victoriametrics.com/stream-aggregation/#deduplication and logs for more details. + Possible solutions: increase deduplication interval; deduplicate smaller number of series; reduce samples' ingestion rate." diff --git a/deployment/docker/docker-compose-cluster.yml b/deployment/docker/docker-compose-cluster.yml index 6194de252..1672f516b 100644 --- a/deployment/docker/docker-compose-cluster.yml +++ b/deployment/docker/docker-compose-cluster.yml @@ -1,4 +1,3 @@ -version: '3.5' services: # Metrics collector. # It scrapes targets defined in --promscrape.config diff --git a/deployment/docker/docker-compose-victorialogs.yml b/deployment/docker/docker-compose-victorialogs.yml index 484a80265..e2e676481 100644 --- a/deployment/docker/docker-compose-victorialogs.yml +++ b/deployment/docker/docker-compose-victorialogs.yml @@ -1,4 +1,3 @@ -version: "3.5" services: # Grafana instance configured with VictoriaLogs as datasource grafana: diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index e444bd654..8be7e6501 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.5" services: # Metrics collector. # It scrapes targets defined in --promscrape.config diff --git a/deployment/docker/vm-datasource/docker-compose-cluster.yml b/deployment/docker/vm-datasource/docker-compose-cluster.yml index 948c6d12c..5c8d806df 100644 --- a/deployment/docker/vm-datasource/docker-compose-cluster.yml +++ b/deployment/docker/vm-datasource/docker-compose-cluster.yml @@ -1,4 +1,3 @@ -version: "3.5" services: grafana: container_name: grafana diff --git a/deployment/docker/vm-datasource/docker-compose.yml b/deployment/docker/vm-datasource/docker-compose.yml index bfbf25bc6..994d09630 100644 --- a/deployment/docker/vm-datasource/docker-compose.yml +++ b/deployment/docker/vm-datasource/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.5" services: grafana: container_name: grafana diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f9093c22e..de8573c75 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -30,6 +30,10 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). ## tip +* FEATURE: [alerts-vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmagent.yml): add new alerting rules `StreamAggrFlushTimeout` and `StreamAggrDedupFlushTimeout` to notify about issues during stream aggregation. +* FEATURE: [dashboards/vmagent](https://grafana.com/grafana/dashboards/12683): add row `Streaming aggregation` with panels related to [streaming aggregation](https://docs.victoriametrics.com/stream-aggregation/) process. + + ## [v1.102.0-rc1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.0-rc1) Released at 2024-06-07