2023-08-29 11:06:55 +02:00
{
"__inputs" : [ ] ,
"__elements" : { } ,
"__requires" : [
{
"type" : "grafana" ,
"id" : "grafana" ,
"name" : "Grafana" ,
2024-04-29 12:10:24 +02:00
"version" : "10.4.2"
2023-08-29 11:06:55 +02:00
} ,
{
"type" : "datasource" ,
"id" : "victoriametrics-datasource" ,
"name" : "VictoriaMetrics" ,
"version" : "1.0.0"
} ,
{
"type" : "panel" ,
"id" : "stat" ,
"name" : "Stat" ,
"version" : ""
} ,
{
"type" : "panel" ,
"id" : "table" ,
"name" : "Table" ,
"version" : ""
} ,
{
"type" : "panel" ,
"id" : "text" ,
"name" : "Text" ,
"version" : ""
} ,
{
"type" : "panel" ,
"id" : "timeseries" ,
"name" : "Time series" ,
"version" : ""
}
] ,
"annotations" : {
"list" : [
{
"builtIn" : 1 ,
"datasource" : {
"type" : "datasource" ,
"uid" : "grafana"
} ,
"enable" : true ,
"hide" : true ,
"iconColor" : "rgba(0, 211, 255, 1)" ,
"name" : "Annotations & Alerts" ,
"target" : {
"limit" : 100 ,
"matchAny" : false ,
"tags" : [ ] ,
"type" : "dashboard"
} ,
"type" : "dashboard"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"enable" : true ,
2024-05-22 16:32:51 +02:00
"expr" : "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(short_version))" ,
2023-08-29 11:06:55 +02:00
"hide" : true ,
"iconColor" : "dark-blue" ,
"name" : "version" ,
"textFormat" : "{{short_version}}" ,
"titleFormat" : "Version change"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"enable" : true ,
2024-05-22 16:32:51 +02:00
"expr" : "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job, instance)" ,
2023-08-29 11:06:55 +02:00
"iconColor" : "dark-yellow" ,
"name" : "restarts" ,
"textFormat" : "{{job}}:{{instance}} restarted"
}
]
} ,
2024-06-10 11:49:00 +02:00
"description" : "Overview for VictoriaMetrics vmagent v1.102.0 or higher" ,
2023-08-29 11:06:55 +02:00
"editable" : true ,
"fiscalYearStartMonth" : 0 ,
"graphTooltip" : 1 ,
"id" : null ,
"links" : [
{
"icon" : "doc" ,
"tags" : [ ] ,
"targetBlank" : true ,
"title" : "vmagent wiki" ,
"tooltip" : "" ,
"type" : "link" ,
2024-04-18 01:36:13 +02:00
"url" : "https://docs.victoriametrics.com/vmagent/"
2023-08-29 11:06:55 +02:00
} ,
{
"icon" : "external link" ,
"tags" : [ ] ,
"targetBlank" : true ,
"title" : "Found a bug?" ,
"type" : "link" ,
"url" : "https://github.com/VictoriaMetrics/VictoriaMetrics/issues"
} ,
{
"icon" : "external link" ,
"tags" : [ ] ,
"targetBlank" : true ,
"title" : "New releases" ,
"type" : "link" ,
"url" : "https://github.com/VictoriaMetrics/VictoriaMetrics/releases"
}
] ,
"liveNow" : false ,
"panels" : [
{
"collapsed" : false ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 0
} ,
"id" : 105 ,
"panels" : [ ] ,
"title" : "Stats" ,
"type" : "row"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-03-04 15:43:31 +01:00
"description" : "Shows the rate of [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) scraped from configured targets." ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
}
]
2024-04-29 12:10:24 +02:00
}
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 3 ,
"w" : 4 ,
"x" : 0 ,
"y" : 1
} ,
"id" : 103 ,
"links" : [
{
"title" : "Troubleshooting" ,
2024-04-18 01:36:13 +02:00
"url" : "https://docs.victoriametrics.com/vmagent/#troubleshooting"
2023-08-29 11:06:55 +02:00
}
] ,
"options" : {
"colorMode" : "value" ,
"graphMode" : "area" ,
"justifyMode" : "auto" ,
"orientation" : "auto" ,
"reduceOptions" : {
"calcs" : [
"last"
] ,
"fields" : "" ,
"values" : false
} ,
2024-02-02 15:41:08 +01:00
"showPercentChange" : false ,
2023-08-29 11:06:55 +02:00
"text" : { } ,
2024-02-02 15:41:08 +01:00
"textMode" : "auto" ,
"wideLayout" : true
2023-08-29 11:06:55 +02:00
} ,
2024-04-29 12:10:24 +02:00
"pluginVersion" : "10.4.2" ,
2023-08-29 11:06:55 +02:00
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
dashboards: remove `path!="/favicon.ico"` filter from `requests rate` graphs
The `path!="/favicon.ico"` filter has little sense, since there are many other special paths,
which may be filtered out - /metrics, /flags, /health, /ping, /robots.txt, /-/healthy, /-/ready, /reload, etc.
See /lib/httpserver/httpserver.go for more details.
It will be hard or impossible to maintain filters for all these paths, so it is better to drop this filter
in order to simplify queries and improve the consistency of these queries.
2023-11-16 19:26:03 +01:00
"expr" : "sum(rate(vm_promscrape_scraped_samples_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))" ,
2023-08-29 11:06:55 +02:00
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Samples scraped/s" ,
"type" : "stat"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-03-04 15:43:31 +01:00
"description" : "Shows the rate of ingested [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)" ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
}
]
2024-04-29 12:10:24 +02:00
}
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 3 ,
"w" : 4 ,
"x" : 4 ,
"y" : 1
} ,
"id" : 102 ,
"links" : [
{
"title" : "Troubleshooting" ,
2024-04-18 01:36:13 +02:00
"url" : "https://docs.victoriametrics.com/vmagent/#troubleshooting"
2023-08-29 11:06:55 +02:00
}
] ,
"options" : {
"colorMode" : "value" ,
"graphMode" : "area" ,
"justifyMode" : "auto" ,
"orientation" : "auto" ,
"reduceOptions" : {
"calcs" : [
"last"
] ,
"fields" : "" ,
"values" : false
} ,
2024-02-02 15:41:08 +01:00
"showPercentChange" : false ,
2023-08-29 11:06:55 +02:00
"text" : { } ,
2024-02-02 15:41:08 +01:00
"textMode" : "auto" ,
"wideLayout" : true
2023-08-29 11:06:55 +02:00
} ,
2024-04-29 12:10:24 +02:00
"pluginVersion" : "10.4.2" ,
2023-08-29 11:06:55 +02:00
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vmagent_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))" ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Samples ingested/s" ,
"type" : "stat"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-02-02 15:48:26 +01:00
"description" : "Shows the number of targets scraped per second." ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
2024-02-02 15:48:26 +01:00
"decimals" : 1 ,
2023-08-29 11:06:55 +02:00
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
}
]
2024-04-29 12:10:24 +02:00
}
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 3 ,
"w" : 4 ,
"x" : 8 ,
"y" : 1
} ,
2024-02-02 15:48:26 +01:00
"id" : 134 ,
2023-08-29 11:06:55 +02:00
"options" : {
"colorMode" : "value" ,
2024-02-02 15:48:26 +01:00
"graphMode" : "none" ,
2023-08-29 11:06:55 +02:00
"justifyMode" : "auto" ,
"orientation" : "auto" ,
"reduceOptions" : {
"calcs" : [
"last"
] ,
"fields" : "" ,
"values" : false
} ,
2024-02-02 15:41:08 +01:00
"showPercentChange" : false ,
2023-08-29 11:06:55 +02:00
"text" : { } ,
2024-02-02 15:41:08 +01:00
"textMode" : "auto" ,
"wideLayout" : true
2023-08-29 11:06:55 +02:00
} ,
2024-04-29 12:10:24 +02:00
"pluginVersion" : "10.4.2" ,
2023-08-29 11:06:55 +02:00
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-02-02 15:48:26 +01:00
"editorMode" : "code" ,
"expr" : "sum(rate(vm_promscrape_scrapes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) " ,
2023-08-29 11:06:55 +02:00
"interval" : "" ,
2024-02-02 15:48:26 +01:00
"legendFormat" : "__auto" ,
"range" : true ,
2023-08-29 11:06:55 +02:00
"refId" : "A"
}
] ,
2024-02-02 15:48:26 +01:00
"title" : "Targets scraped/s" ,
2023-08-29 11:06:55 +02:00
"type" : "stat"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-02-02 15:48:26 +01:00
"description" : "Shows total number of all configured scrape targets in state `up` or `down`.\n\nSee `http://vmagent-host:8429/targets` to get list of all targets. \n" ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
}
]
2024-04-29 12:10:24 +02:00
}
2023-08-29 11:06:55 +02:00
} ,
2024-02-02 15:48:26 +01:00
"overrides" : [
{
"matcher" : {
"id" : "byName" ,
"options" : "down"
} ,
"properties" : [
{
"id" : "thresholds" ,
"value" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 1
}
]
}
}
]
}
]
2023-08-29 11:06:55 +02:00
} ,
"gridPos" : {
"h" : 3 ,
"w" : 4 ,
"x" : 12 ,
"y" : 1
} ,
"id" : 72 ,
"links" : [
{
"title" : "Troubleshooting" ,
2024-04-18 01:36:13 +02:00
"url" : "https://docs.victoriametrics.com/vmagent/#troubleshooting"
2023-08-29 11:06:55 +02:00
}
] ,
"options" : {
"colorMode" : "value" ,
"graphMode" : "area" ,
"justifyMode" : "auto" ,
"orientation" : "auto" ,
"reduceOptions" : {
"calcs" : [
"last"
] ,
"fields" : "" ,
"values" : false
} ,
2024-02-02 15:41:08 +01:00
"showPercentChange" : false ,
2023-08-29 11:06:55 +02:00
"text" : { } ,
2024-02-02 15:41:08 +01:00
"textMode" : "auto" ,
"wideLayout" : true
2023-08-29 11:06:55 +02:00
} ,
2024-04-29 12:10:24 +02:00
"pluginVersion" : "10.4.2" ,
2023-08-29 11:06:55 +02:00
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-02-02 15:48:26 +01:00
"editorMode" : "code" ,
"expr" : "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"up\"})" ,
2023-08-29 11:06:55 +02:00
"interval" : "" ,
"legendFormat" : "up" ,
2024-02-02 15:48:26 +01:00
"range" : true ,
2023-08-29 11:06:55 +02:00
"refId" : "A"
2024-02-02 15:48:26 +01:00
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"down\"})" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "down" ,
"range" : true ,
"refId" : "B"
2023-08-29 11:06:55 +02:00
}
] ,
2024-02-02 15:48:26 +01:00
"title" : "Scrape targets" ,
2023-08-29 11:06:55 +02:00
"type" : "stat"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows number of generated error messages in logs over last 30m. Non-zero value may be a sign of connectivity or missconfiguration errors." ,
"fieldConfig" : {
"defaults" : {
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 1
}
]
} ,
2024-04-29 12:10:24 +02:00
"unit" : "short"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 3 ,
"w" : 4 ,
"x" : 16 ,
"y" : 1
} ,
"id" : 16 ,
"links" : [
{
"targetBlank" : true ,
"title" : "Troubleshooting" ,
2024-04-18 01:36:13 +02:00
"url" : "https://docs.victoriametrics.com/vmagent/#troubleshooting"
2023-08-29 11:06:55 +02:00
}
] ,
"options" : {
"colorMode" : "value" ,
"graphMode" : "area" ,
"justifyMode" : "auto" ,
"orientation" : "auto" ,
"reduceOptions" : {
"calcs" : [
"last"
] ,
"fields" : "" ,
"values" : false
} ,
2024-02-02 15:41:08 +01:00
"showPercentChange" : false ,
2023-08-29 11:06:55 +02:00
"text" : { } ,
2024-02-02 15:41:08 +01:00
"textMode" : "auto" ,
"wideLayout" : true
2023-08-29 11:06:55 +02:00
} ,
2024-04-29 12:10:24 +02:00
"pluginVersion" : "10.4.2" ,
2023-08-29 11:06:55 +02:00
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"expr" : "sum(increase(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[30m]))" ,
"interval" : "" ,
"legendFormat" : "" ,
"refId" : "A"
}
] ,
"title" : "Log errors (30m)" ,
"type" : "stat"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Persistent queue size shows size of pending samples in bytes which hasn't been flushed to remote storage yet. \nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored." ,
"fieldConfig" : {
"defaults" : {
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 10485760
}
]
} ,
2024-04-29 12:10:24 +02:00
"unit" : "bytes"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 3 ,
"w" : 4 ,
"x" : 20 ,
"y" : 1
} ,
"id" : 56 ,
"options" : {
"colorMode" : "value" ,
"graphMode" : "area" ,
"justifyMode" : "auto" ,
"orientation" : "auto" ,
"reduceOptions" : {
"calcs" : [
"last"
] ,
"fields" : "" ,
"values" : false
} ,
2024-02-02 15:41:08 +01:00
"showPercentChange" : false ,
2023-08-29 11:06:55 +02:00
"text" : { } ,
2024-02-02 15:41:08 +01:00
"textMode" : "auto" ,
"wideLayout" : true
2023-08-29 11:06:55 +02:00
} ,
2024-04-29 12:10:24 +02:00
"pluginVersion" : "10.4.2" ,
2023-08-29 11:06:55 +02:00
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"expr" : "sum(vm_persistentqueue_bytes_pending{job=~\"$job\", instance=~\"$instance\"})" ,
"interval" : "" ,
"legendFormat" : "" ,
"refId" : "A"
}
] ,
"title" : "Persistent queue size" ,
"type" : "stat"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "thresholds"
} ,
"custom" : {
"align" : "auto" ,
2024-02-02 15:41:08 +01:00
"cellOptions" : {
"type" : "auto"
} ,
2023-08-29 11:06:55 +02:00
"inspect" : false ,
"minWidth" : 50
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
2024-04-29 12:10:24 +02:00
}
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [
{
"matcher" : {
"id" : "byName" ,
"options" : "Time"
} ,
"properties" : [
{
"id" : "custom.hidden" ,
"value" : true
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "Value"
} ,
"properties" : [
{
"id" : "displayName" ,
"value" : "Count"
}
]
}
]
} ,
"gridPos" : {
"h" : 5 ,
"w" : 8 ,
"x" : 0 ,
"y" : 4
} ,
"id" : 101 ,
"options" : {
2024-02-02 15:41:08 +01:00
"cellHeight" : "sm" ,
2023-08-29 11:06:55 +02:00
"footer" : {
2024-02-02 15:41:08 +01:00
"countRows" : false ,
2023-08-29 11:06:55 +02:00
"fields" : "" ,
"reducer" : [
"sum"
] ,
"show" : false
} ,
"showHeader" : true
} ,
2024-04-29 12:10:24 +02:00
"pluginVersion" : "10.4.2" ,
2023-08-29 11:06:55 +02:00
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)" ,
"format" : "table" ,
"instant" : true ,
"range" : false ,
"refId" : "A"
}
] ,
"type" : "table"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:41:08 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:41:08 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "stepAfter" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"decimals" : 0 ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-04-29 12:10:24 +02:00
"unit" : "short"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 5 ,
"w" : 16 ,
"x" : 8 ,
"y" : 4
} ,
"id" : 13 ,
"options" : {
"legend" : {
"calcs" : [
"lastNotNull"
] ,
"displayMode" : "table" ,
"placement" : "right" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(up{job=~\"$job\", instance=~\"$instance\"}) by (job)" ,
"format" : "time_series" ,
"instant" : false ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"refId" : "A"
}
] ,
"title" : "Uptime" ,
"type" : "timeseries"
} ,
{
"collapsed" : false ,
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 9
} ,
"id" : 24 ,
"panels" : [ ] ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"refId" : "A"
}
] ,
"title" : "Overview" ,
"type" : "row"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-03-04 15:43:31 +01:00
"description" : "Shows in/out [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) rate including push and pull models. \n\nThe out-rate could be different to in-rate because of replication or additional timeseries added by vmagent for every scraped target.\n\nClick on the line and choose Drilldown to show CPU usage per instance\n" ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:41:08 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:41:08 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [
{
"title" : "Drilldown" ,
"url" : "/d/G7Z9GzMGz_vm?viewPanel=123&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}"
}
] ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-04-29 12:10:24 +02:00
"unit" : "short"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [
{
"matcher" : {
"id" : "byRegexp" ,
"options" : "/out .*/"
} ,
"properties" : [
{
"id" : "custom.transform" ,
"value" : "negative-Y"
}
]
}
]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
"y" : 10
} ,
"id" : 5 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "none"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate({__name__=~\"vmagent_rows_inserted_total|vm_promscrape_scraped_samples_sum\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "in {{job}}" ,
"range" : true ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vmagent_remotewrite_block_size_rows_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)" ,
"interval" : "" ,
"legendFormat" : "out {{job}}" ,
"range" : true ,
"refId" : "B"
}
] ,
"title" : "Samples rate ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-10-17 12:12:47 +02:00
"description" : "Shows the persistent queue size of pending samples in bytes >2MB which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\nClick on the line and choose Drilldown to show persistent queue size per instance.\n" ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:41:08 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:41:08 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [
{
"targetBlank" : true ,
"title" : "Drilldown" ,
"url" : "/d/G7Z9GzMGz_vm?viewPanel=125&var-url=${__field.labels.url}&var-ds=$ds&var-instance=$instance&var-job=$job&${__url_time_range}"
}
] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-04-29 12:10:24 +02:00
"unit" : "bytes"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
"y" : 10
} ,
"id" : 17 ,
"links" : [
{
"title" : "Troubleshooting" ,
2024-04-18 01:36:13 +02:00
"url" : "https://docs.victoriametrics.com/vmagent/#troubleshooting"
2023-08-29 11:06:55 +02:00
}
] ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "none"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(vmagent_remotewrite_pending_data_bytes{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}) by (job, url) > 2e6" ,
"interval" : "" ,
"legendFormat" : "{{job}} => {{url}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Persistent queue size ($instance) to ($url)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:48:26 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "bars" ,
"fillOpacity" : 100 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:48:26 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-02-02 15:48:26 +01:00
"color" : "green" ,
"value" : null
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-04-29 12:10:24 +02:00
"unit" : "short"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
"y" : 18
} ,
"id" : 107 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.1.0" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, level) > 0" ,
"format" : "time_series" ,
"hide" : false ,
"interval" : "5m" ,
"intervalFactor" : 1 ,
"legendFormat" : "{{job}} - {{level}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Logging rate" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-03-04 15:43:31 +01:00
"description" : "Shows rate of dropped [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) from persistent queue. vmagent drops samples from queue if in-memory and on-disk queues are full and it is unable to flush them to remote storage.\nThe max size of on-disk queue is configured by `-remoteWrite.maxDiskUsagePerURL` flag." ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:48:26 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:48:26 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-02-02 15:48:26 +01:00
"color" : "green" ,
"value" : null
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-04-29 12:10:24 +02:00
"unit" : "decbytes"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
"y" : 18
} ,
"id" : 49 ,
"links" : [
{
"targetBlank" : true ,
"title" : "Troubleshooting" ,
2024-04-18 01:36:13 +02:00
"url" : "https://docs.victoriametrics.com/vmagent/#troubleshooting"
2023-08-29 11:06:55 +02:00
}
] ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "none"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vm_persistentqueue_bytes_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0" ,
"interval" : "" ,
"legendFormat" : "{{ path }}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Persistent queue dropped rate ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the rate of requests served by vmagent HTTP server." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:48:26 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:48:26 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-06-10 11:49:00 +02:00
"color" : "green" ,
"value" : null
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "short"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
"y" : 26
} ,
"id" : 15 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vmagent_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, path) > 0" ,
"interval" : "" ,
"legendFormat" : "" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Requests rate ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Errors rate shows rate for multiple metrics that track possible errors in vmagent, such as network or parsing errors." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:48:26 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:48:26 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-06-10 11:49:00 +02:00
"color" : "green" ,
"value" : null
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "short"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
"y" : 26
} ,
"id" : 69 ,
"links" : [
{
"targetBlank" : true ,
"title" : "Troubleshooting" ,
2024-04-18 01:36:13 +02:00
"url" : "https://docs.victoriametrics.com/vmagent/#troubleshooting"
2023-08-29 11:06:55 +02:00
}
] ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vmagent_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, protocol) > 0" ,
"interval" : "" ,
"legendFormat" : "requests:{{protocol}} ({{job}})" ,
"range" : true ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vm_protoparser_read_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0" ,
"interval" : "" ,
"legendFormat" : "parse: {{type}} ({{job}})" ,
"range" : true ,
"refId" : "B"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vm_ingestserver_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0" ,
"interval" : "" ,
"legendFormat" : "ingest: {{type}} ({{job}})" ,
"range" : true ,
"refId" : "C"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vm_protoparser_unmarshal_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0" ,
"interval" : "" ,
"legendFormat" : "unmarshal: {{type}} ({{job}})" ,
"range" : true ,
"refId" : "D"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vm_promscrape_dial_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0" ,
"interval" : "" ,
"legendFormat" : "scrape dial ({{job}})" ,
"range" : true ,
"refId" : "E"
}
] ,
"title" : "Errors rate ($instance)" ,
"type" : "timeseries"
} ,
{
"collapsed" : true ,
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 34
} ,
"id" : 45 ,
"panels" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Click on the line and choose Drilldown to show CPU usage per instance" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 16:21:21 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 16:21:21 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [
{
"targetBlank" : true ,
"title" : "Drilldown" ,
"url" : "/d/G7Z9GzMGz_vm?viewPanel=119&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}"
}
] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-04-29 12:10:24 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "percentunit"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 35
2023-08-29 11:06:55 +02:00
} ,
"id" : 109 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.1.0" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)" ,
"format" : "time_series" ,
"interval" : "" ,
"intervalFactor" : 1 ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "CPU ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2023-11-14 09:50:06 +01:00
"description" : "Percentage of used RSS memory (resident).\nThe RSS memory shows the amount of memory recently accessed by the application. It includes anonymous memory and data from recently accessed files (aka page cache).\nThe application's performance will significantly degrade when memory usage is close to 100%.\n\nClick on the line and choose Drilldown to show memory usage per instance" ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 16:21:21 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 16:21:21 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [
{
"targetBlank" : true ,
"title" : "Drilldown" ,
"url" : "/d/G7Z9GzMGz_vm?viewPanel=117&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}"
}
] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green" ,
"value" : null
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "percentunit"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 35
2023-08-29 11:06:55 +02:00
} ,
"id" : 111 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.1.0" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)" ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "RSS memory % usage ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the number of bytes read/write from the storage layer when vmagent has to buffer data on disk or read already buffered data.\n\nClick on the line and choose Drilldown to show CPU usage per instance" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 16:21:21 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 16:21:21 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [
{
"targetBlank" : true ,
"title" : "Drilldown" ,
"url" : "/d/G7Z9GzMGz_vm?viewPanel=121&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}"
}
] ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green" ,
"value" : null
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "bytes"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [
{
"matcher" : {
"id" : "byName" ,
"options" : "read"
} ,
"properties" : [
{
"id" : "custom.transform" ,
"value" : "negative-Y"
}
]
}
]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 43
2023-08-29 11:06:55 +02:00
} ,
"id" : 81 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "none"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0" ,
"format" : "time_series" ,
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 1 ,
"legendFormat" : "read {{job}}" ,
"range" : true ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0" ,
"format" : "time_series" ,
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 1 ,
"legendFormat" : "write {{job}}" ,
"range" : true ,
"refId" : "B"
}
] ,
"title" : "Disk writes/reads ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Network usage shows the bytes rate for data accepted by vmagent and pushed via remotewrite protocol.\nDiscrepancies are possible because of different protocols used for ingesting, scraping and writing data." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 16:21:21 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 16:21:21 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green" ,
"value" : null
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "bps"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [
{
"matcher" : {
"id" : "byRegexp" ,
"options" : "/out.*/"
} ,
"properties" : [
{
"id" : "custom.transform" ,
"value" : "negative-Y"
}
]
}
]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 43
2023-08-29 11:06:55 +02:00
} ,
"id" : 7 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) * 8 \n+ sum(rate(vm_promscrape_conn_bytes_read_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) * 8" ,
"interval" : "" ,
"legendFormat" : "in {{job}}" ,
"range" : true ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vmagent_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) * 8" ,
"interval" : "" ,
"legendFormat" : "out {{job}}" ,
"range" : true ,
"refId" : "B"
}
] ,
"title" : "Network usage ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Panel shows the percentage of open file descriptors in the OS per instance.\nReaching the limit of open files (100%) can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 16:21:21 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 16:21:21 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"decimals" : 5 ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green" ,
"value" : null
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "percentunit"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 51
2023-08-29 11:06:55 +02:00
} ,
"id" : 83 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "max(\n max_over_time(process_open_fds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_max_fds{job=~\"$job\", instance=~\"$instance\"}\n) by(job)" ,
"format" : "time_series" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Open FDs usage % ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 16:21:21 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 16:21:21 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"decimals" : 0 ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green" ,
"value" : null
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "short"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 51
2023-08-29 11:06:55 +02:00
} ,
"id" : 39 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(max_over_time(go_goroutines{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)" ,
"format" : "time_series" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Goroutines ($instance)" ,
"type" : "timeseries"
} ,
2024-02-02 16:21:21 +01:00
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/troubleshooting/ " ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisBorderShow" : false ,
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"insertNulls" : false ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"decimals" : 0 ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green" ,
"value" : null
2024-02-02 16:21:21 +01:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "percentunit"
2024-02-02 16:21:21 +01:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 59
2024-02-02 16:21:21 +01:00
} ,
"id" : 135 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "max(\n rate(go_gc_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n ) by(job)" ,
"format" : "time_series" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "CPU spent on GC ($instance)" ,
"type" : "timeseries"
} ,
2024-09-23 16:54:42 +02:00
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisBorderShow" : false ,
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"insertNulls" : false ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "line"
}
} ,
"decimals" : 0 ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green" ,
"value" : null
} ,
{
"color" : "red" ,
"value" : 0.1
}
]
} ,
"unit" : "s"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
"y" : 59
} ,
"id" : 149 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)" ,
"format" : "time_series" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Go scheduling latency" ,
"type" : "timeseries"
} ,
2023-08-29 11:06:55 +02:00
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 16:21:21 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 16:21:21 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"decimals" : 0 ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green" ,
"value" : null
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "short"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 67
2023-08-29 11:06:55 +02:00
} ,
"id" : 41 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(max_over_time(process_num_threads{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)" ,
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Threads ($instance)" ,
"type" : "timeseries"
}
] ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"refId" : "A"
}
] ,
"title" : "Resource usage" ,
"type" : "row"
} ,
{
"collapsed" : true ,
"datasource" : {
"type" : "datasource" ,
"uid" : "grafana"
} ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 35
} ,
"id" : 94 ,
"panels" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-06-21 11:42:38 +02:00
"description" : "Shows top 10 jobs by the number of new series registered by vmagent over the 5min range. These jobs generate the most of the churn rate." ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-10-02 21:32:11 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "short"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 20
2023-08-29 11:06:55 +02:00
} ,
"id" : 92 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
2024-06-21 11:42:38 +02:00
"expr" : "topk_max(10, sum(sum_over_time(scrape_series_added[5m])) by (job)) > 0" ,
2023-08-29 11:06:55 +02:00
"interval" : "" ,
"legendFormat" : "{{ job }}" ,
"range" : true ,
"refId" : "A"
}
] ,
2024-06-21 11:42:38 +02:00
"title" : "Top 10 jobs by unique samples" ,
2023-08-29 11:06:55 +02:00
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-06-21 11:42:38 +02:00
"description" : "Shows top 10 instances by the number of new series registered by vmagent over the 5min range. These instances generate the most of the churn rate." ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-10-02 21:32:11 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "short"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 20
2023-08-29 11:06:55 +02:00
} ,
"id" : 95 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "topk(10, sum(sum_over_time(scrape_series_added[5m])) by (instance)) > 0" ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
2024-06-21 11:42:38 +02:00
"title" : "Top 10 instances by unique samples" ,
2023-08-29 11:06:55 +02:00
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-10-17 12:12:47 +02:00
"description" : "Shows write saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vmagent or improve the disk throughput." ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"axisSoftMax" : 2 ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "line"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-10-02 21:32:11 +02:00
"color" : "transparent"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 0.9
}
]
} ,
"unit" : "s"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 28
2023-08-29 11:06:55 +02:00
} ,
"id" : 98 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "max(rate(vm_persistentqueue_write_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)" ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Persistent queue write saturation ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-10-17 12:12:47 +02:00
"description" : "Shows read saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with reading data from the disk. In this case, consider to decrease load on the vmagent or improve the disk throughput." ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"axisSoftMax" : 2 ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "line"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-10-02 21:32:11 +02:00
"color" : "transparent"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 0.9
}
]
} ,
"unit" : "s"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 28
2023-08-29 11:06:55 +02:00
} ,
"id" : 99 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "none"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "max(rate(vm_persistentqueue_read_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)" ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Persistent queue read saturation ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the rate of dropped data blocks in cases when remote storage replies with `400 Bad Request` and `409 Conflict` HTTP responses.\n\nSee https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-10-02 21:32:11 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "short"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 36
2023-08-29 11:06:55 +02:00
} ,
"id" : 79 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "none"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(rate(vmagent_remotewrite_packets_dropped_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) by(job, url) > 0" ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Data blocks dropped ($instance) to ($url)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-03-04 15:43:31 +01:00
"description" : "Shows the rate of dropped [samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) due to relabeling. \nMetric tracks drops for `-remoteWrite.relabelConfig` configuration only." ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-10-02 21:32:11 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "short"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 36
2023-08-29 11:06:55 +02:00
} ,
"id" : 18 ,
"links" : [
{
"targetBlank" : true ,
"title" : "Relabeling" ,
2024-04-18 01:36:13 +02:00
"url" : "https://docs.victoriametrics.com/vmagent/#relabeling"
2023-08-29 11:06:55 +02:00
}
] ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "none"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vmagent_remotewrite_relabel_metrics_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, url) > 0" ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "B"
}
] ,
"title" : "Rows dropped by relabeling ($instance) to ($url)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the rate of parsed datapoints from write or scrape requests." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-10-02 21:32:11 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "short"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 44
2023-08-29 11:06:55 +02:00
} ,
"id" : 127 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(rate(vm_protoparser_rows_read_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0" ,
"interval" : "" ,
"legendFormat" : "{{ type }} ({{job}})" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Datapoints rate ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Tracks the rate of dropped invalid rows because of errors while unmarshaling write requests. The exact errors messages will be printed in logs." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-10-02 21:32:11 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "short"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 44
2023-08-29 11:06:55 +02:00
} ,
"id" : 50 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(rate(vm_rows_invalid_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0" ,
"interval" : "" ,
"legendFormat" : "{{type}} ({{job}})" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Invalid datapoints rate ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "thresholds"
} ,
"custom" : {
"align" : "auto" ,
2024-02-02 15:41:08 +01:00
"cellOptions" : {
"type" : "auto"
} ,
2023-08-29 11:06:55 +02:00
"inspect" : false
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2023-10-02 21:32:11 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
}
} ,
"overrides" : [
{
"matcher" : {
"id" : "byName" ,
"options" : "Value"
} ,
"properties" : [
{
"id" : "custom.hidden" ,
"value" : true
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "Time"
} ,
"properties" : [
{
"id" : "custom.hidden" ,
"value" : true
}
]
}
]
} ,
"gridPos" : {
"h" : 7 ,
"w" : 24 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 52
2023-08-29 11:06:55 +02:00
} ,
"id" : 129 ,
"options" : {
"footer" : {
"fields" : "" ,
"reducer" : [
"sum"
] ,
"show" : false
} ,
"showHeader" : true ,
"sortBy" : [
{
"desc" : true ,
"displayName" : "job"
}
]
} ,
2023-10-02 21:32:11 +02:00
"pluginVersion" : "9.2.6" ,
2023-08-29 11:06:55 +02:00
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)" ,
"format" : "table" ,
"instant" : true ,
"legendFormat" : "__auto" ,
"range" : false ,
"refId" : "A"
}
] ,
"title" : "Non-default flags" ,
"transformations" : [
{
"id" : "groupBy" ,
"options" : {
"fields" : {
"instance" : {
"aggregations" : [ ]
} ,
"job" : {
"aggregations" : [ ] ,
"operation" : "groupby"
} ,
"name" : {
"aggregations" : [ ] ,
"operation" : "groupby"
} ,
"value" : {
"aggregations" : [ ] ,
"operation" : "groupby"
}
}
}
}
] ,
"type" : "table"
}
] ,
"targets" : [
{
"datasource" : {
"type" : "datasource" ,
"uid" : "grafana"
} ,
"refId" : "A"
}
] ,
"title" : "Troubleshooting" ,
"type" : "row"
} ,
{
"collapsed" : true ,
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 36
} ,
"id" : 28 ,
"panels" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:48:26 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:48:26 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "short"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 7 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 45
2023-08-29 11:06:55 +02:00
} ,
"id" : 48 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"up\"}) by(job, type) > 0" ,
"format" : "time_series" ,
"interval" : "" ,
"legendFormat" : "{{type}} ({{job}})" ,
"range" : true ,
"refId" : "A"
}
] ,
2023-12-06 08:44:39 +01:00
"title" : "Scrape targets UP(By Type)" ,
2023-08-29 11:06:55 +02:00
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:48:26 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:48:26 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "short"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 7 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 45
2023-08-29 11:06:55 +02:00
} ,
"id" : 76 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(vm_promscrape_targets{job=~\"$job\", instance=~\"$instance\", status=\"down\"}) by(job, type) > 0" ,
"format" : "time_series" ,
"interval" : "" ,
"legendFormat" : "{{type}} ({{job}})" ,
"range" : true ,
"refId" : "A"
}
] ,
2023-12-06 08:44:39 +01:00
"title" : "Scrape targets DOWN(By Type)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:48:26 +01:00
"axisBorderShow" : false ,
2023-12-06 08:44:39 +01:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:48:26 +01:00
"insertNulls" : false ,
2023-12-06 08:44:39 +01:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green"
2023-12-06 08:44:39 +01:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "short"
2023-12-06 08:44:39 +01:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 7 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 52
2023-12-06 08:44:39 +01:00
} ,
"id" : 132 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(vm_promscrape_scrape_pool_targets{job=~\"$job\", instance=~\"$instance\", status=\"up\"}) by(job, scrape_job) > 0" ,
"format" : "time_series" ,
"interval" : "" ,
"legendFormat" : "{{job}}: {{scrape_job}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Scrape targets UP(By Job)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:48:26 +01:00
"axisBorderShow" : false ,
2023-12-06 08:44:39 +01:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:48:26 +01:00
"insertNulls" : false ,
2023-12-06 08:44:39 +01:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green"
2023-12-06 08:44:39 +01:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "short"
2023-12-06 08:44:39 +01:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 7 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 52
2023-12-06 08:44:39 +01:00
} ,
"id" : 133 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(vm_promscrape_scrape_pool_targets{job=~\"$job\", instance=~\"$instance\", status=\"down\"}) by(job, scrape_job) > 0" ,
"format" : "time_series" ,
"interval" : "" ,
"legendFormat" : "{{job}}: {{scrape_job}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Scrape targets DOWN(By Job)" ,
2023-08-29 11:06:55 +02:00
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the number of scrapes per second." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:48:26 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:48:26 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "short"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 59
2023-08-29 11:06:55 +02:00
} ,
"id" : 20 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(rate(vm_promscrape_scrapes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)" ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Scrape rate ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the number of datapoints scraped per second." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:48:26 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:48:26 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "short"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 59
2023-08-29 11:06:55 +02:00
} ,
"id" : 126 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(rate(vm_promscrape_scraped_samples_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)" ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Scraped datapoints rate ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:48:26 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:48:26 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "bytes"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 67
2023-08-29 11:06:55 +02:00
} ,
"id" : 46 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_response_size_bytes_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)" ,
"format" : "time_series" ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Scrape response size 0.99 quantile ($instance)" ,
"type" : "timeseries"
} ,
2024-07-30 14:57:17 +02:00
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisBorderShow" : false ,
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"insertNulls" : false ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green"
2024-07-30 14:57:17 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "s"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 67
2024-07-30 14:57:17 +02:00
} ,
"id" : 148 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)" ,
"format" : "time_series" ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Scrape duration 0.99 quantile ($instance)" ,
"type" : "timeseries"
} ,
2023-08-29 11:06:55 +02:00
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-02-02 15:48:26 +01:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-02-02 15:48:26 +01:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-09-23 16:54:42 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "short"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 75
2023-08-29 11:06:55 +02:00
} ,
"id" : 31 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vm_promscrape_scrapes_failed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0" ,
"interval" : "" ,
"legendFormat" : "scrapes failed ({{job}})" ,
"range" : true ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vm_promscrape_scrapes_timed_out_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0" ,
"interval" : "" ,
"legendFormat" : "timeouts ({{job}})" ,
"range" : true ,
"refId" : "B"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vm_promscrape_scrapes_gunzip_failed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0" ,
"interval" : "" ,
"legendFormat" : "gunzip fails ({{job}})" ,
"range" : true ,
"refId" : "C"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vm_promscrape_dial_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0" ,
"interval" : "" ,
"legendFormat" : "dial fails ({{job}})" ,
"range" : true ,
"refId" : "D"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vm_promscrape_max_scrape_size_exceeded_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "max scrape size exceeded ({{job}})" ,
"range" : true ,
"refId" : "E"
}
] ,
"title" : "Scrape fails ($instance)" ,
"type" : "timeseries"
}
] ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"refId" : "A"
}
] ,
"title" : "Scraping" ,
"type" : "row"
} ,
{
"collapsed" : true ,
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 37
} ,
"id" : 71 ,
"panels" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the rate of write requests served by ingestserver (UDP, TCP connections) and HTTP server." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-02-02 15:41:08 +01:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "short"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 22
2023-08-29 11:06:55 +02:00
} ,
"id" : 73 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(rate(vm_ingestserver_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type, net) > 0" ,
"interval" : "" ,
"legendFormat" : "{{net}}: {{ type }} ({{job}})" ,
"range" : true ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(rate(vmagent_http_requests_total{job=~\"$job\", instance=~\"$instance\", protocol!=\"\"}[$__rate_interval])) by(job, protocol) > 0" ,
"interval" : "" ,
"legendFormat" : "{{ protocol }}: http ({{job}})" ,
"range" : true ,
"refId" : "B"
}
] ,
"title" : "Requests rate ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the rate of rows ingested in vmagent via push protocols." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "short"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 14
2023-08-29 11:06:55 +02:00
} ,
"id" : 131 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(rate(vmagent_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) > 0" ,
"interval" : "" ,
"legendFormat" : "{{ type }} ({{job}})" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Rows rate ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2023-10-02 21:32:11 +02:00
"description" : "Shows how many concurrent inserts are taking place.\n\nIf the number of concurrent inserts hitting the `limit` or is close to the `limit` constantly - it might be a sign of a resource shortage.\n\n If vmagent's CPU usage and remote write connection saturation are at normal level, it might be that `-maxConcurrentInserts` cmd-line flag need to be increased." ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "short"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
2023-09-26 14:26:40 +02:00
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 22
2023-10-02 21:32:11 +02:00
} ,
"id" : 130 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "max_over_time(vm_concurrent_insert_current{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])" ,
"interval" : "" ,
"legendFormat" : "{{instance}} ({{job}})" ,
"range" : true ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "min(vm_concurrent_insert_capacity{job=~\"$job\", instance=~\"$instance\"}) by(job)" ,
"interval" : "" ,
"legendFormat" : "limit ({{job}})" ,
"range" : true ,
"refId" : "B"
}
] ,
"title" : "Concurrent inserts ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the rate of write errors in ingestserver (UDP, TCP connections) and HTTP server." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-10-02 21:32:11 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-10-02 21:32:11 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "green"
2023-10-02 21:32:11 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "short"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 22
2023-08-29 11:06:55 +02:00
} ,
"id" : 77 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"exemplar" : true ,
dashboards: remove `path!="/favicon.ico"` filter from `requests rate` graphs
The `path!="/favicon.ico"` filter has little sense, since there are many other special paths,
which may be filtered out - /metrics, /flags, /health, /ping, /robots.txt, /-/healthy, /-/ready, /reload, etc.
See /lib/httpserver/httpserver.go for more details.
It will be hard or impossible to maintain filters for all these paths, so it is better to drop this filter
in order to simplify queries and improve the consistency of these queries.
2023-11-16 19:26:03 +01:00
"expr" : "sum(rate(vm_ingestserver_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(type, net) > 0" ,
2023-08-29 11:06:55 +02:00
"interval" : "" ,
"legendFormat" : "{{ type }} ({{net}})" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"exemplar" : true ,
"expr" : "sum(rate(vmagent_http_request_errors_total{job=~\"$job\", instance=~\"$instance\", protocol!=\"\"}[$__rate_interval])) by(protocol) > 0" ,
"interval" : "" ,
"legendFormat" : "{{ protocol }} (http)" ,
"refId" : "B"
}
] ,
"title" : "Error rate ($instance)" ,
"type" : "timeseries"
}
] ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"refId" : "A"
}
] ,
"title" : "Ingestion" ,
"type" : "row"
} ,
{
"collapsed" : true ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 38
} ,
2024-06-10 11:49:00 +02:00
"id" : 136 ,
2023-08-29 11:06:55 +02:00
"panels" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-07-01 14:56:17 +02:00
"description" : "Shows the number of matched samples by the aggregation rule. \n\nThe more samples is matched, the more work this aggregation rule does. The matching rule is specified via `match` param.\n\nSee more details in [stream aggregation config](https://docs.victoriametrics.com/stream-aggregation/#stream-aggregation-config). " ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
2024-06-10 11:49:00 +02:00
"axisSoftMin" : 0 ,
2023-08-29 11:06:55 +02:00
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
2024-06-10 11:49:00 +02:00
"showPoints" : "auto" ,
2023-08-29 11:06:55 +02:00
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
2024-07-01 14:56:17 +02:00
}
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 15
2023-08-29 11:06:55 +02:00
} ,
2024-07-01 14:56:17 +02:00
"id" : 146 ,
2023-08-29 11:06:55 +02:00
"options" : {
"legend" : {
2024-07-01 14:56:17 +02:00
"calcs" : [
"min" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
2023-08-29 11:06:55 +02:00
"placement" : "bottom" ,
2024-07-01 14:56:17 +02:00
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
2023-08-29 11:06:55 +02:00
} ,
"tooltip" : {
2024-06-10 11:49:00 +02:00
"mode" : "single" ,
"sort" : "none"
2023-08-29 11:06:55 +02:00
}
} ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
2024-07-05 15:10:37 +02:00
"expr" : "sum(rate(vm_streamaggr_matched_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0" ,
2024-06-10 11:49:00 +02:00
"instant" : false ,
2024-09-03 14:23:35 +02:00
"legendFormat" : "__auto" ,
2023-08-29 11:06:55 +02:00
"range" : true ,
"refId" : "A"
}
] ,
2024-07-01 14:56:17 +02:00
"title" : "Matched samples ($instance)" ,
2023-08-29 11:06:55 +02:00
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-07-01 14:56:17 +02:00
"description" : "The rate of ignored samples during aggregation. \nStream aggregation will drop samples with NaN values, or samples with too old timestamps. See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples " ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
2024-06-10 11:49:00 +02:00
"axisSoftMin" : 0 ,
2023-08-29 11:06:55 +02:00
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
2024-06-10 11:49:00 +02:00
"showPoints" : "auto" ,
2023-08-29 11:06:55 +02:00
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
2024-06-10 11:49:00 +02:00
}
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 15
2023-08-29 11:06:55 +02:00
} ,
2024-06-10 11:49:00 +02:00
"id" : 143 ,
2023-08-29 11:06:55 +02:00
"options" : {
"legend" : {
2024-06-10 11:49:00 +02:00
"calcs" : [ ] ,
"displayMode" : "list" ,
2023-08-29 11:06:55 +02:00
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
2024-06-10 11:49:00 +02:00
"mode" : "single" ,
"sort" : "none"
2023-08-29 11:06:55 +02:00
}
} ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
2024-07-05 15:10:37 +02:00
"expr" : "sum(rate(vm_streamaggr_ignored_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0) without (instance, pod)" ,
2024-07-01 14:56:17 +02:00
"instant" : false ,
2024-09-03 14:23:35 +02:00
"legendFormat" : "__auto" ,
2024-07-01 14:56:17 +02:00
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Ignored samples ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the number of produced samples by the aggregation rule. \n\nNumber of produced samples depend on params like `by`, `without`, `interval`, etc.\n\nSee more details in [stream aggregation config](https://docs.victoriametrics.com/stream-aggregation/#stream-aggregation-config). " ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisBorderShow" : false ,
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"axisSoftMin" : 0 ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"insertNulls" : false ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "auto" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "green"
2024-07-01 14:56:17 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
}
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 23
2024-07-01 14:56:17 +02:00
} ,
"id" : 147 ,
"options" : {
"legend" : {
"calcs" : [
"min" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
app/vmagent/remotewrite: follow-up for f153f54d11250da050aa93bc4fa9b7ba9e144691
- Move the remaining code responsible for stream aggregation initialization from remotewrite.go to streamaggr.go .
This improves code maintainability a bit.
- Properly shut down streamaggr.Aggregators initialized inside remotewrite.CheckStreamAggrConfigs().
This prevents from potential resource leaks.
- Use separate functions for initializing and reloading of global stream aggregation and per-remoteWrite.url stream aggregation.
This makes the code easier to read and maintain. This also fixes INFO and ERROR logs emitted by these functions.
- Add an ability to specify `name` option in every stream aggregation config. This option is used as `name` label
in metrics exposed by stream aggregation at /metrics page. This simplifies investigation of the exposed metrics.
- Add `path` label additionally to `name`, `url` and `position` labels at metrics exposed by streaming aggregation.
This label should simplify investigation of the exposed metrics.
- Remove `match` and `group` labels from metrics exposed by streaming aggregation, since they have little practical applicability:
it is hard to use these labels in query filters and aggregation functions.
- Rename the metric `vm_streamaggr_flushed_samples_total` to less misleading `vm_streamaggr_output_samples_total` .
This metric shows the number of samples generated by the corresponding streaming aggregation rule.
This metric has been added in the commit 861852f2624895e01f93ce196607c72616ce2a94 .
See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6462
- Remove the metric `vm_streamaggr_stale_samples_total`, since it is unclear how it can be used in practice.
This metric has been added in the commit 861852f2624895e01f93ce196607c72616ce2a94 .
See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6462
- Remove Alias and aggrID fields from streamaggr.Options struct, since these fields aren't related to optional params,
which could modify the behaviour of the constructed streaming aggregator.
Convert the Alias field to regular argument passed to LoadFromFile() function, since this argument is mandatory.
- Pass Options arg to LoadFromFile() function by reference, since this structure is quite big.
This also allows passing nil instead of Options when default options are enough.
- Add `name`, `path`, `url` and `position` labels to `vm_streamaggr_dedup_state_size_bytes` and `vm_streamaggr_dedup_state_items_count` metrics,
so they have consistent set of labels comparing to the rest of streaming aggregation metrics.
- Convert aggregator.aggrStates field type from `map[string]aggrState` to `[]aggrOutput`, where `aggrOutput` contains the corresponding
`aggrState` plus all the related metrics (currently only `vm_streamaggr_output_samples_total` metric is exposed with the corresponding
`output` label per each configured output function). This simplifies and speeds up the code responsible for updating per-output
metrics. This is a follow-up for the commit 2eb1bc4f814037ae87ac6556011ae0d3caee6bc8 .
See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6604
- Added missing urls to docs ( https://docs.victoriametrics.com/stream-aggregation/ ) in error messages. These urls help users
figuring out why VictoriaMetrics or vmagent generates the corresponding error messages. The urls were removed for unknown reason
in the commit 2eb1bc4f814037ae87ac6556011ae0d3caee6bc8 .
- Fix incorrect update for `vm_streamaggr_output_samples_total` metric in flushCtx.appendSeriesWithExtraLabel() function.
While at it, reduce memory usage by limiting the maximum number of samples per flush to 10K.
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5467
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6268
2024-07-15 18:01:37 +02:00
"expr" : "sum(rate(vm_streamaggr_output_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0" ,
2024-06-10 11:49:00 +02:00
"instant" : false ,
2024-09-03 14:23:35 +02:00
"legendFormat" : "__auto" ,
2023-08-29 11:06:55 +02:00
"range" : true ,
"refId" : "A"
}
] ,
2024-07-01 14:56:17 +02:00
"title" : "Produced samples ($instance)" ,
2023-08-29 11:06:55 +02:00
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-06-10 11:49:00 +02:00
"description" : "Shows events when deduplication or aggregation couldn't be finished in the configured interval. Such events may result into bad accuracy of produced data.\n\nPossible solutions:\n* increase interval; \n* use match filter matching smaller number of series;\n* reduce samples ingestion rate to stream aggregation" ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
2024-06-10 11:49:00 +02:00
"axisSoftMin" : 0 ,
2023-08-29 11:06:55 +02:00
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
2024-06-10 11:49:00 +02:00
"showPoints" : "auto" ,
2023-08-29 11:06:55 +02:00
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
2024-06-10 11:49:00 +02:00
"min" : -5 ,
2023-08-29 11:06:55 +02:00
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
2024-06-10 11:49:00 +02:00
}
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
2024-07-01 14:56:17 +02:00
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 23
2023-08-29 11:06:55 +02:00
} ,
2024-06-10 11:49:00 +02:00
"id" : 139 ,
2023-08-29 11:06:55 +02:00
"options" : {
"legend" : {
2024-06-10 11:49:00 +02:00
"calcs" : [ ] ,
"displayMode" : "list" ,
2023-08-29 11:06:55 +02:00
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
2024-06-10 11:49:00 +02:00
"mode" : "single" ,
2023-08-29 11:06:55 +02:00
"sort" : "none"
}
} ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
2024-07-05 15:10:37 +02:00
"expr" : "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0" ,
2024-06-10 11:49:00 +02:00
"instant" : false ,
2024-09-03 14:23:35 +02:00
"legendFormat" : "aggregation: {{url}} ({{job}}): {{path}}:{{position}}" ,
2023-08-29 11:06:55 +02:00
"range" : true ,
"refId" : "A"
2024-06-10 11:49:00 +02:00
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
2024-07-05 15:10:37 +02:00
"expr" : "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0" ,
2024-06-10 11:49:00 +02:00
"hide" : false ,
"instant" : false ,
2024-09-03 14:23:35 +02:00
"legendFormat" : "deduplication: {{url}} ({{job}}): {{path}}:{{position}}" ,
2024-06-10 11:49:00 +02:00
"range" : true ,
"refId" : "B"
2023-08-29 11:06:55 +02:00
}
] ,
2024-07-01 14:56:17 +02:00
"title" : "Flush timeouts ($instance)" ,
2023-08-29 11:06:55 +02:00
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-07-01 14:56:17 +02:00
"description" : "Shows the max lag between samples timestamps within one batch passed to the aggregation input.\n\nLower is better.\n\nToo high lag or lag exceeding the interval might be a sign that data was delayed before aggregation or resource insufficiency on aggregator. Samples with high lag may affect accuracy of aggregation.\n\nSee https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples" ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
2024-06-10 11:49:00 +02:00
"axisSoftMin" : 0 ,
2023-08-29 11:06:55 +02:00
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
2024-06-10 11:49:00 +02:00
"showPoints" : "auto" ,
2023-08-29 11:06:55 +02:00
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
2024-06-10 11:49:00 +02:00
"unit" : "s"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
2024-07-01 14:56:17 +02:00
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 31
2023-08-29 11:06:55 +02:00
} ,
2024-06-10 11:49:00 +02:00
"id" : 142 ,
2023-08-29 11:06:55 +02:00
"options" : {
"legend" : {
2024-06-10 11:49:00 +02:00
"calcs" : [ ] ,
"displayMode" : "list" ,
2023-08-29 11:06:55 +02:00
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
2024-06-10 11:49:00 +02:00
"mode" : "single" ,
"sort" : "none"
2023-08-29 11:06:55 +02:00
}
} ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
2024-07-05 15:10:37 +02:00
"expr" : "histogram_quantile(0.99, sum(rate(vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod))" ,
2024-06-10 11:49:00 +02:00
"instant" : false ,
2024-09-03 14:23:35 +02:00
"legendFormat" : "__auto" ,
2023-08-29 11:06:55 +02:00
"range" : true ,
"refId" : "A"
}
] ,
2024-07-01 14:56:17 +02:00
"title" : "Samples lag 0.99 quantile ($instance)" ,
2023-08-29 11:06:55 +02:00
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
2024-07-01 14:56:17 +02:00
"description" : "The 99th percentile of avg flush duration for the aggregated data. \n\nSmaller is better.\n\nAggregation can produce incorrect results ff flush duration exceeds configured deduplication interval. See \"Flush Timeouts\" panel." ,
2023-08-29 11:06:55 +02:00
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
2024-06-10 11:49:00 +02:00
"axisSoftMin" : 0 ,
2023-08-29 11:06:55 +02:00
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
2024-06-10 11:49:00 +02:00
"showPoints" : "auto" ,
2023-08-29 11:06:55 +02:00
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
2024-06-10 11:49:00 +02:00
"mode" : "off"
2023-08-29 11:06:55 +02:00
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "green"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
2024-06-10 11:49:00 +02:00
"value" : 80
2023-08-29 11:06:55 +02:00
}
]
} ,
2024-07-01 14:56:17 +02:00
"unit" : "s"
2023-08-29 11:06:55 +02:00
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
2024-07-01 14:56:17 +02:00
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 31
2023-08-29 11:06:55 +02:00
} ,
2024-07-01 14:56:17 +02:00
"id" : 137 ,
2023-08-29 11:06:55 +02:00
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "list" ,
"placement" : "bottom" ,
2024-06-10 11:49:00 +02:00
"showLegend" : true
2023-08-29 11:06:55 +02:00
} ,
"tooltip" : {
2024-06-10 11:49:00 +02:00
"mode" : "single" ,
2023-08-29 11:06:55 +02:00
"sort" : "none"
}
} ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
2024-07-01 14:56:17 +02:00
"expr" : "histogram_quantile(0.99, rate(vm_streamaggr_dedup_flush_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]))" ,
2024-06-10 11:49:00 +02:00
"instant" : false ,
2024-09-03 14:23:35 +02:00
"legendFormat" : "__auto" ,
2023-08-29 11:06:55 +02:00
"range" : true ,
"refId" : "A"
}
] ,
2024-07-01 14:56:17 +02:00
"title" : "Dedup flush duration 0.99 quantile ($instance)" ,
2023-08-29 11:06:55 +02:00
"type" : "timeseries"
} ,
2024-07-01 14:56:17 +02:00
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the size of Label Compressor in number of entries.\n\nLabels compressor encodes label-value pairs during aggregation to optimise memory usage. It is expected for its size to grow with time and to reset on vmagent restarts.\n\nRapid spikes in Label compressor size might be a sign of significant changes in labels of received samples." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisBorderShow" : false ,
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"axisSoftMin" : 0 ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"insertNulls" : false ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "auto" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "green"
2024-07-01 14:56:17 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
}
} ,
"overrides" : [
{
"matcher" : {
"id" : "byRegexp" ,
"options" : "/bytes.*/"
} ,
"properties" : [
{
"id" : "custom.axisPlacement" ,
"value" : "right"
} ,
{
"id" : "unit" ,
"value" : "bytes"
}
]
}
]
} ,
2023-08-29 11:06:55 +02:00
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 39
2023-08-29 11:06:55 +02:00
} ,
2024-06-10 11:49:00 +02:00
"id" : 141 ,
2023-08-29 11:06:55 +02:00
"options" : {
"legend" : {
2024-06-10 11:49:00 +02:00
"calcs" : [ ] ,
"displayMode" : "list" ,
2023-08-29 11:06:55 +02:00
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
2024-06-10 11:49:00 +02:00
"mode" : "single" ,
"sort" : "none"
}
} ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
2024-07-01 14:56:17 +02:00
"expr" : "max(vm_streamaggr_labels_compressor_items_count{job=~\"$job\",instance=~\"$instance\"}) by(job, instance)" ,
"hide" : false ,
2024-06-10 11:49:00 +02:00
"instant" : false ,
2024-07-01 14:56:17 +02:00
"legendFormat" : "items: {{instance}} ({{job}})" ,
2024-06-10 11:49:00 +02:00
"range" : true ,
"refId" : "A"
2024-07-01 14:56:17 +02:00
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "max(vm_streamaggr_labels_compressor_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)" ,
"hide" : false ,
"instant" : false ,
"legendFormat" : "bytes: {{instance}} ({{job}})" ,
"range" : true ,
"refId" : "B"
2024-06-10 11:49:00 +02:00
}
] ,
2024-07-01 14:56:17 +02:00
"title" : "Labels compressor ($instance)" ,
2024-06-10 11:49:00 +02:00
"type" : "timeseries"
}
] ,
"title" : "Streaming aggregation" ,
"type" : "row"
} ,
{
"collapsed" : true ,
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 39
} ,
"id" : 58 ,
"panels" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the rate of requests to configured remote write endpoints by url and status code.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\n" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisBorderShow" : false ,
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"insertNulls" : false ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"decimals" : 2 ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "green"
2024-06-10 11:49:00 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "short"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 16
2024-06-10 11:49:00 +02:00
} ,
"id" : 60 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(rate(vmagent_remotewrite_requests_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) by(job, url, status_code) > 0" ,
"interval" : "" ,
"legendFormat" : "" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Requests rate ($instance) to ($url)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the global rate for number of written bytes via remote write connections." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisBorderShow" : false ,
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"insertNulls" : false ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "green"
2024-06-10 11:49:00 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "decbytes"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 16
2024-06-10 11:49:00 +02:00
} ,
"id" : 66 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(rate(vmagent_remotewrite_conn_bytes_written_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0" ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Bytes write rate ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows requests retry rate by url. Number of retries is unlimited but protected with delays up to 1m between attempts.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\n" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisBorderShow" : false ,
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"insertNulls" : false ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "green"
2024-06-10 11:49:00 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "short"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 24
2024-06-10 11:49:00 +02:00
} ,
"id" : 61 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "none"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(rate(vmagent_remotewrite_retries_count_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) by(url) > 0" ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Retry rate ($instance) to ($url)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows current number of established connections to remote write endpoints.\n\n" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
"axisBorderShow" : false ,
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"insertNulls" : false ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "green"
2024-06-10 11:49:00 +02:00
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "short"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 24
2024-06-10 11:49:00 +02:00
} ,
"id" : 65 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(max_over_time(vmagent_remotewrite_conns{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)" ,
"interval" : "" ,
"legendFormat" : "__auto" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Connections ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the current limit usage of unique series over an hourly period. Vmagent will start to drop series once the limit is reached.\n\nPlease note, panel will be blank if `remoteWrite.maxHourlySeries` is not set." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-07-01 14:56:17 +02:00
"axisBorderShow" : false ,
2024-06-10 11:49:00 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-07-01 14:56:17 +02:00
"insertNulls" : false ,
2024-06-10 11:49:00 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "line"
}
} ,
"mappings" : [ ] ,
"max" : 100 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "transparent"
2024-06-10 11:49:00 +02:00
} ,
{
"color" : "red" ,
"value" : 0.9
}
]
} ,
"unit" : "percent"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 32
2024-06-10 11:49:00 +02:00
} ,
"id" : 88 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "list" ,
"placement" : "bottom" ,
"showLegend" : false
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "none"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "max(\n vmagent_hourly_series_limit_current_series{job=~\"$job\", instance=~\"$instance\"} \n / \n vmagent_hourly_series_limit_max_series{job=~\"$job\", instance=~\"$instance\"}\n ) by(job) * 100" ,
"interval" : "" ,
"legendFormat" : "{{job}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Hourly series limit" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows saturation of every connection to remote storage. If the threshold of 90% is reached, then the connection is saturated (busy or slow) by more than 90%, so vmagent won't be able to keep up and can start buffering data. \n\nThis usually means that `-remoteWrite.queues` command-line flag must be increased in order to increase the number of connections per each remote storage.\n" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-07-01 14:56:17 +02:00
"axisBorderShow" : false ,
2024-06-10 11:49:00 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "line"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "transparent"
} ,
{
"color" : "red" ,
"value" : 0.9
}
]
} ,
"unit" : "percentunit"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 32
2024-06-10 11:49:00 +02:00
} ,
"id" : 84 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
2023-08-29 11:06:55 +02:00
"sort" : "desc"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "max(\n rate(vmagent_remotewrite_send_duration_seconds_total{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])\n /\n vmagent_remotewrite_queues{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}\n) by(job, url)" ,
"interval" : "" ,
"legendFormat" : "" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Remote write connection saturation ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the current limit usage of unique series over a daily period. Vmagent will start to drop series once the limit is reached.\n\nPlease note, panel will be blank if `remoteWrite.maxDailySeries` is not set." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-07-01 14:56:17 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-07-01 14:56:17 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "line"
}
} ,
"mappings" : [ ] ,
"max" : 100 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
2024-07-30 14:57:17 +02:00
"color" : "transparent"
2023-08-29 11:06:55 +02:00
} ,
{
"color" : "red" ,
"value" : 0.9
}
]
} ,
"unit" : "percentunit"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 40
2023-08-29 11:06:55 +02:00
} ,
"id" : 90 ,
"options" : {
"legend" : {
"calcs" : [ ] ,
"displayMode" : "list" ,
"placement" : "bottom" ,
"showLegend" : false
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "none"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "max(\n vmagent_daily_series_limit_current_series{job=~\"$job\",instance=~\"$instance\"} \n / \n vmagent_daily_series_limit_max_series{job=~\"$job\",instance=~\"$instance\"}\n) by(job)" ,
"interval" : "" ,
"legendFormat" : "{{job}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Daily series limit" ,
"type" : "timeseries"
}
] ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"refId" : "A"
}
] ,
"title" : "Remote write" ,
"type" : "row"
} ,
{
"collapsed" : true ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
2024-06-10 11:49:00 +02:00
"y" : 40
2023-08-29 11:06:55 +02:00
} ,
"id" : 113 ,
"panels" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"gridPos" : {
"h" : 2 ,
"w" : 24 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 25
2023-08-29 11:06:55 +02:00
} ,
"id" : 115 ,
"options" : {
"code" : {
"language" : "plaintext" ,
"showLineNumbers" : false ,
"showMiniMap" : false
} ,
"content" : "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance." ,
"mode" : "markdown"
} ,
2024-06-10 11:49:00 +02:00
"pluginVersion" : "10.4.2" ,
2023-08-29 11:06:55 +02:00
"transparent" : true ,
"type" : "text"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "" ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green"
}
]
} ,
"unit" : "short"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 27
2023-08-29 11:06:55 +02:00
} ,
"id" : 119 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.1.0" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : false ,
"expr" : "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)" ,
"format" : "time_series" ,
"interval" : "" ,
"intervalFactor" : 1 ,
"legendFormat" : "{{instance}} ({{job}})" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "CPU usage ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green"
}
]
} ,
"unit" : "decbytes"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 27
2023-08-29 11:06:55 +02:00
} ,
"id" : 117 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "desc"
}
} ,
"pluginVersion" : "9.1.0" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])" ,
"interval" : "" ,
"legendFormat" : "{{instance}} ({{job}})" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "RSS memory usage ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the persistent queue size of pending samples in bytes which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green"
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "bytes"
} ,
"overrides" : [ ]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 35
2023-08-29 11:06:55 +02:00
} ,
"id" : 125 ,
"links" : [
{
"title" : "Troubleshooting" ,
2024-04-18 01:36:13 +02:00
"url" : "https://docs.victoriametrics.com/vmagent/#troubleshooting"
2023-08-29 11:06:55 +02:00
}
] ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "none"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"exemplar" : true ,
"expr" : "sum(vmagent_remotewrite_pending_data_bytes{job=~\"$job\", instance=~\"$instance\", url=~\"$url\"}) by (instance, url)" ,
"interval" : "" ,
"legendFormat" : "{{instance}} => {{url}}" ,
"range" : true ,
"refId" : "A"
}
] ,
"title" : "Persistent queue size ($instance) to ($url)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows in/out samples rate including push and pull models. \n\nThe out-rate could be different to in-rate because of replication or additional timeseries added by vmagent for every scraped target." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green"
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "short"
} ,
"overrides" : [
{
"matcher" : {
"id" : "byRegexp" ,
"options" : "/out .*/"
} ,
"properties" : [
{
"id" : "custom.transform" ,
"value" : "negative-Y"
}
]
}
]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 12 ,
2024-09-23 16:54:42 +02:00
"y" : 35
2023-08-29 11:06:55 +02:00
} ,
"id" : 123 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "none"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vm_promscrape_scraped_samples_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)\n+ sum(rate(vmagent_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "in {{instance}} {{job}}" ,
"range" : true ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(vmagent_remotewrite_block_size_rows_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)" ,
"interval" : "" ,
"legendFormat" : "out {{instance}} {{job}}" ,
"range" : true ,
"refId" : "B"
}
] ,
"title" : "Samples rate ($instance)" ,
"type" : "timeseries"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"description" : "Shows the number of bytes read/write from the storage layer when vmagent has to buffer data on disk or read already buffered data." ,
"fieldConfig" : {
"defaults" : {
"color" : {
"mode" : "palette-classic"
} ,
"custom" : {
2024-06-10 11:49:00 +02:00
"axisBorderShow" : false ,
2023-08-29 11:06:55 +02:00
"axisCenteredZero" : false ,
"axisColorMode" : "text" ,
"axisLabel" : "" ,
"axisPlacement" : "auto" ,
"barAlignment" : 0 ,
"drawStyle" : "line" ,
"fillOpacity" : 0 ,
"gradientMode" : "none" ,
"hideFrom" : {
"legend" : false ,
"tooltip" : false ,
"viz" : false
} ,
2024-06-10 11:49:00 +02:00
"insertNulls" : false ,
2023-08-29 11:06:55 +02:00
"lineInterpolation" : "linear" ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"scaleDistribution" : {
"type" : "linear"
} ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
} ,
"thresholdsStyle" : {
"mode" : "off"
}
} ,
"links" : [ ] ,
"mappings" : [ ] ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [
{
"color" : "green"
} ,
{
"color" : "red" ,
"value" : 80
}
]
} ,
"unit" : "bytes"
} ,
"overrides" : [
{
"matcher" : {
"id" : "byName" ,
"options" : "read"
} ,
"properties" : [
{
"id" : "custom.transform" ,
"value" : "negative-Y"
}
]
}
]
} ,
"gridPos" : {
"h" : 8 ,
"w" : 12 ,
"x" : 0 ,
2024-09-23 16:54:42 +02:00
"y" : 43
2023-08-29 11:06:55 +02:00
} ,
"id" : 121 ,
"options" : {
"legend" : {
"calcs" : [
"mean" ,
"lastNotNull" ,
"max"
] ,
"displayMode" : "table" ,
"placement" : "bottom" ,
"showLegend" : true ,
"sortBy" : "Last *" ,
"sortDesc" : true
} ,
"tooltip" : {
"mode" : "multi" ,
"sort" : "none"
}
} ,
"pluginVersion" : "9.2.6" ,
"targets" : [
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)" ,
"format" : "time_series" ,
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 1 ,
"legendFormat" : "read {{instance}} {{job}}" ,
"range" : true ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"editorMode" : "code" ,
"expr" : "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job,instance)" ,
"format" : "time_series" ,
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 1 ,
"legendFormat" : "write {{instance}} {{job}}" ,
"range" : true ,
"refId" : "B"
}
] ,
"title" : "Disk writes/reads ($instance)" ,
"type" : "timeseries"
}
] ,
"title" : "Drilldown" ,
"type" : "row"
}
] ,
"refresh" : "" ,
2024-02-02 15:41:08 +01:00
"schemaVersion" : 39 ,
2023-08-29 11:06:55 +02:00
"tags" : [
"vmagent" ,
"victoriametrics"
] ,
"templating" : {
"list" : [
{
"current" : {
2024-02-02 15:48:26 +01:00
"selected" : false ,
2023-08-29 11:06:55 +02:00
"text" : "VictoriaMetrics" ,
2024-02-02 15:48:26 +01:00
"value" : "P4169E866C3094E38"
2023-08-29 11:06:55 +02:00
} ,
"hide" : 0 ,
"includeAll" : false ,
"multi" : false ,
"name" : "ds" ,
"options" : [ ] ,
"query" : "victoriametrics-datasource" ,
"queryValue" : "" ,
"refresh" : 1 ,
"regex" : "" ,
"skipUrlSync" : false ,
"type" : "datasource"
} ,
{
"allValue" : "" ,
"current" : { } ,
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"definition" : "label_values(vm_app_version{version=~\"^vmagent.*\"}, job)" ,
"hide" : 0 ,
"includeAll" : true ,
"multi" : true ,
"name" : "job" ,
"options" : [ ] ,
"query" : {
"query" : "label_values(vm_app_version{version=~\"^vmagent.*\"}, job)" ,
"refId" : "VictoriaMetrics-job-Variable-Query"
} ,
"refresh" : 1 ,
"regex" : "" ,
"skipUrlSync" : false ,
"sort" : 0 ,
"tagValuesQuery" : "" ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
} ,
{
"allValue" : ".*" ,
"current" : { } ,
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"definition" : "label_values(vm_app_version{job=~\"$job\"}, instance)" ,
"hide" : 0 ,
"includeAll" : true ,
"multi" : true ,
"name" : "instance" ,
"options" : [ ] ,
"query" : {
"query" : "label_values(vm_app_version{job=~\"$job\"}, instance)" ,
"refId" : "VictoriaMetrics-instance-Variable-Query"
} ,
"refresh" : 1 ,
"regex" : "" ,
"skipUrlSync" : false ,
"sort" : 0 ,
"tagValuesQuery" : "" ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
} ,
{
"allValue" : ".*" ,
"current" : { } ,
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"definition" : "label_values(vmagent_remotewrite_requests_total{job=~\"$job\", instance=~\"$instance\"}, url)" ,
"description" : "The remote write URLs" ,
"hide" : 0 ,
"includeAll" : true ,
"multi" : true ,
"name" : "url" ,
"options" : [ ] ,
"query" : {
"query" : "label_values(vmagent_remotewrite_requests_total{job=~\"$job\", instance=~\"$instance\"}, url)" ,
"refId" : "StandardVariableQuery"
} ,
"refresh" : 1 ,
"regex" : "" ,
"skipUrlSync" : false ,
"sort" : 0 ,
"type" : "query"
} ,
{
"datasource" : {
"type" : "victoriametrics-datasource" ,
"uid" : "$ds"
} ,
"filters" : [ ] ,
"hide" : 0 ,
"name" : "adhoc" ,
"skipUrlSync" : false ,
"type" : "adhoc"
}
]
} ,
"time" : {
"from" : "now-3h" ,
"to" : "now"
} ,
"timepicker" : {
"refresh_intervals" : [
"10s" ,
"30s" ,
"1m" ,
"5m" ,
"15m" ,
"30m" ,
"1h" ,
"2h" ,
"1d"
]
} ,
"timezone" : "" ,
"title" : "VictoriaMetrics - vmagent (VM)" ,
"uid" : "G7Z9GzMGz_vm" ,
"version" : 1 ,
"weekStart" : ""
2024-10-17 12:12:47 +02:00
}
